#!/usr/bin/env python """Module doctsring Routine Name: write_llps.py Desc: This routine is run from the command line with one or more filenames as arguments (it also accepts -h and --help), the files being UALF data from the LP2000 Lighning Processor. UALF is the Universal ASCII Lightning Format from Vaisala, Inc. It reads UALF data files and re-formats the date and time of the lightning records starting with the current date/time. All output is to stdout/stderr (but can be re-directed). It uses the actual time difference in the data to write out the re-formatted records. This is specific to the WR LP2000 system (aka LLPS or LLP). The initial --fast switch compresses time by a factor of 10000 (otherwise one could grow old waiting for the next output record). This code ignores the nanoseconds field in the LLPS data, thus it does not attempt to round up or down to the nearest second for any time calculations. All time calculations are in (epoch) seconds derived from UTC. Arguments: Name I/O Description ---- --- ----------- filename(s) In One or more LP2000 UALF data files in ascii text, version 0 -h|--help N/A Prints module docstring -v|--verbose N/A Prints extra debug values in the data -f|--fast N/A Speeds up processing by 10000 -p|--pipe N/A Switches to pipe output. You must specify the program to write to as an argument to --pipe, ie, write_llps.py -f --pipe="/bin/cat -" data.txt The program must be in the path, or the full path must be specified; spaces must be quoted. usage: write_llps.py [-hfvp|--help --fast --verbose --pipe] [...] Revision History: Date Name Description ---- ---- ----------- 07-17-03 S.L. Arnold Secondary Implementation with fileinput. 07-18-03 S.L. Arnold Removed fileinput and added --fast switch 07-19-03 S.L. Arnold Fixed CR problem and --verbose switch 07-21-03 S.L. Arnold Added piped output option with no buffering 03-31-04 S.L. Arnold Added regex compile as back-port fix for isdigit """ ##-------------------------------------- import re, string, time, sys from time import time, gmtime, strftime, strptime, mktime, sleep #from string import join def process(line, time_pre, fast, verbose): prefix = string.split(line,"\t",7)[0] ## 1st list element isdigit = re.compile(r"\d+$").match ## define isdigit for 1.5.2 if not isdigit(prefix): ## check for garbage or end of file if prefix == "\r\n" or prefix == "\n": if verbose: print "end of data reached for file: %s" % filename return() ## need to use a return() here else: return print "non-numeric data encountered" if verbose: print "unexpected value on the following line:" print line sys.exit(1) if not prefix: ## check for ULAF Version 0 raise 'UALFVersionError','Input data is not expected UALF version.' if verbose: print "Current pre-loop time: %s seconds" % time_pre time_dilation_factor = 1 if fast: time_dilation_factor = 10000 if not time_pre: ## set time for first record (line) of data line_time = string.split(line,"\t",7) data_time = line_time[1:7] suffix = line_time[7] data_time_str = string.join(data_time) time_current = mktime(strptime(data_time_str,"%Y %m %d %H %M %S")) else: line_time = string.split(line,"\t",7) ## list of string parts data_time = line_time[1:7] ## time data (list of strings) suffix = line_time[7] ## all the rest (string w/ tabs) time_current = time_pre ## set current time (previous record) ## convert list of time elements to string data_time_str = string.join(data_time) ## convert string to tuple and then to seconds time_next = mktime(strptime(data_time_str,"%Y %m %d %H %M %S")) delta = (time_next - time_current) / time_dilation_factor if verbose: print "Delta time difference: %s seconds" % delta sleep(delta) ## wait for next record time_current = time_next ## get current time in UTC and convert to string time_now_str = strftime("%Y %m %d %H %M %S", gmtime(time())) ## convert spaces back to tabs time_now = string.join(string.split(time_now_str),'\t') ## create new tab-delimited string of data new_line = prefix + "\t" + time_now + "\t" + suffix out_list = (new_line, time_current) return out_list def main(argv): import os, getopt, sys global filename # Do the right thing with boolean values for all known Python versions. try: True, False except NameError: (True, False) = (1, 0) try: args, filenames = getopt.getopt(argv[1:], "hfv", ["help", "fast", "verbose", "pipe="]) except getopt.error, msg: args = "dummy" print msg print "Usage: %s [-h|f|v|--pipe=pipe-prog] [ ...]" % (argv[0],) print " Processes old LLPS data and outputs with current time tags." sys.exit(2) fast = False verbose = False pipe = False pipe_program = False for o, a in args: if o in ("-h", "--help"): print __doc__ sys.exit(0) if o in ("-f", "--fast"): fast = True if o in ("-v", "--verbose"): verbose = True for o in args[:]: if o[0] == '--pipe' and o[1] != '': pipe = True pipe_program = o[1] args.remove(o) break elif o[0] == '--pipe' and o[1] == '': print "The --pipe option requires an argument." sys.exit(2) if verbose: print "options =", args print "filenames =", filenames if not filenames: print "Please specify one or more UALF input files" for filename in filenames: try: infile = open(filename, "r") except IOError, err: sys.stderr.write(err) if verbose: print "Please check the name(s) of your input file(s)" os.close(sys.stderr.fileno()) sys.exit(2) time_pre = 0 try: for line in infile.readlines(): ## iterate through the file try: ## catch non-numeric data errors new_line, new_time = process(line, time_pre, fast, verbose) if not pipe: sys.stdout.write(new_line) sys.stdout.flush() else: os.popen(pipe_program, 'w', 0).write(new_line) time_pre = new_time except TypeError: print "non-numeric data encountered" if verbose: print "unexpected value on the following line:" print line sys.exit(1) except ValueError: ## the return from blank line if verbose: ## causes a ValueError here print " Continuing with next available file..." continue ## this assumes a blank line ## only occurs at the end of infile.close() ## a data file if __name__ == "__main__": main(sys.argv)