|
0
|
1 #!/usr/bin/env python
|
|
|
2 import argparse
|
|
|
3 import datetime
|
|
|
4 import decimal
|
|
|
5 import re
|
|
|
6 import shutil
|
|
|
7 import sys
|
|
|
8
|
|
|
9 parser = argparse.ArgumentParser()
|
|
|
10 parser.add_argument('--data_type', dest='data_type', default=None, help='Temperature data type, normals or actuals')
|
|
|
11 parser.add_argument('--input_actuals', dest='input_actuals', default=None, help='Daily actuals temperature data')
|
|
|
12 parser.add_argument('--input_normals', dest='input_normals', default=None, help='30 year normals temperature data')
|
|
|
13 parser.add_argument('--output', dest='output', help='Output dataset'),
|
|
|
14 args = parser.parse_args()
|
|
|
15
|
|
|
16 ACTUALS_HEADER = "LATITUDE,LONGITUDE,DATE,DOY,TMIN,TMAX"
|
|
|
17 NORMALS_HEADER = "stationid,latitude,longitude,elev_m,name,st,mmdd,doy,tmin,tmax"
|
|
|
18
|
|
|
19 def add_error_msg(accumulated_msgs, msg):
|
|
|
20 return "%s\n%s" % (accumulated_msgs, msg)
|
|
|
21
|
|
|
22
|
|
|
23 def empty_value(line_no, label, accumulated_msgs):
|
|
|
24 return add_error_msg(accumulated_msgs, "The required %s value is missing on line %d." % (label, line_no))
|
|
|
25
|
|
|
26
|
|
|
27 def stop_error(msg):
|
|
|
28 sys.exit(msg)
|
|
|
29
|
|
|
30
|
|
|
31 def validate_date_string(line_no, date_string, accumulated_msgs):
|
|
|
32 try:
|
|
|
33 datetime.datetime.strptime(date_string, '%Y-%m-%d')
|
|
|
34 return accumulated_msgs
|
|
|
35 except ValueError:
|
|
|
36 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect date format (%s must be YYYY-MM-DD)." % (line_no, date_string))
|
|
|
37
|
|
|
38
|
|
|
39 def validate_decimal(line_no, decimal_string, accumulated_msgs, label):
|
|
|
40 try:
|
|
|
41 decimal.Decimal(decimal_string)
|
|
|
42 return accumulated_msgs
|
|
|
43 except Exception:
|
|
|
44 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect %s decimal value (%s)." % (line_no, label, decimal_string))
|
|
|
45
|
|
|
46
|
|
|
47 def validate_integer(line_no, integer_string, accumulated_msgs, label):
|
|
|
48 if integer_string.isdigit():
|
|
|
49 return accumulated_msgs
|
|
4
|
50 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect %s integer value (%s)." % (line_no, label, integer_string))
|
|
0
|
51
|
|
|
52
|
|
|
53 def validate_mmdd(line_no, mmdd, accumulated_msgs):
|
|
|
54 try:
|
|
3
|
55 datetime.datetime.strptime(mmdd, '%m-%d')
|
|
0
|
56 return accumulated_msgs
|
|
|
57 except ValueError:
|
|
3
|
58 return add_error_msg(accumulated_msgs, "Line %d contains an incorrect date format (%s must be mm-dd)." % (line_no, mmdd))
|
|
0
|
59
|
|
|
60
|
|
|
61 accumulated_msgs = ""
|
|
|
62 # Parse the input file, skipping the header, and validating
|
|
|
63 # that each data line consists of 31 comma-separated items.
|
|
|
64 if args.data_type == "normals":
|
|
|
65 input_file = args.input_normals
|
|
|
66 header = NORMALS_HEADER
|
|
|
67 last_doy = 0
|
|
|
68 num_normals_rows = 0
|
|
|
69 else:
|
|
|
70 input_file = args.input_actuals
|
|
|
71 header = ACTUALS_HEADER
|
|
|
72 with open(input_file, "r") as ih:
|
|
|
73 for i, line in enumerate(ih):
|
|
|
74 line = line.rstrip("\r\n")
|
|
|
75 if i == 0:
|
|
|
76 if line != header:
|
|
|
77 accumulated_msgs = add_error_msg(accumulated_msgs, "The header is invalid, must be %s" % header)
|
|
5
|
78 continue
|
|
0
|
79 items = line.split(",")
|
|
|
80 if args.data_type == "normals":
|
|
|
81 if i > 367:
|
|
|
82 accumulated_msgs = add_error_msg(accumulated_msgs, "The input file contains more than 367 lines (must be 1 header line and 366 data lines).")
|
|
|
83 stop_error(accumulated_msgs)
|
|
|
84 if len(items) != 10:
|
|
|
85 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains %s columns, (must be 10)." % (i, len(items)))
|
|
|
86 stop_error(accumulated_msgs)
|
|
|
87 stationid = items[0]
|
|
2
|
88 if len(stationid) == 0:
|
|
0
|
89 accumulated_msgs = empty_value(i, "stationid", accumulated_msgs)
|
|
|
90 latitude = items[1]
|
|
|
91 accumulated_msgs = validate_decimal(i, latitude, accumulated_msgs, "latitude")
|
|
|
92 longitude = items[2]
|
|
|
93 accumulated_msgs = validate_decimal(i, longitude, accumulated_msgs, "longitude")
|
|
|
94 elev_m = items[3]
|
|
|
95 accumulated_msgs = validate_decimal(i, elev_m, accumulated_msgs, "elev_m")
|
|
|
96 name = items[4]
|
|
|
97 if len(name) == 0:
|
|
|
98 accumulated_msgs = empty_value(i, "name", accumulated_msgs)
|
|
|
99 st = items[5]
|
|
|
100 if len(st) == 0:
|
|
|
101 accumulated_msgs = empty_value(i, "st", accumulated_msgs)
|
|
|
102 mmdd = items[6]
|
|
|
103 accumulated_msgs = validate_mmdd(i, mmdd, accumulated_msgs)
|
|
|
104 doy = items[7]
|
|
|
105 accumulated_msgs = validate_integer(i, doy, accumulated_msgs, "doy")
|
|
|
106 # Make sure the DOY values are consecutive.
|
|
|
107 try:
|
|
|
108 if int(doy) != (last_doy + 1):
|
|
|
109 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains a DOY (%s) that is not conexcutive." % (i, doy))
|
|
|
110 stop_error(accumulated_msgs)
|
|
|
111 else:
|
|
|
112 last_doy += 1
|
|
|
113 except Exception:
|
|
|
114 # The error for an invalid integer was captured above.
|
|
|
115 pass
|
|
|
116 tmin = items[8]
|
|
|
117 accumulated_msgs = validate_decimal(i, tmin, accumulated_msgs, "tmin")
|
|
|
118 tmax = items[9]
|
|
|
119 accumulated_msgs = validate_decimal(i, tmax, accumulated_msgs, "tmax")
|
|
|
120 num_normals_rows += 1
|
|
|
121 else:
|
|
|
122 if i > 367:
|
|
|
123 accumulated_msgs = add_error_msg(accumulated_msgs, "The input file contains more than 367 lines (must be 1 header line and no more than 366 data lines).")
|
|
|
124 stop_error(accumulated_msgs)
|
|
|
125 if len(items) != 6:
|
|
|
126 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains %s columns, (must be 6)." % (i, len(items)))
|
|
|
127 stop_error(accumulated_msgs)
|
|
|
128 latitude = items[0]
|
|
|
129 accumulated_msgs = validate_decimal(i, latitude, accumulated_msgs, "LATITUDE")
|
|
|
130 longitude = items[1]
|
|
|
131 accumulated_msgs = validate_decimal(i, longitude, accumulated_msgs, "LONGITUDE")
|
|
|
132 date_string = items[2]
|
|
|
133 accumulated_msgs = validate_date_string(line_no, date_string, accumulated_msgs)
|
|
|
134 doy = items[3]
|
|
|
135 accumulated_msgs = validate_integer(i, doy, accumulated_msgs, "doy")
|
|
|
136 # Make sure the DOY values are consecutive.
|
|
|
137 if i==0:
|
|
|
138 try:
|
|
|
139 last_doy = int(doy)
|
|
|
140 except Exception:
|
|
|
141 # The error for an invalid integer was captured above.
|
|
|
142 pass
|
|
|
143 else:
|
|
|
144 try:
|
|
|
145 if int(doy) != (last_doy + 1):
|
|
|
146 accumulated_msgs = add_error_msg(accumulated_msgs, "Line %d contains a DOY (%s) that is not conexcutive." % (i, doy))
|
|
|
147 stop_error(accumulated_msgs)
|
|
|
148 else:
|
|
|
149 last_doy += 1
|
|
|
150 except Exception:
|
|
|
151 # The error for an invalid integer was captured above.
|
|
|
152 pass
|
|
|
153 tmin = items[8]
|
|
|
154 accumulated_msgs = validate_decimal(i, tmin, accumulated_msgs, "tmin")
|
|
|
155 tmax = items[9]
|
|
|
156 accumulated_msgs = validate_decimal(i, tmax, accumulated_msgs, "tmax")
|
|
|
157 if args.data_type == "normals" and num_normals_rows != 367:
|
|
1
|
158 accumulated_msgs = add_error_msg(accumulated_msgs, "The input file contains %d rows, (must be 367)." % num_normals_rows)
|
|
0
|
159
|
|
|
160 if len(accumulated_msgs) > 0:
|
|
|
161 stop_error(accumulated_msgs)
|
|
|
162
|
|
|
163 shutil.copyfile(input_file, args.output)
|
