1 | |
---|
2 | import collections, glob, string, re, json, sys |
---|
3 | from fcc_utils2 import mipTableScan, snlist, tupsort |
---|
4 | from config_c4 import CC_CONFIG_DIR |
---|
5 | |
---|
6 | def uniquify( ll ): |
---|
7 | ll.sort() |
---|
8 | l0 = [ll[0],] |
---|
9 | for l in ll[1:]: |
---|
10 | if l != l0[-1]: |
---|
11 | l0.append(l) |
---|
12 | return l0 |
---|
13 | |
---|
14 | heightRequired = ['tas','tasmax','tasmin','huss','sfcWind','sfcWindmax','wsgsmax','uas','vas'] |
---|
15 | cmip5_ignore = ['depth','depth_c','eta','nsigma','vertices_latitude','vertices_longitude','ztop','ptop','p0','z1','z2','href','k_c','a','a_bnds','ap','ap_bnds','b','b_bnds','sigma','sigma_bnds','zlev','zlev_bnds'] |
---|
16 | cmip5AxesAtts = ['axis', 'bounds_values', 'climatology', 'coords_attrib', 'formula', 'index_only', 'long_name', 'must_call_cmor_grid', 'must_have_bounds', 'out_name', 'positive', 'requested', 'requested_bounds', 'standard_name', 'stored_direction', 'tolerance', 'type', 'units', 'valid_max', 'valid_min', 'value', 'z_bounds_factors', 'z_factors'] |
---|
17 | |
---|
18 | |
---|
19 | NT_mip = collections.namedtuple( 'mip',['label','dir','pattern'] ) |
---|
20 | NT_var = collections.namedtuple( 'var',['name','sn','snStat','realm','units','longName','comment','mip'] ) |
---|
21 | NT_canvari = collections.namedtuple( 'canonicalVariation',['conditions','text', 'ref'] ) |
---|
22 | vlist = [ |
---|
23 | ('uas', |
---|
24 | "eastward_wind", |
---|
25 | 'Eastward Near-Surface Wind Speed', |
---|
26 | 'Eastward Near-Surface Wind', |
---|
27 | 'WRONG LONG NAME (*speed* included)'), |
---|
28 | ('vas', |
---|
29 | "northward_wind", |
---|
30 | 'Northward Near-Surface Wind Speed', |
---|
31 | 'Northward Near-Surface Wind', |
---|
32 | 'WRONG LONG NAME (*speed* included)'), |
---|
33 | ("snc", |
---|
34 | "surface_snow_area_fraction", |
---|
35 | "Snow Area Fraction", |
---|
36 | "Surface Snow Area Fraction", |
---|
37 | "WRONG LONG NAME (*surface* omitted)"), |
---|
38 | ("prsn", |
---|
39 | "snowfall_flux", |
---|
40 | "Solid Precipitation", |
---|
41 | "Snowfall Flux", |
---|
42 | "WRONG LONG NAME"), |
---|
43 | ("tntscpbl", |
---|
44 | "tendency_of_air_temperature_due_to_stratiform_cloud_and_precipitation_and_boundary_layer_mixing", |
---|
45 | "Tendency of Air Temperature due to Stratiform Cloud Condensation and Evaporation", |
---|
46 | "Tendency of Air Temperature Due to Stratiform Cloud and Precipitation and Boundary Layer Mixing", |
---|
47 | 'WRONG LONG NAME'), |
---|
48 | ("tas", |
---|
49 | "air_temperature", |
---|
50 | "Air Temperature", |
---|
51 | "Near-Surface Air Temperature", |
---|
52 | 'WRONG LONG NAME'), |
---|
53 | ("cfadDbze94", |
---|
54 | "histogram_of_equivalent_reflectivity_factor_over_height_above_reference_ellipsoid", |
---|
55 | "CloudSat Radar Reflectivity CFAD", |
---|
56 | "CloudSat Radar Reflectivity", |
---|
57 | "INCONSISTENT LONG NAME"), |
---|
58 | ("cfadLidarsr532", |
---|
59 | "histogram_of_backscattering_ratio_over_height_above_reference_ellipsoid", |
---|
60 | "CALIPSO Scattering Ratio CFAD", |
---|
61 | "CALIPSO Scattering Ratio", |
---|
62 | "INCONSISTENT LONG NAME"), |
---|
63 | ("cl", |
---|
64 | "cloud_area_fraction_in_atmosphere_layer", |
---|
65 | "Cloud Area Fraction", |
---|
66 | "Cloud Area Fraction in Atmosphere Layer", |
---|
67 | "INCONSISTENT LONG NAME"), |
---|
68 | ("clcalipso", |
---|
69 | "cloud_area_fraction_in_atmosphere_layer", |
---|
70 | "CALIPSO Cloud Fraction", |
---|
71 | "CALIPSO Cloud Area Fraction", |
---|
72 | 'WRONG LONG NAME'), |
---|
73 | ("cltisccp", |
---|
74 | "cloud_area_fraction", |
---|
75 | "ISCCP Total Total Cloud Fraction", |
---|
76 | "ISCCP Total Cloud Fraction", |
---|
77 | 'WRONG LONG NAME'), |
---|
78 | ("reffclwc", |
---|
79 | "effective_radius_of_convective_cloud_liquid_water_particle", |
---|
80 | "Convective Cloud Droplet Effective Radius", |
---|
81 | "Hydrometeor Effective Radius of Convective Cloud Liquid Water", |
---|
82 | 'INCONSISTENT LONG NAMES'), |
---|
83 | ("reffclws", |
---|
84 | "effective_radius_of_stratiform_cloud_liquid_water_particle", |
---|
85 | 'Stratiform Cloud Droplet Effective Radius', |
---|
86 | 'Hydrometeor Effective Radius of Stratiform Cloud Liquid Water', |
---|
87 | 'INCONSISTENT LONG NAMES' ) ] |
---|
88 | |
---|
89 | |
---|
90 | class helper: |
---|
91 | |
---|
92 | def __init__(self): |
---|
93 | self.applycv = True |
---|
94 | self.re1 = re.compile( '"(.*)"=="(.*)"' ) |
---|
95 | |
---|
96 | self.cmip5Tables= ['CMIP5_3hr', 'CMIP5_6hrPlev', 'CMIP5_Amon', 'CMIP5_cfDay', 'CMIP5_cfOff', 'CMIP5_day', 'CMIP5_grids', 'CMIP5_Lmon', 'CMIP5_OImon', 'CMIP5_Oyr', |
---|
97 | 'CMIP5_6hrLev', 'CMIP5_aero', ' CMIP5_cf3hr', 'CMIP5_cfMon', 'CMIP5_cfSites', 'CMIP5_fx', 'CMIP5_LImon', 'CMIP5_Oclim', 'CMIP5_Omon' ] |
---|
98 | self.cmip5DefPoint = ['CMIP5_3hr', 'CMIP5_6hrPlev', 'CMIP5_cfOff', 'CMIP5_6hrLev', ' CMIP5_cf3hr', 'CMIP5_cfSites' ] |
---|
99 | |
---|
100 | self.canonvar = [ NT_canvari( (('table','CMIP5_3hr'),), 'This is sampled synoptically.', '' ), |
---|
101 | NT_canvari( (), 'The flux is computed as the mass divided by the area of the grid cell.', 'This is calculated as the convective mass flux divided by the area of the whole grid cell (not just the area of the cloud).' ), |
---|
102 | ] |
---|
103 | |
---|
104 | self.canonvar = [] |
---|
105 | for l in open( 'config/canonicalVariations.txt' ).readlines(): |
---|
106 | if l[0] != '#': |
---|
107 | ix = l.index(':') |
---|
108 | s = string.strip( l[ix:] ) |
---|
109 | r = self.re1.findall( s ) |
---|
110 | assert len(r) == 1, 'Cannot parse: %s' % s |
---|
111 | self.canonvar.append( NT_canvari( (), r[0][0], r[0][1] ) ) |
---|
112 | |
---|
113 | def match(self,a,b): |
---|
114 | if type(a) == type( 'X' ) and type(b) == type( 'X' ): |
---|
115 | a0,b0 = map( lambda x: string.replace(x, '__ABSENT__',''), [a,b] ) |
---|
116 | return string.strip( string.replace(a0, ' ', ' '), '"') == string.strip( string.replace(b0, ' ', ' '), '"') |
---|
117 | else: |
---|
118 | return a == b |
---|
119 | |
---|
120 | def checkCond( self, table, var, conditions ): |
---|
121 | val = True |
---|
122 | for ck, cv in conditions: |
---|
123 | if ck == 'table': |
---|
124 | val &= table == cv |
---|
125 | elif ck == 'var': |
---|
126 | val &= var == cv |
---|
127 | |
---|
128 | return val |
---|
129 | |
---|
130 | |
---|
131 | |
---|
132 | class snsub: |
---|
133 | |
---|
134 | def __init__(self): |
---|
135 | pass |
---|
136 | |
---|
137 | def isFalseSn(self,var,sn): |
---|
138 | if sn == 'mole_concentration_of_molecular_oxygen_in_sea_water': |
---|
139 | return (True,'mole_concentration_of_dissolved_molecular_oxygen_in_sea_water', 'INVALID STANDARD NAME') |
---|
140 | elif var == 'rldscs' and sn == 'downwelling_longwave_flux_in_air_assuming_clear_sky': |
---|
141 | return (True,'surface_downwelling_longwave_flux_in_air_assuming_clear_sky','WRONG STANDARD NAME (should be surface)' ) |
---|
142 | elif var == 'clisccp' and sn == 'cloud_area_fraction_in_atmosphere_layer': |
---|
143 | return (True, 'isccp_cloud_area_fraction', 'WRONG STANDARD NAME (should be isccp)' ) |
---|
144 | return (False,'no match','') |
---|
145 | |
---|
146 | def isFalseLn(self,var,ln): |
---|
147 | for tt in vlist: |
---|
148 | if var == tt[0] and ln == tt[2]: |
---|
149 | return (True, tt[3], tt[4] ) |
---|
150 | return (False,'no match','') |
---|
151 | |
---|
152 | |
---|
153 | class mipCo: |
---|
154 | |
---|
155 | def __init__(self,mips,helper=None): |
---|
156 | self.vl0 = [] |
---|
157 | self.al0 = [] |
---|
158 | self.tl = [] |
---|
159 | self.dl = [] |
---|
160 | self.td = {} |
---|
161 | self.dd = {} |
---|
162 | self.helper = helper |
---|
163 | for mip in mips: |
---|
164 | self._scan(mip) |
---|
165 | |
---|
166 | def _scan(self,mip): |
---|
167 | |
---|
168 | ## dl = glob.glob( '%s%s' % (mip.dir,mip.pattern) ) |
---|
169 | dl = glob.glob( '%s/%s%s' % (CC_CONFIG_DIR, mip.dir,mip.pattern) ) |
---|
170 | dl.sort() |
---|
171 | for d in dl: |
---|
172 | if d[-5:] != 'grids': |
---|
173 | tab = string.split( d, '/')[-1] |
---|
174 | isoceanic = tab[:7] == "CMIP5_O" |
---|
175 | l2 = ms.scan_table( open( d ).readlines(), None, asDict=True, lax=True, tag="x", warn=True) |
---|
176 | l2k = [] |
---|
177 | usedDims = [] |
---|
178 | for k in l2.keys(): |
---|
179 | if k not in cmip5_ignore: |
---|
180 | l2k.append(k) |
---|
181 | if l2[k][0] != 'scalar': |
---|
182 | usedDims += l2[k][0] |
---|
183 | l2k.sort() |
---|
184 | usedDims.sort() |
---|
185 | usedDims = uniquify( usedDims ) |
---|
186 | |
---|
187 | ##self.al0 += ms.adict.keys() |
---|
188 | self.vl0 += l2k |
---|
189 | self.tl.append( [tab,l2, l2k,isoceanic] ) |
---|
190 | self.td[tab] = l2 |
---|
191 | ##self.dd[tab] = ms.adict.copy() |
---|
192 | self.dd[tab] = {} |
---|
193 | for k in ms.adict.keys(): |
---|
194 | if k not in usedDims: |
---|
195 | print "WARNING[X1]: axis %s declared and not used in table %s" % (k,tab) |
---|
196 | for u in usedDims: |
---|
197 | if ms.adict.has_key(u): |
---|
198 | self.dd[tab][u] = ms.adict[u] |
---|
199 | else: |
---|
200 | print 'WARNING[X2]: USED DIMENSION %s not in table %s' % (u,tab) |
---|
201 | ##self.dl.append( [tab, ms.adict.copy()] ) |
---|
202 | self.dl.append( [tab, self.dd[tab].copy() ] ) |
---|
203 | self.al0 += self.dd[tab].keys() |
---|
204 | |
---|
205 | self.vl0.sort() |
---|
206 | self.vl = [] |
---|
207 | self.vl.append( self.vl0[0] ) |
---|
208 | self.vdict = { self.vl[0]:[] } |
---|
209 | for v in self.vl0[1:]: |
---|
210 | if v != self.vl[-1]: |
---|
211 | self.vl.append(v) |
---|
212 | self.vdict[v] = [] |
---|
213 | self.al0.sort() |
---|
214 | self.al = [self.al0[0],] |
---|
215 | self.adict = { self.al[0]:[] } |
---|
216 | for v in self.al0[1:]: |
---|
217 | if v != self.al[-1]: |
---|
218 | self.al.append(v) |
---|
219 | self.adict[v] = [] |
---|
220 | |
---|
221 | for t in self.tl: |
---|
222 | for k in t[2]: |
---|
223 | self.vdict[k].append(t[0]) |
---|
224 | |
---|
225 | ## create list of tables for each dimension. |
---|
226 | for a in self.dl: |
---|
227 | for k in a[1].keys(): |
---|
228 | self.adict[k].append(a[0]) |
---|
229 | |
---|
230 | self.dims = self.adict.keys() |
---|
231 | self.dims.sort() |
---|
232 | self.vars = self.vdict.keys() |
---|
233 | self.vars.sort() |
---|
234 | ##for v in self.vars: |
---|
235 | ##l = self.vdict[v] |
---|
236 | ##l.sort() |
---|
237 | ## print v, l, td[l[0]][v][1].get('standard_name','__NO_STANDARD_NAME__') |
---|
238 | |
---|
239 | class runcheck1: |
---|
240 | def __init__( self, m, thisatts, isAxes=False): |
---|
241 | self.errors = [] |
---|
242 | if not isAxes: |
---|
243 | vars = m.vars |
---|
244 | vdict = m.vdict |
---|
245 | td = m.td |
---|
246 | ix = 1 |
---|
247 | xxx = '' |
---|
248 | else: |
---|
249 | vars = m.dims |
---|
250 | vdict = m.adict |
---|
251 | td = m.dd |
---|
252 | ix = 0 |
---|
253 | xxx = 'dim: ' |
---|
254 | |
---|
255 | self.ix = ix |
---|
256 | self.vars = vars |
---|
257 | self.vdict = vdict |
---|
258 | # dictionary td[tab][var][ix][attribute] |
---|
259 | self.td = td |
---|
260 | vd2 = {} |
---|
261 | for v in vars: |
---|
262 | l = vdict[v] |
---|
263 | l.sort() |
---|
264 | if len(l) > 1: |
---|
265 | for att in thisatts: |
---|
266 | if att != "__name__": |
---|
267 | ##for att in ['standard_name','units']: |
---|
268 | if att == '__dimensions__': |
---|
269 | atl = map( lambda x: string.join( td[x][v][0] ), l ) |
---|
270 | else: |
---|
271 | atl = map( lambda x: td[x][v][ix].get(att,'__ABSENT__'), l ) |
---|
272 | atl.sort() |
---|
273 | av = [atl[0],] |
---|
274 | for a in atl[1:]: |
---|
275 | if a != av[-1]: |
---|
276 | av.append(a) |
---|
277 | if att == 'standard_name': |
---|
278 | for a in av: |
---|
279 | if a not in snl and a not in snla: |
---|
280 | print "ERROR[A1]: ",xxx,"INVALID STANDARD NAME: ",a,v |
---|
281 | self.errors.append( "INVALID STANDARD NAME: %s [%s]" % (a,v) ) |
---|
282 | if len(av) > 1: |
---|
283 | ee = {} |
---|
284 | |
---|
285 | for a in [True,False]: |
---|
286 | ee[a] = [] |
---|
287 | |
---|
288 | isol = [] |
---|
289 | for x in l: |
---|
290 | a = td[x][v][ix].get(att,'__ABSENT__') |
---|
291 | try: |
---|
292 | if att == 'standard_name' or ( att == 'long_name' and vd2[v][0] == 2): |
---|
293 | iso = x[:7] == 'CMIP5_O' |
---|
294 | tt = snsubber.isFalseSn( v, a ) |
---|
295 | elif att == 'long_name': |
---|
296 | tt = snsubber.isFalseLn( v, a ) |
---|
297 | dims = td[x][v][0] |
---|
298 | iso = 'depth0m' in dims |
---|
299 | else: |
---|
300 | iso = False |
---|
301 | tt = (False,) |
---|
302 | ## iso = False |
---|
303 | except: |
---|
304 | print att,v |
---|
305 | raise |
---|
306 | isol.append((iso,x)) |
---|
307 | if tt[0]: |
---|
308 | print 'INFO[Y1]: Substituting ',v,a,tt |
---|
309 | ee[iso].append( tt[1] ) |
---|
310 | else: |
---|
311 | ee[iso].append( a ) |
---|
312 | |
---|
313 | for a in [True,False]: |
---|
314 | ok = True |
---|
315 | if len(ee[a]) > 1 : |
---|
316 | for x in ee[a][1:]: |
---|
317 | if x != ee[a][0]: |
---|
318 | ok = False |
---|
319 | |
---|
320 | if not ok: |
---|
321 | print xxx,'E001: Multiple values : ',att,v,ee |
---|
322 | for t in isol: |
---|
323 | if t[0] == a: |
---|
324 | tab = t[1] |
---|
325 | if att in ['standard_name','long_name']: |
---|
326 | print "E002",xxx,tab,td[tab][v][ix].get('standard_name','__ABSENT__'),td[tab][v][ix].get('long_name','__ABSENT__') |
---|
327 | else: |
---|
328 | print "E003",xxx,tab,td[tab][v][ix].get(att,'__ABSENT__') |
---|
329 | |
---|
330 | if att == "standard_name": |
---|
331 | vd2[v] = (2,[ee[True],ee[False]] ) |
---|
332 | else: |
---|
333 | if att == "standard_name": |
---|
334 | tt = snsubber.isFalseSn( v, av[0] ) |
---|
335 | if tt[0]: |
---|
336 | print 'INFO[A2]: Substituting ',v,av[0],tt |
---|
337 | vd2[v] = (1, tt[1]) |
---|
338 | else: |
---|
339 | vd2[v] = (1, av) |
---|
340 | elif len(l) == 1: |
---|
341 | tab = vdict[v][0] |
---|
342 | a = td[tab][v][ix].get('standard_name','__ABSENT__') |
---|
343 | tt = snsubber.isFalseSn( v, a ) |
---|
344 | if tt[0]: |
---|
345 | print 'INFO[A3]: Substituting ',v,a,tt |
---|
346 | vd2[v] = (1, tt[1]) |
---|
347 | else: |
---|
348 | vd2[v] = (1, a) |
---|
349 | ##print 'MULTIPLE VALUES: ',v,att,av |
---|
350 | else: |
---|
351 | print "WARNING[X4]: ",xxx, 'Zero length element: %s' % v |
---|
352 | |
---|
353 | def chkDims( self, reqh=None): |
---|
354 | pass |
---|
355 | |
---|
356 | class typecheck1: |
---|
357 | def __init__( self, m, thisatts,helper=None): |
---|
358 | self.type2Atts = ['positive','comment', 'long_name', 'modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'flag_meanings', 'flag_values'] |
---|
359 | self.type3Atts = ['positive','long_name','modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'flag_meanings', 'flag_values'] |
---|
360 | self.type4Atts = ['positive','modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'flag_meanings', 'flag_values'] |
---|
361 | self.type2Atts = ['positive','comment', 'long_name', 'modeling_realm', 'out_name', 'standard_name', 'type', 'units'] |
---|
362 | self.type3Atts = ['positive','long_name','modeling_realm', 'out_name', 'standard_name', 'type', 'units'] |
---|
363 | self.type4Atts = ['positive','modeling_realm', 'standard_name', 'type', 'units'] |
---|
364 | self.m = m |
---|
365 | vars = m.vars |
---|
366 | vdict = m.vdict |
---|
367 | self.helper=helper |
---|
368 | td = m.td |
---|
369 | vd2 = {} |
---|
370 | type1, type2, type3, type4, type5 = [[],[],[],[],[],] |
---|
371 | vd2 = {} |
---|
372 | for v in vars: |
---|
373 | l = vdict[v] |
---|
374 | dl = map( lambda x: string.join( td[x][v][0] ), l ) |
---|
375 | vd2[v] = str( uniquify( dl ) ) |
---|
376 | json.dump( vd2, open( 'mipInfo.json', 'w' ) ) |
---|
377 | for v in vars: |
---|
378 | l = vdict[v] |
---|
379 | l.sort() |
---|
380 | if len(l) == 1: |
---|
381 | type1.append(v) |
---|
382 | elif len(l) > 1: |
---|
383 | adict = {} |
---|
384 | for att in thisatts: |
---|
385 | if att == '__dimensions__': |
---|
386 | atl = map( lambda x: (string.join( td[x][v][0] ),x), l ) |
---|
387 | else: |
---|
388 | atl = map( lambda x: (td[x][v][1].get(att,'__ABSENT__'),x), l ) |
---|
389 | atl.sort( tupsort(0).cmp ) |
---|
390 | a0 = atl[0][0] |
---|
391 | if a0 == None: |
---|
392 | a0 = "" |
---|
393 | av = [a0,] |
---|
394 | for a,tab in atl[1:]: |
---|
395 | if a == None: |
---|
396 | a = "" |
---|
397 | if a != av[-1]: |
---|
398 | if self.helper != None and self.helper.applycv: |
---|
399 | thisok=False |
---|
400 | pmatch = False |
---|
401 | for cond,src,targ in self.helper.canonvar: |
---|
402 | if string.find(a,src) != -1 or string.find(av[-1],src) != -1: |
---|
403 | ##print 'Potential match ---- ',a |
---|
404 | ##print src,'###',targ |
---|
405 | ##print av[-1] |
---|
406 | pmatch = True |
---|
407 | if self.helper.checkCond( tab, v, cond ): |
---|
408 | if self.helper.match(string.replace( a, src, targ ), av[-1]) or self.helper.match(string.replace( av[-1], src, targ ), a): |
---|
409 | thisok = True |
---|
410 | if thisok: |
---|
411 | print 'INFO[typecheck]: conditional match found', tab, v |
---|
412 | else: |
---|
413 | if pmatch: |
---|
414 | ##print '########### no matvh found' |
---|
415 | pass |
---|
416 | av.append(a) |
---|
417 | else: |
---|
418 | av.append(a) |
---|
419 | adict[att] = av |
---|
420 | |
---|
421 | ## check for type 2 |
---|
422 | tval = None |
---|
423 | ##if adict[ 'positive' ] == ['__ABSENT__']: |
---|
424 | if all( map( lambda x: len(adict[x]) == 1, self.type2Atts )): |
---|
425 | tval = 2 |
---|
426 | elif all( map( lambda x: len(adict[x]) == 1, self.type3Atts )): |
---|
427 | tval = 3 |
---|
428 | elif all( map( lambda x: len(adict[x]) == 1, self.type4Atts )): |
---|
429 | tval = 4 |
---|
430 | else: |
---|
431 | l = map( lambda x: '%s:%s, ' % (x,len(adict[x]) ), self.type2Atts ) |
---|
432 | if tval == 2: |
---|
433 | type2.append( v) |
---|
434 | elif tval == 3: |
---|
435 | type3.append( v) |
---|
436 | elif tval == 4: |
---|
437 | type4.append( v) |
---|
438 | else: |
---|
439 | type5.append(v) |
---|
440 | xx = float( len(vars) ) |
---|
441 | print "INFO[XXX]", string.join( map( lambda x: '%s (%5.1f%%);' % (x,x/xx*100), [len(type1), len(type2), len(type3), len(type4), len(type5)] ) ) |
---|
442 | self.type1 = type1 |
---|
443 | self.type2 = type2 |
---|
444 | self.type3 = type3 |
---|
445 | self.type4 = type4 |
---|
446 | self.type5 = type5 |
---|
447 | |
---|
448 | def exportHtml( self, typecode ): |
---|
449 | |
---|
450 | allAtts = ['__dimensions__', 'cell_methods', 'comment', 'long_name', 'modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'valid_max', 'valid_min', 'positive', 'ok_max_mean_abs', 'ok_min_mean_abs', 'flag_meanings', 'flag_values'] |
---|
451 | fixedType1Tmpl = """:%(standard_name)s [%(units)s]</h3> |
---|
452 | %(__dimensions__)s<br/> |
---|
453 | %(long_name)s<br/> |
---|
454 | realm: %(modeling_realm)s; out_name: %(out_name)s; type: %(type)s, <br/> |
---|
455 | cell_methods: %(cell_methods)s; comment: %(comment)s<br/> |
---|
456 | """ |
---|
457 | fixedType2TmplA = """:%(standard_name)s [%(units)s]</h3> |
---|
458 | %(long_name)s<br/> |
---|
459 | realm: %(modeling_realm)s; out_name: %(out_name)s; type: %(type)s, <br/> |
---|
460 | comment: %(comment)s<br/> |
---|
461 | """ |
---|
462 | fixedType2TmplB = "<li>%s [%s]: %s -- (%s,%s,%s,%s)</li>\n" |
---|
463 | |
---|
464 | fixedType3TmplA = """:%(standard_name)s [%(units)s]</h3> |
---|
465 | realm: %(modeling_realm)s; out_name: %(out_name)s; type: %(type)s <br/> |
---|
466 | """ |
---|
467 | fixedType3TmplB = "<li>%s [%s]: %s: %s [%s]</li>\n" |
---|
468 | fixedType4TmplB = "<li>%s [%s]: %s [%s]</li>\n" |
---|
469 | fixedType5TmplA = """ [%(units)s]</h3> |
---|
470 | out_name: %(out_name)s; type: %(type)s <br/> |
---|
471 | """ |
---|
472 | fixedType5TmplB = "<li>%s [%s]: %s, %s [%s]: %s, %s:: %s, %s</li>\n" |
---|
473 | |
---|
474 | if typecode == 1: |
---|
475 | oo = open( 'type1.html', 'w' ) |
---|
476 | self.type1.sort() |
---|
477 | ee = {} |
---|
478 | for v in self.type1: |
---|
479 | tab = self.m.vdict[v][0] |
---|
480 | if not ee.has_key(tab): |
---|
481 | ee[tab] = [] |
---|
482 | ee[tab].append( v ) |
---|
483 | keys = ee.keys() |
---|
484 | keys.sort() |
---|
485 | for k in keys: |
---|
486 | oo.write( '<h2>Table %s</h2>\n' % k ) |
---|
487 | for v in ee[k]: |
---|
488 | try: |
---|
489 | etmp = {} |
---|
490 | for a in allAtts: |
---|
491 | etmp[a] = self.m.td[k][v][1].get( a, 'unset' ) |
---|
492 | etmp['__dimensions__'] = string.join( self.m.td[k][v][0] ) |
---|
493 | oo.write( '<h3>' + v + (fixedType1Tmpl % etmp) ) |
---|
494 | except: |
---|
495 | print k, self.m.td[k][v][1].keys() |
---|
496 | raise |
---|
497 | oo.close() |
---|
498 | elif typecode == 2: |
---|
499 | oo = open( 'type2.html', 'w' ) |
---|
500 | self.type2.sort() |
---|
501 | oo.write( '<h2>Variables with fixed attributes</h2>\n' ) |
---|
502 | for v in self.type2: |
---|
503 | l = self.m.vdict[v] |
---|
504 | etmp = {} |
---|
505 | for a in allAtts: |
---|
506 | etmp[a] = self.m.td[l[0]][v][1].get( a, 'unset' ) |
---|
507 | oo.write( '<h3>' + v + (fixedType2TmplA % etmp) ) |
---|
508 | oo.write( '<ul>\n' ) |
---|
509 | for t in l: |
---|
510 | dims = string.join( self.m.td[t][v][0] ) |
---|
511 | sa = tuple( [t,dims,] + map( lambda x: self.m.td[t][v][1].get( x, 'unset' ), ['cell_methods','valid_max', 'valid_min', 'ok_max_mean_abs', 'ok_min_mean_abs'] ) ) |
---|
512 | oo.write( fixedType2TmplB % sa ) |
---|
513 | oo.write( '</ul>\n' ) |
---|
514 | oo.close() |
---|
515 | |
---|
516 | elif typecode in [3,4,5]: |
---|
517 | oo = open( 'type%s.html' % typecode, 'w' ) |
---|
518 | thistype,h2,al,tmplA,tmplB = { 3:(self.type3,"Variables with varying comment",['long_name','comment','cell_methods'], fixedType3TmplA, fixedType3TmplB), |
---|
519 | 4:(self.type4,"Variables with varying long_name",['long_name','cell_methods'],fixedType3TmplA, fixedType4TmplB), |
---|
520 | 5:(self.type5,"Remaining variables",['standard_name','long_name','out_name', 'modeling_realm','positive','type','units'],fixedType5TmplA, fixedType5TmplB) }[typecode] |
---|
521 | ##['positive','modeling_realm', 'out_name', 'standard_name', 'type', 'units'] |
---|
522 | thistype.sort() |
---|
523 | oo.write( '<h2>%s</h2>\n' % h2 ) |
---|
524 | for v in thistype: |
---|
525 | l = self.m.vdict[v] |
---|
526 | etmp = {} |
---|
527 | for a in allAtts: |
---|
528 | etmp[a] = self.m.td[l[0]][v][1].get( a, 'unset' ) |
---|
529 | oo.write( '<h3>' + v + (tmplA % etmp) ) |
---|
530 | oo.write( '<ul>\n' ) |
---|
531 | for t in l: |
---|
532 | dims = string.join( self.m.td[t][v][0] ) |
---|
533 | sa = tuple( [t,dims,] + map( lambda x: self.m.td[t][v][1].get( x, 'unset' ), al ) ) |
---|
534 | oo.write( tmplB % sa ) |
---|
535 | oo.write( '</ul>\n' ) |
---|
536 | oo.close() |
---|
537 | |
---|
538 | class run(object): |
---|
539 | |
---|
540 | def __init__(self): |
---|
541 | pass |
---|
542 | |
---|
543 | def run(self): |
---|
544 | self.m = mipCo( mips ) |
---|
545 | self.json() |
---|
546 | al = [] |
---|
547 | for k0 in self.m.dd.keys(): |
---|
548 | for k1 in self.m.dd[k0].keys(): |
---|
549 | al += self.m.dd[k0][k1][0].keys() |
---|
550 | ald = uniquify( al ) |
---|
551 | ald.sort() |
---|
552 | i = ald.index('standard_name') |
---|
553 | ald.pop(i) |
---|
554 | ald = ['standard_name', ] + ald |
---|
555 | self.ald = ald |
---|
556 | |
---|
557 | return self.m,ald |
---|
558 | |
---|
559 | def getTupList(self): |
---|
560 | vl = [] |
---|
561 | keys = self.m.vdict.keys() |
---|
562 | keys.sort() |
---|
563 | for k in keys: |
---|
564 | for t in self.m.vdict[k]: |
---|
565 | ##NT_var = collections.namedtuple( 'mip',['name','sn','snStat','realm','units','longName','comment'] ) |
---|
566 | sn, r, units, ln, c = map( lambda x: self.m.td[t][k][1].get(x,None), ['standard_name','modeling_realm','units','long_name','comment'] ) |
---|
567 | mipid = string.split(t,'_')[0] |
---|
568 | if c == '': |
---|
569 | c = None |
---|
570 | v = NT_var( k, sn, 'exists', r, units, ln, c,mipid ) |
---|
571 | vl.append(v) |
---|
572 | self.tupList = vl |
---|
573 | return vl |
---|
574 | |
---|
575 | def compTupList(self): |
---|
576 | tl1 = uniquify(self.tupList) |
---|
577 | tl2 = [tl1[0],] |
---|
578 | ### these lines comment on all differences of variables with the same name, including differences in comments. |
---|
579 | for t in tl1[1:]: |
---|
580 | if t[:7] == tl2[-1][:7]: |
---|
581 | pass |
---|
582 | elif t[:3] == tl2[-1][:3] and t[4:6] == tl2[-1][4:6]: |
---|
583 | if (t.mip == 'CMIP5' and tl2[-1].mip == 'CCMI1') or (t.mip == 'CCMI1' and tl2[-1].mip == 'CMIP5'): |
---|
584 | tl2[-1] = t |
---|
585 | else: |
---|
586 | print 'What to do??' |
---|
587 | print tl2[-1] |
---|
588 | print t |
---|
589 | else: |
---|
590 | tl2.append(t) |
---|
591 | return tl1, tl2 |
---|
592 | |
---|
593 | def runchecks(self): |
---|
594 | self.v = runcheck1( self.m, self.ald, isAxes=False ) |
---|
595 | ## check consistency of dimension |
---|
596 | self.r2 = runcheck1( self.m, self.ald, isAxes=True ) |
---|
597 | return self.v, self.r2 |
---|
598 | |
---|
599 | def json(self): |
---|
600 | keys = self.m.adict.keys() |
---|
601 | keys.sort() |
---|
602 | fh = open( 'axes_json.txt', 'w' ) |
---|
603 | for k in keys: |
---|
604 | ee = self.m.dd[self.m.adict[k][0]][k][0] |
---|
605 | ee["__name__"] = k |
---|
606 | fh.write( json.dumps( ee ) + '\n' ) |
---|
607 | fh.close() |
---|
608 | |
---|
609 | ms = mipTableScan() |
---|
610 | snc = snlist() |
---|
611 | snl, snla = snc.gen_sn_list( ) |
---|
612 | snsubber = snsub() |
---|
613 | |
---|
614 | mips = { "cmip5":NT_mip( 'cmip5','cmip5_vocabs/mip/', 'CMIP5_*' ), |
---|
615 | "ccmi":NT_mip( 'ccmi', 'ccmi_vocabs/mip/', 'CCMI1_*'), |
---|
616 | "pmip":NT_mip( 'pmip', 'pmip3_vocabs/mip/', 'PMIP3_*'), |
---|
617 | "cordex":NT_mip( 'cordex', 'cordex_vocabs/mip/', 'CORDEX_*'), |
---|
618 | "specs":NT_mip( 'specs', 'specs_vocabs/mip/', 'SPECS_*') } |
---|
619 | |
---|
620 | mipl = mips.keys() |
---|
621 | mipl = ['specs'] |
---|
622 | mips = map( lambda x: mips[x], mipl ) |
---|
623 | r = run() |
---|
624 | |
---|
625 | m,ald = r.run() |
---|
626 | |
---|
627 | tl = r.getTupList() |
---|
628 | tl1,tl2 = r.compTupList() |
---|
629 | v,r2 = r.runchecks() |
---|
630 | |
---|
631 | allatts = ms.al |
---|
632 | thisatts = ['standard_name','units','long_name','__dimensions__'] |
---|
633 | ## need to have standard name first. |
---|
634 | for a in allatts: |
---|
635 | if a not in thisatts: |
---|
636 | thisatts.append(a) |
---|
637 | |
---|
638 | h = helper() |
---|
639 | s = typecheck1( m, thisatts, helper=h) |
---|
640 | s.exportHtml( 1 ) |
---|
641 | s.exportHtml( 2 ) |
---|
642 | s.exportHtml( 3 ) |
---|
643 | s.exportHtml( 4 ) |
---|
644 | s.exportHtml( 5 ) |
---|