Context Navigation

source: CCCC/trunk/ceda_cc/extractMipInfo.py @ 326

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/extractMipInfo.py@326

Revision 326, 22.0 KB checked in by mjuckes, 7 years ago (diff)
drastically reduced content of c4.py -- moved code to c4_run.py

Line
1
2	import collections, glob, string, re, json, sys
3	from fcc_utils2 import mipTableScan, snlist, tupsort
4	from config_c4 import CC_CONFIG_DIR
5
6	def uniquify( ll ):
7	ll.sort()
8	l0 = [ll[0],]
9	for l in ll[1:]:
10	if l != l0[-1]:
11	l0.append(l)
12	return l0
13
14	heightRequired = ['tas','tasmax','tasmin','huss','sfcWind','sfcWindmax','wsgsmax','uas','vas']
15	cmip5_ignore = ['depth','depth_c','eta','nsigma','vertices_latitude','vertices_longitude','ztop','ptop','p0','z1','z2','href','k_c','a','a_bnds','ap','ap_bnds','b','b_bnds','sigma','sigma_bnds','zlev','zlev_bnds']
16	cmip5AxesAtts = ['axis', 'bounds_values', 'climatology', 'coords_attrib', 'formula', 'index_only', 'long_name', 'must_call_cmor_grid', 'must_have_bounds', 'out_name', 'positive', 'requested', 'requested_bounds', 'standard_name', 'stored_direction', 'tolerance', 'type', 'units', 'valid_max', 'valid_min', 'value', 'z_bounds_factors', 'z_factors']
17
18
19	NT_mip = collections.namedtuple( 'mip',['label','dir','pattern'] )
20	NT_var = collections.namedtuple( 'var',['name','sn','snStat','realm','units','longName','comment','mip'] )
21	NT_canvari = collections.namedtuple( 'canonicalVariation',['conditions','text', 'ref'] )
22	vlist = [
23	('uas',
24	"eastward_wind",
25	'Eastward Near-Surface Wind Speed',
26	'Eastward Near-Surface Wind',
27	'WRONG LONG NAME (speed included)'),
28	('vas',
29	"northward_wind",
30	'Northward Near-Surface Wind Speed',
31	'Northward Near-Surface Wind',
32	'WRONG LONG NAME (speed included)'),
33	("snc",
34	"surface_snow_area_fraction",
35	"Snow Area Fraction",
36	"Surface Snow Area Fraction",
37	"WRONG LONG NAME (surface omitted)"),
38	("prsn",
39	"snowfall_flux",
40	"Solid Precipitation",
41	"Snowfall Flux",
42	"WRONG LONG NAME"),
43	("tntscpbl",
44	"tendency_of_air_temperature_due_to_stratiform_cloud_and_precipitation_and_boundary_layer_mixing",
45	"Tendency of Air Temperature due to Stratiform Cloud Condensation and Evaporation",
46	"Tendency of Air Temperature Due to Stratiform Cloud and Precipitation and Boundary Layer Mixing",
47	'WRONG LONG NAME'),
48	("tas",
49	"air_temperature",
50	"Air Temperature",
51	"Near-Surface Air Temperature",
52	'WRONG LONG NAME'),
53	("cfadDbze94",
54	"histogram_of_equivalent_reflectivity_factor_over_height_above_reference_ellipsoid",
55	"CloudSat Radar Reflectivity CFAD",
56	"CloudSat Radar Reflectivity",
57	"INCONSISTENT LONG NAME"),
58	("cfadLidarsr532",
59	"histogram_of_backscattering_ratio_over_height_above_reference_ellipsoid",
60	"CALIPSO Scattering Ratio CFAD",
61	"CALIPSO Scattering Ratio",
62	"INCONSISTENT LONG NAME"),
63	("cl",
64	"cloud_area_fraction_in_atmosphere_layer",
65	"Cloud Area Fraction",
66	"Cloud Area Fraction in Atmosphere Layer",
67	"INCONSISTENT LONG NAME"),
68	("clcalipso",
69	"cloud_area_fraction_in_atmosphere_layer",
70	"CALIPSO Cloud Fraction",
71	"CALIPSO Cloud Area Fraction",
72	'WRONG LONG NAME'),
73	("cltisccp",
74	"cloud_area_fraction",
75	"ISCCP Total Total Cloud Fraction",
76	"ISCCP Total Cloud Fraction",
77	'WRONG LONG NAME'),
78	("reffclwc",
79	"effective_radius_of_convective_cloud_liquid_water_particle",
80	"Convective Cloud Droplet Effective Radius",
81	"Hydrometeor Effective Radius of Convective Cloud Liquid Water",
82	'INCONSISTENT LONG NAMES'),
83	("reffclws",
84	"effective_radius_of_stratiform_cloud_liquid_water_particle",
85	'Stratiform Cloud Droplet Effective Radius',
86	'Hydrometeor Effective Radius of Stratiform Cloud Liquid Water',
87	'INCONSISTENT LONG NAMES' ) ]
88
89
90	class helper:
91
92	def __init__(self):
93	self.applycv = True
94	self.re1 = re.compile( '"(.)"=="(.)"' )
95
96	self.cmip5Tables= ['CMIP5_3hr', 'CMIP5_6hrPlev', 'CMIP5_Amon', 'CMIP5_cfDay', 'CMIP5_cfOff', 'CMIP5_day', 'CMIP5_grids', 'CMIP5_Lmon', 'CMIP5_OImon', 'CMIP5_Oyr',
97	'CMIP5_6hrLev', 'CMIP5_aero', ' CMIP5_cf3hr', 'CMIP5_cfMon', 'CMIP5_cfSites', 'CMIP5_fx', 'CMIP5_LImon', 'CMIP5_Oclim', 'CMIP5_Omon' ]
98	self.cmip5DefPoint = ['CMIP5_3hr', 'CMIP5_6hrPlev', 'CMIP5_cfOff', 'CMIP5_6hrLev', ' CMIP5_cf3hr', 'CMIP5_cfSites' ]
99
100	self.canonvar = [ NT_canvari( (('table','CMIP5_3hr'),), 'This is sampled synoptically.', '' ),
101	NT_canvari( (), 'The flux is computed as the mass divided by the area of the grid cell.', 'This is calculated as the convective mass flux divided by the area of the whole grid cell (not just the area of the cloud).' ),
102	]
103
104	self.canonvar = []
105	for l in open( 'config/canonicalVariations.txt' ).readlines():
106	if l[0] != '#':
107	ix = l.index(':')
108	s = string.strip( l[ix:] )
109	r = self.re1.findall( s )
110	assert len(r) == 1, 'Cannot parse: %s' % s
111	self.canonvar.append( NT_canvari( (), r[0][0], r[0][1] ) )
112
113	def match(self,a,b):
114	if type(a) == type( 'X' ) and type(b) == type( 'X' ):
115	a0,b0 = map( lambda x: string.replace(x, '__ABSENT__',''), [a,b] )
116	return string.strip( string.replace(a0, ' ', ' '), '"') == string.strip( string.replace(b0, ' ', ' '), '"')
117	else:
118	return a == b
119
120	def checkCond( self, table, var, conditions ):
121	val = True
122	for ck, cv in conditions:
123	if ck == 'table':
124	val &= table == cv
125	elif ck == 'var':
126	val &= var == cv
127
128	return val
129
130
131
132	class snsub:
133
134	def __init__(self):
135	pass
136
137	def isFalseSn(self,var,sn):
138	if sn == 'mole_concentration_of_molecular_oxygen_in_sea_water':
139	return (True,'mole_concentration_of_dissolved_molecular_oxygen_in_sea_water', 'INVALID STANDARD NAME')
140	elif var == 'rldscs' and sn == 'downwelling_longwave_flux_in_air_assuming_clear_sky':
141	return (True,'surface_downwelling_longwave_flux_in_air_assuming_clear_sky','WRONG STANDARD NAME (should be surface)' )
142	elif var == 'clisccp' and sn == 'cloud_area_fraction_in_atmosphere_layer':
143	return (True, 'isccp_cloud_area_fraction', 'WRONG STANDARD NAME (should be isccp)' )
144	return (False,'no match','')
145
146	def isFalseLn(self,var,ln):
147	for tt in vlist:
148	if var == tt[0] and ln == tt[2]:
149	return (True, tt[3], tt[4] )
150	return (False,'no match','')
151
152
153	class mipCo:
154
155	def __init__(self,mips,helper=None):
156	self.vl0 = []
157	self.al0 = []
158	self.tl = []
159	self.dl = []
160	self.td = {}
161	self.dd = {}
162	self.helper = helper
163	for mip in mips:
164	self._scan(mip)
165
166	def _scan(self,mip):
167
168	## dl = glob.glob( '%s%s' % (mip.dir,mip.pattern) )
169	dl = glob.glob( '%s/%s%s' % (CC_CONFIG_DIR, mip.dir,mip.pattern) )
170	dl.sort()
171	for d in dl:
172	if d[-5:] != 'grids':
173	tab = string.split( d, '/')[-1]
174	isoceanic = tab[:7] == "CMIP5_O"
175	l2 = ms.scan_table( open( d ).readlines(), None, asDict=True, lax=True, tag="x", warn=True)
176	l2k = []
177	usedDims = []
178	for k in l2.keys():
179	if k not in cmip5_ignore:
180	l2k.append(k)
181	if l2[k][0] != 'scalar':
182	usedDims += l2[k][0]
183	l2k.sort()
184	usedDims.sort()
185	usedDims = uniquify( usedDims )
186
187	##self.al0 += ms.adict.keys()
188	self.vl0 += l2k
189	self.tl.append( [tab,l2, l2k,isoceanic] )
190	self.td[tab] = l2
191	##self.dd[tab] = ms.adict.copy()
192	self.dd[tab] = {}
193	for k in ms.adict.keys():
194	if k not in usedDims:
195	print "WARNING[X1]: axis %s declared and not used in table %s" % (k,tab)
196	for u in usedDims:
197	if ms.adict.has_key(u):
198	self.dd[tab][u] = ms.adict[u]
199	else:
200	print 'WARNING[X2]: USED DIMENSION %s not in table %s' % (u,tab)
201	##self.dl.append( [tab, ms.adict.copy()] )
202	self.dl.append( [tab, self.dd[tab].copy() ] )
203	self.al0 += self.dd[tab].keys()
204
205	self.vl0.sort()
206	self.vl = []
207	self.vl.append( self.vl0[0] )
208	self.vdict = { self.vl[0]:[] }
209	for v in self.vl0[1:]:
210	if v != self.vl[-1]:
211	self.vl.append(v)
212	self.vdict[v] = []
213	self.al0.sort()
214	self.al = [self.al0[0],]
215	self.adict = { self.al[0]:[] }
216	for v in self.al0[1:]:
217	if v != self.al[-1]:
218	self.al.append(v)
219	self.adict[v] = []
220
221	for t in self.tl:
222	for k in t[2]:
223	self.vdict[k].append(t[0])
224
225	## create list of tables for each dimension.
226	for a in self.dl:
227	for k in a[1].keys():
228	self.adict[k].append(a[0])
229
230	self.dims = self.adict.keys()
231	self.dims.sort()
232	self.vars = self.vdict.keys()
233	self.vars.sort()
234	##for v in self.vars:
235	##l = self.vdict[v]
236	##l.sort()
237	## print v, l, td[l[0]][v][1].get('standard_name','__NO_STANDARD_NAME__')
238
239	class runcheck1:
240	def __init__( self, m, thisatts, isAxes=False):
241	self.errors = []
242	if not isAxes:
243	vars = m.vars
244	vdict = m.vdict
245	td = m.td
246	ix = 1
247	xxx = ''
248	else:
249	vars = m.dims
250	vdict = m.adict
251	td = m.dd
252	ix = 0
253	xxx = 'dim: '
254
255	self.ix = ix
256	self.vars = vars
257	self.vdict = vdict
258	# dictionary td[tab][var][ix][attribute]
259	self.td = td
260	vd2 = {}
261	for v in vars:
262	l = vdict[v]
263	l.sort()
264	if len(l) > 1:
265	for att in thisatts:
266	if att != "__name__":
267	##for att in ['standard_name','units']:
268	if att == '__dimensions__':
269	atl = map( lambda x: string.join( td[x][v][0] ), l )
270	else:
271	atl = map( lambda x: td[x][v][ix].get(att,'__ABSENT__'), l )
272	atl.sort()
273	av = [atl[0],]
274	for a in atl[1:]:
275	if a != av[-1]:
276	av.append(a)
277	if att == 'standard_name':
278	for a in av:
279	if a not in snl and a not in snla:
280	print "ERROR[A1]: ",xxx,"INVALID STANDARD NAME: ",a,v
281	self.errors.append( "INVALID STANDARD NAME: %s [%s]" % (a,v) )
282	if len(av) > 1:
283	ee = {}
284
285	for a in [True,False]:
286	ee[a] = []
287
288	isol = []
289	for x in l:
290	a = td[x][v][ix].get(att,'__ABSENT__')
291	try:
292	if att == 'standard_name' or ( att == 'long_name' and vd2[v][0] == 2):
293	iso = x[:7] == 'CMIP5_O'
294	tt = snsubber.isFalseSn( v, a )
295	elif att == 'long_name':
296	tt = snsubber.isFalseLn( v, a )
297	dims = td[x][v][0]
298	iso = 'depth0m' in dims
299	else:
300	iso = False
301	tt = (False,)
302	## iso = False
303	except:
304	print att,v
305	raise
306	isol.append((iso,x))
307	if tt[0]:
308	print 'INFO[Y1]: Substituting ',v,a,tt
309	ee[iso].append( tt[1] )
310	else:
311	ee[iso].append( a )
312
313	for a in [True,False]:
314	ok = True
315	if len(ee[a]) > 1 :
316	for x in ee[a][1:]:
317	if x != ee[a][0]:
318	ok = False
319
320	if not ok:
321	print xxx,'E001: Multiple values : ',att,v,ee
322	for t in isol:
323	if t[0] == a:
324	tab = t[1]
325	if att in ['standard_name','long_name']:
326	print "E002",xxx,tab,td[tab][v][ix].get('standard_name','__ABSENT__'),td[tab][v][ix].get('long_name','__ABSENT__')
327	else:
328	print "E003",xxx,tab,td[tab][v][ix].get(att,'__ABSENT__')
329
330	if att == "standard_name":
331	vd2[v] = (2,[ee[True],ee[False]] )
332	else:
333	if att == "standard_name":
334	tt = snsubber.isFalseSn( v, av[0] )
335	if tt[0]:
336	print 'INFO[A2]: Substituting ',v,av[0],tt
337	vd2[v] = (1, tt[1])
338	else:
339	vd2[v] = (1, av)
340	elif len(l) == 1:
341	tab = vdict[v][0]
342	a = td[tab][v][ix].get('standard_name','__ABSENT__')
343	tt = snsubber.isFalseSn( v, a )
344	if tt[0]:
345	print 'INFO[A3]: Substituting ',v,a,tt
346	vd2[v] = (1, tt[1])
347	else:
348	vd2[v] = (1, a)
349	##print 'MULTIPLE VALUES: ',v,att,av
350	else:
351	print "WARNING[X4]: ",xxx, 'Zero length element: %s' % v
352
353	def chkDims( self, reqh=None):
354	pass
355
356	class typecheck1:
357	def __init__( self, m, thisatts,helper=None):
358	self.type2Atts = ['positive','comment', 'long_name', 'modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'flag_meanings', 'flag_values']
359	self.type3Atts = ['positive','long_name','modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'flag_meanings', 'flag_values']
360	self.type4Atts = ['positive','modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'flag_meanings', 'flag_values']
361	self.type2Atts = ['positive','comment', 'long_name', 'modeling_realm', 'out_name', 'standard_name', 'type', 'units']
362	self.type3Atts = ['positive','long_name','modeling_realm', 'out_name', 'standard_name', 'type', 'units']
363	self.type4Atts = ['positive','modeling_realm', 'standard_name', 'type', 'units']
364	self.m = m
365	vars = m.vars
366	vdict = m.vdict
367	self.helper=helper
368	td = m.td
369	vd2 = {}
370	type1, type2, type3, type4, type5 = [[],[],[],[],[],]
371	vd2 = {}
372	for v in vars:
373	l = vdict[v]
374	dl = map( lambda x: string.join( td[x][v][0] ), l )
375	vd2[v] = str( uniquify( dl ) )
376	json.dump( vd2, open( 'mipInfo.json', 'w' ) )
377	for v in vars:
378	l = vdict[v]
379	l.sort()
380	if len(l) == 1:
381	type1.append(v)
382	elif len(l) > 1:
383	adict = {}
384	for att in thisatts:
385	if att == '__dimensions__':
386	atl = map( lambda x: (string.join( td[x][v][0] ),x), l )
387	else:
388	atl = map( lambda x: (td[x][v][1].get(att,'__ABSENT__'),x), l )
389	atl.sort( tupsort(0).cmp )
390	a0 = atl[0][0]
391	if a0 == None:
392	a0 = ""
393	av = [a0,]
394	for a,tab in atl[1:]:
395	if a == None:
396	a = ""
397	if a != av[-1]:
398	if self.helper != None and self.helper.applycv:
399	thisok=False
400	pmatch = False
401	for cond,src,targ in self.helper.canonvar:
402	if string.find(a,src) != -1 or string.find(av[-1],src) != -1:
403	##print 'Potential match ---- ',a
404	##print src,'###',targ
405	##print av[-1]
406	pmatch = True
407	if self.helper.checkCond( tab, v, cond ):
408	if self.helper.match(string.replace( a, src, targ ), av[-1]) or self.helper.match(string.replace( av[-1], src, targ ), a):
409	thisok = True
410	if thisok:
411	print 'INFO[typecheck]: conditional match found', tab, v
412	else:
413	if pmatch:
414	##print '########### no matvh found'
415	pass
416	av.append(a)
417	else:
418	av.append(a)
419	adict[att] = av
420
421	## check for type 2
422	tval = None
423	##if adict[ 'positive' ] == ['__ABSENT__']:
424	if all( map( lambda x: len(adict[x]) == 1, self.type2Atts )):
425	tval = 2
426	elif all( map( lambda x: len(adict[x]) == 1, self.type3Atts )):
427	tval = 3
428	elif all( map( lambda x: len(adict[x]) == 1, self.type4Atts )):
429	tval = 4
430	else:
431	l = map( lambda x: '%s:%s, ' % (x,len(adict[x]) ), self.type2Atts )
432	if tval == 2:
433	type2.append( v)
434	elif tval == 3:
435	type3.append( v)
436	elif tval == 4:
437	type4.append( v)
438	else:
439	type5.append(v)
440	xx = float( len(vars) )
441	print "INFO[XXX]", string.join( map( lambda x: '%s (%5.1f%%);' % (x,x/xx*100), [len(type1), len(type2), len(type3), len(type4), len(type5)] ) )
442	self.type1 = type1
443	self.type2 = type2
444	self.type3 = type3
445	self.type4 = type4
446	self.type5 = type5
447
448	def exportHtml( self, typecode ):
449
450	allAtts = ['__dimensions__', 'cell_methods', 'comment', 'long_name', 'modeling_realm', 'out_name', 'standard_name', 'type', 'units', 'valid_max', 'valid_min', 'positive', 'ok_max_mean_abs', 'ok_min_mean_abs', 'flag_meanings', 'flag_values']
451	fixedType1Tmpl = """:%(standard_name)s [%(units)s]</h3>
452	%(__dimensions__)s<br/>
453	%(long_name)s<br/>
454	realm: %(modeling_realm)s; out_name: %(out_name)s; type: %(type)s, <br/>
455	cell_methods: %(cell_methods)s; comment: %(comment)s<br/>
456	"""
457	fixedType2TmplA = """:%(standard_name)s [%(units)s]</h3>
458	%(long_name)s<br/>
459	realm: %(modeling_realm)s; out_name: %(out_name)s; type: %(type)s, <br/>
460	comment: %(comment)s<br/>
461	"""
462	fixedType2TmplB = "<li>%s [%s]: %s -- (%s,%s,%s,%s)</li>\n"
463
464	fixedType3TmplA = """:%(standard_name)s [%(units)s]</h3>
465	realm: %(modeling_realm)s; out_name: %(out_name)s; type: %(type)s <br/>
466	"""
467	fixedType3TmplB = "<li>%s [%s]: %s: %s [%s]</li>\n"
468	fixedType4TmplB = "<li>%s [%s]: %s [%s]</li>\n"
469	fixedType5TmplA = """ [%(units)s]</h3>
470	out_name: %(out_name)s; type: %(type)s <br/>
471	"""
472	fixedType5TmplB = "<li>%s [%s]: %s, %s [%s]: %s, %s:: %s, %s</li>\n"
473
474	if typecode == 1:
475	oo = open( 'type1.html', 'w' )
476	self.type1.sort()
477	ee = {}
478	for v in self.type1:
479	tab = self.m.vdict[v][0]
480	if not ee.has_key(tab):
481	ee[tab] = []
482	ee[tab].append( v )
483	keys = ee.keys()
484	keys.sort()
485	for k in keys:
486	oo.write( '<h2>Table %s</h2>\n' % k )
487	for v in ee[k]:
488	try:
489	etmp = {}
490	for a in allAtts:
491	etmp[a] = self.m.td[k][v][1].get( a, 'unset' )
492	etmp['__dimensions__'] = string.join( self.m.td[k][v][0] )
493	oo.write( '<h3>' + v + (fixedType1Tmpl % etmp) )
494	except:
495	print k, self.m.td[k][v][1].keys()
496	raise
497	oo.close()
498	elif typecode == 2:
499	oo = open( 'type2.html', 'w' )
500	self.type2.sort()
501	oo.write( '<h2>Variables with fixed attributes</h2>\n' )
502	for v in self.type2:
503	l = self.m.vdict[v]
504	etmp = {}
505	for a in allAtts:
506	etmp[a] = self.m.td[l[0]][v][1].get( a, 'unset' )
507	oo.write( '<h3>' + v + (fixedType2TmplA % etmp) )
508	oo.write( '<ul>\n' )
509	for t in l:
510	dims = string.join( self.m.td[t][v][0] )
511	sa = tuple( [t,dims,] + map( lambda x: self.m.td[t][v][1].get( x, 'unset' ), ['cell_methods','valid_max', 'valid_min', 'ok_max_mean_abs', 'ok_min_mean_abs'] ) )
512	oo.write( fixedType2TmplB % sa )
513	oo.write( '</ul>\n' )
514	oo.close()
515
516	elif typecode in [3,4,5]:
517	oo = open( 'type%s.html' % typecode, 'w' )
518	thistype,h2,al,tmplA,tmplB = { 3:(self.type3,"Variables with varying comment",['long_name','comment','cell_methods'], fixedType3TmplA, fixedType3TmplB),
519	4:(self.type4,"Variables with varying long_name",['long_name','cell_methods'],fixedType3TmplA, fixedType4TmplB),
520	5:(self.type5,"Remaining variables",['standard_name','long_name','out_name', 'modeling_realm','positive','type','units'],fixedType5TmplA, fixedType5TmplB) }[typecode]
521	##['positive','modeling_realm', 'out_name', 'standard_name', 'type', 'units']
522	thistype.sort()
523	oo.write( '<h2>%s</h2>\n' % h2 )
524	for v in thistype:
525	l = self.m.vdict[v]
526	etmp = {}
527	for a in allAtts:
528	etmp[a] = self.m.td[l[0]][v][1].get( a, 'unset' )
529	oo.write( '<h3>' + v + (tmplA % etmp) )
530	oo.write( '<ul>\n' )
531	for t in l:
532	dims = string.join( self.m.td[t][v][0] )
533	sa = tuple( [t,dims,] + map( lambda x: self.m.td[t][v][1].get( x, 'unset' ), al ) )
534	oo.write( tmplB % sa )
535	oo.write( '</ul>\n' )
536	oo.close()
537
538	class run(object):
539
540	def __init__(self):
541	pass
542
543	def run(self):
544	self.m = mipCo( mips )
545	self.json()
546	al = []
547	for k0 in self.m.dd.keys():
548	for k1 in self.m.dd[k0].keys():
549	al += self.m.dd[k0][k1][0].keys()
550	ald = uniquify( al )
551	ald.sort()
552	i = ald.index('standard_name')
553	ald.pop(i)
554	ald = ['standard_name', ] + ald
555	self.ald = ald
556
557	return self.m,ald
558
559	def getTupList(self):
560	vl = []
561	keys = self.m.vdict.keys()
562	keys.sort()
563	for k in keys:
564	for t in self.m.vdict[k]:
565	##NT_var = collections.namedtuple( 'mip',['name','sn','snStat','realm','units','longName','comment'] )
566	sn, r, units, ln, c = map( lambda x: self.m.td[t][k][1].get(x,None), ['standard_name','modeling_realm','units','long_name','comment'] )
567	mipid = string.split(t,'_')[0]
568	if c == '':
569	c = None
570	v = NT_var( k, sn, 'exists', r, units, ln, c,mipid )
571	vl.append(v)
572	self.tupList = vl
573	return vl
574
575	def compTupList(self):
576	tl1 = uniquify(self.tupList)
577	tl2 = [tl1[0],]
578	### these lines comment on all differences of variables with the same name, including differences in comments.
579	for t in tl1[1:]:
580	if t[:7] == tl2[-1][:7]:
581	pass
582	elif t[:3] == tl2[-1][:3] and t[4:6] == tl2[-1][4:6]:
583	if (t.mip == 'CMIP5' and tl2[-1].mip == 'CCMI1') or (t.mip == 'CCMI1' and tl2[-1].mip == 'CMIP5'):
584	tl2[-1] = t
585	else:
586	print 'What to do??'
587	print tl2[-1]
588	print t
589	else:
590	tl2.append(t)
591	return tl1, tl2
592
593	def runchecks(self):
594	self.v = runcheck1( self.m, self.ald, isAxes=False )
595	## check consistency of dimension
596	self.r2 = runcheck1( self.m, self.ald, isAxes=True )
597	return self.v, self.r2
598
599	def json(self):
600	keys = self.m.adict.keys()
601	keys.sort()
602	fh = open( 'axes_json.txt', 'w' )
603	for k in keys:
604	ee = self.m.dd[self.m.adict[k][0]][k][0]
605	ee["__name__"] = k
606	fh.write( json.dumps( ee ) + '\n' )
607	fh.close()
608
609	ms = mipTableScan()
610	snc = snlist()
611	snl, snla = snc.gen_sn_list( )
612	snsubber = snsub()
613
614	mips = { "cmip5":NT_mip( 'cmip5','cmip5_vocabs/mip/', 'CMIP5_*' ),
615	"ccmi":NT_mip( 'ccmi', 'ccmi_vocabs/mip/', 'CCMI1_*'),
616	"pmip":NT_mip( 'pmip', 'pmip3_vocabs/mip/', 'PMIP3_*'),
617	"cordex":NT_mip( 'cordex', 'cordex_vocabs/mip/', 'CORDEX_*'),
618	"specs":NT_mip( 'specs', 'specs_vocabs/mip/', 'SPECS_*') }
619
620	mipl = mips.keys()
621	mipl = ['specs']
622	mips = map( lambda x: mips[x], mipl )
623	r = run()
624
625	m,ald = r.run()
626
627	tl = r.getTupList()
628	tl1,tl2 = r.compTupList()
629	v,r2 = r.runchecks()
630
631	allatts = ms.al
632	thisatts = ['standard_name','units','long_name','__dimensions__']
633	## need to have standard name first.
634	for a in allatts:
635	if a not in thisatts:
636	thisatts.append(a)
637
638	h = helper()
639	s = typecheck1( m, thisatts, helper=h)
640	s.exportHtml( 1 )
641	s.exportHtml( 2 )
642	s.exportHtml( 3 )
643	s.exportHtml( 4 )
644	s.exportHtml( 5 )

Note: See TracBrowser for help on using the repository browser.

Download in other formats: