source: CMIP6dreqbuild/trunk/srcMisc/dreqSX.py @ 681

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CMIP6dreqbuild/trunk/srcMisc/dreqSX.py@756
Revision 681, 6.1 KB checked in by mjuckes, 4 years ago (diff)

near release

Line 
1
2import shelve, string, collections
3from utils_wb import workbook, uniCleanFunc
4import cmip5so
5from sets import Set
6
7vdate = "20160309"
8vdate2 = "20160601"
9
10## writen by dreq_consol_tables
11shvg = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'r' )
12shnv = shelve.open( 'dreq_consol_tables_nv_shelve_v%s' % vdate, 'r' )
13shnvo = shelve.open( 'dreq_consol_tables_nvom_shelve_v%s' % vdate, 'r' )
14## from vlsc5b --- includes new names from OMIP
15shold = shelve.open( 'dreq_consol_tables_old_shelve_v20150713', 'r' )
16shlist = [shvg,shnv,shold]
17
18##MIP columns: recordid, ['comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'standard_name', 'deflate', 'long_name', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', '#cell_methods', 'valid_max', 'positive'] , var, mip table, dims
19
20##ee0: Short name of group      Variable short name     Table   Frequency       Description extension (optional)        Shape   Levels  Time mean, point or climatology Mask (optional), priority, mip, uuid
21## vgrp, var, table, freq, comment, shape, levels, tform, mask, priority, mip, recordId
22## e.g. 'PMIP-Lmon', u'rh_c14', u'new', u'mon', '', u'XYT', '', '', '', 2, 'PMIP', '21de1a34-1597-11e5-861b-ac72891c3257')
23#####
24###['group','var','table','frequency','shape',''....
25### Want to map shape onto a "feature type" which will reference a set of dimensions.
26##XYT: latitude, longitude, time
27##'4 element vector' : seaice transect x time.
28## KZT: RFMIP: not clear what K is.
29## '2D vector field': from SIMIP, vector as in list: extra dimension is a list of thickness categories (
30
31##eenv:
32## MIP  standard name check     Review comments Shape etc       Short name      CF standard_name        standard name status    Native grid     units   Long Name       description/comments    Priority        associated observational dataset, ..., recordID (last)
33##e.g. [2, u'FAFMIP', -1.0, '', u'yr|XYZ|mean', u'ocontemppsmadvect', u'tendency_of_sea_water_conservative_temperature_expressed_as_heat_content_due_to_parameterized_submesoscale_advection', -1.0, u'ocean', u'W/m2', '', u'Level 2: priority unclear', 2.0, '', '', '', '', 'b1feb8a2-1598-11e5-92ad-ac72891c3257']
34
35##eeold:
36##e.g. ['ef36eefae77f82455c0a925167a0d4ca', 'rss', 'Net shortwave surface radiation', 'surface_net_downward_shortwave_flux', 'unset', 'W m-2', 'SPECS_day', 'mv.001.0094', 'mv.001.0394', '', '']
37##ee_xref
38##hash, uuid list -- list of mip tables for each parameter key (ee_old)
39
40ee_xref = {}
41ee_xref_bck = collections.defaultdict( list )
42ee_miptables_old = collections.defaultdict( list )
43ee_miptables_vars_old = collections.defaultdict( list )
44ee_ovars_old = collections.defaultdict( list )
45ee_mip = {}
46ee0 = {}
47eenv = {}
48eenvo = {}
49## dictionary to carry short names of old variables.
50eeold = {}
51###for ee,shv in [(ee0,shvg),(eenv,shnv),(eeold,shold)]:
52##for ee,shv in [(ee0,shvg),(eenv,shnv)]:
53for k in shnv:
54  eenv[k] = shnv[k]
55for k in shnvo:
56  eenvo[k] = shnvo[k]
57ccl = collections.defaultdict( int )
58for k in shvg:
59  ee0[k] = shvg[k]
60  ccl[len(ee0[k])] += 1
61
62print 'INFO.0000: tables ... groups ....',ccl
63 
64
65rmod = 'csv'
66eeoldl = collections.defaultdict( list )
67if rmod == 'xls':
68  for ee,fn in [(ee_xref,'vlsc5_xref_v%s.xls' % vdate),(ee_mip,'vlsc5_mip_v%s.xls' % vdate),(eeold,'vlsc5b_v%s.xls' % vdate)]:
69    wb = workbook( fn )
70    sh = wb.book.sheet_by_name( 'Sheet1' )
71    for i in range(sh.nrows):
72      r = map( lambda x: string.strip(str(uniCleanFunc(x.value))), sh.row(i) )
73      ee[r[0]] = r
74else:
75  for ee,fn,sep,xxx,nrmin in [(ee_xref,'vlsc5_xref_v%s.csv' % vdate2,',',False,5),(ee_mip,'vlsc5_mip_v%s.csv' % vdate2, '\t',False,24),(eeoldl,'vlsc5b_v%s.csv' % vdate2, '\t',True,9)]:
76    for i in open(fn).readlines():
77      r = map( lambda x: string.strip(str(uniCleanFunc(x))), string.split( string.strip(i), sep ) )
78      while len(r) < nrmin:
79        r.append( '' )
80      if xxx:
81        ee[r[0]].append( r )
82      else:
83        ee[r[0]] = r
84
85ss = set()
86for k in ee_mip.keys():
87 if len( ee_mip[k][:] ) != 24:
88   print 'ERROR.015.0003: Bad record length %s: %s' % (k,ee_mip[k])
89 mipt = ee_mip[k][22]
90 ss.add( mipt )
91
92for h in eeoldl.keys():
93  if len( eeoldl[h] ) == 1:
94    eeold[h] = eeoldl[h][0]
95  else:
96    ll = []
97    for k in range( len( eeoldl[h] ) ):
98      if eeoldl[h][k][8] == '':
99        ll.append(k)
100    if len(ll) == 0:
101      print 'ERROR.004.0001: no unforwarded record in vlsc5b for h=%s' % h
102      eeold[h] = eeoldl[h][0]
103    elif len(ll) > 1:
104      print 'ERROR.004.0002: multiple unforwarded records in vlsc5b for h=%s (%s)' % (h,eeoldl[h][ll[0]][1])
105      eeold[h] = eeoldl[h][ll[0]]
106    else:
107      eeold[h] = eeoldl[h][ll[0]]
108
109k2m = []
110km = []
111for k in ee_xref.keys():
112  if eeold.has_key(k):
113    if eeold[k][1][-4:] == "Clim":
114      print 'bad xref to clim: ',eeold[k]
115    for k2 in ee_xref[k][1:]:
116      if k2 != '':
117        if ee_mip.has_key(k2):
118          ee_xref_bck[k2].append(k)
119        else:
120          k2m.append( (k2,k) )
121  else:
122    km.append(k)
123
124##
125## had bug here due to extension of ee_mip to include dimensions.
126##
127ee_byvar = collections.defaultdict(list)
128for k in ee_mip.keys():
129  tid = ee_mip[k][-2]
130  ee_miptables_old[tid].append(  ee_mip[k][0] )
131  ee_miptables_vars_old[tid].append(  ee_mip[k][15] )
132  ee_ovars_old['%s.%s' % (tid,ee_mip[k][15]) ].append(  k )
133  ee_byvar[ ee_mip[k][15] ].append( k )
134
135## check cmip5 vs. standard output
136s1 = Set()
137cmip5ByVar = collections.defaultdict( list )
138for k in ee_miptables_vars_old.keys():
139  if k[:6] == 'CMIP5_':
140    s2 = Set( ee_miptables_vars_old[k] )
141    kk = k[6:]
142    s1.add( kk )
143    assert cmip5so.so.has_key(kk ), 'Table %s not found in standard output' % kk
144    lnf = []
145    for v in ee_miptables_vars_old[k]:
146      if not cmip5so.so[kk].a.has_key(v):
147         lnf.append(v)
148    if len(lnf) > 0:
149      print 'ERROR.010.0001: Variables not found in %s:: %s' % (kk,str(lnf))
150
151    lnf = []
152    for v in cmip5so.so[kk].a.keys():
153      if not v in s2:
154         lnf.append(v)
155    if len(lnf) > 0:
156      print 'ERROR.010.0002: SO Variables not found in %s:: %s' % (kk,str(lnf))
157
158for s in cmip5so.so.keys():
159  if not s in s1:
160    print 'ERROR.010.0003: Table %s not in old mip tables' % s
161  for v in cmip5so.so[s].a.keys():
162    cmip5ByVar[v].append(s)
163   
164
165for s in shlist:
166  s.close()
Note: See TracBrowser for help on using the repository browser.