source: CMIP6dreqbuild/trunk/srcMisc/dreqSX.py @ 926

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CMIP6dreqbuild/trunk/srcMisc/dreqSX.py@1157
Revision 926, 6.0 KB checked in by mjuckes, 3 years ago (diff)

stripped sx203.py from sx202.py

Line 
1
2import shelve, string, collections
3from utils_wb import workbook, uniCleanFunc
4from sets import Set
5
6vdate = "20160309"
7vdate2 = "20160601"
8
9## writen by dreq_consol_tables
10shvg = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'r' )
11shnv = shelve.open( 'dreq_consol_tables_nv_shelve_v%s' % vdate, 'r' )
12shnvo = shelve.open( 'dreq_consol_tables_nvom_shelve_v%s' % vdate, 'r' )
13## from vlsc5b --- includes new names from OMIP
14shlist = [shvg,shnv]
15
16##MIP columns: recordid, ['comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'standard_name', 'deflate', 'long_name', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', '#cell_methods', 'valid_max', 'positive'] , var, mip table, dims
17
18##ee0: Short name of group      Variable short name     Table   Frequency       Description extension (optional)        Shape   Levels  Time mean, point or climatology Mask (optional), priority, mip, uuid
19## vgrp, var, table, freq, comment, shape, levels, tform, mask, priority, mip, recordId
20## e.g. 'PMIP-Lmon', u'rh_c14', u'new', u'mon', '', u'XYT', '', '', '', 2, 'PMIP', '21de1a34-1597-11e5-861b-ac72891c3257')
21#####
22###['group','var','table','frequency','shape',''....
23### Want to map shape onto a "feature type" which will reference a set of dimensions.
24##XYT: latitude, longitude, time
25##'4 element vector' : seaice transect x time.
26## KZT: RFMIP: not clear what K is.
27## '2D vector field': from SIMIP, vector as in list: extra dimension is a list of thickness categories (
28
29##eenv:
30## MIP  standard name check     Review comments Shape etc       Short name      CF standard_name        standard name status    Native grid     units   Long Name       description/comments    Priority        associated observational dataset, ..., recordID (last)
31##e.g. [2, u'FAFMIP', -1.0, '', u'yr|XYZ|mean', u'ocontemppsmadvect', u'tendency_of_sea_water_conservative_temperature_expressed_as_heat_content_due_to_parameterized_submesoscale_advection', -1.0, u'ocean', u'W/m2', '', u'Level 2: priority unclear', 2.0, '', '', '', '', 'b1feb8a2-1598-11e5-92ad-ac72891c3257']
32
33##eeold:
34##e.g. ['ef36eefae77f82455c0a925167a0d4ca', 'rss', 'Net shortwave surface radiation', 'surface_net_downward_shortwave_flux', 'unset', 'W m-2', 'SPECS_day', 'mv.001.0094', 'mv.001.0394', '', '']
35##ee_xref
36##hash, uuid list -- list of mip tables for each parameter key (ee_old)
37
38ee_xref = {}
39ee_xref_bck = collections.defaultdict( list )
40ee_miptables_old = collections.defaultdict( list )
41ee_miptables_vars_old = collections.defaultdict( list )
42ee_ovars_old = collections.defaultdict( list )
43ee_mip = {}
44ee0 = {}
45eenv = {}
46eenvo = {}
47## dictionary to carry short names of old variables.
48eeold = {}
49for k in shnv:
50  eenv[k] = shnv[k]
51for k in shnvo:
52  eenvo[k] = shnvo[k]
53ccl = collections.defaultdict( int )
54for k in shvg:
55  ee0[k] = shvg[k]
56  ccl[len(ee0[k])] += 1
57
58print 'INFO.0000: tables ... groups ....',ccl
59 
60
61rmod = 'csv'
62eeoldl = collections.defaultdict( list )
63if rmod == 'xls':
64  for ee,fn in [(ee_xref,'vlsc5_xref_v%s.xls' % vdate),(ee_mip,'vlsc5_mip_v%s.xls' % vdate),(eeold,'vlsc5b_v%s.xls' % vdate)]:
65    wb = workbook( fn )
66    sh = wb.book.sheet_by_name( 'Sheet1' )
67    for i in range(sh.nrows):
68      r = map( lambda x: string.strip(str(uniCleanFunc(x.value))), sh.row(i) )
69      ee[r[0]] = r
70else:
71  for ee,fn,sep,xxx,nrmin in [(ee_xref,'vlsc5_xref_v%s.csv' % vdate2,',',False,5),(ee_mip,'vlsc5_mip_v%s.csv' % vdate2, '\t',False,24),(eeoldl,'vlsc5b_v%s.csv' % vdate2, '\t',True,9)]:
72    for i in open(fn).readlines():
73      r = map( lambda x: string.strip(str(uniCleanFunc(x))), string.split( string.strip(i), sep ) )
74      while len(r) < nrmin:
75        r.append( '' )
76      if xxx:
77        ee[r[0]].append( r )
78      else:
79        ee[r[0]] = r
80
81ss = set()
82for k in ee_mip.keys():
83 if len( ee_mip[k][:] ) != 24:
84   print 'ERROR.015.0003: Bad record length %s: %s' % (k,ee_mip[k])
85 mipt = ee_mip[k][22]
86 ss.add( mipt )
87
88for h in eeoldl.keys():
89  if len( eeoldl[h] ) == 1:
90    eeold[h] = eeoldl[h][0]
91  else:
92    ll = []
93    for k in range( len( eeoldl[h] ) ):
94      if eeoldl[h][k][8] == '':
95        ll.append(k)
96    if len(ll) == 0:
97      print 'ERROR.004.0001: no unforwarded record in vlsc5b for h=%s' % h
98      eeold[h] = eeoldl[h][0]
99    elif len(ll) > 1:
100      print 'ERROR.004.0002: multiple unforwarded records in vlsc5b for h=%s (%s)' % (h,eeoldl[h][ll[0]][1])
101      eeold[h] = eeoldl[h][ll[0]]
102    else:
103      eeold[h] = eeoldl[h][ll[0]]
104
105k2m = []
106km = []
107for k in ee_xref.keys():
108  if eeold.has_key(k):
109    if eeold[k][1][-4:] == "Clim":
110      print 'bad xref to clim: ',eeold[k]
111    for k2 in ee_xref[k][1:]:
112      if k2 != '':
113        if ee_mip.has_key(k2):
114          ee_xref_bck[k2].append(k)
115        else:
116          k2m.append( (k2,k) )
117  else:
118    km.append(k)
119
120##
121## had bug here due to extension of ee_mip to include dimensions.
122##
123ee_byvar = collections.defaultdict(list)
124for k in ee_mip.keys():
125  tid = ee_mip[k][-2]
126  ee_miptables_old[tid].append(  ee_mip[k][0] )
127  ee_miptables_vars_old[tid].append(  ee_mip[k][15] )
128  ee_ovars_old['%s.%s' % (tid,ee_mip[k][15]) ].append(  k )
129  ee_byvar[ ee_mip[k][15] ].append( k )
130
131doCmipVar = False
132if doCmipVar:
133 import cmip5so
134## check cmip5 vs. standard output
135 s1 = Set()
136 cmip5ByVar = collections.defaultdict( list )
137 for k in ee_miptables_vars_old.keys():
138  if k[:6] == 'CMIP5_':
139    s2 = Set( ee_miptables_vars_old[k] )
140    kk = k[6:]
141    s1.add( kk )
142    assert cmip5so.so.has_key(kk ), 'Table %s not found in standard output' % kk
143    lnf = []
144    for v in ee_miptables_vars_old[k]:
145      if not cmip5so.so[kk].a.has_key(v):
146         lnf.append(v)
147    if len(lnf) > 0:
148      print 'ERROR.010.0001: Variables not found in %s:: %s' % (kk,str(lnf))
149
150    lnf = []
151    for v in cmip5so.so[kk].a.keys():
152      if not v in s2:
153         lnf.append(v)
154    if len(lnf) > 0:
155      print 'ERROR.010.0002: SO Variables not found in %s:: %s' % (kk,str(lnf))
156
157 for s in cmip5so.so.keys():
158  if not s in s1:
159    print 'ERROR.010.0003: Table %s not in old mip tables' % s
160  for v in cmip5so.so[s].a.keys():
161    cmip5ByVar[v].append(s)
162   
163for s in shlist:
164  s.close()
Note: See TracBrowser for help on using the repository browser.