1 | |
---|
2 | import shelve, string, collections |
---|
3 | from utils_wb import workbook, uniCleanFunc |
---|
4 | import cmip5so |
---|
5 | from sets import Set |
---|
6 | |
---|
7 | vdate = "20150827" |
---|
8 | |
---|
9 | sh = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'r' ) |
---|
10 | shnv = shelve.open( 'dreq_consol_tables_nv_shelve_v%s' % vdate, 'r' ) |
---|
11 | ## from vlsc5b --- includes new names from OMIP |
---|
12 | shold = shelve.open( 'dreq_consol_tables_old_shelve_v20150713', 'r' ) |
---|
13 | shlist = [sh,shnv,shold] |
---|
14 | |
---|
15 | ##MIP columns: recordid, ['comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'standard_name', 'deflate', 'long_name', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', '#cell_methods', 'valid_max', 'positive'] , var, mip table, dims |
---|
16 | |
---|
17 | ##ee0: Short name of group Variable short name Table Frequency Description extension (optional) Shape Levels Time mean, point or climatology Mask (optional), priority, mip, uuid |
---|
18 | ## vgrp, var, table, freq, comment, shape, levels, tform, mask, priority, mip, recordId |
---|
19 | ## e.g. 'PMIP-Lmon', u'rh_c14', u'new', u'mon', '', u'XYT', '', '', '', 2, 'PMIP', '21de1a34-1597-11e5-861b-ac72891c3257') |
---|
20 | ##### |
---|
21 | ###['group','var','table','frequency','shape',''.... |
---|
22 | ### Want to map shape onto a "feature type" which will reference a set of dimensions. |
---|
23 | ##XYT: latitude, longitude, time |
---|
24 | ##'4 element vector' : seaice transect x time. |
---|
25 | ## KZT: RFMIP: not clear what K is. |
---|
26 | ## '2D vector field': from SIMIP, vector as in list: extra dimension is a list of thickness categories ( |
---|
27 | |
---|
28 | ##eenv: |
---|
29 | ## MIP standard name check Review comments Shape etc Short name CF standard_name standard name status Native grid units Long Name description/comments Priority associated observational dataset, ..., recordID (last) |
---|
30 | ##e.g. [2, u'FAFMIP', -1.0, '', u'yr|XYZ|mean', u'ocontemppsmadvect', u'tendency_of_sea_water_conservative_temperature_expressed_as_heat_content_due_to_parameterized_submesoscale_advection', -1.0, u'ocean', u'W/m2', '', u'Level 2: priority unclear', 2.0, '', '', '', '', 'b1feb8a2-1598-11e5-92ad-ac72891c3257'] |
---|
31 | |
---|
32 | ##eeold: |
---|
33 | ##e.g. ['ef36eefae77f82455c0a925167a0d4ca', 'rss', 'Net shortwave surface radiation', 'surface_net_downward_shortwave_flux', 'unset', 'W m-2', 'SPECS_day', 'mv.001.0094', 'mv.001.0394', '', ''] |
---|
34 | ##ee_xref |
---|
35 | ##hash, uuid list -- list of mip tables for each parameter key (ee_old) |
---|
36 | |
---|
37 | ee_xref = {} |
---|
38 | ee_xref_bck = collections.defaultdict( list ) |
---|
39 | ee_miptables_old = collections.defaultdict( list ) |
---|
40 | ee_miptables_vars_old = collections.defaultdict( list ) |
---|
41 | ee_ovars_old = collections.defaultdict( list ) |
---|
42 | ee_mip = {} |
---|
43 | ee0 = {} |
---|
44 | eenv = {} |
---|
45 | ## dictionary to carry short names of old variables. |
---|
46 | eeold = {} |
---|
47 | ###for ee,shv in [(ee0,sh),(eenv,shnv),(eeold,shold)]: |
---|
48 | for ee,shv in [(ee0,sh),(eenv,shnv)]: |
---|
49 | for k in shv.keys(): |
---|
50 | ee[k] = shv[k] |
---|
51 | |
---|
52 | rmod = 'csv' |
---|
53 | eeoldl = collections.defaultdict( list ) |
---|
54 | if rmod == 'xls': |
---|
55 | for ee,fn in [(ee_xref,'vlsc5_xref_v20150826.xls'),(ee_mip,'vlsc5_mip_v20150826.xls'),(eeold,'vlsc5b_v20150826.xls')]: |
---|
56 | wb = workbook( fn ) |
---|
57 | sh = wb.book.sheet_by_name( 'Sheet1' ) |
---|
58 | for i in range(sh.nrows): |
---|
59 | r = map( lambda x: string.strip(str(uniCleanFunc(x.value))), sh.row(i) ) |
---|
60 | ee[r[0]] = r |
---|
61 | else: |
---|
62 | for ee,fn,sep,xxx in [(ee_xref,'vlsc5_xref_v20150826.csv',',',False),(ee_mip,'vlsc5_mip_v20150826.csv', '\t',False),(eeoldl,'vlsc5b_v20150826.csv', '\t',True)]: |
---|
63 | for i in open(fn).readlines(): |
---|
64 | r = map( lambda x: string.strip(str(uniCleanFunc(x))), string.split( string.strip(i), sep ) ) |
---|
65 | while len(r) < 9: |
---|
66 | r.append( '' ) |
---|
67 | if xxx: |
---|
68 | ee[r[0]].append( r ) |
---|
69 | else: |
---|
70 | ee[r[0]] = r |
---|
71 | |
---|
72 | |
---|
73 | for h in eeoldl.keys(): |
---|
74 | if len( eeoldl[h] ) == 1: |
---|
75 | eeold[h] = eeoldl[h][0] |
---|
76 | else: |
---|
77 | ll = [] |
---|
78 | for k in range( len( eeoldl[h] ) ): |
---|
79 | if eeoldl[h][k][8] == '': |
---|
80 | ll.append(k) |
---|
81 | if len(ll) == 0: |
---|
82 | print 'ERROR.004.0001: no unforwarded record in vlsc5b for h=%s' % h |
---|
83 | eeold[h] = eeoldl[h][0] |
---|
84 | elif len(ll) > 1: |
---|
85 | print 'ERROR.004.0002: multiple unforwarded records in vlsc5b for h=%s (%s)' % (h,eeoldl[h][ll[0]][1]) |
---|
86 | eeold[h] = eeoldl[h][ll[0]] |
---|
87 | else: |
---|
88 | eeold[h] = eeoldl[h][ll[0]] |
---|
89 | |
---|
90 | k2m = [] |
---|
91 | km = [] |
---|
92 | for k in ee_xref.keys(): |
---|
93 | if eeold.has_key(k): |
---|
94 | for k2 in ee_xref[k][1:]: |
---|
95 | if k2 != '': |
---|
96 | if ee_mip.has_key(k2): |
---|
97 | ee_xref_bck[k2].append(k) |
---|
98 | else: |
---|
99 | k2m.append( (k2,k) ) |
---|
100 | else: |
---|
101 | km.append(k) |
---|
102 | |
---|
103 | ## |
---|
104 | ## had bug here due to extension of ee_mip to include dimensions. |
---|
105 | ## |
---|
106 | ee_byvar = collections.defaultdict(list) |
---|
107 | for k in ee_mip.keys(): |
---|
108 | tid = ee_mip[k][-2] |
---|
109 | ee_miptables_old[tid].append( ee_mip[k][0] ) |
---|
110 | ee_miptables_vars_old[tid].append( ee_mip[k][-3] ) |
---|
111 | ee_ovars_old['%s.%s' % (tid,ee_mip[k][-3]) ].append( k ) |
---|
112 | ee_byvar[ ee_mip[k][-3] ].append( k ) |
---|
113 | |
---|
114 | ## check cmip5 vs. standard output |
---|
115 | s1 = Set() |
---|
116 | for k in ee_miptables_vars_old.keys(): |
---|
117 | if k[:6] == 'CMIP5_': |
---|
118 | s2 = Set( ee_miptables_vars_old[k] ) |
---|
119 | kk = k[6:] |
---|
120 | s1.add( kk ) |
---|
121 | assert cmip5so.so.has_key(kk ), 'Table %s not found in standard output' % kk |
---|
122 | lnf = [] |
---|
123 | for v in ee_miptables_vars_old[k]: |
---|
124 | if not cmip5so.so[kk].a.has_key(v): |
---|
125 | lnf.append(v) |
---|
126 | if len(lnf) > 0: |
---|
127 | print 'ERROR.010.0001: Variables not found in %s:: %s' % (kk,str(lnf)) |
---|
128 | |
---|
129 | lnf = [] |
---|
130 | for v in cmip5so.so[kk].a.keys(): |
---|
131 | if not v in s2: |
---|
132 | lnf.append(v) |
---|
133 | if len(lnf) > 0: |
---|
134 | print 'ERROR.010.0002: SO Variables not found in %s:: %s' % (kk,str(lnf)) |
---|
135 | |
---|
136 | for s in cmip5so.so.keys(): |
---|
137 | if not s in s1: |
---|
138 | print 'ERROR.010.0002: Table %s not in old mip tables' % s |
---|
139 | |
---|
140 | for s in shlist: |
---|
141 | s.close() |
---|