1 | """Import data request from shelves and pu onto compliant XML document. |
---|
2 | ------------------------------------------------------------------------ |
---|
3 | A new uuid is generated for elements of the "revisedtabitem" section/table (because it is not present in the shelve). |
---|
4 | |
---|
5 | NOTES |
---|
6 | ----- |
---|
7 | July 27th: there is an unresolved problem about preserving uuids when updates are provided in source format ..... |
---|
8 | """ |
---|
9 | |
---|
10 | ### Experiment group dictionary added, and experiment set of named tuples -- making it possible to identify experiments in each mip and group. |
---|
11 | ### Should be able to use this in review to make reference groups ... |
---|
12 | |
---|
13 | ### |
---|
14 | ### Appear to be getting core variables into a clean format |
---|
15 | ### Some filtering to avoit duplicates arising from two approaches to OMIP, flagged as ERROR.090 in standard output. |
---|
16 | ### Corrected by omitting 2nd reference |
---|
17 | ### |
---|
18 | |
---|
19 | import xlrd, xml, os, sys, string, shelve, json |
---|
20 | import utils_wb, uuid |
---|
21 | import xml.dom, xml.dom.minidom |
---|
22 | import collections, string, re |
---|
23 | import choiceCfg |
---|
24 | import importWbMods |
---|
25 | import dreq_utils |
---|
26 | from utils_wb import uniCleanFunc |
---|
27 | import ivg |
---|
28 | |
---|
29 | mipTabMaps = json.load( open('inputs/mipTableMaps.txt', 'r' ) ) |
---|
30 | |
---|
31 | #up: |
---|
32 | varPositive = {'rlus':'up', 'rlut':'up', 'rlutcs':'up', 'hfls':'up', 'hfss':'up', 'mcu':'up', 'rsus':'up', 'rsut':'up', 'rsutcs':'up', 'rlds':'down', 'rsds':'down' } |
---|
33 | |
---|
34 | refMips = ['AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] |
---|
35 | ## |
---|
36 | ## refCmv.refUid[tab][lab] = k |
---|
37 | ## |
---|
38 | refCmv = ivg.refCmv() |
---|
39 | extRefCmvMode = 'r' |
---|
40 | extRefCmv = shelve.open( 'sh/extRefCmv' ) |
---|
41 | extRefCmvInv = {} |
---|
42 | nd = 0 |
---|
43 | sd = set() |
---|
44 | sd2 = set() |
---|
45 | for k in extRefCmv: |
---|
46 | k1 = extRefCmv[k] |
---|
47 | tab,var = k.split( '.' ) |
---|
48 | if k1 in extRefCmvInv: |
---|
49 | tab2, var2 = extRefCmvInv[k1].split('.') |
---|
50 | if var == var2 and (tab == mipTabMaps.get( tab2, None) or tab2 == mipTabMaps.get( tab, None) ): |
---|
51 | pass |
---|
52 | else: |
---|
53 | print 'FATAL: Duplicate key in extRefCmv: %s [%s] %s' % (k,k1,extRefCmvInv[k1]) |
---|
54 | sd.add(k) |
---|
55 | sd2.add(extRefCmvInv[k1]) |
---|
56 | nd += 1 |
---|
57 | extRefCmvInv[k1] = k |
---|
58 | assert nd == 0, 'FATAL ERRORS FOUND ... see standard output: %s, %s' % (str( sd ), str( sd2 ) ) |
---|
59 | |
---|
60 | extRefCmvMapped = {} |
---|
61 | def chkRefCmv( ntab, lab, u0, msg ): |
---|
62 | useExtRef = True |
---|
63 | thattab = mipTabMaps.get( ntab, ntab ) |
---|
64 | thistab = ntab |
---|
65 | if thistab in refCmv.refUid and lab in refCmv.refUid[thistab]: |
---|
66 | uid = refCmv.refUid[thistab][lab] |
---|
67 | print 'INFO.uid0.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,uid) |
---|
68 | if uid != u0: |
---|
69 | extRefCmvMapped[u0] = uid |
---|
70 | elif thattab in refCmv.refUid and lab in refCmv.refUid[thattab]: |
---|
71 | uid = refCmv.refUid[thattab][lab] |
---|
72 | print 'INFO.uid1.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,uid) |
---|
73 | if uid != u0: |
---|
74 | extRefCmvMapped[u0] = uid |
---|
75 | else: |
---|
76 | kk = str( '%s.%s' % (thistab,lab) ) |
---|
77 | kkk = str( '%s.%s' % (thattab,lab) ) |
---|
78 | if kk in extRefCmv: |
---|
79 | if useExtRef: |
---|
80 | uid = extRefCmv[kk] |
---|
81 | print 'INFO.uid.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,uid) |
---|
82 | if uid != u0: |
---|
83 | extRefCmvMapped[u0] = uid |
---|
84 | else: |
---|
85 | u1 = extRefCmv[kk] |
---|
86 | print 'INFO.uid.%s: uid in extRefCmv ignored: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,u1) |
---|
87 | uid = u0 |
---|
88 | elif kkk in extRefCmv: |
---|
89 | if useExtRef: |
---|
90 | uid = extRefCmv[kkk] |
---|
91 | print 'INFO.uid4.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,uid) |
---|
92 | if uid != u0: |
---|
93 | extRefCmvMapped[u0] = uid |
---|
94 | else: |
---|
95 | u1 = extRefCmv[kkk] |
---|
96 | print 'INFO.uid4.%s: uid in extRefCmv ignored: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,u1) |
---|
97 | uid = u0 |
---|
98 | else: |
---|
99 | if extRefCmvMode != 'r': |
---|
100 | extRefCmv[kk] = u0 |
---|
101 | print 'WARN.uid.%s: uid not found: %s, %s (%s)' % (msg,thistab,lab,u0) |
---|
102 | uid = u0 |
---|
103 | return uid |
---|
104 | |
---|
105 | |
---|
106 | siSurfFluxes = ['sidmasstranx', 'sidmasstrany', 'siforcecoriolx', 'siforcecorioly', 'siforcetiltx', 'siforcetilty', 'siforceintstrx', 'siforceintstry', 'sistrxdtop', 'sistrydtop', 'sistrxdbot', 'sistrydbot', 'siu', 'siv'] |
---|
107 | oceanVars = ['fg13co2','fg14co2'] |
---|
108 | realmTab = {'landice':['LImongre','LImonant','LIyrgre','LIyrant','LIfxgre','LIfxant','ImonGre','ImonAnt','IyrGre','IyrAnt','IfxGre','IfxAnt'], |
---|
109 | 'atmos':['6hrLev','6hrPlev','6hrPevpt','em1hrclimmon','em3hr','em1hr','em3hrpt','emDayZ','emDaypt','emMonZ']} |
---|
110 | ## emYr, emMon, emFy, emMonZ .... need checking |
---|
111 | knownRealmA = [ 'LS3MIP [LWday]', 'LS3MIP [LEday]', 'CFMIP [cf1hrClimMon]', 'HighResMIP [3hr_cloud]', 'CFMIP [cf3hr_sim_new]', 'C4MIP [L_3hr]', 'DAMIP [DAMIP_day]', 'DAMIP [DAMIP_3hr_p2]', 'DynVar [DYVR_daily_c]', 'PMIP [PMIP-6hr]', 'HighResMIP [1hrLev]'] |
---|
112 | knownRealmO = [ 'DAMIP [DAMIP_Omon_p2]', 'FAFMIP [fafOyr]'] |
---|
113 | varRealmMaps = {'land':['areacellg','orog'], |
---|
114 | 'landIce land':['snc'], |
---|
115 | 'landIce':['icem', 'sftflf', 'sftgrf','acabf','snicefreez', 'snicem', 'hfgeoubed', 'lithk', 'topg','areacelli','lialb', 'libmassbffl', 'libmassbfgr', 'licalvf', 'lifmassbf', 'litempbotfl', 'litempbotgr', 'litemptop', 'mrroIs','strbasemag', 'topg', 'xvelbase', 'xvelmean', 'xvelsurf', 'yvelbase', 'yvelmean', 'yvelsurf', 'zvelbase', 'zvelsurf','snmIs','sncIs','tsnIs','sbl'], |
---|
116 | 'atmos':['prra','prsn','prrc','sbl','tas','prCrop', 'tasmaxCrop', 'tasminCrop','hfls', 'hfss','rlds', 'rlus', 'rsds', 'rsus'] } |
---|
117 | varRealmMapsInv = {} |
---|
118 | for k in varRealmMaps: |
---|
119 | for v in varRealmMaps[k]: |
---|
120 | varRealmMapsInv[ v ] = k |
---|
121 | |
---|
122 | class replaceItems(object): |
---|
123 | def __init__(self): |
---|
124 | self.v = collections.defaultdict( list ) |
---|
125 | self.items = {} |
---|
126 | |
---|
127 | WbMods = importWbMods.run() |
---|
128 | tableUtils = ivg.tableUtils() |
---|
129 | |
---|
130 | nt__trace = collections.namedtuple( 'trace', ['GROUPITEM_ta','SPATIALSHAPE_01'] ) |
---|
131 | trace = nt__trace( False, False ) |
---|
132 | |
---|
133 | tab2freq = {u'CMIP5_cfOff':'subhr', u'CORDEX_mon':'mon', u'SPECS_day':'day', u'CMIP5_day':'day', \ |
---|
134 | u'PMIP3_OImon':'mon', u'CORDEX_day':'day', u'CMIP5_LImon':'mon', u'CMIP5_OImon':'mon', \ |
---|
135 | u'CMIP5_Lmon':'mon', u'CMIP5_3hr':'3hr', u'CMIP5_Omon':'mon', u'PMIP3_OIclim':'monClim', \ |
---|
136 | u'PMIP3_fx':'fx', u'CORDEX_fx':'fx', u'PMIP3_LImon':'mon', u'CMIP5_6hrPlev':'6hr', u'PMIP3_Lmon':'mon', \ |
---|
137 | u'PMIP3_Amon':'mon', u'SPECS_Omon':'mon', u'CCMI1_fixed':'fx', u'PMIP3_Aclim':'monClim', u'CMIP5_6hrLev':'6hr', \ |
---|
138 | u'CMIP5_Oclim':'monClim', u'PMIP3_LIclim':'monClim', u'CCMI1_monthly':'mon', u'CMIP5_fx':'fx', \ |
---|
139 | u'CMIP5_cfDay':'day', u'CORDEX_6h':'6hr', u'PMIP3_day':'day', u'SPECS_OImon':'mon', u'CMIP5_cfMon':'mon', \ |
---|
140 | u'CORDEX_sem':'monClim', u'SPECS_6hr':'6hr', u'CMIP5_cfSites':'subhr', u'CCMI1_hourly':'hr', u'CMIP5_aero':'mon', \ |
---|
141 | u'CMIP5_Amon':'mon', u'PMIP3_Omon':'mon', u'CCMI1_daily':'day', u'SPECS_fx':'fx', u'PMIP3_Lclim':'monClim', \ |
---|
142 | u'aermonthly':'mon', u'aermonthlyz':'mon', u'aerhourly':'hr', 'aerfixed':'fx', 'aerdaily':'day', |
---|
143 | u'DCPP-day':'day', \ |
---|
144 | u'Odec':'dec', \ |
---|
145 | u'PMIP3_Oclim':'monClim', u'SPECS_Amon':'mon', u'SPECS_Lmon':'mon', u'CMIP5_cf3hr':'3hr', u'CORDEX_3h':'3hr', \ |
---|
146 | 'OImon':'mon', 'Omon_oth':'mon', 'Omon':'mon', 'Oyr':'yr', 'Omon_3d':'mon', \ |
---|
147 | u'CCMI1_annual':'yr', u'CMIP5_Oyr':'yr', \ |
---|
148 | 'Oclim':'monClim', \ |
---|
149 | 'cfMon':'mon', 'emMonZ':'mon', \ |
---|
150 | '3hr':'3hr', 'cfOff':'subhr', 'OMIP-Omon':'mon', '6hrLev':'6hr', 'fx':'fx', 'OMIP-Oyr':'yr', 'cfDay':'day', 'Lmon':'day', \ |
---|
151 | u'cf3hr':'3hr', u'Amon':'mon', u'aero':'mon', u'aero_3d':'mon', u'6hrPlev':'6hr', u'aero_oth':'mon', \ |
---|
152 | u'cf3hr_grid':'3hr', u'LImon':'mon', u'cfSites':'subhr', u'day_oth':'day', u'cfDay_2d':'day', u'day':'day' } |
---|
153 | |
---|
154 | freq2tab = collections.defaultdict( list ) |
---|
155 | for k in tab2freq: |
---|
156 | freq2tab[tab2freq[k]].append( k ) |
---|
157 | |
---|
158 | tableNameMap = {'CMIP5_3hr':'3hr', 'CMIP5_6hrLev':'6hrLev', 'CMIP5_6hrPlev':'6hrPlev', 'CMIP5_Amon':'Amon', 'CMIP5_LImon':'LImon', 'CMIP5_Lmon':'Lmon', 'CMIP5_OImon':'CMIP5_OImon', 'CMIP5_Oclim':'Oclim', 'CMIP5_Omon':'CMIP5_Omon', 'CMIP5_aero':'aero', 'CMIP5_cf3hr':'cf3hr', 'CMIP5_cfDay':'cfDay', 'CMIP5_cfMon':'cfMon', 'CMIP5_cfOff':'cfOff', 'CMIP5_cfSites':'cfsites', 'CMIP5_day':'day', 'CMIP5_fx':'fx', 'OMIP.Omon':'Omon', 'OMIP.Oyr':'Oyr', 'OMIP.day':'Oday', 'OMIP.fx':'Ofx' } |
---|
159 | |
---|
160 | reportedRedundantTables = set() |
---|
161 | redundantTables = ['aero','CMIP5_Oyr','CMIP5_Omon','CMIP5_OImon','CORDEX_3h', 'CORDEX_6h', 'CORDEX_fx', 'CORDEX_mon', 'CORDEX_sem', 'PMIP3_Aclim', 'PMIP3_Amon', 'PMIP3_LIclim', 'PMIP3_LImon', 'PMIP3_Lclim', 'PMIP3_Lmon', 'PMIP3_OIclim', 'PMIP3_OImon', 'PMIP3_Oclim', 'PMIP3_Omon', 'PMIP3_day', 'PMIP3_fx', |
---|
162 | 'SPECS_Lmon', 'SPECS_Amon', 'SPECS_6hr', 'SPECS_day', 'SPECS_fx', 'SPECS_Omon', 'SPECS_OImon', |
---|
163 | 'CORDEX_day', |
---|
164 | 'CCMI1_annual','CCMI1_daily','CCMI1_fixed','CCMI1_hourly','CCMI1_monthly'] |
---|
165 | |
---|
166 | ## mapping of variable groups representing tables .... |
---|
167 | mappedTables = {'Omon':'OMIP-Omon', 'OMIP-Oyr':'Oyr', 'OImon':'SIMIP-seaicemon' } |
---|
168 | mappedTables = {'Omon':'OMIP-Omon', 'CMIP5-Oyr':'OMIP-Oyr', 'OImon':'SIMIP-seaicemon', 'CMIP5-Omon':'OMIP-Omon' } |
---|
169 | ## mappedTables = { 'OImon':'SIMIP-seaicemon' } |
---|
170 | ## 'aeromonthly':'aermonthly' } |
---|
171 | |
---|
172 | freqmap = {'daily':'day', 'Annual':'yr', 'Timestep':'subhr', '1day':'day', '1mon':'mon', 'month':'mon', 'year':'yr', 'monthly':'mon', 'Day':'day', '6h':'6hr', '3 hourly':'3hr', '3 Hourly':'3hr' } |
---|
173 | |
---|
174 | nt_expt = collections.namedtuple( 'expt', ['uid', 'label', 'mip', 'egid' ] ) |
---|
175 | realmdef = { '__all__':['CFMIP [cfMonExtra]', 'PMIP [PMIP-Omon]', 'LS3MIP [LCmon]', 'C4MIP [C_Ocean_T2]', 'C4MIP [C_Ocean_T1]', 'LUMIP [Lyr_Lut]', 'PMIP [PMIP-LIclim]', 'HighResMIP [1hr_strat]', 'SIMIP [seaiceday]', 'C4MIP [C_LandT1]', 'CFMIP [cfDay_2d_new]', 'HighResMIP [6hrPlev_intense]', 'HighResMIP [Amon_diag]', 'DCPP [DCPP-mon]', 'HighResMIP [Amon_conv]', 'SIMIP [seaicemon]', 'DAMIP [new_monthly]', 'FAFMIP [fafOyrB]', 'HighResMIP [Amon_ext]', 'DynVar [DYVR_daily]', 'PMIP [PMIP-Aclim]', 'HighResMIP [3hr_extr]', 'C4MIP [C_Basic]', 'DCPP [DCPP-day]', 'CFMIP [cfMon_3dstd_new]', 'FAFMIP [fafOmon]', 'ISMIP6 [new_fx]', 'LS3MIP [LEday]', 'PMIP [PMIP-aeroclim]', 'PMIP [PMIP-aero]', 'DCPP [DCPP-6hr]', 'DynVar [DYVR_monthly]', 'ISMIP6 [icesheetmon]', 'CFMIP [aeroDay_2d]', 'ISMIP6 [new_LImon]', 'ISMIP6 [new_Omon]', 'CFMIP [cfDayExtra]', 'LS3MIP [LWday]', 'PMIP [PMIP-Oclim]', 'ISMIP6 [icesheetyear]', 'LUMIP [Lmon_Lut]', 'HighResMIP [1ts]', 'PMIP [PMIP-OIclim]', 'HighResMIP [6hrPlev_extr_dr]', 'VolMIP [VIRF]', 'C4MIP [C_LandT2]', 'C4MIP [C_Hist]', 'FAFMIP [fafOmonB]', 'RFMIP [aero_irf]', 'PMIP [PMIP-Lmon]', 'C4MIP [L_day]', 'PMIP [PMIP-Lclim]', 'PMIP [PMIP-Amon]', 'GeoMIP [aeroGeo]', 'HighResMIP [6hrPlev_extr]', 'PMIP [PMIP-day]', 'HighResMIP [Amon_sparc]', 'CFMIP [cfSites_new]', 'CFMIP [cf3hr_sim]'], |
---|
176 | 'atmos':['CFMIP [cfMonExtra]', 'HighResMIP [1hr_strat]', 'CFMIP [cfDay_2d_new]', 'HighResMIP [6hrPlev_intense]', 'HighResMIP [Amon_diag]', 'DCPP [DCPP-mon]', 'HighResMIP [Amon_conv]', 'DAMIP [new_monthly]', 'HighResMIP [Amon_ext]', 'DynVar [DYVR_daily]', 'PMIP [PMIP-Aclim]', 'HighResMIP [3hr_extr]', 'DCPP [DCPP-day]', 'CFMIP [cfMon_3dstd_new]', 'PMIP [PMIP-aeroclim]', 'PMIP [PMIP-aero]', 'DCPP [DCPP-6hr]', 'DynVar [DYVR_monthly]', 'CFMIP [aeroDay_2d]', 'CFMIP [cfDayExtra]', 'HighResMIP [1ts]', 'HighResMIP [6hrPlev_extr_dr]', 'VolMIP [VIRF]', 'RFMIP [aero_irf]', 'PMIP [PMIP-Amon]', 'GeoMIP [aeroGeo]', 'HighResMIP [6hrPlev_extr]', 'PMIP [PMIP-day]', 'HighResMIP [Amon_sparc]', 'CFMIP [cfSites_new]', 'CFMIP [cf3hr_sim]'], |
---|
177 | 'ocean':['PMIP [PMIP-Omon]', 'C4MIP [C_Ocean_T1]', 'C4MIP [C_Ocean_T2]', 'FAFMIP [fafOyrB]', 'FAFMIP [fafOmon]', 'ISMIP6 [new_Omon]', 'PMIP [PMIP-OIclim]', 'FAFMIP [fafOmonB]', 'PMIP [PMIP-Oclim]'], |
---|
178 | 'land':['LS3MIP [LCmon]', 'C4MIP [C_Hist]', 'C4MIP [L_day]', 'PMIP [PMIP-Lclim]', 'C4MIP [C_Basic]', 'LUMIP [Lmon_Lut]', 'LUMIP [Lyr_Lut]', 'C4MIP [C_LandT1]', 'PMIP [PMIP-Lmon]', 'C4MIP [C_LandT2]'], |
---|
179 | 'seaIce':['SIMIP [seaicemon]', 'SIMIP [seaiceday]' ], |
---|
180 | 'landIce':['ISMIP6 [new_LImon]', 'PMIP [PMIP-LIclim]', 'ISMIP6 [icesheetyear]', 'ISMIP6 [new_fx]', 'ISMIP6 [icesheetmon]'], |
---|
181 | '__unset__':['LS3MIP [LWday]', 'LS3MIP [LEday]'] } |
---|
182 | |
---|
183 | realmdefix = {} |
---|
184 | for k in realmdef.keys(): |
---|
185 | if k != '__all__': |
---|
186 | for s in realmdef[k]: |
---|
187 | assert not realmdefix.has_key(s), 'Duplicate targ value in realmdef: %s [%s]' % (s,k) |
---|
188 | realmdefix[s] = k |
---|
189 | |
---|
190 | for k in realmdef['__all__']: |
---|
191 | assert realmdefix.has_key(k), 'Key %s not found in realmdefix' % k |
---|
192 | |
---|
193 | class cls_s1(object): |
---|
194 | def __init__(self): |
---|
195 | self.a = collections.defaultdict( list ) |
---|
196 | |
---|
197 | def mycc(s): |
---|
198 | s = string.replace( s, '&', '' ) |
---|
199 | s = string.replace( s, '.', '-' ) |
---|
200 | if len(s) > 0 and s[-1] == '+': |
---|
201 | s = s[:-1] + 'ETC' |
---|
202 | ## camelcase the input string |
---|
203 | s1 = string.lower( string.replace(string.strip(str( s) ), '-', ' ') ) |
---|
204 | return string.replace( string.capwords( s1 ), ' ', '' ) |
---|
205 | |
---|
206 | empty=re.compile('^$') |
---|
207 | |
---|
208 | def test( x,m): |
---|
209 | if not x: |
---|
210 | print m |
---|
211 | return x |
---|
212 | |
---|
213 | class lcm(object): |
---|
214 | def __init__(self,a,b): |
---|
215 | self.a = {} |
---|
216 | self.b = {} |
---|
217 | for i in a: |
---|
218 | self.a[string.lower(i)] = i |
---|
219 | for i in b: |
---|
220 | self.b[string.lower(i)] = i |
---|
221 | |
---|
222 | class main(object): |
---|
223 | |
---|
224 | def __init__(self,src,rq,doRepl=False,run=False,schemaMode='dreq'): |
---|
225 | self.schemaMode = schemaMode |
---|
226 | self.requestlinkuid = set() |
---|
227 | self.src=src |
---|
228 | self.vgcheck = {} |
---|
229 | self.rqlPreset = {} |
---|
230 | fok = [test(os.path.isfile(src),'%s not found' % src), ] |
---|
231 | assert all( fok), 'Required input file(s) missing' |
---|
232 | self.fix = WbMods.fix |
---|
233 | self.spsh = WbMods.ls |
---|
234 | self.spsh.addTime() |
---|
235 | self.cmDims = WbMods.cm |
---|
236 | self.cmvMap002 = dict() |
---|
237 | self.importWbMods = WbMods |
---|
238 | self.rq = rq |
---|
239 | self.deferredLabMap = {} |
---|
240 | self.struidmapextra = {} |
---|
241 | |
---|
242 | ## groupItems:grps |
---|
243 | for sh in [rq.expt,rq.exgp,rq.objec,rq.refti, rq.revti, rq.grps, rq.rqli, rq.rqit]: |
---|
244 | print sh['__info__'] |
---|
245 | |
---|
246 | self.repl = collections.defaultdict( list ) |
---|
247 | self.replItems = {} |
---|
248 | self.rep = {'cmv':replaceItems()} |
---|
249 | ##self.deferredRequestLinkClone = collections.defaultdict( set ) |
---|
250 | |
---|
251 | self.err0010 = collections.defaultdict( int ) |
---|
252 | self.defaultP = {} |
---|
253 | self.remo = {} |
---|
254 | self.upda = {} |
---|
255 | self.insert = {} |
---|
256 | self.rqLnks = collections.OrderedDict() |
---|
257 | self.exptSet = set() |
---|
258 | self.exptGrpDict = {} |
---|
259 | |
---|
260 | self.objectives = collections.defaultdict( dict ) |
---|
261 | self.objectiveLinks = collections.defaultdict( cls_s1 ) |
---|
262 | self.experiments = {} |
---|
263 | self.experimentGrp = {} |
---|
264 | self.exptPntUid = {} |
---|
265 | self.exptPnt = collections.defaultdict( list ) |
---|
266 | self.mips = set() |
---|
267 | self.dsortdd = {} |
---|
268 | |
---|
269 | if doRepl: |
---|
270 | self.importRepl() |
---|
271 | self.importRepl(rfile='CMVreplace.csv',mode='cmv') |
---|
272 | self.importRemove() |
---|
273 | self.importUpdate() |
---|
274 | self.importInsert() |
---|
275 | |
---|
276 | self.e15_10 = 0 |
---|
277 | self.doc = xml.dom.minidom.parse( self.src ) |
---|
278 | self.vocabs = xml.dom.minidom.parse( 'vocabs.xml' ) |
---|
279 | self.skey = {'experiment':rq.expt, 'exptgroup':rq.exgp, \ |
---|
280 | 'objective':rq.objec, 'var':None,'ovar':rq.refti, 'groupitem':rq.grps, \ |
---|
281 | 'revisedtabitem_xxx':rq.revti, 'requestlink':rq.rqli, 'requestitem':rq.rqit, \ |
---|
282 | 'requestvargroup':rq.rqvg, 'tablesection':rq.rqsect, 'requestvar':rq.revti} |
---|
283 | |
---|
284 | self.newImport = False |
---|
285 | self.prepVar() |
---|
286 | self.writeTimeSlice() |
---|
287 | self.writeMip() |
---|
288 | self.writeMcfg() |
---|
289 | self.writeProcNotes() |
---|
290 | self.writeCmDim() |
---|
291 | self.writeRvg() |
---|
292 | self.prep(rq) |
---|
293 | self.cmvAudit(0) |
---|
294 | self.prepRequestItem() |
---|
295 | self.reviewExpt() |
---|
296 | if run: |
---|
297 | self.run(rq) |
---|
298 | self.writeRelations() |
---|
299 | self.writeRvx() |
---|
300 | if self.schemaMode == 'dreq2': |
---|
301 | self.write2() |
---|
302 | self.writeMipTables() |
---|
303 | self.writeCellMethods() |
---|
304 | self.writeVar() |
---|
305 | self.writeSn() |
---|
306 | self.finish() |
---|
307 | self.finish2() |
---|
308 | extRefCmv.close() |
---|
309 | |
---|
310 | def prepRequestItem(self): |
---|
311 | thissh = self.skey['requestitem'] |
---|
312 | ## 'mip', 'tab', 'expt','rlid','ny', 'nexmax', 'nenmax', 'nymax','treset','info' |
---|
313 | s1 = collections.defaultdict( set ) |
---|
314 | for k in thissh: |
---|
315 | if k[0] != '_': |
---|
316 | ll = thissh[k][:] |
---|
317 | ku = str(ll[2]) |
---|
318 | s1[ll[0]].add(ll[9]) |
---|
319 | if len( ku) == 1: |
---|
320 | print 'SEVERE.rqi.00088: ',ll |
---|
321 | ## |
---|
322 | self.exptPnt[ku].append( str(k) ) |
---|
323 | |
---|
324 | thissh = self.skey['experiment'] |
---|
325 | for k in thissh: |
---|
326 | if k[0] != '_': |
---|
327 | ll = thissh[k][:] |
---|
328 | thisl = str( ll[2] ) |
---|
329 | if ll[4] in ['SolarMIP']: |
---|
330 | print 'ERROR: %s experiment: %s' % (ll[4],str(ll)) |
---|
331 | else: |
---|
332 | self.mips.add( str( ll[4] ) ) |
---|
333 | self.experiments[thisl] = str( ll[0] ) |
---|
334 | self.exptSet.add( nt_expt._make( [str( ll[0] ), thisl, str( ll[4] ), str( ll[1] ) ] ) ) |
---|
335 | |
---|
336 | thissh = self.skey['exptgroup'] |
---|
337 | for k in thissh: |
---|
338 | if k[0] != '_': |
---|
339 | ll = thissh[k][:] |
---|
340 | thisl = str( ll[1] ) |
---|
341 | self.exptGrpDict[thisl] = str( ll[1] ) |
---|
342 | self.experimentGrp[thisl] = str( ll[0] ) |
---|
343 | |
---|
344 | mm = [] |
---|
345 | for m in self.mips: |
---|
346 | if m not in self.mipUids: |
---|
347 | mm.append(m) |
---|
348 | assert len(mm) == 0, 'Not all mips found: %s: %s' % (str(mm),str(self.mipUids)) |
---|
349 | |
---|
350 | def writeProcNotes(self): |
---|
351 | xx = self.vocabs.getElementsByTagName( 'processingOptions' ) |
---|
352 | assert len(xx) == 1, 'Expecting one element named "processingOptions", found %s' % len(xx) |
---|
353 | iDoc = xx[0] |
---|
354 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
---|
355 | thiss = self.doc.getElementsByTagName( 'tags' )[0] |
---|
356 | dil = thiss.getElementsByTagName('item') |
---|
357 | for d in dil: |
---|
358 | thiss.removeChild(d) |
---|
359 | for i in iDoc.getElementsByTagName('item'): |
---|
360 | item = self.doc.createElement( 'item' ) |
---|
361 | thisl = i.getAttribute( 'label' ) |
---|
362 | for k in ['title','description','label']: |
---|
363 | item.setAttribute( k, i.getAttribute( k ) ) |
---|
364 | uid = 'TAG:%s' % thisl |
---|
365 | item.setAttribute( 'uid', uid ) |
---|
366 | thiss.appendChild( item ) |
---|
367 | |
---|
368 | def writeRelations(self): |
---|
369 | thiss = self.doc.getElementsByTagName( 'varRelations' )[0] |
---|
370 | dil = thiss.getElementsByTagName('item') |
---|
371 | for d in dil: |
---|
372 | thiss.removeChild(d) |
---|
373 | for i in self.fix.relations: |
---|
374 | item = self.doc.createElement( 'item' ) |
---|
375 | for j in range( len( self.fix.relationinfo ) ): |
---|
376 | item.setAttribute( self.fix.relationinfo[j], i[j] ) |
---|
377 | thiss.appendChild( item ) |
---|
378 | |
---|
379 | thiss = self.doc.getElementsByTagName( 'varRelLnk' )[0] |
---|
380 | dil = thiss.getElementsByTagName('item') |
---|
381 | for d in dil: |
---|
382 | thiss.removeChild(d) |
---|
383 | cc = collections.defaultdict(int) |
---|
384 | for i in self.fix.relationlinks: |
---|
385 | item = self.doc.createElement( 'item' ) |
---|
386 | tvp = i[1] |
---|
387 | ix = cc[tvp] |
---|
388 | cc[tvp] += 1 |
---|
389 | for j in range( len( self.fix.relationlinkinfo ) ): |
---|
390 | item.setAttribute( self.fix.relationlinkinfo[j], i[0][j] ) |
---|
391 | |
---|
392 | if tvp in self.cmvUidByTVP: |
---|
393 | vid = self.cmvUidByTVP[ tvp ][ix] |
---|
394 | item.setAttribute( 'rid', vid ) |
---|
395 | print 'INFO.093.00011: found: %s' % str(tvp) |
---|
396 | else: |
---|
397 | t,v,p = tvp |
---|
398 | if ('CMIP5_%s' % t,v,p) in self.cmvUidByTVP: |
---|
399 | if ix >= len(self.cmvUidByTVP[ ('CMIP5_%s' % t,v,p) ] ): |
---|
400 | print 'WARN.093.00011b: not found: %s, %s' % (str(tvp),ix) |
---|
401 | else: |
---|
402 | vid = self.cmvUidByTVP[ ('CMIP5_%s' % t,v,p) ][ix] |
---|
403 | item.setAttribute( 'rid', vid ) |
---|
404 | print 'INFO.093.00011a: found: %s' % str(tvp) |
---|
405 | elif t[-2:] in ['pt','zm'] and ( t[:-2],v,p) in self.cmvUidByTVP: |
---|
406 | vid = self.cmvUidByTVP[ ( t[:-2],v,p) ][ix] |
---|
407 | item.setAttribute( 'rid', vid ) |
---|
408 | print 'INFO.093.00011a: found: %s' % str(tvp) |
---|
409 | else: |
---|
410 | print 'WARN.093.00011: not found: %s' % str(tvp) |
---|
411 | thiss.appendChild( item ) |
---|
412 | |
---|
413 | def writeCellMethods(self): |
---|
414 | thiss = self.doc.getElementsByTagName( 'cellMethods' )[0] |
---|
415 | dil = thiss.getElementsByTagName('item') |
---|
416 | for d in dil: |
---|
417 | thiss.removeChild(d) |
---|
418 | ss = set() |
---|
419 | for k in WbMods.cellm.cminfo.keys(): |
---|
420 | t,l,u,d = WbMods.cellm.cminfo[k] |
---|
421 | item = self.doc.createElement( 'item' ) |
---|
422 | item.setAttribute( 'label', l ) |
---|
423 | item.setAttribute( 'title', t ) |
---|
424 | item.setAttribute( 'cell_methods', k ) |
---|
425 | item.setAttribute( 'uid', u ) |
---|
426 | if d != None: |
---|
427 | item.setAttribute( 'description', d ) |
---|
428 | thiss.appendChild( item ) |
---|
429 | |
---|
430 | def writeMipTables(self): |
---|
431 | self.cfgUids = set() |
---|
432 | xx = self.vocabs.getElementsByTagName( 'table' ) |
---|
433 | assert len(xx) == 1, 'Expecting one element named "table", found %s' % len(xx) |
---|
434 | iDoc = xx[0] |
---|
435 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
---|
436 | thiss = self.doc.getElementsByTagName( 'miptable' )[0] |
---|
437 | dil = thiss.getElementsByTagName('item') |
---|
438 | for d in dil: |
---|
439 | thiss.removeChild(d) |
---|
440 | ss = set() |
---|
441 | for i in iDoc.getElementsByTagName('item'): |
---|
442 | thisl = i.getAttribute( 'label' ) |
---|
443 | thisll = string.replace( thisl, '_', '-' ) |
---|
444 | if thisl in self.tables: |
---|
445 | item = self.doc.createElement( 'item' ) |
---|
446 | ss.add(thisl) |
---|
447 | item.setAttribute( 'label', thisll ) |
---|
448 | item.setAttribute( 'uid', 'MIPtable::%s' % thisl ) |
---|
449 | for k in ['title','description','frequency']: |
---|
450 | item.setAttribute( k, i.getAttribute( k ) ) |
---|
451 | thiss.appendChild( item ) |
---|
452 | else: |
---|
453 | print 'WARN.mtid.002: ignoring mip table record for %s' % thisl |
---|
454 | for lab in self.tables: |
---|
455 | if lab not in ss: |
---|
456 | thisll = string.replace( lab, '_', '-' ) |
---|
457 | print 'WARN.mtid.001: creating dummy mip table record for %s' % lab |
---|
458 | item = self.doc.createElement( 'item' ) |
---|
459 | item.setAttribute( 'label', thisll ) |
---|
460 | item.setAttribute( 'title', lab ) |
---|
461 | item.setAttribute( 'frequency', '???' ) |
---|
462 | item.setAttribute( 'uid', 'MIPtable::%s' % lab ) |
---|
463 | thiss.appendChild( item ) |
---|
464 | |
---|
465 | ################################################################## |
---|
466 | |
---|
467 | def writeMcfg(self): |
---|
468 | self.cfgUids = set() |
---|
469 | xx = self.vocabs.getElementsByTagName( 'modelConfig' ) |
---|
470 | assert len(xx) == 1, 'Expecting one element named "modelConfig", found %s' % len(xx) |
---|
471 | iDoc = xx[0] |
---|
472 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
---|
473 | thiss = self.doc.getElementsByTagName( 'modelConfig' )[0] |
---|
474 | dil = thiss.getElementsByTagName('item') |
---|
475 | for d in dil: |
---|
476 | thiss.removeChild(d) |
---|
477 | for i in iDoc.getElementsByTagName('item'): |
---|
478 | item = self.doc.createElement( 'item' ) |
---|
479 | thisl = i.getAttribute( 'label' ) |
---|
480 | for k in ['title','MIPs','usage','range','type']: |
---|
481 | item.setAttribute( k, i.getAttribute( k ) ) |
---|
482 | uid = str( thisl ) |
---|
483 | item.setAttribute( 'uid', uid ) |
---|
484 | item.setAttribute( 'label', uid ) |
---|
485 | self.cfgUids.add( uid ) |
---|
486 | thiss.appendChild( item ) |
---|
487 | |
---|
488 | def writeCmDim(self): |
---|
489 | thiss = self.doc.getElementsByTagName( 'grids' )[0] |
---|
490 | dil = thiss.getElementsByTagName('item') |
---|
491 | for d in dil: |
---|
492 | thiss.removeChild(d) |
---|
493 | kk = ['tables','label','altLabel','description','standardName','title','axis','units','isIndex','coords','bounds','direction','valid_min','valid_max','type','positive','value','boundsValues','requested','boundsRequested','tolRequested','isGrid' ] |
---|
494 | for j in self.cmDims.ss.keys(): |
---|
495 | item = self.doc.createElement( 'item' ) |
---|
496 | for i in range(len(kk)): |
---|
497 | if kk[i] in ['valid_min','valid_max']: |
---|
498 | if str( self.cmDims.ss[j][i] ) != '': |
---|
499 | item.setAttribute( kk[i], str( self.cmDims.ss[j][i] ) ) |
---|
500 | else: |
---|
501 | item.setAttribute( kk[i], str( self.cmDims.ss[j][i] ) ) |
---|
502 | uid = 'dim:%s' % str( self.cmDims.ss[j][1] ) |
---|
503 | item.setAttribute( 'uid', uid ) |
---|
504 | thiss.appendChild( item ) |
---|
505 | |
---|
506 | def writeVar(self): |
---|
507 | thiss = self.doc.getElementsByTagName( 'var' )[0] |
---|
508 | dil = thiss.getElementsByTagName('item') |
---|
509 | for d in dil: |
---|
510 | thiss.removeChild(d) |
---|
511 | #<item description="dummyAt" id="001.001.001" label="example-01" procComment="dummyAt" procnote="dummyAt" prov="dummyAt" provmip="dummyAt" sn="dummyAt" title="dummy title string" uid="fa349c44-9ccb-11e5-8176-5404a60d96b5" units="dummyAt"/> |
---|
512 | kk= ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp','replaced_by'] |
---|
513 | self.usedSn = set() |
---|
514 | for uid in self.importWbMods.vars.d1: |
---|
515 | item = self.doc.createElement( 'item' ) |
---|
516 | thisr = self.importWbMods.vars.d1[uid] |
---|
517 | sn = thisr[7] |
---|
518 | self.usedSn.add( sn ) |
---|
519 | if sn.find( ' ' ) != -1: |
---|
520 | print 'WARN.sn.00001: space in sn: ',sn |
---|
521 | thisr[7] = sn.replace( ' ', '_' ) |
---|
522 | if sn.find( '?' ) != -1: |
---|
523 | print 'WARN.sn.00002: question in sn: ',sn |
---|
524 | thisr[7] = sn.replace( '?', '' ) |
---|
525 | |
---|
526 | for i in range( len(kk) -2 ): |
---|
527 | item.setAttribute( kk[i], thisr[i] ) |
---|
528 | if len( self.importWbMods.vars.c1[ thisr[0] ] ) == 1: |
---|
529 | id = thisr[0] |
---|
530 | else: |
---|
531 | id = '%s.%s' % (thisr[0],self.importWbMods.vars.c1[ thisr[0] ].index( uid ) ) |
---|
532 | print 'INFO.090.00050: setting id for non-unique variable name: %s' % id |
---|
533 | item.setAttribute( 'id', id ) |
---|
534 | thiss.appendChild( item ) |
---|
535 | |
---|
536 | def writeRvg(self): |
---|
537 | thiss = self.doc.getElementsByTagName( 'requestVarGroup' )[0] |
---|
538 | dil = thiss.getElementsByTagName('item') |
---|
539 | for d in dil: |
---|
540 | thiss.removeChild(d) |
---|
541 | #<item description="dummyAt" id="001.001.001" label="example-01" procComment="dummyAt" procnote="dummyAt" prov="dummyAt" provmip="dummyAt" sn="dummyAt" title="dummy title string" uid="fa349c44-9ccb-11e5-8176-5404a60d96b5" units="dummyAt"/> |
---|
542 | kk= ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp','replaced_by'] |
---|
543 | kk= ['uid', 'label', 'title', 'mip', 'ref', 'refNote'] |
---|
544 | for uid in self.importWbMods.vg.vg: |
---|
545 | item = self.doc.createElement( 'item' ) |
---|
546 | thisr = self.importWbMods.vg.vg[uid] |
---|
547 | for i in range( len(kk)): |
---|
548 | item.setAttribute( kk[i], thisr[i] ) |
---|
549 | ##item.setAttribute( 'id', id ) |
---|
550 | thiss.appendChild( item ) |
---|
551 | |
---|
552 | def writeRvx(self): |
---|
553 | thiss = self.doc.getElementsByTagName( 'requestVar' )[0] |
---|
554 | |
---|
555 | ## 'vid', 'title', 'label', 'priority', 'vgid', 'mip' |
---|
556 | cols = self.rq.rqvarx['__cols__'] |
---|
557 | nn = 0 |
---|
558 | for k in self.rq.rqvarx.keys(): |
---|
559 | if k[0] != '_': |
---|
560 | rec = self.rq.rqvarx[k] |
---|
561 | print 'INFO.writeRvx.01001: ',k,rec |
---|
562 | item = self.doc.createElement( 'item' ) |
---|
563 | item.setAttribute( 'uid', k ) |
---|
564 | for i in range(6): |
---|
565 | item.setAttribute( cols[i], rec[i] ) |
---|
566 | vid = item.getAttribute( 'vid' ) |
---|
567 | |
---|
568 | if vid in self.cmvMap002: |
---|
569 | vid = self.cmvMap002[vid] |
---|
570 | item.setAttribute( 'vid', vid ) |
---|
571 | if vid in extRefCmvMapped: |
---|
572 | vid = extRefCmvMapped[vid] |
---|
573 | item.setAttribute( 'vid', vid ) |
---|
574 | |
---|
575 | if vid not in self.cmvByUid: |
---|
576 | print 'ERROR.fixcmv.0005: attempt to use invalid vid (writeRvx): ',vid,k,rec |
---|
577 | thiss.appendChild( item ) |
---|
578 | nn += 1 |
---|
579 | print 'INFO.writeRvx.00001: %s records written' % nn |
---|
580 | |
---|
581 | def writeSn(self): |
---|
582 | import loadcf |
---|
583 | cf = loadcf.cf() |
---|
584 | thiss = self.doc.getElementsByTagName( 'standardname' )[0] |
---|
585 | dil = thiss.getElementsByTagName('item') |
---|
586 | for d in dil: |
---|
587 | thiss.removeChild(d) |
---|
588 | |
---|
589 | for i in sorted( cf.names.keys() ): |
---|
590 | d,u = cf.names[i] |
---|
591 | self.appendSnItem(thiss,i,d,u) |
---|
592 | |
---|
593 | for i in sorted( cf.alias.keys() ): |
---|
594 | if i in self.usedSn: |
---|
595 | print 'WARN.aliassn.00001: standard name alias: %s' % i |
---|
596 | sn = cf.alias[i] |
---|
597 | d,u = cf.names[sn] |
---|
598 | self.appendSnItem(thiss,i,d,u) |
---|
599 | |
---|
600 | def appendSnItem(self,thiss,sn,desc,units): |
---|
601 | item = self.doc.createElement( 'item' ) |
---|
602 | uid = str(sn) |
---|
603 | if uid.find( ' ' ) != -1: |
---|
604 | print 'SEVERE.sn.00001: BAD SN: %s' % uid |
---|
605 | uid = uid.replace( ' ', '' ) |
---|
606 | title = string.capwords( string.replace( uid, '_', ' ' ) ) |
---|
607 | label = string.replace( title, ' ', '' ) |
---|
608 | for w in ['Of','In','On','By']: |
---|
609 | ww = ' %s ' % w |
---|
610 | title = title.replace( ww, string.lower( ww ) ) |
---|
611 | item.setAttribute( 'uid', uid ) |
---|
612 | item.setAttribute( 'label', label ) |
---|
613 | item.setAttribute( 'title', title ) |
---|
614 | item.setAttribute( 'units', str(units) ) |
---|
615 | item.setAttribute( 'description', uniCleanFunc( desc ) ) |
---|
616 | thiss.appendChild( item ) |
---|
617 | |
---|
618 | def writeTimeSlice(self): |
---|
619 | xx = self.vocabs.getElementsByTagName( 'timeSlice' ) |
---|
620 | assert len(xx) == 1, 'Expecting one element named "activity", found %s' % len(xx) |
---|
621 | iDoc = xx[0] |
---|
622 | thiss = self.doc.getElementsByTagName( 'timeSlice' )[0] |
---|
623 | dil = thiss.getElementsByTagName('item') |
---|
624 | for d in dil: |
---|
625 | thiss.removeChild(d) |
---|
626 | |
---|
627 | for i in iDoc.getElementsByTagName('item'): |
---|
628 | item = self.doc.createElement( 'item' ) |
---|
629 | label = i.getAttribute( 'label' ) |
---|
630 | for k in ['label','title','type']: |
---|
631 | item.setAttribute( k, i.getAttribute( k ) ) |
---|
632 | for k in ['child']: |
---|
633 | if i.getAttribute( k ) != '': |
---|
634 | item.setAttribute( k, i.getAttribute( k ) ) |
---|
635 | for k in ['start', 'end', 'step', 'sliceLen', 'nyears']: |
---|
636 | if i.getAttribute( k ) != '': |
---|
637 | v = str( int( float( i.getAttribute( k ) ) ) ) |
---|
638 | item.setAttribute( k, v ) |
---|
639 | item.setAttribute( 'uid', '_slice_%s' % label ) |
---|
640 | thiss.appendChild( item ) |
---|
641 | |
---|
642 | def writeMip(self): |
---|
643 | self.mipUids = set() |
---|
644 | xx = self.vocabs.getElementsByTagName( 'activity' ) |
---|
645 | assert len(xx) == 1, 'Expecting one element named "activity", found %s' % len(xx) |
---|
646 | mipDoc = xx[0] |
---|
647 | ## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/> |
---|
648 | thiss = self.doc.getElementsByTagName( 'mip' )[0] |
---|
649 | dil = thiss.getElementsByTagName('item') |
---|
650 | for d in dil: |
---|
651 | thiss.removeChild(d) |
---|
652 | for i in mipDoc.getElementsByTagName('item'): |
---|
653 | item = self.doc.createElement( 'item' ) |
---|
654 | mip = i.getAttribute( 'label' ) |
---|
655 | for k in ['title','url']: |
---|
656 | item.setAttribute( k, i.getAttribute( k ) ) |
---|
657 | uid = string.replace( mip, ' ', '' ) |
---|
658 | item.setAttribute( 'uid', uid ) |
---|
659 | item.setAttribute( 'label', uid ) |
---|
660 | self.mipUids.add( uid ) |
---|
661 | thiss.appendChild( item ) |
---|
662 | |
---|
663 | if "DECK" not in self.mipUids: |
---|
664 | item = self.doc.createElement( 'item' ) |
---|
665 | item.setAttribute( 'uid', 'DECK' ) |
---|
666 | item.setAttribute( 'url', '' ) |
---|
667 | item.setAttribute( 'label', 'DECK' ) |
---|
668 | item.setAttribute( 'title', 'DECK - set of standard CMIP runs' ) |
---|
669 | self.mipUids.add( "DECK" ) |
---|
670 | thiss.appendChild( item ) |
---|
671 | |
---|
672 | def prepVar(self): |
---|
673 | thissh = self.skey['var'] |
---|
674 | thatsh = self.skey['ovar'] |
---|
675 | self.varsxx = None |
---|
676 | self.varsxxx = None |
---|
677 | dothis = False |
---|
678 | if self.newImport: |
---|
679 | #['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov', 'priority0', 'rlm0'] |
---|
680 | self.vars = collections.defaultdict( list ) |
---|
681 | for k in thissh: |
---|
682 | if k[0] != '_': |
---|
683 | ll = thissh[k][:] |
---|
684 | thisl = str( ll[2] ) |
---|
685 | self.vars[thisl].append( k ) |
---|
686 | else: |
---|
687 | if dothis: |
---|
688 | self.varsxx = collections.defaultdict( list ) |
---|
689 | for k in thissh: |
---|
690 | if k[0] != '_': |
---|
691 | ll = thissh[k][:] |
---|
692 | self.varsxx[str( ll[0] )].append( k ) |
---|
693 | else: |
---|
694 | self.varsxxx = collections.defaultdict( list ) |
---|
695 | for k in thatsh: |
---|
696 | if k[0] != '_': |
---|
697 | ll = thatsh[k][:] |
---|
698 | thisl = str( ll[21] ) |
---|
699 | self.varsxxx[thisl].append( k ) |
---|
700 | |
---|
701 | def prep(self,rq): |
---|
702 | ## keys here match section element names in schema |
---|
703 | #for k in [ 'var','ovar','groupitem', 'revisedtabitem']: |
---|
704 | ## lists for 3 new sections |
---|
705 | ### uuid, spid, tmid, cell_methods, cell_measures, description, procNote, prov |
---|
706 | structure = [] |
---|
707 | self.structInfo = {} |
---|
708 | ### uuid, label, dimensions, shape, levels, levelFlag |
---|
709 | spatialshp = [] |
---|
710 | ### uuid, label, dimension, tstyle |
---|
711 | temporalshp = [] |
---|
712 | ### cmor variables |
---|
713 | self.cmv = [] |
---|
714 | self.rqvList = [] |
---|
715 | self.rqvgList = [] |
---|
716 | self.cmvUidByFL = collections.defaultdict( cls_s1 ) |
---|
717 | self.cmvUidByTab = collections.defaultdict( cls_s1 ) |
---|
718 | self.cmvUidByTVP = collections.defaultdict( list ) |
---|
719 | self.cmvByLab = collections.defaultdict( dict ) |
---|
720 | self.cmvs = set() |
---|
721 | self.cmvinfo = ('uid','stid','label','title','description','vid', \ |
---|
722 | 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', \ |
---|
723 | 'deflate', 'valid_min', 'flag_values', 'modeling_realm', 'valid_max', 'positive', 'mipTable', \ |
---|
724 | 'prov', 'provNote', 'frequency', 'rowIndex', 'coordinates','defaultPriority','mipTableSection','processing') |
---|
725 | self.cmvinfo2 = ('uid','stid','label','title','description','vid', \ |
---|
726 | 'deflate_level', 'shuffle', 'type', \ |
---|
727 | 'deflate', 'modeling_realm', 'positive', 'mipTable', 'mipTableSection', \ |
---|
728 | 'prov', 'provNote', 'frequency', 'rowIndex', 'defaultPriority','processing') |
---|
729 | self.rqvinfo = ('uid','label','vid','gpid','priority','mip','table' ) |
---|
730 | |
---|
731 | |
---|
732 | thissh = self.skey['var'] |
---|
733 | self.varUidByName = self.importWbMods.vars.c1 |
---|
734 | ##self.varUidByName = collections.defaultdict( list ) |
---|
735 | if self.newImport: |
---|
736 | oo = open('refDefaultP.txt', 'w' ) |
---|
737 | for k in thissh: |
---|
738 | if k[0] != '_': |
---|
739 | ##['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov', 'priority0'] |
---|
740 | thisuuid = str(k) |
---|
741 | self.defaultP[thisuuid] = thissh[k][-1] |
---|
742 | oo.write( '%s\t%s\t%s\t%s\n' % (k, thissh[k][0], thissh[k][2], thissh[k][-1] ) ) |
---|
743 | ##self.varUidByName[thissh[k][0]].append( thisuuid ) |
---|
744 | oo.close() |
---|
745 | else: |
---|
746 | ## ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp'] |
---|
747 | self.vars = collections.defaultdict( list ) |
---|
748 | self.vars1 = collections.defaultdict( list ) |
---|
749 | for uid in self.importWbMods.vars.d1: |
---|
750 | self.defaultP[uid] = self.importWbMods.vars.d1[uid][-1] |
---|
751 | ## collection indexed by short_name |
---|
752 | self.vars1[self.importWbMods.vars.d1[uid][0]].append( uid ) |
---|
753 | ## collection indexed by standard_name |
---|
754 | self.vars[self.importWbMods.vars.d1[uid][7]].append( uid ) |
---|
755 | |
---|
756 | ### request variables |
---|
757 | self.rqv = [] |
---|
758 | self.realmByVar = collections.defaultdict( list ) |
---|
759 | for ksect in [ 'ovar','groupitem']: |
---|
760 | thissh = self.skey[ksect] |
---|
761 | if ksect == 'ovar': |
---|
762 | setu1 = set() |
---|
763 | kk = ['uid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid', 'processing', 'gpid','rowIndex','ssect', 'priority'] |
---|
764 | id = kk.index('dimensions') |
---|
765 | ivid = kk.index('vid') |
---|
766 | thisnt = collections.namedtuple( 'ovar', kk ) |
---|
767 | ov = [] |
---|
768 | dset = set() |
---|
769 | ds2 = set() |
---|
770 | ovuu = collections.defaultdict( set ) |
---|
771 | ovll = collections.defaultdict( list ) |
---|
772 | for k in thissh: |
---|
773 | if k[0] != '_': |
---|
774 | rr = thissh[k][:] |
---|
775 | rr[-3] = int( rr[-3] ) |
---|
776 | if rr[id] == 's|c|a|l|a|r': |
---|
777 | print 'ERROR.060.0001: bad dimension string detected and corrected: %s' % rr[id] |
---|
778 | rr[id] = '' |
---|
779 | if string.find( rr[23], 'plevs' ) != -1: |
---|
780 | print 'INFO.qqqq01: ',rr[21:24] |
---|
781 | rr[23] = string.replace(rr[23], 'plevs', 'plev19') |
---|
782 | thisdim = rr[id] |
---|
783 | if thisdim.find( ' ' ) != -1: |
---|
784 | thisdim = '|'.join( [x.strip() for x in thisdim.split() ] ) |
---|
785 | rr[id] = thisdim |
---|
786 | |
---|
787 | tt = thisnt._make( rr ) |
---|
788 | if tt.uid in setu1: |
---|
789 | print 'ERROR.099.0002: duplicate uuid: %s: %s' % (uuid,str(rr)) |
---|
790 | if str(tt.vid) == '__var_not_found_012__': |
---|
791 | print 'ERROR.099.00102: bad vid string: %s [%s]' % (tt.vid,tt.out_name) |
---|
792 | |
---|
793 | if tt.out_name == 'clisccp': |
---|
794 | print 'INFO.clisccp.00002: ',rr,k |
---|
795 | if tt.out_name == 'intppcalc': |
---|
796 | print 'INFO.intppcalc.00002: ',rr,k |
---|
797 | if tt.out_name == 'ch4': |
---|
798 | print 'INFO.ch4.00002: ',rr,k |
---|
799 | |
---|
800 | if str(tt.vid) not in self.importWbMods.vars.d1: |
---|
801 | if tt.out_name in self.importWbMods.vars.c1: |
---|
802 | if len( self.importWbMods.vars.c1[ tt.out_name ] ) == 1: |
---|
803 | rr[ivid] = self.importWbMods.vars.c1[ tt.out_name ][0] |
---|
804 | tt = thisnt._make( rr ) |
---|
805 | print 'INFO.099.00103: bad vid string replaced: %s [%s]' % (tt.vid,tt.out_name) |
---|
806 | else: |
---|
807 | print 'ERROR.099.00104: ambiguous bad vid string: %s [%s]' % (tt.vid,tt.out_name) |
---|
808 | else: |
---|
809 | print 'ERROR.099.00105: bad vid string: %s [%s]' % (tt.vid,tt.out_name) |
---|
810 | setu1.add(tt.uid) |
---|
811 | ov.append( tt ) |
---|
812 | |
---|
813 | dset.add( thisdim ) |
---|
814 | if tt.out_name in ['al','albs']: |
---|
815 | print 'INFO.albs.00001: ',thisdim, rr |
---|
816 | if tt.dimensions == 'location|time1': |
---|
817 | print 'INFO.dimensions.00001',rr |
---|
818 | if tt.out_name == 'msftmyz': |
---|
819 | print 'INFO.msftmyz.00002: ',self.cmv[-1] |
---|
820 | |
---|
821 | ds2.add( (tt.dimensions,tt.cell_methods,tt.cell_measures,tt.flag_values, tt.flag_meanings) ) |
---|
822 | ovuu[tt.vid].add( tt.modeling_realm ) |
---|
823 | ovll[tt.vid].append( tt ) |
---|
824 | if rr[22][-3:] in ['mon','day','3hr']: |
---|
825 | fr = rr[22][-3:] |
---|
826 | elif rr[22][-2:] in [ 'yr','fx']: |
---|
827 | fr = rr[22][-2:] |
---|
828 | else: |
---|
829 | fr = tab2freq[rr[22]] |
---|
830 | |
---|
831 | prov = '%s ((isd.003))' % tt.mipTable |
---|
832 | if tt.mipTable[:4] == 'OMIP': |
---|
833 | prov = 'CMIP6 [OMIP] ((isd.001))' |
---|
834 | elif string.find( tt.mipTable, '_' ) != -1: |
---|
835 | prov = '%s [%s] ((isd.002))' % tuple( string.split( tt.mipTable, '_' ) ) |
---|
836 | |
---|
837 | if tt.var[:5] == 'xx..ccmi_': |
---|
838 | pass |
---|
839 | assert False, 'blind pass disabled .....' |
---|
840 | else: |
---|
841 | lab = tt.var |
---|
842 | if tt.out_name != tt.var: |
---|
843 | print 'WARN.outname.00001: out name issue.... ',tt.out_name,tt.var |
---|
844 | if lab in {'*','include Oyr 3D tracers'}: |
---|
845 | lab = '-copy-' |
---|
846 | print 'ERROR.copy.00090: ',tt |
---|
847 | ## |
---|
848 | ## insert conditional on redundant tables here ... |
---|
849 | ## |
---|
850 | if tt.mipTable in redundantTables: |
---|
851 | if tt.mipTable not in reportedRedundantTables: |
---|
852 | print 'INFO.000.11111: redundant table spotted: ',tt.mipTable |
---|
853 | reportedRedundantTables.add( tt.mipTable ) |
---|
854 | else: |
---|
855 | if lab != '-copy-' and string.find( lab, '-' ) != -1: |
---|
856 | print 'ERROR.207.00010: bad character in label: %s' % lab |
---|
857 | lab = string.replace( lab, '-', '' ) |
---|
858 | if tt.ssect != '': |
---|
859 | ssect,sectid = tt.ssect |
---|
860 | else: |
---|
861 | ssect = '' |
---|
862 | sectid = None |
---|
863 | |
---|
864 | uid = chkRefCmv( tt.mipTable, lab, tt.uid, 'cmv.01009' ) |
---|
865 | if uid in self.cmvs: |
---|
866 | print 'ERROR.090.09010: duplicate cmv uid %s : %s' % (uid, str(tt.__dict__) ) |
---|
867 | self.cmvs.add( uid ) |
---|
868 | self.cmv.append( (uid, (tt.dimensions,tt.cell_methods,tt.cell_measures,tt.flag_values,tt.flag_meanings), \ |
---|
869 | lab, tt.title, tt.comment, tt.vid, \ |
---|
870 | tt.deflate_level, tt.shuffle, tt.ok_max_mean_abs, tt.flag_meanings, tt.type, tt.ok_min_mean_abs, \ |
---|
871 | tt.deflate, tt.valid_min, tt.flag_values, tt.modeling_realm, tt.valid_max, tt.positive, tt.mipTable, \ |
---|
872 | prov, ssect, fr, tt.rowIndex, '',int(tt.priority), ssect, tt.processing) ) |
---|
873 | if tt.ok_max_mean_abs != '': |
---|
874 | try: |
---|
875 | xx = float( tt.ok_max_mean_abs ) |
---|
876 | except: |
---|
877 | print 'FAILED to take float of ok_max_mean_abs -.....' |
---|
878 | print tt.__dict__ |
---|
879 | raise |
---|
880 | if tt.out_name == 'intppcalc': |
---|
881 | print 'INFO.intppcalc.00003: ',self.cmv[-1] |
---|
882 | if tt.out_name == 'msftmyz': |
---|
883 | print 'INFO.msftmyz.00003: ',self.cmv[-1] |
---|
884 | if tt.processing != '': |
---|
885 | print 'INFO.processing.0001: ',lab,tt.title,tt.processing |
---|
886 | |
---|
887 | self.realmByVar[ lab ].append( tt.modeling_realm ) |
---|
888 | vgid = tt.gpid |
---|
889 | self.rqv.append( (str( uuid.uuid1() ),lab, uid, vgid, tt.priority,'CMIP5',tt.mipTable) ) |
---|
890 | if sectid not in (None, '_na_'): |
---|
891 | self.rqv.append( (str( uuid.uuid1() ),lab, uid, sectid, tt.priority,'CMIP5',tt.mipTable) ) |
---|
892 | if lab == 'somint': |
---|
893 | print 'INFO.somint.0002:', self.rqv[-1] |
---|
894 | if lab == 'sisnconc': |
---|
895 | print 'INFO.sisnconc.0002:', self.rqv[-1] |
---|
896 | thisd = {} |
---|
897 | for k9 in range(len(self.rqvinfo)): |
---|
898 | thisd[self.rqvinfo[k9]] = self.rqv[-1][k9] |
---|
899 | self.rqvList.append( (thisd, 'prep:thisd' ) ) |
---|
900 | |
---|
901 | self.dsort( dset ) |
---|
902 | print self.dsortdd[ksect][0].keys() |
---|
903 | |
---|
904 | elif ksect == 'groupitem': |
---|
905 | kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'priority', 'mip', 'uid', 'oldv', 'rowIndex', 'cmorv', 'new', 'gpid', 'vkey', 'vid'] |
---|
906 | thisnt = collections.namedtuple( 'groupitem', kk ) |
---|
907 | il = kk.index( 'levels' ) |
---|
908 | iu = kk.index( 'uid' ) |
---|
909 | idx = kk.index( 'descriptionEx' ) |
---|
910 | ixvid = kk.index( 'vid' ) |
---|
911 | ixvar = kk.index( 'var' ) |
---|
912 | ixcmv = kk.index( 'cmorv' ) |
---|
913 | ixp = kk.index( 'priority' ) |
---|
914 | dset = set() |
---|
915 | dsgpi = set() |
---|
916 | gi = [] |
---|
917 | s1 = set() |
---|
918 | sm = set() |
---|
919 | ## |
---|
920 | nov = 0 |
---|
921 | setu = set() |
---|
922 | for k in thissh.keys(): |
---|
923 | if k[0] != '_': |
---|
924 | rr = thissh[k][:] |
---|
925 | if trace.GROUPITEM_ta and rr[ixvar] == 'ta': |
---|
926 | print 'INFO.045.00004: groupitem: ',rr |
---|
927 | |
---|
928 | if rr[ixvar] == 'snc': |
---|
929 | print 'INFO.snc.0010:',rr |
---|
930 | ## |
---|
931 | ## convert floats back to integer |
---|
932 | ## |
---|
933 | if type(rr[ixp]) == type(1.): |
---|
934 | rr[ixp] = int(rr[ixp]) |
---|
935 | ## |
---|
936 | if rr[ixcmv] == '': |
---|
937 | rr[ixcmv] = rr[1] |
---|
938 | tt = thisnt._make( rr ) |
---|
939 | |
---|
940 | if tt.var == 'hus': |
---|
941 | print 'INFO.hus.0001: ',tt.__dict__ |
---|
942 | if str(tt.mip) != 'OMIP': |
---|
943 | gi.append( tt ) |
---|
944 | sm.add( tt.mask ) |
---|
945 | |
---|
946 | smdd = {} |
---|
947 | dsn = {'floating ice shelf':u'floating ice sheet', u'ocean':u'sea', 'yes':'seaIce' } |
---|
948 | d1 = {u'where land over land':(False,'area: mean where land over land'), \ |
---|
949 | u'land':(False,'area: mean where land'), \ |
---|
950 | u'hourly max':(False,'area: mean time: mean within hours time: maximum over hours'), \ |
---|
951 | u'grounded ice sheet':(True,'area: mean where grounded_ice_sheet'), \ |
---|
952 | u'ice sheet':(True,'area: mean where ice_sheet'), \ |
---|
953 | u'floating ice sheet':(True,'area: mean where floating_ice_shelf'), \ |
---|
954 | u'floating land ice':(True,'area: mean where floating_ice_shelf'), \ |
---|
955 | u'where snow on sea ice':(True,'area: mean where snow over sea_ice'), \ |
---|
956 | u'where melt ponds':(True,'area: mean where melt_pond over sea_ice'), \ |
---|
957 | u'where ridge':(True,'area: mean where sea_ice_ridge over sea_ice'), \ |
---|
958 | u'where sea ice':(True,'area: mean where sea_ice'), \ |
---|
959 | u'floating ice shelf':(True,'area: mean where floating_ice_sheet'), \ |
---|
960 | u'seaIce':(True,'area: mean where sea_ice over sea'), \ |
---|
961 | u'where sea ice over sea':(True,'area: mean where sea_ice over sea'), \ |
---|
962 | u'sea':(False,'area: mean where sea') } |
---|
963 | |
---|
964 | for s in sm: |
---|
965 | if s in ['', 'no', u'none',u'Southern hemisphere', u'Northern hemisphere']: |
---|
966 | smdd[s] = (False,'') |
---|
967 | else: |
---|
968 | s1 = string.strip(s) |
---|
969 | try: |
---|
970 | smdd[s] = d1[ dsn.get( s1,s1 ) ] |
---|
971 | except: |
---|
972 | print s, dsn.get( s1,s ) |
---|
973 | raise |
---|
974 | |
---|
975 | eeu = dict() |
---|
976 | |
---|
977 | ccchk = ivg.checkCmv(gi) |
---|
978 | nrep = 0 |
---|
979 | |
---|
980 | for tt in gi: |
---|
981 | tabsect = '' |
---|
982 | gr = 'a' |
---|
983 | if tt.mip == 'OMIP' \ |
---|
984 | or any( [string.find(tt.table,i) != -1 for i in ['Omon','Oyr','Oclim','OImon','Ocean'] ] ) \ |
---|
985 | or any( [string.find(tt.group,i) != -1 for i in ['Omon','Oyr','Oclim','OImon','Ocean'] ] ) \ |
---|
986 | or tt.cmorv in oceanVars: |
---|
987 | gr = 'o' |
---|
988 | elif tt.cmorv in siSurfFluxes: |
---|
989 | gr = "M" |
---|
990 | print 'INFO.simip.00010: ', tt.cmorv |
---|
991 | |
---|
992 | if self.varsxx != None: |
---|
993 | if tt.cmorv in self.varsxx: |
---|
994 | sss = set() |
---|
995 | for k in self.varsxx[tt.cmorv]: |
---|
996 | sss.add(self.skey['var'][k][-1] ) |
---|
997 | sss = list(sss) |
---|
998 | if len(sss) == 1: |
---|
999 | if sss[0] == 'ocean': |
---|
1000 | gr = 'o' |
---|
1001 | elif sss[0] == 'atmos': |
---|
1002 | gr = 'a' |
---|
1003 | print 'INFO:lllw:', tt.cmorv, sss, gr |
---|
1004 | elif self.varsxxx != None: |
---|
1005 | if tt.cmorv in self.varsxxx: |
---|
1006 | sss = set() |
---|
1007 | for k in self.varsxxx[tt.cmorv]: |
---|
1008 | sss.add(self.skey['ovar'][k][21] ) |
---|
1009 | sss = list(sss) |
---|
1010 | if len(sss) == 1: |
---|
1011 | if sss[0] == 'ocean': |
---|
1012 | gr = 'o' |
---|
1013 | elif sss[0] == 'atmos': |
---|
1014 | gr = 'a' |
---|
1015 | print 'INFO:lllw:', tt.cmorv, sss, gr |
---|
1016 | |
---|
1017 | |
---|
1018 | if tt.cmorv in ['talknat', 'sltbasin','clisccp','fCLandToOcean']: |
---|
1019 | print 'INFO.llll: ',tt,gr |
---|
1020 | ## |
---|
1021 | ## adding label for mip here messes up referencing later ... |
---|
1022 | ## |
---|
1023 | if tt.mip in [ 'SIMIP','LUMIP']: |
---|
1024 | grprov = tt.mip |
---|
1025 | else: |
---|
1026 | grprov = "" |
---|
1027 | |
---|
1028 | if tt.cmorv[-3:] in ['500','700','200','300','850','000']: |
---|
1029 | print 'INFO.022.0002: Setting p... coordinate for ',tt |
---|
1030 | if tt.cmorv[-5:] in [ 't2000']: |
---|
1031 | thisc = 'depth2000m' |
---|
1032 | elif tt.cmorv[-4:] in [ 't300', 't700']: |
---|
1033 | thisc = 'depth%sm' % tt.cmorv[-3:] |
---|
1034 | elif tt.cmorv[-3:] == '000': |
---|
1035 | assert tt.cmorv[-4:] == '1000',str(tt) |
---|
1036 | thisc = 'p%s' % tt.cmorv[-4:] |
---|
1037 | else: |
---|
1038 | thisc = 'p%s' % tt.cmorv[-3:] |
---|
1039 | thissh = '%s|%s' % (tt.shape, thisc) |
---|
1040 | else: |
---|
1041 | thissh = tt.shape |
---|
1042 | |
---|
1043 | if thissh.find( ':' ) != -1: |
---|
1044 | thissh, realmsuggest = thissh.split( ':' ) |
---|
1045 | elif tt.cmorv in varRealmMapsInv: |
---|
1046 | realmsuggest = varRealmMapsInv[tt.cmorv] |
---|
1047 | else: |
---|
1048 | realmsuggest = None |
---|
1049 | |
---|
1050 | ### need to check where variables are, perhaps, refering to "ovar" |
---|
1051 | ##kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'misc', 'mip', 'uid', 'rowIndex', 'new', 'gpid', 'vkey', 'vid'] |
---|
1052 | prov = '%s [%s]' % (tt.mip,tt.group) |
---|
1053 | provNote = '' |
---|
1054 | lab = tt.cmorv |
---|
1055 | if lab in {'*','include Oyr 3D tracers'}: |
---|
1056 | lab = '-copy-' |
---|
1057 | elif lab == 'wap (omega)': |
---|
1058 | lab = 'wap' |
---|
1059 | provNote = 'wap (omega)' |
---|
1060 | lab = string.replace( string.strip(lab), '_', '-' ) |
---|
1061 | fr = freqmap.get( tt.freq, tt.freq ) |
---|
1062 | assert tt.group not in ['icesheetmon', 'icesheetyear','icesheetfx'], 'Unexpected tt.group: %s' % str(tt) |
---|
1063 | if tt.group in ['LImonant','LImongre','LIyrant','LIyrgre','LIfxant','LIfxgre']: |
---|
1064 | gr = 'g' |
---|
1065 | ntab = tt.group |
---|
1066 | elif tt.group in ['ImonAnt','ImonGre','IyrAnt','IyrGre','IfxAnt','IfxGre']: |
---|
1067 | gr = 'g' |
---|
1068 | ntab = tt.group |
---|
1069 | elif tt.group in ['icesheetmonant','icesheetmongre','icesheetfxant','icesheetfxgre']: |
---|
1070 | gr = 'g' |
---|
1071 | ntab = 'LI' + tt.group[8:] |
---|
1072 | elif tt.group in ['icesheetyearant','icesheetyeargre']: |
---|
1073 | gr = 'g' |
---|
1074 | ntab = 'LIyr' + tt.group[-3:] |
---|
1075 | elif tt.table in ['Omon','Oyr','day'] and tt.shape == '=': |
---|
1076 | ntab = tt.table |
---|
1077 | print 'INFO.table.02002: new request var for existing CMOR var: %s, %s' % (tt.table,lab) |
---|
1078 | else: |
---|
1079 | ntab = 'em%s' % string.capwords( fr ) |
---|
1080 | |
---|
1081 | if fr == '6hr' and tt.levels in ['7h',4.0, '4']: |
---|
1082 | ntab = '6hrPlev' |
---|
1083 | print 'INFO.033.00001: adding to 6hrPlev ....',lab, tt.__dict__ |
---|
1084 | |
---|
1085 | elif tt.gpid in self.importWbMods.vg.vg and self.importWbMods.vg.vg[tt.gpid][1] == 'ISMIP6-LImon-01': |
---|
1086 | ##kk= ['uid', 'label', 'title', 'mip', 'ref', 'refNote'] |
---|
1087 | |
---|
1088 | ntab = 'LImon' |
---|
1089 | tabsect = 'new' |
---|
1090 | print 'INFO.limon.0001: Adding %s to LImon/new' % lab |
---|
1091 | |
---|
1092 | if thissh == 'YZT' and ntab[-1] != 'Z': |
---|
1093 | ntab += 'Z' |
---|
1094 | |
---|
1095 | dset.add( (thissh, tt.levels,tt.tstyle,gr) ) |
---|
1096 | if 'area: mean where ice_sheet' == smdd[tt.mask][1] and gr == 'o': |
---|
1097 | print 'SEVERE.structure.30001: ',tt.__dict__,gr |
---|
1098 | dsgpi.add( (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask], 'None', 'None', grprov) ) |
---|
1099 | # |
---|
1100 | # picking out re-purposed CMIP5 variables |
---|
1101 | # |
---|
1102 | if ovuu.has_key(tt.vid) and len( ovuu[tt.vid] ) == 1: |
---|
1103 | print 'INFO.ovuu.00001: lookup started %s, %s' % (tt.cmorv,tt.group) |
---|
1104 | if len( ovuu[tt.vid] ) > 1: |
---|
1105 | print 'WARN.ovuu.00002: multiple realms: %s, %s: %s' % (tt.cmorv,tt.group, str( ovuu[tt.vid] ) ) |
---|
1106 | tt1 = ovll[tt.vid][0] |
---|
1107 | thisrealm = tt1.modeling_realm |
---|
1108 | ttl = ovll[tt.vid][0].title |
---|
1109 | p1 = ovll[tt.vid][0].priority |
---|
1110 | #tt1 = ovuu[tt.vid] |
---|
1111 | if thisrealm == 'seaIce': |
---|
1112 | ntab = 'OImon' |
---|
1113 | if tt.priority in [1,2,3]: |
---|
1114 | p = tt.priority |
---|
1115 | else: |
---|
1116 | p = int(p1) |
---|
1117 | |
---|
1118 | if gr == 'o' and thisrealm == 'atmos': |
---|
1119 | thisrealm = 'ocean' |
---|
1120 | |
---|
1121 | if string.find( tt.descriptionEx, '//' ) != -1: |
---|
1122 | ttl += ' ' + string.split( tt.descriptionEx, '/' )[0] |
---|
1123 | |
---|
1124 | uid = chkRefCmv( ntab, lab, tt.uid, 'cmv.01001' ) |
---|
1125 | if uid != tt.uid: |
---|
1126 | self.cmvMap002[tt.uid] = uid |
---|
1127 | |
---|
1128 | thiscmv = (uid, (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask],tt1.flag_values,tt1.flag_meanings), |
---|
1129 | lab, ttl, tt.descriptionEx, tt1.vid, \ |
---|
1130 | tt1.deflate_level, tt1.shuffle, tt1.ok_max_mean_abs, tt1.flag_meanings, tt1.type, tt1.ok_min_mean_abs, \ |
---|
1131 | tt1.deflate, tt1.valid_min, tt1.flag_values, thisrealm, tt1.valid_max, tt1.positive, ntab, \ |
---|
1132 | prov, provNote, tt.freq, tt.rowIndex, '', int(p),tabsect, '') |
---|
1133 | |
---|
1134 | if type(tt1.valid_max) == type(1.) and tt1.valid_max <= tt1.valid_min: |
---|
1135 | print 'ERROR.valmx.00001: valid_max below/equal valid_min: ',tt1 |
---|
1136 | self.realmByVar[ lab ].append( thisrealm ) |
---|
1137 | thisrqv = [str( uuid.uuid1() ),lab, uid, tt.gpid, p, tt.mip, tt.group] |
---|
1138 | if lab in [ 'sisnconc','mrro','snc']: |
---|
1139 | print 'INFO.%s.0001a:' % lab, thisrqv |
---|
1140 | nov += 1 |
---|
1141 | else: |
---|
1142 | print 'INFO.ovuu.00003: lookup started %s, %s' % (tt.cmorv,tt.group) |
---|
1143 | thisrealm = '' |
---|
1144 | if realmsuggest != None: |
---|
1145 | thisrealm = realmsuggest |
---|
1146 | elif realmdefix.has_key( prov ): |
---|
1147 | thisrealm = realmdefix[prov] |
---|
1148 | else: |
---|
1149 | print 'WARN:070.00001: realm not found for %s,%s' % (lab,prov) |
---|
1150 | if gr == 'o' and thisrealm == 'atmos': |
---|
1151 | thisrealm = 'ocean' |
---|
1152 | |
---|
1153 | if thisrealm == 'seaIce': |
---|
1154 | if fr == 'mon': |
---|
1155 | ntab = 'OImon' |
---|
1156 | elif fr == 'day': |
---|
1157 | ntab = 'SIday' |
---|
1158 | else: |
---|
1159 | raise |
---|
1160 | |
---|
1161 | if tt.priority in [1,2,3]: |
---|
1162 | p = tt.priority |
---|
1163 | ptr = 1 |
---|
1164 | elif self.defaultP.get( tt.vid, 102) in [1,2,3]: |
---|
1165 | p = self.defaultP.get( tt.vid, 102) |
---|
1166 | ptr = 2 |
---|
1167 | else: |
---|
1168 | p = 1 |
---|
1169 | print 'WARN:070.00002: priority defaulting to 1 for %s,%s' % (lab,prov) |
---|
1170 | ptr = 3 |
---|
1171 | |
---|
1172 | if lab != '-copy-' and string.find( lab, '-' ) != -1: |
---|
1173 | print 'ERROR.207.00020: bad character in label: %s' % lab |
---|
1174 | lab = string.replace( lab, '-', '' ) |
---|
1175 | ttl = '__from %s' % tt.table |
---|
1176 | if lab != '-copy-': |
---|
1177 | if tt.vid in self.importWbMods.vars.d1: |
---|
1178 | ttl = self.importWbMods.vars.d1[tt.vid][1] |
---|
1179 | if string.find( tt.descriptionEx, '//' ) != -1: |
---|
1180 | ttl += ' ' + string.split( tt.descriptionEx, '/' )[0] |
---|
1181 | |
---|
1182 | if ntab in refCmv.refUid[ntab] and lab in refCmv.refUid[ntab]: |
---|
1183 | uid = refCmv.refUid[ntab][lab] |
---|
1184 | else: |
---|
1185 | kk = str( '%s.%s' % (ntab,lab) ) |
---|
1186 | if kk in extRefCmv: |
---|
1187 | uid = extRefCmv[kk] |
---|
1188 | print 'INFO.cmv.01003: uid found in extRedCmv: %s, %s (%s)' % (ntab,lab,uid) |
---|
1189 | else: |
---|
1190 | if tt.uid in extRefCmvInv: |
---|
1191 | print 'SEVERE.cmv.09909: uid issue %s [%s] %s' % (kk,tt.uid,extRefCmvInv[tt.uid]) |
---|
1192 | if extRefCmvMode != 'r': |
---|
1193 | extRefCmv[kk] = tt.uid |
---|
1194 | print 'ERROR.cmv.01003: uid not found: %s, %s (%s)' % (ntab,lab,tt.uid) |
---|
1195 | uid = tt.uid |
---|
1196 | |
---|
1197 | if tt.levels in ['7h',]: |
---|
1198 | if lab.find( tt.levels) == -1: |
---|
1199 | print 'WARN.cmv.00880: extending label %s --> %s (%s)' % (lab,lab+tt.levels, str( tt.__dict__)) |
---|
1200 | lab += tt.levels |
---|
1201 | uid = chkRefCmv( ntab, lab, tt.uid, 'cmv.01011' ) |
---|
1202 | if uid in eeu: |
---|
1203 | print 'WARN.cmv.90001: duplicate cmv uid: ',uid,tt.__dict__,eeu[uid] |
---|
1204 | thiscmv = (uid, (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask], 'None', 'None'), lab, ttl, tt.descriptionEx, tt.vid, \ |
---|
1205 | 0, 0, None, 0, '', None, \ |
---|
1206 | 0,None,0, thisrealm, None, '', ntab, \ |
---|
1207 | prov, provNote, tt.freq, tt.rowIndex, '', str(p), tabsect, '') |
---|
1208 | self.realmByVar[ lab ].append( thisrealm ) |
---|
1209 | |
---|
1210 | thisrqv = [str( uuid.uuid1() ),lab, uid, tt.gpid, p, tt.mip, tt.group] |
---|
1211 | if lab in ['mrro', 'sisnconc','snc']: |
---|
1212 | print 'INFO.%s.0001b:' % lab, thisrqv, ptr, thiscmv |
---|
1213 | if trace.GROUPITEM_ta and lab == 'ta': |
---|
1214 | print 'INFO.045.00005: groupitem: ',thisrqv |
---|
1215 | |
---|
1216 | if ccchk.uidReplace.has_key(thiscmv[0]): |
---|
1217 | nrep += 1 |
---|
1218 | print 'ERROR.cmv.09005: duplicate cmv uuid: %s: %s' % (uid, str(tt.__dict__) ) |
---|
1219 | thisrqv[2] = ccchk.uidReplace[thiscmv[0]] |
---|
1220 | else: |
---|
1221 | if uid in setu1: |
---|
1222 | print 'ERROR.cmv.09001: duplicate cmv uuid: %s: %s ' % (uid,str(tt.__dict__)) |
---|
1223 | elif uid not in eeu: |
---|
1224 | eeu[uid] = tt.__dict__ |
---|
1225 | assert len(thiscmv) == len( self.cmvinfo ), 'BAD CMV specification length: %s' % str(cmv) |
---|
1226 | self.cmv.append( thiscmv ) |
---|
1227 | print 'INFO.cmv.x0001: append cmv: ',tt.__dict__ |
---|
1228 | self.rqv.append( thisrqv ) |
---|
1229 | if thisrqv[1] == 'somint': |
---|
1230 | print 'INFO.somint.0003',thisrqv |
---|
1231 | ## Variables marked with shape '?' from PMIP are actually defined by OMIP .... |
---|
1232 | if tt.shape == '?': |
---|
1233 | print 'ERROR.801.00001: shape not set: %s, %s, %s' % ( tt.shape,tt.mip,tt.var) |
---|
1234 | |
---|
1235 | ##['', u'none', u'land', u'grounded ice sheet', u'floating ice sheet', u'floating ice shelf', u'ocean', u'ice sheet', u'sea', u'Southern hemisphere', u'Northern hemisphere'] |
---|
1236 | self.dsort( dset,mode=ksect ) |
---|
1237 | print 'INFO.088.00001: nrep=%s' % nrep |
---|
1238 | print 'INFO.088.00002: nov=%s [%s]' % (nov,len(gi)) |
---|
1239 | |
---|
1240 | nrep = [] |
---|
1241 | for c in self.cmv: |
---|
1242 | u,v,f,t,p,pn = (c[0],c[2],c[21],c[18],c[19],c[20]) |
---|
1243 | self.cmvUidByFL[f].a[v].append( u ) |
---|
1244 | self.cmvUidByTab[t].a[v].append( u ) |
---|
1245 | if v[0] != '-': |
---|
1246 | if (t,v,p) in self.cmvUidByTVP: |
---|
1247 | nrep.append( (t,v,p) ) |
---|
1248 | self.cmvUidByTVP[ (t,v,p) ].append( u ) |
---|
1249 | self.cmvByLab[v][ u ] = (f,t) |
---|
1250 | if len( nrep) > 0: |
---|
1251 | print 'WARN.108.00088: unresolved .... %s' % len(nrep) |
---|
1252 | for n in nrep: |
---|
1253 | print n |
---|
1254 | |
---|
1255 | ####################################################################################### |
---|
1256 | ################# |
---|
1257 | std = set() |
---|
1258 | ssd = set() |
---|
1259 | esd = {} |
---|
1260 | |
---|
1261 | for ksect in [ 'ovar','groupitem']: |
---|
1262 | #### create structure list for spatial and temporal dimensions |
---|
1263 | for t in self.dsortdd[ksect][0].keys(): |
---|
1264 | tdim, tt = self.dsort_u1( self.dsortdd[ksect][0][t], self.dsortdd[ksect][1] ) |
---|
1265 | |
---|
1266 | std.add( tdim ) |
---|
1267 | ssd.add( tt ) |
---|
1268 | |
---|
1269 | etd = self.spsh.etd |
---|
1270 | for s in std: |
---|
1271 | assert s in etd, 'Temporal dimension record not found for %s (see importWbMods.loadSpatial)' % s |
---|
1272 | |
---|
1273 | kk = 0 |
---|
1274 | self.spidInfo = {} |
---|
1275 | for s in ssd: |
---|
1276 | d,lfl,nl = s |
---|
1277 | if d == 'longitude|latitude|plev7': |
---|
1278 | print 'INFO.plev7.0002: resetting plev7 to plev7c' |
---|
1279 | d = 'longitude|latitude|plev7c' |
---|
1280 | if d == 'longitude|latitude|': |
---|
1281 | print 'INFO.plev7.0003: removing trailing |' |
---|
1282 | d = 'longitude|latitude' |
---|
1283 | if d not in WbMods.ls.ss: |
---|
1284 | print 'SEVERE.spatial.0001: dimensions not found in WbMods.ls: %s' % d |
---|
1285 | assert False, 'Spatial shapes must be specified in spatialShape_ref.xls: %s' % d |
---|
1286 | u = str( uuid.uuid1() ) |
---|
1287 | lab = 'ssd-%3.3i' % kk |
---|
1288 | description = 'ssd.%3.3i' % kk |
---|
1289 | else: |
---|
1290 | u = WbMods.ls.ss[d][4] |
---|
1291 | lab = WbMods.ls.ss[d][0] |
---|
1292 | description = WbMods.ls.ss[d][1] |
---|
1293 | print 'INFO.spatial.0002: setting spatial shape uid: ',u,lab,description |
---|
1294 | |
---|
1295 | kk+=1 |
---|
1296 | esd[s] = (u,lab,description,d,lfl,nl) |
---|
1297 | self.spidInfo[u] = (u,lab,description,d,lfl,nl) |
---|
1298 | |
---|
1299 | ## clumsy here .. uid copied from spatialShape_ref.xls |
---|
1300 | esd['Y-na'] = ('6dc25718-a24a-11e5-a29d-5404a60d96b5','extra','extra','latitude','false',1) |
---|
1301 | self.dimsTime = etd |
---|
1302 | self.dimsTimeInfo = ['uid','label','title','dimensions'] |
---|
1303 | self.dimsSpace = esd |
---|
1304 | self.dimsSpaceInfo = ['uid','label','title','dimensions','levelFlag','levels', 'dimids'] |
---|
1305 | |
---|
1306 | sshp = set() |
---|
1307 | eshp = set() |
---|
1308 | self.lls = ivg.structureList() |
---|
1309 | nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tmid','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] ) |
---|
1310 | kk = 0 |
---|
1311 | strdd = {} |
---|
1312 | ee3 = {} |
---|
1313 | for dims,cmet,cmea,flgv, flgm in ds2: |
---|
1314 | tdim, tt = self.dsort_u1( self.dsortdd['ovar'][0][dims], self.dsortdd['ovar'][1] ) |
---|
1315 | d1,odim,d2,crd = self.dsortdd['ovar'][0][dims] |
---|
1316 | sdim,lfl,nl = tt |
---|
1317 | u = str( uuid.uuid1() ) |
---|
1318 | spid = esd[tt][0] |
---|
1319 | splab = esd[tt][1] |
---|
1320 | tmid = etd[tdim][0] |
---|
1321 | tlab = etd[tdim][1] |
---|
1322 | kk += 1 |
---|
1323 | label = 'str-a%3.3i' % kk |
---|
1324 | proc = '' |
---|
1325 | prov = 'CMIP5/OMIP' |
---|
1326 | desc = '' |
---|
1327 | ##print 'INFO.zzz01: ',dims, crd |
---|
1328 | t0 = (dims,cmet,cmea,flgv,flgm) |
---|
1329 | if cmet in ['None',None]: |
---|
1330 | if tlab == 'time-mean': |
---|
1331 | cmet = 'time: mean' |
---|
1332 | elif tlab == 'time-point': |
---|
1333 | cmet = 'time: point' |
---|
1334 | else: |
---|
1335 | cmet = '' |
---|
1336 | if cmea in ['None',None]: |
---|
1337 | cmea = '' |
---|
1338 | |
---|
1339 | if flgv in ['None',None,u'None']: |
---|
1340 | flgv = '' |
---|
1341 | if flgm in ['None',None,u'None']: |
---|
1342 | flgm = '' |
---|
1343 | t1 = (dims,cmet,cmea,flgv,flgm) |
---|
1344 | ##errc,u,label = self.lls.add( [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] ) |
---|
1345 | errc,u,label = self.lls.add2( [u,label,splab,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] ) |
---|
1346 | print 'INFO.structure.00011:',t0,t1,u,label,spid,splab |
---|
1347 | self.structInfo[u] = (label,cmet,spid,tmid) |
---|
1348 | strdd[t0] = u |
---|
1349 | strdd[t1] = u |
---|
1350 | |
---|
1351 | |
---|
1352 | ttt = [ {u'Instantaneous (end of year)', 'instantaneous', 'point', 'Synoptic', u'synoptic', u'time: point'}, |
---|
1353 | {'Mean', u'time: mean', u'Time mean', u'daily mean', u'mean', u'time mean', u'time: day', u'time: mean', u'weighted time mean', 'Cumulative annual fraction'}, set(), {''}, 'Daily max', 'Daily min'] |
---|
1354 | cmetl = ['time: point', 'time: mean', 'time: mean within years time: mean over years ', '', 'time: maximum', 'time: minimum'] |
---|
1355 | ##cmetlsi = ['time: point', ('time: mean area: mean where sea_ice over sea','time: mean where sea_ice area: mean where sea_ice'), 'time: mean within years time: mean over years ', ''] |
---|
1356 | ee1 = {} |
---|
1357 | for shp,lvls,sty,gr,tmsk, flgv, flgm, prov in dsgpi: |
---|
1358 | t = (shp,lvls,sty,gr) |
---|
1359 | ttx = (shp,lvls,sty,gr,tmsk, flgv, flgm, prov) |
---|
1360 | if "t2000" in shp: |
---|
1361 | print 'CCCC',shp,lvls,sty,gr,tmsk, flgv, flgm, prov |
---|
1362 | tdim, tt = self.dsort_u1( self.dsortdd['groupitem'][0][t], self.dsortdd['groupitem'][1] ) |
---|
1363 | d1,odim,d2,crd = self.dsortdd['groupitem'][0][t] |
---|
1364 | tflm, msk = tmsk |
---|
1365 | sdim,lfl,nl = tt |
---|
1366 | u = str( uuid.uuid1() ) |
---|
1367 | spid = esd[tt][0] |
---|
1368 | splab = WbMods.ls.labByUid[ spid ] |
---|
1369 | tmid = etd[tdim][0] |
---|
1370 | tlab = etd[tdim][1] |
---|
1371 | print 'INFO.spatial.6666: %s, %s, %s' % (splab,spid,str(tt)) |
---|
1372 | |
---|
1373 | if "XY" in splab: |
---|
1374 | if gr == "M": |
---|
1375 | cmea = "--MODEL" |
---|
1376 | else: |
---|
1377 | cmea = 'area: areacell%s' % gr |
---|
1378 | else: |
---|
1379 | cmea = '' |
---|
1380 | |
---|
1381 | cmet = '' |
---|
1382 | for i in range(4): |
---|
1383 | if sty in ttt[i]: |
---|
1384 | if i == 1 and prov == 'SIMIP': |
---|
1385 | if sty == u'weighted time mean': |
---|
1386 | cmet = 'area: time: mean where sea_ice' |
---|
1387 | elif sty == u'partial mean over sea': |
---|
1388 | cmet = 'area: mean where sea_ice over sea time: mean' |
---|
1389 | else: |
---|
1390 | cmet = 'area: time: mean' |
---|
1391 | elif prov == 'LUMIP' and shp == 'XYLT': |
---|
1392 | print 'INFO.lumip.00505: ',shp,lvls,sty,gr,tmsk, flgv, flgm, prov |
---|
1393 | if sty == 'Time mean': |
---|
1394 | cmet = 'area: time: mean where landuse' |
---|
1395 | elif sty == 'Cumulative annual fraction': |
---|
1396 | cmet = 'area: mean time: sum' |
---|
1397 | elif sty == 'Instantaneous (end of year)': |
---|
1398 | cmet = 'area: mean where landUse time: point' |
---|
1399 | else: |
---|
1400 | cmet = cmetl[i] |
---|
1401 | if cmet == '': |
---|
1402 | cmet = msk |
---|
1403 | elif msk != '': |
---|
1404 | if "time:" in msk: |
---|
1405 | print 'INFO.cmet.05005: using time: info in msk: ',msk,cmet |
---|
1406 | cmet = msk |
---|
1407 | else: |
---|
1408 | cmet = msk + ' ' + cmet |
---|
1409 | kk += 1 |
---|
1410 | label = 'str-b%3.3i' % kk |
---|
1411 | proc = '' |
---|
1412 | if prov == '': |
---|
1413 | prov = 'CMIP5, endorsed MIPs' |
---|
1414 | desc = '' |
---|
1415 | dims = tt[0] |
---|
1416 | if tdim != '': |
---|
1417 | dims += '|' + tdim |
---|
1418 | if odim == 'spectband': |
---|
1419 | print 'INFO.spectband.00002: ',tt, tdim, cmet, cmea, label, u |
---|
1420 | |
---|
1421 | if strdd.has_key( (dims,cmet,cmea,'','') ) and crd == '' and odim == '': |
---|
1422 | u = strdd[ (dims,cmet,cmea,'','') ] |
---|
1423 | ##print 'INFO.zzzy03: ',tt, tdim, cmet, cmea, label, u |
---|
1424 | elif ee1.has_key( (tmid, spid, odim, cmet, cmea, crd) ): |
---|
1425 | u = ee1[(tmid, spid, odim, cmet, cmea, crd)] |
---|
1426 | ##print 'INFO.zzzy04: ', tt, tdim, cmet, label, u |
---|
1427 | else: |
---|
1428 | errc,u,label = self.lls.add2( [u,label,splab,tlab, odim, crd, cmet, cmea, '', '', desc, proc, prov] ) |
---|
1429 | print 'INFO.zzzy02: ',tt, tdim, cmet, label, u, ttx |
---|
1430 | |
---|
1431 | ee1[(tmid, spid, odim, cmet, cmea, crd)] = u |
---|
1432 | |
---|
1433 | if sty == u'Synoptic': |
---|
1434 | print 'yyyy: ',(shp,lvls,sty,gr,tmsk,flgv,flgm) |
---|
1435 | strdd[(shp,lvls,sty,gr,tmsk,flgv,flgm)] = u |
---|
1436 | if flgv == 'None' and flgm == 'None': |
---|
1437 | strdd[(shp,lvls,sty,gr,tmsk,'','')] = u |
---|
1438 | self.structInfo[u] = (label,cmet,spid,tmid) |
---|
1439 | |
---|
1440 | ##nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tmid','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] ) |
---|
1441 | ##self.lls.close() |
---|
1442 | self.strdd = strdd |
---|
1443 | |
---|
1444 | def cmvAudit(self,nn): |
---|
1445 | nrep = [] |
---|
1446 | self.cmvByUid = {} |
---|
1447 | for c in self.cmv: |
---|
1448 | u,v,f,t,p,pn = (c[0],c[2],c[21],c[18],c[19],c[20]) |
---|
1449 | self.cmvUidByFL[f].a[v].append( u ) |
---|
1450 | self.cmvUidByTab[t].a[v].append( u ) |
---|
1451 | if v[0] != '-': |
---|
1452 | if (t,v,p) in self.cmvUidByTVP: |
---|
1453 | nrep.append( (t,v,p) ) |
---|
1454 | self.cmvUidByTVP[ (t,v,p) ].append( u ) |
---|
1455 | self.cmvByLab[v][ u ] = (f,t) |
---|
1456 | self.cmvByUid[u] = c |
---|
1457 | if len( nrep) > 0: |
---|
1458 | print 'WARN.108.00088: unresolved .... %s' % len(nrep) |
---|
1459 | for n in nrep: |
---|
1460 | print n |
---|
1461 | topop = [] |
---|
1462 | cc1 = collections.defaultdict( int ) |
---|
1463 | cc2 = collections.defaultdict( int ) |
---|
1464 | for k in self.fix.fixcmv: |
---|
1465 | vid = str( self.fix.fixcmv[k][4] ) |
---|
1466 | fn = str( self.fix.fixcmv[k][6] ) |
---|
1467 | cc1[fn] += 1 |
---|
1468 | if vid not in self.cmvByUid: |
---|
1469 | print 'WARN.fixcmv.0001: invalid vid %s [%s]' % ( vid, str( self.fix.fixcmv[k] ) ) |
---|
1470 | topop.append( k ) |
---|
1471 | cc2[fn] += 1 |
---|
1472 | if len(topop) > 0: |
---|
1473 | print 'WARN.fixcmv.0009: %s invalid vid value popped from fixcmv [%s]' % ( len( topop ), len( self.fix.fixcmv ) ) |
---|
1474 | for k in topop: |
---|
1475 | self.fix.fixcmv.pop( k ) |
---|
1476 | for f in sorted( cc2.keys() ): |
---|
1477 | print 'WARN.fixcmv.0010: %s invalid vid value popped from fixcmv:%s [%s]' % (cc2[f],f,cc1[f]) |
---|
1478 | |
---|
1479 | def dsort_u1(self,eed,sde): |
---|
1480 | tdim = eed[2] |
---|
1481 | sdim = eed[0] |
---|
1482 | nl = sde[sdim] |
---|
1483 | lfl = 'true' |
---|
1484 | if nl == -1: |
---|
1485 | lfl = 'false' |
---|
1486 | nl = 0 |
---|
1487 | elif nl == -2: |
---|
1488 | nl = 1 |
---|
1489 | pass |
---|
1490 | return (tdim, (sdim,lfl,nl) ) |
---|
1491 | |
---|
1492 | |
---|
1493 | def dsort( self, dset, mode='ovar' ): |
---|
1494 | """Sort the dimensions string into spatial, temporal and other sections, |
---|
1495 | returns a dictionary of tuples: spatial dimensions, other, temporal, coords and a 2nd, keyyed on spatial dimensions with a level count indicator. """ |
---|
1496 | vdims = {'plev7':7, 'plev':4, 'plevs':19, 'plev3':3,'plev8':8,'alt40':40,'alev1':1, 'alevhalf':-1,'olevel':-1,'rho':-1,'sdepth':-1,'alevel':-1} |
---|
1497 | odims = {'sza5','vegtype','scatratio','dbze','typepdec','vgidx','tau','effectRad'} |
---|
1498 | dl = sorted( list(dset) ) |
---|
1499 | if mode == 'ovar': |
---|
1500 | eed = {} |
---|
1501 | sss = [set(),set(),set(),set()] |
---|
1502 | for s in dl: |
---|
1503 | if s[:4] == 'time': |
---|
1504 | bits = ['',s[4:]] |
---|
1505 | else: |
---|
1506 | bits = string.split(s, '|time', maxsplit=1 ) |
---|
1507 | if len(bits) == 1: |
---|
1508 | ts = '' |
---|
1509 | else: |
---|
1510 | ts = 'time' |
---|
1511 | |
---|
1512 | b0 = bits[0] |
---|
1513 | b1 = '' |
---|
1514 | bb = string.split(bits[0],'|') |
---|
1515 | if bb[-1] in odims: |
---|
1516 | if bb[-1] == 'tauxxx': |
---|
1517 | b0 = bits[0][:-4] |
---|
1518 | ts = 'tau|time' |
---|
1519 | else: |
---|
1520 | b0 = string.join( bb[:-1], '|' ) |
---|
1521 | b1 = bb[-1] |
---|
1522 | |
---|
1523 | if len(bits) == 1: |
---|
1524 | tup = (b0,b1,'','') |
---|
1525 | elif len(bits) == 2: |
---|
1526 | b2 = string.split( bits[1], '|', maxsplit=1 ) |
---|
1527 | if len(b2) == 2: |
---|
1528 | tup = (b0,b1, ts + b2[0], b2[1]) |
---|
1529 | else: |
---|
1530 | tup = (b0,b1, ts + bits[1], '') |
---|
1531 | eed[s] = tup |
---|
1532 | for k in range(4): |
---|
1533 | sss[k].add(tup[k]) |
---|
1534 | |
---|
1535 | lcount = {} |
---|
1536 | for s in sss[0]: |
---|
1537 | bits = string.split( s, '|' ) |
---|
1538 | nl = -2 |
---|
1539 | thisb = None |
---|
1540 | for b in bits: |
---|
1541 | if vdims.has_key(b): |
---|
1542 | assert nl == -2, 'Duplicate vertical level count in %s' % s |
---|
1543 | nl = vdims[b] |
---|
1544 | thisb = b |
---|
1545 | lcount[s] = nl |
---|
1546 | if nl == -1 and s == 'latitude|basin': |
---|
1547 | print 'SEVERE: error in level deduction' |
---|
1548 | print s, bits, nl, thisb |
---|
1549 | raise |
---|
1550 | else: |
---|
1551 | ## input is a set of tuples, shape, levels, tstyle, o/a |
---|
1552 | ## aim is to extract spatial, other, temporal dimensions + coords. |
---|
1553 | sss = [set(),set(),set()] |
---|
1554 | ttt = [ ('time1', u'Instantaneous (end of year)', 'instantaneous', 'point', 'Synoptic', u'synoptic', u'time: point'), |
---|
1555 | ('Daily max', 'Daily min', 'time', 'Mean', u'time: mean', u'Time mean', u'daily mean', u'mean', u'time mean', u'time: day', u'time: mean', u'weighted time mean', 'Cumulative annual fraction','partial mean over sea'), |
---|
1556 | (), |
---|
1557 | ('diurnal','time3'), ('',)] |
---|
1558 | ttdd = {} |
---|
1559 | kkk = ['time1','time','time2','time3',''] |
---|
1560 | for i in range(5): |
---|
1561 | for k in ttt[i]: |
---|
1562 | ttdd[k] = kkk[i] |
---|
1563 | |
---|
1564 | ##lll = {[u'1.0', u'14.0', u'16.0', u'17.0', u'27.0', u'3.0', u'36.0', u'4.0', u'40.0', u'7.0', ]:'flt', [u'17 (or 23 )', u'17 (or 23)', u'17 (or23)']:'obs', [u'Model levels or 27Plevs', u'all', u'all model levels', u'all model levels above 400hPa', u'all*', u'integrated over depth'], [u'single level',], [u'soil levels',], [u'surface',], [[u'100 m',] } |
---|
1565 | dds = {'2D':'XYT', 'XY':'XYT', '':'?', '2D ':'XYT', 'XYZ':'XYZT', } |
---|
1566 | ## 'K'?? 'XYK' |
---|
1567 | ## |
---|
1568 | ## |
---|
1569 | ## spatial dimensions, other, time, coordinates, wild card in spatial dimensions. |
---|
1570 | ## |
---|
1571 | dd = {'XKT':('latitude|basin','','time','',False), \ |
---|
1572 | 'XYZKT':('longitude|latitude|%s','tau','time','',True), \ |
---|
1573 | 'XYZKmT':('longitude|latitude|%s','misrBands','time','',True), \ |
---|
1574 | 'XYZKKT':('longitude|latitude|%s','effectRadXX|tau','time','',True), \ |
---|
1575 | 'XYZKKlT':('longitude|latitude|%s','effectRadLi|tau','time','',True), \ |
---|
1576 | 'XYZKKiT':('longitude|latitude|%s','effectRadIc|tau','time','',True), \ |
---|
1577 | 'XYZKdbT':('longitude|latitude|%s','dbze','time','',True), \ |
---|
1578 | 'XYZKscT':('longitude|latitude|%s','scatratio','time','',True), \ |
---|
1579 | 'XYZT':('longitude|latitude|%s','','time','',True), \ |
---|
1580 | 'XYST':('longitude|latitude|sdepth','','time','',False), \ |
---|
1581 | 'XYSnT':('longitude|latitude|snowdepth','','time','',False), \ |
---|
1582 | 'XYS1T':('longitude|latitude','sdepth1','time','',False), \ |
---|
1583 | 'XYH1T':('longitude|latitude','height100','time','',False), \ |
---|
1584 | 'XYZT*':('longitude|latitude|%s','','time','',True), \ |
---|
1585 | 'KZT':('alevel','spectband','time','',False), \ |
---|
1586 | 'Ksp':('','spectband','','',False), \ |
---|
1587 | 'XYKspT':('longitude|latitude','spectband','time','',False), \ |
---|
1588 | 'XYZKspT':('longitude|latitude|%s','spectband','time','',True), \ |
---|
1589 | 'XYT':('longitude|latitude','','time','',False), \ |
---|
1590 | '2DV':('longitude|latitude','','time','',False), \ |
---|
1591 | 'XYKT':('longitude|latitude','sza5','time','',False), \ |
---|
1592 | 'XYLT':('longitude|latitude','landUse','time','',False), \ |
---|
1593 | 'XYant':('xant|yant','','time','',False), \ |
---|
1594 | 'XYgre':('xgre|ygre','','time','',False), \ |
---|
1595 | 'XYKszT':('longitude|latitude','sza5','time','',False), \ |
---|
1596 | 'YZT':('latitude|%s','','time','',True), \ |
---|
1597 | ## snowband was an error ... all the variables should have been on ice bands ... easier to correct here. |
---|
1598 | 'XYKsn':('longitude|latitude','iceband','time','',False), \ |
---|
1599 | 'XYKic':('longitude|latitude','iceband','time','',False), \ |
---|
1600 | 'K4':('siline','','time','',False), \ |
---|
1601 | 'scalar':('','','time','',False), \ |
---|
1602 | '':('','','time','',False), \ |
---|
1603 | 'sheet':('icesheet','','time','',False), \ |
---|
1604 | 'ZST1':('alevel','site','time','',False), \ |
---|
1605 | 'BasinYT':('latitude|basin','','time','',False), \ |
---|
1606 | '?':( '?', '?','?','?',False) |
---|
1607 | } |
---|
1608 | |
---|
1609 | for d in dl: |
---|
1610 | for k in range(3): |
---|
1611 | sss[k].add( d[k] ) |
---|
1612 | |
---|
1613 | ls0 = set() |
---|
1614 | ls1 = set() |
---|
1615 | ## |
---|
1616 | ## sort shapes |
---|
1617 | ## |
---|
1618 | for s0 in sss[0]: |
---|
1619 | if string.find( s0, '|' ) != -1: |
---|
1620 | s,crd0 = string.split( s0, '|' ) |
---|
1621 | else: |
---|
1622 | s,crd0 = s0,None |
---|
1623 | |
---|
1624 | if not dd.has_key( dds.get(s,s) ): |
---|
1625 | print 'ERROR.shape.0001: NO SOLUTION FOUND FOR SHAPE: "%s"' % s |
---|
1626 | else: |
---|
1627 | tt = dd[dds.get(s,s)] |
---|
1628 | print 'INFO.structure.00077: mapping %s to %s' % (s,str(tt)) |
---|
1629 | if tt[-1]: |
---|
1630 | ls0.add( s0 ) |
---|
1631 | ## |
---|
1632 | ## look at shapes which require additional information about vertical coord |
---|
1633 | ## |
---|
1634 | for d in dl: |
---|
1635 | if d[0] in ls0: |
---|
1636 | ls1.add( d[1] ) |
---|
1637 | |
---|
1638 | lset = {'soil levels':'sdepth', 'all model levels above 400hPa':'alevel'} |
---|
1639 | lsdd = {} |
---|
1640 | print 'ls1....',ls1 |
---|
1641 | for l in ls1: |
---|
1642 | ii = 0 |
---|
1643 | if l in ['all','all*','all model levels']: |
---|
1644 | ii = -1 |
---|
1645 | this = '%slevel' |
---|
1646 | elif lset.has_key(l): |
---|
1647 | ii = -1 |
---|
1648 | this = lset[l] |
---|
1649 | elif l == '': |
---|
1650 | ii = -2 |
---|
1651 | this = l |
---|
1652 | elif l == '7h': |
---|
1653 | ii = 7 |
---|
1654 | this = 'plev7h' |
---|
1655 | elif l == '3h': |
---|
1656 | ii = 3 |
---|
1657 | this = 'plev3h' |
---|
1658 | else: |
---|
1659 | ii = int( float(l) ) |
---|
1660 | if ii in [40,16]: |
---|
1661 | this = 'alt%s' % ii |
---|
1662 | elif ii > 1.5: |
---|
1663 | if ii > 35: |
---|
1664 | ii = 39 |
---|
1665 | ## |
---|
1666 | ## merge 14 and 17 level specifications to a 19 level coordinate |
---|
1667 | ## |
---|
1668 | if ii in [14,17]: |
---|
1669 | ii = 19 |
---|
1670 | |
---|
1671 | this = 'plev%s' % ii |
---|
1672 | if ii == 7: |
---|
1673 | this += 'c' |
---|
1674 | |
---|
1675 | else: |
---|
1676 | this = '' |
---|
1677 | lsdd[l] = (ii,this) |
---|
1678 | ##['', 1.0, u'all', 3.0, 4.0, u'soil levels', 7.0, 40.0, u'all model levels', u'17 (or 23)', u'all*', 14.0, u'Model levels or 27Plevs', 16.0, 17.0, u'all model levels above 400hPa', u'17 (or23)', 36.0, u'17 (or 23 )', 27.0] |
---|
1679 | |
---|
1680 | lcount = {} |
---|
1681 | eed = {} |
---|
1682 | for d in dl: |
---|
1683 | s0 = d[0] |
---|
1684 | if string.find( s0, '|' ) != -1: |
---|
1685 | s,crd0 = string.split( s0, '|' ) |
---|
1686 | else: |
---|
1687 | s,crd0 = s0,None |
---|
1688 | if not dd.has_key( dds.get(s,s) ): |
---|
1689 | print 'NO SOLUTION FOUND FOR SHAPE: "%s"' % s |
---|
1690 | else: |
---|
1691 | tt = dd[dds.get(s,s)] |
---|
1692 | if tt[-1]: |
---|
1693 | n,zz = lsdd[d[1]] |
---|
1694 | if len(zz) > 0 and zz[0] == '%': |
---|
1695 | zz = zz % d[3] |
---|
1696 | sc = tt[0] % zz |
---|
1697 | if sc[-1] == "|": |
---|
1698 | print 'WARN.080.05000: Bad dimension found %s [%s] %s %s' % (sc,str(tt), str((n,zz)), str(d) ) |
---|
1699 | else: |
---|
1700 | sc = tt[0] |
---|
1701 | if (string.find( sc, 'alevel' ) != -1) or (string.find( sc, 'olevel' ) != -1): |
---|
1702 | n = -1 |
---|
1703 | else: |
---|
1704 | n = -2 |
---|
1705 | |
---|
1706 | lcount[sc] = n |
---|
1707 | if n == -1 and sc == 'latitude|basin': |
---|
1708 | print 'SEVERE [2]: error in level deduction' |
---|
1709 | print sc, tt[0], tt[-1], n |
---|
1710 | raise |
---|
1711 | if crd0 != None: |
---|
1712 | tt3 = tt[3] + crd0 |
---|
1713 | else: |
---|
1714 | tt3 = tt[3] |
---|
1715 | |
---|
1716 | if d[2] == '' and tt[2] not in ['','?']: |
---|
1717 | d2 = tt[2] |
---|
1718 | else: |
---|
1719 | d2 = d[2] |
---|
1720 | print 'INFO:qqq05: ',d, sc, tt, ttdd.get( d2, '**%s**' % d2), tt3 |
---|
1721 | if d2 not in ttdd: |
---|
1722 | print 'ERROR.time.00800: ',d2,ttdd.keys() |
---|
1723 | eed[d] = (sc,tt[1],ttdd.get( d2, '**%s**' % d2),tt3) |
---|
1724 | print 'INFO.ddddd.00002:',d,eed[d] |
---|
1725 | self.dsortdd[mode] = (eed,lcount) |
---|
1726 | |
---|
1727 | def run(self,rq): |
---|
1728 | |
---|
1729 | xx = self.doc.getElementsByTagName( 'main' ) |
---|
1730 | assert len(xx) == 1, 'Expecting one element named "main", found %s' % len(xx) |
---|
1731 | main = xx[0] |
---|
1732 | ## main = self.doc.childNodes[0] |
---|
1733 | xsn = [] |
---|
1734 | ff = {} |
---|
1735 | for c in main.childNodes: |
---|
1736 | tag = c.nodeName |
---|
1737 | if tag != '#text': |
---|
1738 | xsn.append(tag) |
---|
1739 | il = [] |
---|
1740 | for i in c.childNodes: |
---|
1741 | if i.nodeName == 'item': |
---|
1742 | il.append(i) |
---|
1743 | xx = dict( il[0].attributes.items() ) |
---|
1744 | ff[string.lower(tag)] = (c,xx) |
---|
1745 | |
---|
1746 | print xsn |
---|
1747 | print ff.keys() |
---|
1748 | self.ff = ff |
---|
1749 | |
---|
1750 | ## keys here match section element names in schema |
---|
1751 | ## dec 2015 -- removing 'var' from this list |
---|
1752 | assert self.schemaMode == 'dreq2', 'Old schema mode not supported here' |
---|
1753 | sectlist = [ 'experiment','exptgroup','objective','requestlink', 'requestitem','requestvargroup','tablesection','requestvar' ] |
---|
1754 | sectlist = [ 'experiment','exptgroup','objective','requestlink', 'requestitem','tablesection','requestvar' ] |
---|
1755 | |
---|
1756 | for k in sectlist: |
---|
1757 | thissh = self.skey[k] |
---|
1758 | ## set labmod False for section in which label coercion is not wanted |
---|
1759 | labmod = True |
---|
1760 | lll = [] |
---|
1761 | for i in thissh.keys(): |
---|
1762 | if i[0] != '_': |
---|
1763 | lll.append(i) |
---|
1764 | idk = 0 |
---|
1765 | if len(lll) > 0: |
---|
1766 | ## |
---|
1767 | ## remove example items, xcept where this has been done in previous function |
---|
1768 | ## |
---|
1769 | if k not in ['xxxx']: |
---|
1770 | dil = ff[k][0].getElementsByTagName('item') |
---|
1771 | for d in dil: |
---|
1772 | ff[k][0].removeChild(d) |
---|
1773 | |
---|
1774 | vare = {} |
---|
1775 | nfr = set() |
---|
1776 | for i in lll: |
---|
1777 | ll = map( uniCleanFunc, thissh[i] ) |
---|
1778 | assert len(ll) == len( thissh[i] ), 'Lost list elements after uniClean' |
---|
1779 | idk += 1 |
---|
1780 | ################################################################# |
---|
1781 | ######### NEED TO CLEAN THIS IF ITEMS FROM OTHER SECTIONS ARE FILTERED |
---|
1782 | ####################################################################### |
---|
1783 | dothis = True |
---|
1784 | if k not in ['var','experiment']: |
---|
1785 | item = self.doc.createElement( 'item' ) |
---|
1786 | |
---|
1787 | if k == 'objective': |
---|
1788 | ## - mip; tab; expt; rlid; ny |
---|
1789 | labmod = False |
---|
1790 | kk = ['mip', 'label', 'title', 'description'] |
---|
1791 | ##thisl = string.replace(string.strip(str( ll[1] ) ), '-', ' ') |
---|
1792 | ## camelcase this label. |
---|
1793 | ##thisl = string.replace( string.capwords( thisl ), ' ', '' ) |
---|
1794 | thisl = mycc( ll[1] ) |
---|
1795 | mip = str( ll[0] ) |
---|
1796 | self.objectives[mip][thisl] = i |
---|
1797 | for j in range(len(kk)): |
---|
1798 | thisv = str( ll[j] ) |
---|
1799 | item.setAttribute( kk[j], thisv ) |
---|
1800 | item.setAttribute( 'uid', str(i) ) |
---|
1801 | elif k == 'experiment': |
---|
1802 | labmod = False |
---|
1803 | kk = ['uid','egid','label','description','mip','mcfg','tier','nstart','starty','endy','yps','ensz','ntot','comment'] |
---|
1804 | thisl = str( ll[2] ) |
---|
1805 | ## self.mips.add( str( ll[4] ) ) |
---|
1806 | if ll[4] in ['SolarMIP']: |
---|
1807 | dothis = False |
---|
1808 | else: |
---|
1809 | item = self.doc.createElement( 'item' ) |
---|
1810 | for j in range(len(kk)): |
---|
1811 | if kk[j] in ['tier','ensz']: |
---|
1812 | if type(ll[j]) == type( 1 ): |
---|
1813 | thisv = str( ll[j] ) |
---|
1814 | else: |
---|
1815 | thisv = string.join( [str(x) for x in ll[j]] ) |
---|
1816 | else: |
---|
1817 | thisv = str( uniCleanFunc(ll[j]) ) |
---|
1818 | item.setAttribute( kk[j], thisv ) |
---|
1819 | if kk[j] == 'yps': |
---|
1820 | if type(ll[j]) == type([]): |
---|
1821 | print 'ERROR.yps.00001: ', ll |
---|
1822 | item.setAttribute( 'yps', '25' ) |
---|
1823 | ################################################################################################## |
---|
1824 | elif k == 'exptgroup': |
---|
1825 | labmod = False |
---|
1826 | kk = ['uid','label','tierMin','ntot'] |
---|
1827 | thisl = str( ll[1] ) |
---|
1828 | for j in range(len(kk)): |
---|
1829 | thisv = str( ll[j] ) |
---|
1830 | item.setAttribute( kk[j], thisv ) |
---|
1831 | |
---|
1832 | ################################################################################################## |
---|
1833 | elif k == 'groupitem': |
---|
1834 | kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'misc', 'mip', 'uid', 'new', 'gpid', 'vkey', 'vid'] |
---|
1835 | assert len(kk) == len(ll), 'length mismatch, %s %s' % (len(kk),len(ll)) |
---|
1836 | # - group; table; freq; descriptionEx; shape; levels; tstyle; mask; mip; mip2; uuid; new; vid |
---|
1837 | ix_gpid = kk.index( 'gpid' ) |
---|
1838 | thisl = string.strip(str( ll[1] ) ) |
---|
1839 | for j in range(len(kk)): |
---|
1840 | if j != 1: |
---|
1841 | ##item.setAttribute( kk[j], str( ll[j] ) ) |
---|
1842 | if kk[j] == 'misc': |
---|
1843 | item.setAttribute( 'priority', str( ll[j] ) ) |
---|
1844 | else: |
---|
1845 | item.setAttribute( kk[j], str( ll[j] ) ) |
---|
1846 | item.setAttribute( 'title', thisl ) |
---|
1847 | if string.find(thisl, ' ') != -1: |
---|
1848 | thisl = string.split(thisl)[0] |
---|
1849 | thisuuid = ll[11] |
---|
1850 | |
---|
1851 | gpid = str(ll[ix_gpid]) |
---|
1852 | if not self.importWbMods.vg.vg.has_key(gpid): |
---|
1853 | print 'ERROR.015.0010: gpid not found %s' % str(ll) |
---|
1854 | self.e15_10 += 1 |
---|
1855 | if self.repl.has_key( thisuuid ): |
---|
1856 | for tag,old,new in self.repl[thisuuid]: |
---|
1857 | thisold = item.getAttribute( tag ) |
---|
1858 | assert thisold == old, 'Attempt to replace value which is not present' |
---|
1859 | item.setAttribute( tag, new ) |
---|
1860 | |
---|
1861 | #################################### from revised CMIP5 tables ### request variables referencing core tables |
---|
1862 | elif k in ['requestvar']: |
---|
1863 | #### need to fill gaps in variable groups --- or do better job upstream ########## !!!!!!!!!!!!!!!! |
---|
1864 | ## - table; mip; uuid; priority |
---|
1865 | ## |
---|
1866 | ## variable group ids pulled through via mapping above .. very messy |
---|
1867 | ## still missing some OMIP matches .... OMIP revised tables are not explicitly requested anywhere. |
---|
1868 | ## |
---|
1869 | kk = ['var', 'table', 'mip', 'vid', 'priority','vgid'] |
---|
1870 | ##vgmaps = {'Oyr':'OMIP-Oyr','OImon':'SIMIP-seaicemon'} |
---|
1871 | |
---|
1872 | vgid = str( ll[5] ) |
---|
1873 | doThis = True |
---|
1874 | if vgid == '__unset__01__': |
---|
1875 | dothis = False |
---|
1876 | else: |
---|
1877 | thisl = string.strip(str( ll[0] ) ) |
---|
1878 | if len( thisl ) > 6 and thisl[-6:] == 'Adjust': |
---|
1879 | dothis = False |
---|
1880 | |
---|
1881 | if dothis: |
---|
1882 | thisuuid = str( uuid.uuid1() ) |
---|
1883 | thisd = {} |
---|
1884 | |
---|
1885 | if ll[2] not in refMips: |
---|
1886 | print 'SEVERE.mip.000001: bad mip label in request item: %s --- %s' % (ll[2],str(ll)) |
---|
1887 | |
---|
1888 | for j in range(len(kk)): |
---|
1889 | if j != 0 and kk[j] != 'table': |
---|
1890 | item.setAttribute( kk[j], str( ll[j] ) ) |
---|
1891 | thisd[kk[j]] = str( ll[j] ) |
---|
1892 | vid = str( ll[3] ) |
---|
1893 | if vid in extRefCmvMapped: |
---|
1894 | print 'INFO.isd005.0002: mapping vid: ',vid,extRefCmvMapped[vid] |
---|
1895 | vid = extRefCmvMapped[vid] |
---|
1896 | |
---|
1897 | print 'INFO.isd005.0001: ',thisl, uid, vgid |
---|
1898 | item.setAttribute( 'title', thisl + ' ((isd.005))' ) |
---|
1899 | item.setAttribute( 'uid', thisuuid ) |
---|
1900 | item.setAttribute( 'vgid', vgid ) |
---|
1901 | item.setAttribute( 'vid', vid ) |
---|
1902 | |
---|
1903 | thisd['title'] = thisl + ' ((isd.005))' |
---|
1904 | thisd['uid'] = thisuuid |
---|
1905 | if vid in self.cmvMap002: |
---|
1906 | print 'INFO.vid.00001: resetting requestvar vid: %s ---> %s' % (vid,self.cmvMap002[vid]) |
---|
1907 | vid = self.cmvMap002[vid] |
---|
1908 | thisd['vid'] = vid |
---|
1909 | thisd['vgid'] = vgid |
---|
1910 | |
---|
1911 | ################################## |
---|
1912 | elif k == 'requestlink': |
---|
1913 | ## - uuid; mip; tab; objective; grid; gridreq; comment |
---|
1914 | kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'ref', 'refNote', 'refid'] |
---|
1915 | |
---|
1916 | uid = str( ll[0] ) |
---|
1917 | mip = string.strip(str( ll[1] ) ) |
---|
1918 | obj = string.strip(str( ll[3] ) ) |
---|
1919 | if mip == "GMMIP": |
---|
1920 | bits = [ mycc(x) for x in string.split( obj, ' ' ) ] |
---|
1921 | else: |
---|
1922 | bits = [ mycc(x) for x in string.split( obj, ',' ) ] |
---|
1923 | |
---|
1924 | ## |
---|
1925 | ## look for objectives specified in experiment column .... |
---|
1926 | ## |
---|
1927 | |
---|
1928 | for b in bits: |
---|
1929 | self.objectiveLinks[mip].a[b].append( str( ll[0] ) ) |
---|
1930 | self.requestlinkuid.add( str(ll[0] ) ) |
---|
1931 | self.rqlPreset[str(ll[0])] = ll[9] |
---|
1932 | ttl = string.strip(str( ll[2] ) ) |
---|
1933 | self.rqLnks[str(ll[0])] = ll[:] |
---|
1934 | dothis = False |
---|
1935 | |
---|
1936 | ################################## |
---|
1937 | elif k == 'requestitem': |
---|
1938 | ## - mip; tab; expt; rlid; ny |
---|
1939 | ## defer so that cross-check of "expt" can be done .. or scan this earlier .... |
---|
1940 | kk = ['mip', 'tab', 'expt', 'rlid', 'ny', 'nexmax', 'nenmax', 'nymax','treset','info'] |
---|
1941 | finishedClones = False |
---|
1942 | ku = str(ll[2]) |
---|
1943 | if self.exptPntUid[ku][1] == 'list' and len( self.exptPntUid[ku][2] ) > 0: |
---|
1944 | cloning = True |
---|
1945 | clones = set( self.exptPntUid[ku][2] ) |
---|
1946 | kcl = 0 |
---|
1947 | else: |
---|
1948 | cloning = False |
---|
1949 | |
---|
1950 | while not finishedClones: |
---|
1951 | thisl = mycc( '%s %s' % (ll[0],ll[1]) ) |
---|
1952 | if str( ll[3] ) not in self.rqlPreset: |
---|
1953 | print 'ERROR.090.02000: requestlink not found .. ',ll |
---|
1954 | preset = '-99' |
---|
1955 | else: |
---|
1956 | preset = str( self.rqlPreset[ str( ll[3] ) ] ) |
---|
1957 | if str( ll[3] ) not in self.requestlinkuid: |
---|
1958 | print 'SEVERE.005.00005: rlid not found: %s' % str(ll) |
---|
1959 | for j in range(len(kk)-1): |
---|
1960 | if kk[j] == 'ny': |
---|
1961 | if ll[j] == '': |
---|
1962 | thisv = 0 |
---|
1963 | else: |
---|
1964 | try: |
---|
1965 | thisv = str( int(ll[j]) ) |
---|
1966 | except: |
---|
1967 | print 'SEVERE: failed to set "ny": %s' % str(ll) |
---|
1968 | thisv = 999 |
---|
1969 | else: |
---|
1970 | thisv = str( ll[j] ) |
---|
1971 | if not (kk[j] == 'treset' and thisv in ['None','']): |
---|
1972 | item.setAttribute( kk[j], thisv ) |
---|
1973 | item.setAttribute( 'title', '%s, %s, %s' % (ll[0],ll[1],ll[2]) ) |
---|
1974 | item.setAttribute( 'uid', str(i) ) |
---|
1975 | item.setAttribute( 'preset', preset ) |
---|
1976 | if cloning: |
---|
1977 | ku = clones.pop() |
---|
1978 | if kcl != 0: |
---|
1979 | item.setAttribute( 'uid', str(uuid.uuid1()) ) |
---|
1980 | kcl += 1 |
---|
1981 | ##print 'INFO.clone.00001: ',ku, kcl |
---|
1982 | if ku in self.experiments: |
---|
1983 | esid = self.experiments[ku] |
---|
1984 | esidmsg = 'Experiment %s' % ku |
---|
1985 | elif ku in self.exptPntUid: |
---|
1986 | esid = self.exptPntUid[ku][0] |
---|
1987 | esidmsg = str(self.exptPntUid[ku]) |
---|
1988 | else: |
---|
1989 | print 'ERROR.esid.00001:', ku, ' Not Found' |
---|
1990 | esidmsg = 'ERROR: Experiment %s Not Found' % ku |
---|
1991 | esid = str(uuid.uuid1()) |
---|
1992 | |
---|
1993 | item.setAttribute( 'esid', esid ) |
---|
1994 | item.setAttribute( 'esidComment', esidmsg ) |
---|
1995 | if type(ll[9]) in [type(''), type(u'')]: |
---|
1996 | kkk = ll[9].strip( ) |
---|
1997 | if kkk[:3] == '_sl': |
---|
1998 | item.setAttribute( 'tslice', kkk ) |
---|
1999 | elif kkk not in ['','all']: |
---|
2000 | print 'ERROR.tslice.0007: unrecognised slice: %s' % kkk |
---|
2001 | |
---|
2002 | finishedClones = (not cloning) or ( len( clones) == 0 ) |
---|
2003 | if not finishedClones: |
---|
2004 | ff[k][0].appendChild( item ) |
---|
2005 | item = self.doc.createElement( 'item' ) |
---|
2006 | ################################## |
---|
2007 | elif k in ['requestvargroup','tablesection']: |
---|
2008 | if k == 'requestvargroup': |
---|
2009 | kk = ['uid', 'mip', 'label', 'title', 'ref', 'refNote'] |
---|
2010 | else: |
---|
2011 | kk = ['uid', 'gpid', 'mip', 'label', 'title', 'ref', 'refNote'] |
---|
2012 | thisl = string.replace( str( ll[2] ), '.', '-' ) |
---|
2013 | omitrvg = {'C4MIP-Omon','C4MIP-Oclim','HighResMIP-OImon','HighResMIP-Oclim'} |
---|
2014 | if thisl in omitrvg: |
---|
2015 | dothis = False |
---|
2016 | else: |
---|
2017 | thisd = {} |
---|
2018 | for j in range(len(kk)): |
---|
2019 | item.setAttribute( kk[j], str( ll[j] ) ) |
---|
2020 | thisd[kk[j]] = str( ll[j] ) |
---|
2021 | #### |
---|
2022 | if dothis: |
---|
2023 | if labmod: |
---|
2024 | if string.find( thisl, '_' ) != -1: |
---|
2025 | thisl = string.replace( thisl, '_', '-' ) |
---|
2026 | if string.find( thisl, '*' ) != -1: |
---|
2027 | thisl = string.replace( thisl, '*', '-' ) |
---|
2028 | if string.find( thisl, '!' ) != -1: |
---|
2029 | thisl = string.replace( thisl, '!', '-' ) |
---|
2030 | if string.find( thisl, ' ' ) != -1: |
---|
2031 | thisl = string.replace( thisl, ' ', '-' ) |
---|
2032 | if string.find( thisl, ':' ) != -1: |
---|
2033 | thisl = string.replace( thisl, ':', '-' ) |
---|
2034 | item.setAttribute( 'label', thisl ) |
---|
2035 | ff[k][0].appendChild( item ) |
---|
2036 | if k in ['requestvargroup','tablesection']: |
---|
2037 | thisd['label'] = thisl |
---|
2038 | self.rqvgList.append( thisd ) |
---|
2039 | if len(nfr) > 0: |
---|
2040 | print 'ERROR:nfr: ',nfr |
---|
2041 | |
---|
2042 | def fixfixcmv(self): |
---|
2043 | """deal with changing cmv ids relative to fixcmv""" |
---|
2044 | ##self.cmvinfo = ('uid','stid','label','title','description','vid', \ |
---|
2045 | ##'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', \ |
---|
2046 | ##'deflate', 'valid_min', 'flag_values', 'modeling_realm', 'valid_max', 'positive', 'mipTable', \ |
---|
2047 | ##'prov', 'provNote', 'frequency', 'rowIndex', 'coordinates','defaultPriority','mipTableSection') |
---|
2048 | ff = {} |
---|
2049 | for v in self.cmv: |
---|
2050 | kkk = '%s.%s.%s' % (v[2],v[18],v[19]) |
---|
2051 | if kkk in self.fix.fixcmvx: |
---|
2052 | for k in self.fix.fixcmvx[kkk]: |
---|
2053 | if k != v[0]: |
---|
2054 | print 'WARN.cmv.00040: changed uid:', k, v |
---|
2055 | ff[k] = v[0] |
---|
2056 | ## ['label','uid','table','mode','uid2','prov'] |
---|
2057 | |
---|
2058 | for vid in self.fix.fixcmv.keys(): |
---|
2059 | r = list( self.fix.fixcmv[vid] ) |
---|
2060 | print r |
---|
2061 | if r[1] in ff or r[4] in ff: |
---|
2062 | if r[1] in ff: |
---|
2063 | r[1] = ff[r[1]] |
---|
2064 | if r[4] in ff: |
---|
2065 | r[4] = ff[r[4]] |
---|
2066 | self.fix.fixcmv[r[1]] = r |
---|
2067 | |
---|
2068 | |
---|
2069 | def write2(self): |
---|
2070 | """write sections for cmor variable, request variable, structure, spatial and temporal dimensions""" |
---|
2071 | spsResets = collections.defaultdict(list) |
---|
2072 | doneStructures = set() |
---|
2073 | doneMessages = set() |
---|
2074 | self.fixfixcmv() |
---|
2075 | self.spLab2Uid = {} |
---|
2076 | self.tmLab2Uid = {} |
---|
2077 | self.tmUid2Lab = {} |
---|
2078 | self.tmsp2str = {} |
---|
2079 | self.tables = set() |
---|
2080 | for ksect in [ u'spatialshape', u'temporalshape', u'structure', u'cmorvar', u'requestvar', u'requestlink']: |
---|
2081 | ##for ksect in [ u'cmorvar', u'spatialshape', u'temporalshape', u'structure']: |
---|
2082 | dil = self.ff[ksect][0].getElementsByTagName('item') |
---|
2083 | if ksect == 'requestvar': |
---|
2084 | for item in dil: |
---|
2085 | vid = item.getAttribute( 'vid' ) |
---|
2086 | if vid in self.fix.fixcmv: |
---|
2087 | if self.fix.fixcmv[vid][3] == u'replace': |
---|
2088 | vid1 = str( self.fix.fixcmv[vid][4] ) |
---|
2089 | if vid1 not in self.cmvByUid: |
---|
2090 | print 'ERROR.fixcmv.0002: attempt to use invalid vid: ',item.getAttribute( 'label' ),vid1,vid in self.cmvByUid |
---|
2091 | else: |
---|
2092 | item.setAttribute( 'vid', vid1 ) |
---|
2093 | print 'INFO.fixcmv.00020: replacing link in requestVar %s: %s' % (item.getAttribute( 'uid' ),vid1) |
---|
2094 | else: |
---|
2095 | for d in dil: |
---|
2096 | self.ff[ksect][0].removeChild(d) |
---|
2097 | |
---|
2098 | if ksect == u'cmorvar': |
---|
2099 | ktable = self.cmvinfo.index('mipTable') |
---|
2100 | kfreq = self.cmvinfo.index('frequency') |
---|
2101 | klabel = self.cmvinfo.index('label') |
---|
2102 | |
---|
2103 | s1 = dict() |
---|
2104 | stab = set() |
---|
2105 | tmidpt = self.tmLab2Uid['time-point'] |
---|
2106 | self._cmvLabByUid = {} |
---|
2107 | for r in self.cmv: |
---|
2108 | if r[0] in self.rep['cmv'].items: |
---|
2109 | print 'INFO.090.09099: detected cmv replace request: ',r |
---|
2110 | elif r[0] in s1: |
---|
2111 | print 'ERROR.090.09005: duplicate key at ',r,s1[r[0]] |
---|
2112 | else: |
---|
2113 | s1[ r[0] ] = (r[ktable],r[kfreq],r[klabel]) |
---|
2114 | stab.add( r[ktable] ) |
---|
2115 | u0 = r[0] |
---|
2116 | l0 = r[2] |
---|
2117 | # |
---|
2118 | # identify structures for AerChemMIP tables |
---|
2119 | # |
---|
2120 | if len( r[1] ) == 2: |
---|
2121 | spid = self.spLab2Uid[r[1][0]] |
---|
2122 | if r[1][1] == 'mean': |
---|
2123 | tmid = self.tmLab2Uid['time-mean'] |
---|
2124 | tmode = 'mean' |
---|
2125 | else: |
---|
2126 | tmid = self.tmLab2Uid['time-point'] |
---|
2127 | tmode = 'point' |
---|
2128 | |
---|
2129 | if (tmid,spid,tmode) in self.lls.tmsp2str: |
---|
2130 | stid = self.lls.tmsp2str[ (tmid,spid,tmode) ] |
---|
2131 | else: |
---|
2132 | print 'SEVERE.stid.0001: no structure record for ',tmid,spid,tmode,r[1][0] |
---|
2133 | stid = '__stid_not_found__' |
---|
2134 | print 'INFO.klklkl: AerChemMIP structure .... ',r[1], spid |
---|
2135 | else: |
---|
2136 | if r[1] in self.strdd: |
---|
2137 | stid = self.strdd[r[1]] |
---|
2138 | else: |
---|
2139 | for k in self.strdd.keys(): |
---|
2140 | if k[0] == r[1][0]: |
---|
2141 | print '*** ',k |
---|
2142 | else: |
---|
2143 | print '+++ ',k |
---|
2144 | print 'SEVERE.999.00099: strdd key not found: %s' % str(r[1]) |
---|
2145 | print r |
---|
2146 | stid = '__struct_not_found_001__' |
---|
2147 | |
---|
2148 | stinfo = self.structInfo.get( stid, ['','','',''] ) |
---|
2149 | spinfo = self.spidInfo.get( stinfo[2] , ['','','','',''] ) |
---|
2150 | tmid = stinfo[3] |
---|
2151 | if "plev7h" in spinfo[3]: |
---|
2152 | lsfx = '7h' |
---|
2153 | elif "plev7c" in spinfo[3]: |
---|
2154 | lsfx = '' |
---|
2155 | elif "plev4" in spinfo[3]: |
---|
2156 | lsfx = '4' |
---|
2157 | elif "plev23" in spinfo[3] and r[2] == 'tntlw' : |
---|
2158 | ##print 'plev7h found',r,spinfo |
---|
2159 | lsfx = '23' |
---|
2160 | else: |
---|
2161 | lsfx = '' |
---|
2162 | |
---|
2163 | if r[2] == 'clisccp': |
---|
2164 | print 'INFO.clisccp.00001:',r |
---|
2165 | |
---|
2166 | if r[0] in self.fix.fixcmv: |
---|
2167 | ## print 'INFO.fixcmv.00010: omitting record %s, %s, %s' % (str( self.fix.fixcmv[r[0]] ), r[0],r[2] ) |
---|
2168 | pass |
---|
2169 | else: |
---|
2170 | item = self.doc.createElement( 'item' ) |
---|
2171 | for k in range(len(self.cmvinfo)): |
---|
2172 | n = self.cmvinfo[k] |
---|
2173 | if n in self.cmvinfo2: |
---|
2174 | if n == 'mipTable' and str(r[k]) == 'CMIP5_Oyr': |
---|
2175 | print 'ERROR.047.0002: ',r |
---|
2176 | if n == "vid": |
---|
2177 | if str(r[k]) == '__var_not_found_012__': |
---|
2178 | print 'ERROR.099.00202: bad vid string: %s [%s]' % (r[k],str(r[:3])) |
---|
2179 | vid = str(r[k]) |
---|
2180 | if vid in self.importWbMods.vars.vmap: |
---|
2181 | ov = vid |
---|
2182 | vid = self.importWbMods.vars.vmap[vid] |
---|
2183 | print 'INFO.040.04444: mapping',ov,vid |
---|
2184 | |
---|
2185 | if n == "rowIndex": |
---|
2186 | ix = str(r[k]) |
---|
2187 | elif n == "defaultPriority": |
---|
2188 | priority = str(r[k]) |
---|
2189 | elif n == "title": |
---|
2190 | title = str(r[k]) |
---|
2191 | |
---|
2192 | if n == "uid": |
---|
2193 | if r[k].find( 'ea313ee8-a0de-11e6-bc63-ac72891c3257' ) != -1: |
---|
2194 | print 'INFO.uid.9999: ',r |
---|
2195 | item.setAttribute( n, r[k].strip() ) |
---|
2196 | elif n == "stid": |
---|
2197 | item.setAttribute( n, stid ) |
---|
2198 | elif n in ["mipTableSection","processing"]: |
---|
2199 | if str(r[k]) != '': |
---|
2200 | item.setAttribute( n, str(r[k]) ) |
---|
2201 | elif n == "label": |
---|
2202 | thisl = string.replace( r[k], '_', '' ) |
---|
2203 | if len(lsfx) > 0: |
---|
2204 | if thisl[ - len( lsfx): ] != lsfx: |
---|
2205 | thisl += lsfx |
---|
2206 | item.setAttribute( n, thisl ) |
---|
2207 | elif n in ["valid_min","valid_max","ok_max_mean_abs","ok_min_mean_abs"]: |
---|
2208 | if str(r[k]) != "" and r[k] not in [None, 'None']: |
---|
2209 | item.setAttribute( n, str(r[k]) ) |
---|
2210 | elif n == "frequency": |
---|
2211 | fr = freqmap.get( str(r[k]), str(r[k]) ) |
---|
2212 | item.setAttribute( n, fr ) |
---|
2213 | elif n == "mipTable": |
---|
2214 | tn = tableNameMap.get( str(r[k]), str(r[k]) ) |
---|
2215 | if tn == 'em6hr': |
---|
2216 | if 'alev' in spinfo[3]: |
---|
2217 | tn = '6hrLev' |
---|
2218 | else: |
---|
2219 | if tmid == tmidpt: |
---|
2220 | tn = '6hrPlevpt' |
---|
2221 | else: |
---|
2222 | tn = '6hrPlev' |
---|
2223 | print 'INFO.6hrPlev.0001:', tmid, tmidpt, tmid == tmidpt, r,tn, self.tmUid2Lab[tmid] |
---|
2224 | item.setAttribute( n, tn ) |
---|
2225 | else: |
---|
2226 | item.setAttribute( n, str(r[k]) ) |
---|
2227 | |
---|
2228 | ### reset table name for OImon ... correcting for frequency as well |
---|
2229 | ### |
---|
2230 | omitThis = False |
---|
2231 | mtable = item.getAttribute( 'mipTable' ) |
---|
2232 | prov = item.getAttribute( 'prov' ) |
---|
2233 | realm = item.getAttribute( 'modeling_realm' ) |
---|
2234 | if realm in ['','__unset__']: |
---|
2235 | if prov in knownRealmA: |
---|
2236 | item.setAttribute( 'modeling_realm', 'atmos' ) |
---|
2237 | elif prov in knownRealmO: |
---|
2238 | item.setAttribute( 'modeling_realm', 'ocean' ) |
---|
2239 | |
---|
2240 | mtable0 = mtable |
---|
2241 | if mtable0 == 'OImon': |
---|
2242 | if r[kfreq] == 'mon': |
---|
2243 | mtable = 'SImon' |
---|
2244 | else: |
---|
2245 | mtable = 'SIday' |
---|
2246 | elif mtable0 in ['em6hr','em3hr','emDay','emMon']: |
---|
2247 | if stid in self.structInfo: |
---|
2248 | if string.find( self.structInfo[stid][1], 'time: point' ) != -1 or tmid == tmidpt: |
---|
2249 | mtable += 'pt' |
---|
2250 | if string.find( self.spatialDims[ self.structInfo[stid][2] ], 'longitude' ) == -1 and mtable[-1] != 'Z': |
---|
2251 | mtable += 'Z' |
---|
2252 | else: |
---|
2253 | print 'WARN.090.09999: structure uid not found',stid, r |
---|
2254 | elif mtable0 == 'Amon' and thisl[-6:] == 'Adjust': |
---|
2255 | print 'INFO.adjust.0001: %s moved to AmonAdj .. and omitted' % thisl |
---|
2256 | mtable = 'AmonAdj' |
---|
2257 | omitThis = True |
---|
2258 | if mtable != r[ktable]: |
---|
2259 | item.setAttribute( 'mipTable', mtable ) |
---|
2260 | item.setAttribute( 'mtid', 'MIPtable::%s' % mtable ) |
---|
2261 | self.tables.add( mtable ) |
---|
2262 | |
---|
2263 | thisuid = r[0] |
---|
2264 | if tableUtils.getSubGroup(spinfo[3],mtable,thisl,ix,title,thisuid,priority): |
---|
2265 | item.setAttribute( 'subGroup', tableUtils.subGroup ) |
---|
2266 | |
---|
2267 | if str(vid) not in self.importWbMods.vars.d1: |
---|
2268 | |
---|
2269 | if thisl in self.importWbMods.vars.c1 or (thisl[-2:] in ['17','19','23','27','7c','7h'] and thisl[:-2] in self.importWbMods.vars.c1): |
---|
2270 | if thisl in self.importWbMods.vars.c1: |
---|
2271 | thisn = thisl |
---|
2272 | else: |
---|
2273 | thisn = thisl[:-2] |
---|
2274 | |
---|
2275 | if len( self.importWbMods.vars.c1[ thisn ] ) == 1: |
---|
2276 | newvid = self.importWbMods.vars.c1[ thisn ][0] |
---|
2277 | item.setAttribute( 'vid', newvid ) |
---|
2278 | print 'INFO.099.00110: bad vid string replaced: %s [%s]' % (newvid,thisl) |
---|
2279 | else: |
---|
2280 | print 'ERROR.099.00111: ambiguous bad vid string: %s [%s]' % (vid,thisl) |
---|
2281 | else: |
---|
2282 | print 'ERROR.099.00112: bad vid string: %s [%s]' % (vid,thisl) |
---|
2283 | elif thisl == 'H2p': |
---|
2284 | print 'WARN.h2p.00001: H2p vid recording as OK here .... %s' % vid |
---|
2285 | if self.repl.has_key( thisuid ): |
---|
2286 | for tag,old,new in self.repl[thisuid]: |
---|
2287 | thisold = item.getAttribute( tag ) |
---|
2288 | if thisold != old: |
---|
2289 | ##print 'WARN.yyy: Attempt to replace value which is not present' |
---|
2290 | print 'WARN.050.00051: Failed attempt to replace values in cmv: %s old:%s new:%s' % (tag,old,new) |
---|
2291 | else: |
---|
2292 | if tag == 'vid' and new not in self.importWbMods.vars.d1: |
---|
2293 | print 'ERROR.050.00052: Attempt to replave values in cmv with invalid vid: %s old:%s new:%s' % (tag,old,new) |
---|
2294 | else: |
---|
2295 | item.setAttribute( tag, new ) |
---|
2296 | print 'INFO.050.00050: replacing values in cmv: %s old:%s new:%s' % (tag,old,new) |
---|
2297 | |
---|
2298 | if thisuid in self.deferredLabMap: |
---|
2299 | lab0,lab1 = self.deferredLabMap[thisuid] |
---|
2300 | labold = item.getAttribute( 'label' ) |
---|
2301 | if labold != lab0: |
---|
2302 | print 'ERROR.labmap.0001: attempt to change wrong label: ',lab0,lab1,labold |
---|
2303 | else: |
---|
2304 | item.setAttribute( 'label', lab1 ) |
---|
2305 | print 'INFO.labmap.0001: attempt to change label: ',lab0,lab1 |
---|
2306 | |
---|
2307 | ### final check on structure .. but won't work if tuple not found ... but why is structure not there?? |
---|
2308 | stid = item.getAttribute( 'stid' ) |
---|
2309 | if stid in self.struidmapextra: |
---|
2310 | mid = ('INFO.structure2',stid) |
---|
2311 | if mid not in doneMessages: |
---|
2312 | print 'INFO.structure: mapping [2]: %s to %s' % (stid,self.struidmapextra[stid]) |
---|
2313 | doneMessages.add(mid) |
---|
2314 | item.setAttribute( 'stid', self.struidmapextra[stid] ) |
---|
2315 | stid = self.struidmapextra[stid] |
---|
2316 | if stid in WbMods.str.uidMap: |
---|
2317 | mid = ('INFO.structure',stid) |
---|
2318 | if mid not in doneMessages: |
---|
2319 | print 'INFO.structure: mapping: %s to %s' % (stid,WbMods.str.uidMap[stid]) |
---|
2320 | doneMessages.add(mid) |
---|
2321 | if stid in self.struidmapextra: |
---|
2322 | assert self.struidmapextra[stid] == WbMods.str.uidMap[stid], 'CONFLICT in stid mappings: %s, %s, %s' % (stid,self.struidmapextra[stid], WbMods.str.uidMap[stid]) |
---|
2323 | item.setAttribute( 'stid', WbMods.str.uidMap[stid] ) |
---|
2324 | |
---|
2325 | ## append CMORvar record |
---|
2326 | u1 = item.getAttribute('uid') |
---|
2327 | if u1 in extRefCmvMapped: |
---|
2328 | print 'WARN.uidmap.00004: mapping CMORvar uid %s --> %s' % (u1,extRefCmvMapped[u1] ) |
---|
2329 | item.setAttribute('uid',extRefCmvMapped[u1]) |
---|
2330 | l1 = item.getAttribute( 'label') |
---|
2331 | #up: rlus rlut rlutcs hfls hfss mcu rsus rsut rsutcs down:rlds rsds |
---|
2332 | if l1 in varPositive: |
---|
2333 | item.setAttribute( 'positive', varPositive[l1] ) |
---|
2334 | print 'INFO.positive.00001: setting positive [%s]: %s' % (l1,varPositive[l1]) |
---|
2335 | if l1 != l0: |
---|
2336 | print 'WARN.cmvlab.00030: changed cmv label: %s %s %s %s' % (l0,l1,u0,u1) |
---|
2337 | self._cmvLabByUid[item.getAttribute('uid')] = item.getAttribute( 'label') |
---|
2338 | if not omitThis: |
---|
2339 | self.ff[ksect][0].appendChild( item ) |
---|
2340 | print 'INFO.tables.0001: ',stab |
---|
2341 | |
---|
2342 | elif ksect == u'structure': |
---|
2343 | ers = {} |
---|
2344 | for kk in spsResets: |
---|
2345 | if len( spsResets[kk] ) > 0: |
---|
2346 | utarg = spsResets[kk][0][0] |
---|
2347 | for r in spsResets[kk][1:]: |
---|
2348 | ers[r[0]] = utarg |
---|
2349 | ##flds = self.structList[0][1]._fields |
---|
2350 | flds = self.lls.structAttr |
---|
2351 | for u,nt in self.lls.ll: |
---|
2352 | smode = 'new' |
---|
2353 | if smode == 'new': |
---|
2354 | d1 = self.lls.asDict( u ) |
---|
2355 | tlab = d1['time label'] |
---|
2356 | ##print 'INFO.stid.00072: %s, %s' % (u,str( d1 )) |
---|
2357 | else: |
---|
2358 | ##print 'INFO.stid.00071: %s, %s' % (u,str( nt.__dict__ )) |
---|
2359 | d1 = {} |
---|
2360 | for f in flds: |
---|
2361 | if f == 'spid' and nt.__dict__[f] in ers: |
---|
2362 | d1[f] = ers[ nt.__dict__[f] ] |
---|
2363 | print 'INFO.801.00011: updating target spid .... ', d1[f] |
---|
2364 | elif f == 'coords' and 't2000' in nt.__dict__[f]: |
---|
2365 | d1[f] = str( nt.__dict__[f] ) |
---|
2366 | elif f == 'coords' and nt.__dict__[f][:4] in ['plev']: |
---|
2367 | cnew = 'p' + nt.__dict__[f][4:] |
---|
2368 | d1[f] = str( cnew ) |
---|
2369 | |
---|
2370 | print 'INFO.801.00013: updating target coords .... %s --> %s ' % ( nt.__dict__[f], cnew ) |
---|
2371 | elif f == 'uid': |
---|
2372 | d1[f] = u |
---|
2373 | elif f == 'tlab': |
---|
2374 | tlab = nt.tlab |
---|
2375 | tmid = self.tmLab2Uid[tlab] |
---|
2376 | d1['tmid'] = tmid |
---|
2377 | |
---|
2378 | else: |
---|
2379 | d1[f] = str( nt.__dict__[f] ) |
---|
2380 | if d1.get( 'cell_measures', '' ) == '@OPT': |
---|
2381 | print 'WARN.structure.00077: changing cell_measures @OPT to --OPT',str(d1 ) |
---|
2382 | d1['cell_measures'] = '--OPT' |
---|
2383 | |
---|
2384 | if d1['spid'] in WbMods.ls.labByUid: |
---|
2385 | slab = WbMods.ls.labByUid[ d1['spid'] ] |
---|
2386 | print 'INFO.801.00061: spid found in structure record ... %s\n%s' % (d1['spid'],slab) |
---|
2387 | else: |
---|
2388 | slab = '?' |
---|
2389 | print 'SEVERE.801.00060: bad spid found in structure record ... %s\n%s' % (d1['spid'],str(d1)) |
---|
2390 | cm = d1['cell_methods'] |
---|
2391 | if cm == 'null': |
---|
2392 | if tlab == 'time-point': |
---|
2393 | cm = 'time: point' |
---|
2394 | d1['cell_methods'] = cm |
---|
2395 | elif tlab == 'time-mean': |
---|
2396 | cm = 'time: mean' |
---|
2397 | d1['cell_methods'] = cm |
---|
2398 | elif cm in WbMods.cellm.cmmap: |
---|
2399 | cm0 = cm |
---|
2400 | cm = WbMods.cellm.cmmap[cm] |
---|
2401 | print 'INFO.cmmap.00001: ',cm0,cm |
---|
2402 | d1['cell_methods'] = cm |
---|
2403 | |
---|
2404 | cc = d1['coords'] |
---|
2405 | oo = d1['odims'] |
---|
2406 | if cc not in ['','?']: |
---|
2407 | print 'INFO.coords.00001: >%s<' % cc |
---|
2408 | d1['cids'] = 'dim:%s' % cc |
---|
2409 | if oo not in ['','?']: |
---|
2410 | ooc = string.join( ['dim:%s' % x for x in oo.split( '|' )] ) |
---|
2411 | print 'INFO.odims.00001: >%s<' % ooc |
---|
2412 | d1['cids'] = ooc |
---|
2413 | t = (tlab,slab,d1['odims'],d1['coords'],d1['cell_methods'],d1['cell_measures'],d1['flag_meanings']) |
---|
2414 | |
---|
2415 | dothis = True |
---|
2416 | if t not in WbMods.str.lookup: |
---|
2417 | print 'WARN.structure.0040: tuple not in lookup: ',t,d1['uid'],d1['spid'],d1 |
---|
2418 | else: |
---|
2419 | print 'WARN.structure.0041: tuple in lookup ignored: ',t,d1['uid'],d1['spid'],d1 |
---|
2420 | lab = d1['label'] |
---|
2421 | if lab in WbMods.str.labMap: |
---|
2422 | print 'INFO.structure.0050: omitting structure record ....',t,lab, d1['uid'] |
---|
2423 | lab2 = WbMods.str.labMap[lab] |
---|
2424 | dothis = False |
---|
2425 | self.struidmapextra[d1['uid']] = WbMods.str.uidByLab[lab2] |
---|
2426 | print 'INFO.structure.0062: setting stid mapping ....',t,d1['uid'],WbMods.str.uidByLab[lab2], d1['label'], lab2 |
---|
2427 | |
---|
2428 | lab = d1['label'] |
---|
2429 | if lab in WbMods.str.mods: |
---|
2430 | for k in WbMods.str.mods[lab]: |
---|
2431 | print 'INFO.structure.0055: modifying attributes: %s: %s: %s' % (lab,k,WbMods.str.mods[lab][k]) |
---|
2432 | d1[k] = WbMods.str.mods[lab][k] |
---|
2433 | if k == 'cell_methods': |
---|
2434 | cm = WbMods.str.mods[lab][k] |
---|
2435 | if cm in WbMods.cellm.cminfo: |
---|
2436 | d1['cmid'] = WbMods.cellm.cminfo[cm][2] |
---|
2437 | else: |
---|
2438 | print 'SEVERE.structure.0055: cmid for new cell method not found: %s' % (cm) |
---|
2439 | |
---|
2440 | if dothis and d1['uid'] in doneStructures: |
---|
2441 | print 'WARN.structure.05021: repeat uid in structure definitions ....%s' % d1['uid'] |
---|
2442 | if dothis and d1['uid'] not in doneStructures and d1['uid'] not in WbMods.str.uidMap: |
---|
2443 | if cm not in WbMods.cellm.cminfo: |
---|
2444 | print 'WARN.cellm.0001: Cell methods not found: ',cm, d1['uid'] |
---|
2445 | else: |
---|
2446 | d1['cmid'] = WbMods.cellm.cminfo[cm][2] |
---|
2447 | |
---|
2448 | doneStructures.add( d1['uid'] ) |
---|
2449 | item = self.doc.createElement( 'item' ) |
---|
2450 | for k in d1: |
---|
2451 | if k not in ['time_label','spatial_label', 'time label','spatial label']: |
---|
2452 | item.setAttribute( k.replace( ' ','_'), d1[k] ) |
---|
2453 | self.ff[ksect][0].appendChild( item ) |
---|
2454 | else: |
---|
2455 | if not dothis: |
---|
2456 | print 'INFO.structure.09001: omitting %s' % d1['label'] |
---|
2457 | elif d1['uid'] in doneStructures: |
---|
2458 | print 'INFO.structure.09002: omitting %s' % d1['label'] |
---|
2459 | elif d1['uid'] in WbMods.str.uidMap: |
---|
2460 | print 'INFO.structure.09003: omitting %s' % d1['label'] |
---|
2461 | ### |
---|
2462 | elif ksect == u'spatialshape': |
---|
2463 | |
---|
2464 | ## self.dimsSpaceInfo = ['uid','label','title','dimensions','levelFlag','levels'] |
---|
2465 | self.spatialDims = {} |
---|
2466 | for k in self.dimsSpace.keys(): |
---|
2467 | rr = list( self.dimsSpace[k] ) |
---|
2468 | kk = rr[3] |
---|
2469 | if kk == 'scalar': |
---|
2470 | kk = '' |
---|
2471 | rr = list(rr) |
---|
2472 | rr[3] = '' |
---|
2473 | elif kk == 'longitude|latitude|plev': |
---|
2474 | kk = 'longitude|latitude|plev4' |
---|
2475 | rr = list(rr) |
---|
2476 | rr[3] = kk |
---|
2477 | elif kk == 'longitude|latitude|plev7': |
---|
2478 | kk = 'longitude|latitude|plev7c' |
---|
2479 | print 'INFO.plev7.00001: plev7 --> plev7c' |
---|
2480 | rr = list(rr) |
---|
2481 | rr[3] = kk |
---|
2482 | self.spatialDims[rr[0]] = rr[3] |
---|
2483 | bits = [x for x in rr[3].split( '|' ) if x not in ['','?']] |
---|
2484 | rr.append( string.join( ['dim:%s' % x for x in bits] ) ) |
---|
2485 | if kk not in spsResets: |
---|
2486 | item = self.doc.createElement( 'item' ) |
---|
2487 | for i in range(len(self.dimsSpaceInfo)): |
---|
2488 | item.setAttribute( self.dimsSpaceInfo[i], str( rr[i] ) ) |
---|
2489 | ##assert rr[3] in self.spsh.ss, 'FATAL: spatial shape record not recognised: %s' % str(rr) |
---|
2490 | if rr[3] in self.spsh.ss: |
---|
2491 | lab,ttl,lf,li,u = self.spsh.ss[rr[3]] |
---|
2492 | if trace.SPATIALSHAPE_01: |
---|
2493 | print 'INFO.035.0001: new spatial shape label: %s, %s' % (lab, ttl) |
---|
2494 | if ttl[:4] != 'ssd.': |
---|
2495 | item.setAttribute( 'title', ttl ) |
---|
2496 | if len(lab) < 4 or lab[:4] != 'ssd-': |
---|
2497 | item.setAttribute( 'label', lab ) |
---|
2498 | item.setAttribute( 'levels', li ) |
---|
2499 | item.setAttribute( 'uid', u ) |
---|
2500 | item.setAttribute( 'levelFlag', string.lower(lf) ) |
---|
2501 | else: |
---|
2502 | print 'ERROR.055.00001: spatial shape record not recognised: %s' % str(rr) |
---|
2503 | print 'INFO.801.00120: new shape for %s' % str(rr) |
---|
2504 | thisl = item.getAttribute( 'label' ) |
---|
2505 | assert thisl not in self.spLab2Uid.keys(), 'Duplicate spatial shape label: %s' % thisl |
---|
2506 | self.spLab2Uid[thisl] = item.getAttribute( 'uid' ) |
---|
2507 | self.ff[ksect][0].appendChild( item ) |
---|
2508 | else: |
---|
2509 | print 'INFO.801.00121: duplicate shape for %s' % str(rr) |
---|
2510 | spsResets[kk].append( rr ) |
---|
2511 | |
---|
2512 | elif ksect == u'temporalshape': |
---|
2513 | for k in self.dimsTime.keys(): |
---|
2514 | rr = self.dimsTime[k] |
---|
2515 | item = self.doc.createElement( 'item' ) |
---|
2516 | for i in range(len(self.dimsTimeInfo)): |
---|
2517 | item.setAttribute( self.dimsTimeInfo[i], str( rr[i] ) ) |
---|
2518 | if self.dimsTimeInfo[i] == 'dimensions': |
---|
2519 | item.setAttribute( 'dimid', 'dim:%s' % rr[i] ) |
---|
2520 | self.ff[ksect][0].appendChild( item ) |
---|
2521 | self.tmLab2Uid[ item.getAttribute( 'label' ) ] = item.getAttribute( 'uid' ) |
---|
2522 | self.tmUid2Lab[ item.getAttribute( 'uid' ) ] = item.getAttribute( 'label' ) |
---|
2523 | elif ksect == u'requestvar': |
---|
2524 | ##kk = ['var', 'table', 'mip', 'vid', 'priority'] |
---|
2525 | kk = ['label', 'table', 'mip', 'vid', 'priority','vgid','title','uid'] |
---|
2526 | kktr = {'vgid':'gpid', 'title':'label'} |
---|
2527 | kkf = 0 |
---|
2528 | for t in self.rqv: |
---|
2529 | if t[5][:6] == 'SPECS_': |
---|
2530 | print 'SEVERE.specs.00001: mis-assigned mip: ',t |
---|
2531 | if t[1][-6:] == 'Adjust': |
---|
2532 | doThis = False |
---|
2533 | else: |
---|
2534 | item = self.doc.createElement( 'item' ) |
---|
2535 | |
---|
2536 | for i in range(len(kk)): |
---|
2537 | k = kktr.get( kk[i], kk[i] ) |
---|
2538 | v = t[ self.rqvinfo.index(k) ] |
---|
2539 | if k == 'priority': |
---|
2540 | item.setAttribute( kk[i], str( int(v) ) ) |
---|
2541 | elif kk[i] == 'label': |
---|
2542 | thisl = string.replace( v, '_', '' ) |
---|
2543 | item.setAttribute( kk[i], thisl ) |
---|
2544 | elif kk[i] == 'title': |
---|
2545 | item.setAttribute( kk[i], str( v ) + ' ((isd.006))' ) |
---|
2546 | elif kk[i] == 'table': |
---|
2547 | pass |
---|
2548 | else: |
---|
2549 | item.setAttribute( kk[i], str( v ) ) |
---|
2550 | |
---|
2551 | vid = item.getAttribute( 'vid' ) |
---|
2552 | if vid in self.fix.fixcmv: |
---|
2553 | if self.fix.fixcmv[vid][3] == u'replace': |
---|
2554 | vidn = str( self.fix.fixcmv[vid][4] ) |
---|
2555 | if vidn not in self._cmvLabByUid: |
---|
2556 | print 'SEVERE.fixcmv.00040: attempt to redirect request to non-existant variable: %s,%s,%s::%s' % (vid,vidn,thisl,str(self.fix.fixcmv[vid]) ) |
---|
2557 | else: |
---|
2558 | labn = self.cmvByUid[vidn][2] |
---|
2559 | if vid in self.cmvByUid: |
---|
2560 | lab = self.cmvByUid[vid][2] |
---|
2561 | if lab != labn: |
---|
2562 | print 'WARN.fixcmv.00001: label change: %s -- %s (%s --> %s)' % (lab,labn,vid,vidn) |
---|
2563 | elif labn != thisl: |
---|
2564 | print 'WARN.fixcmv.00002: label change [2]: %s -- %s (%s --> %s)' % (thisl,labn,vid,vidn) |
---|
2565 | item.setAttribute( 'vid', vidn ) |
---|
2566 | vid = vidn |
---|
2567 | print 'INFO.fixcmv.00050: replacing link in requestVar %s %s %s' % (str(t),vidn,labn) |
---|
2568 | |
---|
2569 | newd = None |
---|
2570 | if vid in self.importWbMods.mmsi.ss: |
---|
2571 | print 'INFO.046.0002: mip map found: ', self.importWbMods.mmsi.ss[vid] |
---|
2572 | mm = self.importWbMods.mmsi.ss[vid] |
---|
2573 | ##newd = self._editRequestVar01( item, mm, t, {} ) |
---|
2574 | |
---|
2575 | vid = item.getAttribute( 'vid' ) |
---|
2576 | if vid in self.cmvMap002: |
---|
2577 | vid = self.cmvMap002[vid] |
---|
2578 | item.setAttribute( 'vid', vid ) |
---|
2579 | |
---|
2580 | if vid in extRefCmvMapped: |
---|
2581 | vid = extRefCmvMapped[vid] |
---|
2582 | item.setAttribute( 'vid', vid ) |
---|
2583 | |
---|
2584 | if vid not in self.cmvByUid: |
---|
2585 | print 'ERROR.reqvar.000001: vid not found: %s' % str(t) |
---|
2586 | else: |
---|
2587 | lab0 = self.cmvByUid[vid][2] |
---|
2588 | if vid not in self._cmvLabByUid: |
---|
2589 | print 'WARN.cmvlab.00020: no _cmv.. record ... %s %s: %s' % (vid,lab0,str( self.cmvByUid[vid] ) ) |
---|
2590 | else: |
---|
2591 | lab = self._cmvLabByUid[vid] |
---|
2592 | if lab != lab0: |
---|
2593 | print 'WARN.cmvlab.00010: difference in records ... %s %s: %s' % (vid,lab,str( self.cmvByUid[vid] ) ) |
---|
2594 | if lab != thisl: |
---|
2595 | ##print 'INFO.reqvar.00001: inconsistent requestvar label: %s --> %s (%s)' % (thisl, lab, vid) |
---|
2596 | print 'INFO.reqvar.00001: changing requestvar label: %s --> %s' % (thisl, lab) |
---|
2597 | item.setAttribute( 'label', lab ) |
---|
2598 | |
---|
2599 | tm = item.getAttribute( 'mip' ) |
---|
2600 | |
---|
2601 | did = '%s__%s' % (t[2],t[3]) |
---|
2602 | if self.vgcheck.has_key(did): |
---|
2603 | print 'ERROR.090.09002: Duplicate request ....: ',t,self.vgcheck[did] |
---|
2604 | else: |
---|
2605 | self.vgcheck[did] = t |
---|
2606 | self.ff[ksect][0].appendChild( item ) |
---|
2607 | ## |
---|
2608 | ###################################################################################### |
---|
2609 | ## |
---|
2610 | elif ksect == u'requestlink': |
---|
2611 | ## - uuid; mip; tab; objective; grid; gridreq; comment |
---|
2612 | kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'ref', 'refNote', 'refid'] |
---|
2613 | kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'refid'] |
---|
2614 | ##kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt','opar','ref', 'refNote', 'refid'] |
---|
2615 | |
---|
2616 | for k,ll in self.rqLnks.items(): |
---|
2617 | if k[0] != '_': |
---|
2618 | assert len(ll) == len(kk), 'Bad record length in requestLink' |
---|
2619 | thisl = string.strip(str( ll[2] ) ) |
---|
2620 | uid = str( ll[0] ).strip() |
---|
2621 | item = self.doc.createElement( 'item' ) |
---|
2622 | for j in range(len(kk)): |
---|
2623 | thiskk = kk[j] |
---|
2624 | ## preset moved to request item .... so it can also be used on a per-experiment basis |
---|
2625 | if thiskk != 'preset': |
---|
2626 | if kk[j] == 'comment': |
---|
2627 | v = utils_wb.uniCleanFunc( ll[j] ) |
---|
2628 | else: |
---|
2629 | v = str( ll[j] ) |
---|
2630 | |
---|
2631 | ### redirect to new variable group if specified in mappings |
---|
2632 | ## |
---|
2633 | item.setAttribute( thiskk, v ) |
---|
2634 | |
---|
2635 | item.setAttribute( 'label', thisl.replace( '.', '-' ) ) |
---|
2636 | item.setAttribute( 'title', thisl ) |
---|
2637 | self.ff[ksect][0].appendChild( item ) |
---|
2638 | ################################## |
---|
2639 | |
---|
2640 | def _editRequestVar01( self, item, mm, t, thisd, vid=None ): |
---|
2641 | if vid == None: |
---|
2642 | vid = mm[0].vid2 |
---|
2643 | item.setAttribute( 'vid', vid ) |
---|
2644 | item.setAttribute( 'label', mm[0].lab2 ) |
---|
2645 | thisd['vid'] = vid |
---|
2646 | thisd['label'] = mm[0].lab2 |
---|
2647 | print 'INFO.fffff.00001:', mm[0].lab2, mm[0].vid2 |
---|
2648 | othd = None |
---|
2649 | if len( mm ) > 1: |
---|
2650 | othd = thisd.copy() |
---|
2651 | assert len( mm ) < 3, 'Unable to deal with triple clone' |
---|
2652 | cln = item.cloneNode(True) |
---|
2653 | uid = str( uuid.uuid1() ) |
---|
2654 | cln.setAttribute( 'uid', uid ) |
---|
2655 | cln.setAttribute( 'label', mm[1].lab2 ) |
---|
2656 | othd['uid'] = uid |
---|
2657 | othd['label'] = mm[1].lab2 |
---|
2658 | self.ff['requestvar'][0].appendChild( cln ) |
---|
2659 | |
---|
2660 | print 'INFO.046.0003: request variable split:', t[1] |
---|
2661 | |
---|
2662 | def finishExptGrp(self): |
---|
2663 | """finish exptGroup section and create exptgrouplink section""" |
---|
2664 | this = self.doc.getElementsByTagName('exptGroup')[0] |
---|
2665 | ### kk = ['uid','label','tierMin','ntot'] |
---|
2666 | for k in self.exptPntUid.keys(): |
---|
2667 | u,typ = self.exptPntUid[k] |
---|
2668 | ## if not isg: |
---|
2669 | ##item = self.doc.createElement( 'item' ) |
---|
2670 | ##item.setAttribute( uid, u ) |
---|
2671 | ##this = self.doc.getElementsByTagName('exptGroupLink')[0] |
---|
2672 | ##dil = this.getElementsByTagName('item') |
---|
2673 | ##for d in dil: |
---|
2674 | ##this.removeChild(d) |
---|
2675 | ##this.appendChild( item ) |
---|
2676 | |
---|
2677 | def reviewExpt(self): |
---|
2678 | """Prepare cross-linking information to populate requestitem, exptgroup and exptgrouplink""" |
---|
2679 | sh = shelve.open( '../framework/sh_links/exptMappings', 'r' ) |
---|
2680 | extraMappings={ 'LS3MIP-1, LS3MIP-2':['LmipH', 'LmipF'], \ |
---|
2681 | 'LULCC_hist, CPL_1pctDF, LND_DF, ATM_DF, CPL_DF, CPL_noLULCC_hist_conc (LUMIP)':['LULCC_hist', 'CPL_1pctDF', 'LND_DF', 'ATM_DF', 'CPL_DF', 'CPL_noLULCC_hist_conc'] \ |
---|
2682 | } |
---|
2683 | |
---|
2684 | ## |
---|
2685 | ## extra fix for HighResMIP .... after encoding in template made consistent with other MIPs |
---|
2686 | ## |
---|
2687 | shm = {} |
---|
2688 | for k in sh.keys(): |
---|
2689 | if string.find( k, ' ' ) != -1 and k[:4] == 'High': |
---|
2690 | k1 = string.split( k)[0] |
---|
2691 | assert k1 not in shm, 'Duplicate abbreviated key: %s [%s and %s]' % (k1,k,shm[k1]) |
---|
2692 | shm[k1] = k |
---|
2693 | ks = sorted( self.exptPnt.keys() ) |
---|
2694 | self.exptPntMap = collections.defaultdict( list ) |
---|
2695 | self.exptPntUid = {} |
---|
2696 | ner = 0 |
---|
2697 | ngr = 0 |
---|
2698 | nmr = 0 |
---|
2699 | nuns = 0 |
---|
2700 | nls = 0 |
---|
2701 | for k0 in ks: |
---|
2702 | k = string.replace( string.replace( k0, '\n', ' ' ), ' ', ' ') |
---|
2703 | thisl = None |
---|
2704 | if sh.has_key(k): |
---|
2705 | thisl = sh[k] |
---|
2706 | elif shm.has_key(k): |
---|
2707 | thisl = sh[shm[k]] |
---|
2708 | elif extraMappings.has_key(k): |
---|
2709 | thisl = extraMappings[k] |
---|
2710 | elif self.experiments.has_key(k): |
---|
2711 | thisl = [k,] |
---|
2712 | else: |
---|
2713 | thisl = [k,] |
---|
2714 | print 'WARN.080.00010: no mapping founds for experiment %s' % k |
---|
2715 | |
---|
2716 | |
---|
2717 | if thisl != None: |
---|
2718 | thisu = None |
---|
2719 | thisls = set( thisl ) |
---|
2720 | linkType = None |
---|
2721 | for k2 in thisls: |
---|
2722 | ok = True |
---|
2723 | if self.experiments.has_key(k2): |
---|
2724 | ner += 1 |
---|
2725 | linkType = 'expt' |
---|
2726 | thisu = self.experiments[k2] |
---|
2727 | elif self.experimentGrp.has_key(k2): |
---|
2728 | ngr += 1 |
---|
2729 | thisu = self.experimentGrp[k2] |
---|
2730 | linkType = 'grp' |
---|
2731 | elif k2 in self.mips: |
---|
2732 | nmr += 1 |
---|
2733 | thisu = k2 |
---|
2734 | linkType = 'mip' |
---|
2735 | else: |
---|
2736 | k2c = dreq_utils.labcoerce(k2) |
---|
2737 | if self.experimentGrp.has_key(k2c): |
---|
2738 | ngr += 1 |
---|
2739 | thisu = self.experimentGrp[k2c] |
---|
2740 | linkType = 'grp' |
---|
2741 | else: |
---|
2742 | ok = False |
---|
2743 | print 'ERROR.080.00999: target experiment not found: %s [%s]' % (k2,k) |
---|
2744 | if not ok: |
---|
2745 | self.exptPntMap[k].append( k2 ) |
---|
2746 | ## |
---|
2747 | ## store uid of expt group to be pointed at. |
---|
2748 | ## |
---|
2749 | if len(thisls) == 1 and thisu != None: |
---|
2750 | self.exptPntUid[k0] = (thisu, linkType, thisls) |
---|
2751 | else: |
---|
2752 | self.exptPntUid[k0] = (str( uuid.uuid1() ), 'list', tuple(thisls) ) |
---|
2753 | print 'INFO.expt.00001: ',k0, thisl, thisls, self.exptPntUid[k0] |
---|
2754 | nls += 1 |
---|
2755 | else: |
---|
2756 | nuns += 1 |
---|
2757 | self.exptPntUid[k0] = (None,None) |
---|
2758 | |
---|
2759 | sh.close() |
---|
2760 | print 'INFO.080.00010: Expt refs: %s, Expt group refs: %s, MIP expt set refs: %s, unset: %s, lists: %s' % (ner,ngr,nmr, nuns, nls) |
---|
2761 | |
---|
2762 | def reviewObj(self): |
---|
2763 | ## review objectives |
---|
2764 | ## |
---|
2765 | ol = [] |
---|
2766 | fnd1 = set() |
---|
2767 | for m in self.objectives.keys(): |
---|
2768 | not_used = [] |
---|
2769 | used = set() |
---|
2770 | not_found = [] |
---|
2771 | found = [] |
---|
2772 | alist = ['All','All?','all'] |
---|
2773 | aaa = [self.objectiveLinks[m].a.has_key(x) for x in ['All','All?','all'] ] |
---|
2774 | us1 = set() |
---|
2775 | for k in self.objectiveLinks[m].a: |
---|
2776 | x = None |
---|
2777 | if k == 'AllExceptPaleorcms': |
---|
2778 | x = self.objectives[m].keys() |
---|
2779 | x.pop( x.index( 'Paleorcms' ) ) |
---|
2780 | elif k == 'AllExceptNaturalVariability': |
---|
2781 | x = self.objectives[m].keys() |
---|
2782 | x.pop( x.index( 'NaturalVariability' ) ) |
---|
2783 | |
---|
2784 | if x != None: |
---|
2785 | fnd1.add(k) |
---|
2786 | for i in x: |
---|
2787 | used.add(i) |
---|
2788 | us1.add(i) |
---|
2789 | for u in self.objectiveLinks[m].a[k]: |
---|
2790 | ol.append( (m,self.objectives[m][i], u ) ) |
---|
2791 | |
---|
2792 | allused = (m != 'GMMIP') and any( aaa ) |
---|
2793 | for o in self.objectives[m].keys(): |
---|
2794 | if allused or self.objectiveLinks[m].a.has_key(o): |
---|
2795 | used.add(o) |
---|
2796 | for u in self.objectiveLinks[m].a[o]: |
---|
2797 | ol.append( (m,self.objectives[m][o], u ) ) |
---|
2798 | for k in alist: |
---|
2799 | if self.objectiveLinks[m].a.has_key(k): |
---|
2800 | for u in self.objectiveLinks[m].a[k]: |
---|
2801 | ol.append( (m,self.objectives[m][o], u ) ) |
---|
2802 | elif o not in us1: |
---|
2803 | not_used.append(o) |
---|
2804 | |
---|
2805 | for o in self.objectiveLinks[m].a.keys(): |
---|
2806 | if (o in {'All','All?'} and len(self.objectives[m].keys()) > 0 ) or self.objectives[m].has_key(o) or o in fnd1: |
---|
2807 | found.append(o) |
---|
2808 | else: |
---|
2809 | not_found.append(o) |
---|
2810 | if len(not_used) > 0 or len(not_found) > 0: |
---|
2811 | print 'WARN.080.00002: %s: OK %s: unused: %s: not found:%s' % (m,str(used),str(not_used),str(not_found)) |
---|
2812 | else: |
---|
2813 | print 'INFO.080.00002: %s: all objectives matched' % m |
---|
2814 | self.objList = ol |
---|
2815 | |
---|
2816 | |
---|
2817 | def finish(self): |
---|
2818 | ## varChoice section |
---|
2819 | self.choices() |
---|
2820 | ## |
---|
2821 | self.reviewObj() |
---|
2822 | ## |
---|
2823 | return |
---|
2824 | #d1 = {} |
---|
2825 | #for i in self.rqvgList: |
---|
2826 | #d1[i['uid']] = i |
---|
2827 | #this = self.doc.getElementsByTagName('requestVarGroup')[0] |
---|
2828 | #dil = this.getElementsByTagName('item') |
---|
2829 | kkk = 0 |
---|
2830 | kk0 = [] |
---|
2831 | ss = {} |
---|
2832 | #for item in dil: |
---|
2833 | #uid = item.getAttribute( 'uid' ) |
---|
2834 | #refn = item.getAttribute( 'refNote' ) |
---|
2835 | #bits = string.split( refn, '_' ) |
---|
2836 | #if len(bits) == 2: |
---|
2837 | #if string.find(bits[0], '.' ) != -1: |
---|
2838 | #bits[0] = string.split( bits[0], '.')[-1] |
---|
2839 | #ss[ tuple( bits ) ] = uid |
---|
2840 | #if uid not in d1: |
---|
2841 | #kk0.append( 'Record for uid %s not found' % uid ) |
---|
2842 | #else: |
---|
2843 | #lab = item.getAttribute( 'label' ) |
---|
2844 | #if d1[uid]['label'] == lab: |
---|
2845 | #kkk += 1 |
---|
2846 | #else: |
---|
2847 | #kk0.append( 'Mismatch in label for %s' % uid ) |
---|
2848 | #print 'INFO.rvg.00001: %s records checked OK; %s failed' % (kkk,len(kk0)) |
---|
2849 | notNeeded = ( ('Oyr','3dtr'), ('Amon','2d'), ('cf3hr','grid')) |
---|
2850 | ##nnf = 0 |
---|
2851 | dt1 = {} |
---|
2852 | for s in tableUtils.sgset: |
---|
2853 | a,b = string.split( s, '_' ) |
---|
2854 | if (a,b) not in notNeeded: |
---|
2855 | if (a,b) in ss: |
---|
2856 | ## |
---|
2857 | ## create lookup --- dt1[<sub-group id>] = <requestVarGroup uid> |
---|
2858 | ## |
---|
2859 | dt1[s] = ss[(a,b)] |
---|
2860 | else: |
---|
2861 | print 'WARN.rvg.00002: %s %s not found' % (a,b) |
---|
2862 | nnf += 1 |
---|
2863 | |
---|
2864 | assert nnf == 0, 'Not all sub-groups found' |
---|
2865 | ### |
---|
2866 | ### read through request vars ... and look to see how many refer to subgroups identified in dt1: |
---|
2867 | ### how many of these are identified in tableUtils, how many in tableUtils are missing? |
---|
2868 | ### |
---|
2869 | dt1vs = {} |
---|
2870 | for s in dt1: |
---|
2871 | dt1vs[ dt1[s] ] = s |
---|
2872 | |
---|
2873 | ##print 'INFO.rvg.00009: dt1vs: %s' % str(dt1vs) |
---|
2874 | this = self.doc.getElementsByTagName('requestVar')[0] |
---|
2875 | dil = this.getElementsByTagName('item') |
---|
2876 | nf0 = 0 |
---|
2877 | nf1 = 0 |
---|
2878 | nf2 = 0 |
---|
2879 | ss0 = set() |
---|
2880 | sn0 = collections.defaultdict( int ) |
---|
2881 | sn1 = collections.defaultdict( int ) |
---|
2882 | sn2 = collections.defaultdict( int ) |
---|
2883 | for item in dil: |
---|
2884 | vgid = item.getAttribute( 'vgid' ) |
---|
2885 | if vgid in dt1vs: |
---|
2886 | s = dt1vs[vgid] |
---|
2887 | nf0 += 1 |
---|
2888 | sn0[s] += 1 |
---|
2889 | vid = item.getAttribute( 'vid' ) |
---|
2890 | ss0.add( vid ) |
---|
2891 | if vid in tableUtils.vdict: |
---|
2892 | nf1 += 1 |
---|
2893 | sn1[s] += 1 |
---|
2894 | else: |
---|
2895 | this.removeChild(item) |
---|
2896 | |
---|
2897 | for vid in tableUtils.vdict: |
---|
2898 | if vid not in ss0: |
---|
2899 | var,ttl,mip,tab,sg,priority = tableUtils.vdict[vid] |
---|
2900 | a,b = string.split( sg, '_' ) |
---|
2901 | if (a,b) not in notNeeded: |
---|
2902 | nf2 += 1 |
---|
2903 | sn2[ tableUtils.vdict[vid][4] ] += 1 |
---|
2904 | assert sg in dt1, 'Cannot locate requestVarGroupd for %s: %s' % (sg,str(dt1.keys())) |
---|
2905 | vgid = dt1[sg] |
---|
2906 | item = self.doc.createElement( 'item' ) |
---|
2907 | item.setAttribute( 'uid', str(uuid.uuid1()) ) |
---|
2908 | item.setAttribute( 'vid', vid ) |
---|
2909 | item.setAttribute( 'vgid', vgid ) |
---|
2910 | item.setAttribute( 'title', '%s ((isd.009))' % var ) |
---|
2911 | item.setAttribute( 'label', var ) |
---|
2912 | item.setAttribute( 'mip', mip ) |
---|
2913 | item.setAttribute( 'priority', priority ) |
---|
2914 | this.appendChild( item ) |
---|
2915 | |
---|
2916 | print 'INFO.rvg.00010: variables found in sg rvgs: %s' % nf0 |
---|
2917 | print 'INFO.rvg.00011: of these, number in tableUtils: %s' % nf1 |
---|
2918 | print 'INFO.rvg.00012: variables in tableUtils not in sg rvgs: %s [from %s]' % (nf2,len(tableUtils.vdict.keys())) |
---|
2919 | for k in sorted( sn0.keys() ): |
---|
2920 | print 'INFO.rvg.00020: # in tableUtils [%s] %s [from %s]' % (k,sn0[k],sn1[k]) |
---|
2921 | for k in sorted( sn2.keys() ): |
---|
2922 | print 'INFO.rvg.00021: # in tableUtils [%s] not found %s' % (k,sn2[k]) |
---|
2923 | |
---|
2924 | |
---|
2925 | def finish2(self): |
---|
2926 | ## |
---|
2927 | ## objectiveLink |
---|
2928 | ## |
---|
2929 | this = self.doc.getElementsByTagName('objectiveLink')[0] |
---|
2930 | dil = this.getElementsByTagName('item') |
---|
2931 | for d in dil: |
---|
2932 | this.removeChild(d) |
---|
2933 | for t in self.objList: |
---|
2934 | item = self.doc.createElement( 'item' ) |
---|
2935 | item.setAttribute( 'label', t[0] ) |
---|
2936 | item.setAttribute( 'oid', t[1] ) |
---|
2937 | item.setAttribute( 'rid', t[2] ) |
---|
2938 | thisuuid = str( uuid.uuid1() ) |
---|
2939 | item.setAttribute( 'uid', thisuuid ) |
---|
2940 | this.appendChild( item ) |
---|
2941 | ## |
---|
2942 | ## remarks |
---|
2943 | ## |
---|
2944 | this = self.doc.getElementsByTagName('remarks')[0] |
---|
2945 | dil = this.getElementsByTagName('item') |
---|
2946 | for d in dil: |
---|
2947 | this.removeChild(d) |
---|
2948 | |
---|
2949 | ##mn = self.doc.getElementsByTagName('main')[0] |
---|
2950 | for item in self.doc.getElementsByTagName('item'): |
---|
2951 | for k,v in item.attributes.items(): |
---|
2952 | if type(v) not in ( type( u'x' ), type( 'x' ) ): |
---|
2953 | print 'TYPE ERROR %s, %s: %s' % (item.parentNode.tagName, k,v) |
---|
2954 | txt = self.doc.toprettyxml(indent='\t', newl='\n', encoding=None) |
---|
2955 | if self.schemaMode == 'dreq2': |
---|
2956 | ofn = 'trial2_20150831.xml' |
---|
2957 | else: |
---|
2958 | ofn = 'trial_20150831.xml' |
---|
2959 | oo = open( ofn, 'w' ) |
---|
2960 | lines = string.split( txt, '\n' ) |
---|
2961 | for line in lines: |
---|
2962 | l = utils_wb.uniCleanFunc( string.strip(line) ) |
---|
2963 | if empty.match(l): |
---|
2964 | continue |
---|
2965 | else: |
---|
2966 | oo.write(l + '\n') |
---|
2967 | oo.close() |
---|
2968 | |
---|
2969 | def choices(self): |
---|
2970 | """Populate the 'varChoice' section, using the information in the 'choices' dictionary in 'choicecfg'""" |
---|
2971 | |
---|
2972 | ccfg = choiceCfg.choices( self ) |
---|
2973 | self.ccfg = ccfg |
---|
2974 | ## |
---|
2975 | ## choices |
---|
2976 | ## |
---|
2977 | this = self.doc.getElementsByTagName('varChoice')[0] |
---|
2978 | dil = this.getElementsByTagName('item') |
---|
2979 | for d in dil: |
---|
2980 | this.removeChild(d) |
---|
2981 | for k in ccfg.choices.keys(): |
---|
2982 | item = self.doc.createElement( 'item' ) |
---|
2983 | cc = ccfg.choices[k] |
---|
2984 | for kk in cc.keys(): |
---|
2985 | item.setAttribute( kk, cc[kk] ) |
---|
2986 | this.appendChild( item ) |
---|
2987 | ##print 'INFO.zzz.00008: Appending ',cc[kk] |
---|
2988 | |
---|
2989 | this = self.doc.getElementsByTagName('varChoiceLinkR')[0] |
---|
2990 | dil = this.getElementsByTagName('item') |
---|
2991 | for d in dil: |
---|
2992 | this.removeChild(d) |
---|
2993 | for cc in ccfg.varChoiceLnk: |
---|
2994 | item = self.doc.createElement( 'item' ) |
---|
2995 | ##print 'INFO.zzz', cc |
---|
2996 | for i in range( len( ccfg.varChoiceLnkInfo ) ): |
---|
2997 | item.setAttribute( ccfg.varChoiceLnkInfo[i], cc[i] ) |
---|
2998 | this.appendChild( item ) |
---|
2999 | |
---|
3000 | this = self.doc.getElementsByTagName('varChoiceLinkC')[0] |
---|
3001 | dil = this.getElementsByTagName('item') |
---|
3002 | for d in dil: |
---|
3003 | this.removeChild(d) |
---|
3004 | for cc in ccfg.varChoiceLnkCfg: |
---|
3005 | item = self.doc.createElement( 'item' ) |
---|
3006 | for i in range( len( ccfg.varChoiceLnkCfgInfo ) ): |
---|
3007 | item.setAttribute( ccfg.varChoiceLnkCfgInfo[i], cc[i] ) |
---|
3008 | this.appendChild( item ) |
---|
3009 | |
---|
3010 | ##self.varChoiceLnkCfgInfo = [ 'uid', 'label', 'title', 'vid', 'cid', 'cfgid', 'cfg' |
---|
3011 | |
---|
3012 | def importRepl(self,rfile='uuidreplace.csv',mode='var'): |
---|
3013 | for l in open(rfile).readlines(): |
---|
3014 | bits = string.split( string.strip(l), '\t' ) |
---|
3015 | assert len(bits) == 7, 'Could not parse item replacement file' |
---|
3016 | if mode == 'var': |
---|
3017 | self.repl[ bits[3] ].append( (bits[2],bits[0],bits[1] ) ) |
---|
3018 | self.replItems[bits[0]] = bits[1] |
---|
3019 | elif mode == 'cmv': |
---|
3020 | self.rep['cmv'].v[ bits[3] ].append( (bits[2],bits[0],bits[1] ) ) |
---|
3021 | self.rep['cmv'].items[bits[0]] = bits[1] |
---|
3022 | |
---|
3023 | |
---|
3024 | def importRemove(self,rfile='uuidremove.csv'): |
---|
3025 | for l in open(rfile).readlines(): |
---|
3026 | self.remo[ string.strip(l) ] = 1 |
---|
3027 | |
---|
3028 | def importUpdate(self,rfile='uuidupdate.csv'): |
---|
3029 | for l in open(rfile).readlines(): |
---|
3030 | bits = string.split( string.strip(l), '\t' ) |
---|
3031 | tgs = string.strip( bits[2] ) |
---|
3032 | if string.find( tgs, ' ' ) == -1: |
---|
3033 | tags = [tgs,] |
---|
3034 | else: |
---|
3035 | tags = string.split( tgs ) |
---|
3036 | self.upda[ bits[0] ] = {'comment':bits[1], 'tags':tags, 'label':bits[3], 'title':bits[4] } |
---|
3037 | |
---|
3038 | def importInsert(self,rfile='uuidinsert.csv'): |
---|
3039 | for l in open(rfile).readlines(): |
---|
3040 | bits = string.split( string.strip(l), '\t' ) |
---|
3041 | if bits[0] == 'unique': |
---|
3042 | self.insert[bits[1]] = bits[3] |
---|
3043 | |
---|
3044 | mode = 'dreq2' |
---|
3045 | sampleXml = '../framework/out/%sSample.xml' % mode |
---|
3046 | from scansh02 import rq |
---|
3047 | |
---|
3048 | m = main(sampleXml, rq,run=True, schemaMode=mode, doRepl=False) |
---|
3049 | |
---|