Context Navigation

source: CMIP6dreqbuild/trunk/src/workbook/isd3b.py @ 924

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CMIP6dreqbuild/trunk/src/workbook/isd3b.py@924

Revision 924, 130.7 KB checked in by mjuckes, 4 years ago (diff)
intermediate -- with dup cmv problem

Line
1	"""Import data request from shelves and pu onto compliant XML document.
2	------------------------------------------------------------------------
3	A new uuid is generated for elements of the "revisedtabitem" section/table (because it is not present in the shelve).
4
5	NOTES
6	-----
7	July 27th: there is an unresolved problem about preserving uuids when updates are provided in source format .....
8	"""
9
10	### Experiment group dictionary added, and experiment set of named tuples -- making it possible to identify experiments in each mip and group.
11	### Should be able to use this in review to make reference groups ...
12
13	###
14	### Appear to be getting core variables into a clean format
15	### Some filtering to avoit duplicates arising from two approaches to OMIP, flagged as ERROR.090 in standard output.
16	### Corrected by omitting 2nd reference
17	###
18
19	import xlrd, xml, os, sys, string, shelve, json
20	import utils_wb, uuid
21	import xml.dom, xml.dom.minidom
22	import collections, string, re
23	import choiceCfg
24	import importWbMods
25	import dreq_utils
26	from utils_wb import uniCleanFunc
27	import ivg
28
29	mipTabMaps = json.load( open('inputs/mipTableMaps.txt', 'r' ) )
30
31	#up:
32	varPositive = {'rlus':'up', 'rlut':'up', 'rlutcs':'up', 'hfls':'up', 'hfss':'up', 'mcu':'up', 'rsus':'up', 'rsut':'up', 'rsutcs':'up', 'rlds':'down', 'rsds':'down' }
33
34	refMips = ['AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB']
35	##
36	## refCmv.refUid[tab][lab] = k
37	##
38	refCmv = ivg.refCmv()
39	extRefCmvMode = 'r'
40	extRefCmv = shelve.open( 'sh/extRefCmv' )
41	extRefCmvInv = {}
42	nd = 0
43	sd = set()
44	sd2 = set()
45	for k in extRefCmv:
46	k1 = extRefCmv[k]
47	tab,var = k.split( '.' )
48	if k1 in extRefCmvInv:
49	tab2, var2 = extRefCmvInv[k1].split('.')
50	if var == var2 and (tab == mipTabMaps.get( tab2, None) or tab2 == mipTabMaps.get( tab, None) ):
51	pass
52	else:
53	print 'FATAL: Duplicate key in extRefCmv: %s [%s] %s' % (k,k1,extRefCmvInv[k1])
54	sd.add(k)
55	sd2.add(extRefCmvInv[k1])
56	nd += 1
57	extRefCmvInv[k1] = k
58	assert nd == 0, 'FATAL ERRORS FOUND ... see standard output: %s, %s' % (str( sd ), str( sd2 ) )
59
60	extRefCmvMapped = {}
61	def chkRefCmv( ntab, lab, u0, msg ):
62	useExtRef = True
63	thattab = mipTabMaps.get( ntab, ntab )
64	thistab = ntab
65	if thistab in refCmv.refUid and lab in refCmv.refUid[thistab]:
66	uid = refCmv.refUid[thistab][lab]
67	print 'INFO.uid0.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,uid)
68	if uid != u0:
69	extRefCmvMapped[u0] = uid
70	elif thattab in refCmv.refUid and lab in refCmv.refUid[thattab]:
71	uid = refCmv.refUid[thattab][lab]
72	print 'INFO.uid1.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,uid)
73	if uid != u0:
74	extRefCmvMapped[u0] = uid
75	else:
76	kk = str( '%s.%s' % (thistab,lab) )
77	kkk = str( '%s.%s' % (thattab,lab) )
78	if kk in extRefCmv:
79	if useExtRef:
80	uid = extRefCmv[kk]
81	print 'INFO.uid.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,uid)
82	if uid != u0:
83	extRefCmvMapped[u0] = uid
84	else:
85	u1 = extRefCmv[kk]
86	print 'INFO.uid.%s: uid in extRefCmv ignored: %s, %s (%s) --> {%s}' % (msg,thistab,lab,u0,u1)
87	uid = u0
88	elif kkk in extRefCmv:
89	if useExtRef:
90	uid = extRefCmv[kkk]
91	print 'INFO.uid4.%s: uid found in extRefCmv: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,uid)
92	if uid != u0:
93	extRefCmvMapped[u0] = uid
94	else:
95	u1 = extRefCmv[kkk]
96	print 'INFO.uid4.%s: uid in extRefCmv ignored: %s, %s (%s) --> {%s}' % (msg,thattab,lab,u0,u1)
97	uid = u0
98	else:
99	if extRefCmvMode != 'r':
100	extRefCmv[kk] = u0
101	print 'WARN.uid.%s: uid not found: %s, %s (%s)' % (msg,thistab,lab,u0)
102	uid = u0
103	return uid
104
105
106	siSurfFluxes = ['sidmasstranx', 'sidmasstrany', 'siforcecoriolx', 'siforcecorioly', 'siforcetiltx', 'siforcetilty', 'siforceintstrx', 'siforceintstry', 'sistrxdtop', 'sistrydtop', 'sistrxdbot', 'sistrydbot', 'siu', 'siv']
107	oceanVars = ['fg13co2','fg14co2']
108	realmTab = {'landice':['LImongre','LImonant','LIyrgre','LIyrant','LIfxgre','LIfxant','ImonGre','ImonAnt','IyrGre','IyrAnt','IfxGre','IfxAnt'],
109	'atmos':['6hrLev','6hrPlev','6hrPevpt','em1hrclimmon','em3hr','em1hr','em3hrpt','emDayZ','emDaypt','emMonZ']}
110	## emYr, emMon, emFy, emMonZ .... need checking
111	knownRealmA = [ 'LS3MIP [LWday]', 'LS3MIP [LEday]', 'CFMIP [cf1hrClimMon]', 'HighResMIP [3hr_cloud]', 'CFMIP [cf3hr_sim_new]', 'C4MIP [L_3hr]', 'DAMIP [DAMIP_day]', 'DAMIP [DAMIP_3hr_p2]', 'DynVar [DYVR_daily_c]', 'PMIP [PMIP-6hr]', 'HighResMIP [1hrLev]']
112	knownRealmO = [ 'DAMIP [DAMIP_Omon_p2]', 'FAFMIP [fafOyr]']
113	varRealmMaps = {'land':['areacellg','orog'],
114	'landIce land':['snc'],
115	'landIce':['icem', 'sftflf', 'sftgrf','acabf','snicefreez', 'snicem', 'hfgeoubed', 'lithk', 'topg','areacelli','lialb', 'libmassbffl', 'libmassbfgr', 'licalvf', 'lifmassbf', 'litempbotfl', 'litempbotgr', 'litemptop', 'mrroIs','strbasemag', 'topg', 'xvelbase', 'xvelmean', 'xvelsurf', 'yvelbase', 'yvelmean', 'yvelsurf', 'zvelbase', 'zvelsurf','snmIs','sncIs','tsnIs','sbl'],
116	'atmos':['prra','prsn','prrc','sbl','tas','prCrop', 'tasmaxCrop', 'tasminCrop','hfls', 'hfss','rlds', 'rlus', 'rsds', 'rsus'] }
117	varRealmMapsInv = {}
118	for k in varRealmMaps:
119	for v in varRealmMaps[k]:
120	varRealmMapsInv[ v ] = k
121
122	class replaceItems(object):
123	def __init__(self):
124	self.v = collections.defaultdict( list )
125	self.items = {}
126
127	WbMods = importWbMods.run()
128	tableUtils = ivg.tableUtils()
129
130	nt__trace = collections.namedtuple( 'trace', ['GROUPITEM_ta','SPATIALSHAPE_01'] )
131	trace = nt__trace( False, False )
132
133	tab2freq = {u'CMIP5_cfOff':'subhr', u'CORDEX_mon':'mon', u'SPECS_day':'day', u'CMIP5_day':'day', \
134	u'PMIP3_OImon':'mon', u'CORDEX_day':'day', u'CMIP5_LImon':'mon', u'CMIP5_OImon':'mon', \
135	u'CMIP5_Lmon':'mon', u'CMIP5_3hr':'3hr', u'CMIP5_Omon':'mon', u'PMIP3_OIclim':'monClim', \
136	u'PMIP3_fx':'fx', u'CORDEX_fx':'fx', u'PMIP3_LImon':'mon', u'CMIP5_6hrPlev':'6hr', u'PMIP3_Lmon':'mon', \
137	u'PMIP3_Amon':'mon', u'SPECS_Omon':'mon', u'CCMI1_fixed':'fx', u'PMIP3_Aclim':'monClim', u'CMIP5_6hrLev':'6hr', \
138	u'CMIP5_Oclim':'monClim', u'PMIP3_LIclim':'monClim', u'CCMI1_monthly':'mon', u'CMIP5_fx':'fx', \
139	u'CMIP5_cfDay':'day', u'CORDEX_6h':'6hr', u'PMIP3_day':'day', u'SPECS_OImon':'mon', u'CMIP5_cfMon':'mon', \
140	u'CORDEX_sem':'monClim', u'SPECS_6hr':'6hr', u'CMIP5_cfSites':'subhr', u'CCMI1_hourly':'hr', u'CMIP5_aero':'mon', \
141	u'CMIP5_Amon':'mon', u'PMIP3_Omon':'mon', u'CCMI1_daily':'day', u'SPECS_fx':'fx', u'PMIP3_Lclim':'monClim', \
142	u'aermonthly':'mon', u'aermonthlyz':'mon', u'aerhourly':'hr', 'aerfixed':'fx', 'aerdaily':'day',
143	u'DCPP-day':'day', \
144	u'Odec':'dec', \
145	u'PMIP3_Oclim':'monClim', u'SPECS_Amon':'mon', u'SPECS_Lmon':'mon', u'CMIP5_cf3hr':'3hr', u'CORDEX_3h':'3hr', \
146	'OImon':'mon', 'Omon_oth':'mon', 'Omon':'mon', 'Oyr':'yr', 'Omon_3d':'mon', \
147	u'CCMI1_annual':'yr', u'CMIP5_Oyr':'yr', \
148	'Oclim':'monClim', \
149	'cfMon':'mon', 'emMonZ':'mon', \
150	'3hr':'3hr', 'cfOff':'subhr', 'OMIP-Omon':'mon', '6hrLev':'6hr', 'fx':'fx', 'OMIP-Oyr':'yr', 'cfDay':'day', 'Lmon':'day', \
151	u'cf3hr':'3hr', u'Amon':'mon', u'aero':'mon', u'aero_3d':'mon', u'6hrPlev':'6hr', u'aero_oth':'mon', \
152	u'cf3hr_grid':'3hr', u'LImon':'mon', u'cfSites':'subhr', u'day_oth':'day', u'cfDay_2d':'day', u'day':'day' }
153
154	freq2tab = collections.defaultdict( list )
155	for k in tab2freq:
156	freq2tab[tab2freq[k]].append( k )
157
158	tableNameMap = {'CMIP5_3hr':'3hr', 'CMIP5_6hrLev':'6hrLev', 'CMIP5_6hrPlev':'6hrPlev', 'CMIP5_Amon':'Amon', 'CMIP5_LImon':'LImon', 'CMIP5_Lmon':'Lmon', 'CMIP5_OImon':'CMIP5_OImon', 'CMIP5_Oclim':'Oclim', 'CMIP5_Omon':'CMIP5_Omon', 'CMIP5_aero':'aero', 'CMIP5_cf3hr':'cf3hr', 'CMIP5_cfDay':'cfDay', 'CMIP5_cfMon':'cfMon', 'CMIP5_cfOff':'cfOff', 'CMIP5_cfSites':'cfsites', 'CMIP5_day':'day', 'CMIP5_fx':'fx', 'OMIP.Omon':'Omon', 'OMIP.Oyr':'Oyr', 'OMIP.day':'Oday', 'OMIP.fx':'Ofx' }
159
160	reportedRedundantTables = set()
161	redundantTables = ['aero','CMIP5_Oyr','CMIP5_Omon','CMIP5_OImon','CORDEX_3h', 'CORDEX_6h', 'CORDEX_fx', 'CORDEX_mon', 'CORDEX_sem', 'PMIP3_Aclim', 'PMIP3_Amon', 'PMIP3_LIclim', 'PMIP3_LImon', 'PMIP3_Lclim', 'PMIP3_Lmon', 'PMIP3_OIclim', 'PMIP3_OImon', 'PMIP3_Oclim', 'PMIP3_Omon', 'PMIP3_day', 'PMIP3_fx',
162	'SPECS_Lmon', 'SPECS_Amon', 'SPECS_6hr', 'SPECS_day', 'SPECS_fx', 'SPECS_Omon', 'SPECS_OImon',
163	'CORDEX_day',
164	'CCMI1_annual','CCMI1_daily','CCMI1_fixed','CCMI1_hourly','CCMI1_monthly']
165
166	## mapping of variable groups representing tables ....
167	mappedTables = {'Omon':'OMIP-Omon', 'OMIP-Oyr':'Oyr', 'OImon':'SIMIP-seaicemon' }
168	mappedTables = {'Omon':'OMIP-Omon', 'CMIP5-Oyr':'OMIP-Oyr', 'OImon':'SIMIP-seaicemon', 'CMIP5-Omon':'OMIP-Omon' }
169	## mappedTables = { 'OImon':'SIMIP-seaicemon' }
170	## 'aeromonthly':'aermonthly' }
171
172	freqmap = {'daily':'day', 'Annual':'yr', 'Timestep':'subhr', '1day':'day', '1mon':'mon', 'month':'mon', 'year':'yr', 'monthly':'mon', 'Day':'day', '6h':'6hr', '3 hourly':'3hr', '3 Hourly':'3hr' }
173
174	nt_expt = collections.namedtuple( 'expt', ['uid', 'label', 'mip', 'egid' ] )
175	realmdef = { '__all__':['CFMIP [cfMonExtra]', 'PMIP [PMIP-Omon]', 'LS3MIP [LCmon]', 'C4MIP [C_Ocean_T2]', 'C4MIP [C_Ocean_T1]', 'LUMIP [Lyr_Lut]', 'PMIP [PMIP-LIclim]', 'HighResMIP [1hr_strat]', 'SIMIP [seaiceday]', 'C4MIP [C_LandT1]', 'CFMIP [cfDay_2d_new]', 'HighResMIP [6hrPlev_intense]', 'HighResMIP [Amon_diag]', 'DCPP [DCPP-mon]', 'HighResMIP [Amon_conv]', 'SIMIP [seaicemon]', 'DAMIP [new_monthly]', 'FAFMIP [fafOyrB]', 'HighResMIP [Amon_ext]', 'DynVar [DYVR_daily]', 'PMIP [PMIP-Aclim]', 'HighResMIP [3hr_extr]', 'C4MIP [C_Basic]', 'DCPP [DCPP-day]', 'CFMIP [cfMon_3dstd_new]', 'FAFMIP [fafOmon]', 'ISMIP6 [new_fx]', 'LS3MIP [LEday]', 'PMIP [PMIP-aeroclim]', 'PMIP [PMIP-aero]', 'DCPP [DCPP-6hr]', 'DynVar [DYVR_monthly]', 'ISMIP6 [icesheetmon]', 'CFMIP [aeroDay_2d]', 'ISMIP6 [new_LImon]', 'ISMIP6 [new_Omon]', 'CFMIP [cfDayExtra]', 'LS3MIP [LWday]', 'PMIP [PMIP-Oclim]', 'ISMIP6 [icesheetyear]', 'LUMIP [Lmon_Lut]', 'HighResMIP [1ts]', 'PMIP [PMIP-OIclim]', 'HighResMIP [6hrPlev_extr_dr]', 'VolMIP [VIRF]', 'C4MIP [C_LandT2]', 'C4MIP [C_Hist]', 'FAFMIP [fafOmonB]', 'RFMIP [aero_irf]', 'PMIP [PMIP-Lmon]', 'C4MIP [L_day]', 'PMIP [PMIP-Lclim]', 'PMIP [PMIP-Amon]', 'GeoMIP [aeroGeo]', 'HighResMIP [6hrPlev_extr]', 'PMIP [PMIP-day]', 'HighResMIP [Amon_sparc]', 'CFMIP [cfSites_new]', 'CFMIP [cf3hr_sim]'],
176	'atmos':['CFMIP [cfMonExtra]', 'HighResMIP [1hr_strat]', 'CFMIP [cfDay_2d_new]', 'HighResMIP [6hrPlev_intense]', 'HighResMIP [Amon_diag]', 'DCPP [DCPP-mon]', 'HighResMIP [Amon_conv]', 'DAMIP [new_monthly]', 'HighResMIP [Amon_ext]', 'DynVar [DYVR_daily]', 'PMIP [PMIP-Aclim]', 'HighResMIP [3hr_extr]', 'DCPP [DCPP-day]', 'CFMIP [cfMon_3dstd_new]', 'PMIP [PMIP-aeroclim]', 'PMIP [PMIP-aero]', 'DCPP [DCPP-6hr]', 'DynVar [DYVR_monthly]', 'CFMIP [aeroDay_2d]', 'CFMIP [cfDayExtra]', 'HighResMIP [1ts]', 'HighResMIP [6hrPlev_extr_dr]', 'VolMIP [VIRF]', 'RFMIP [aero_irf]', 'PMIP [PMIP-Amon]', 'GeoMIP [aeroGeo]', 'HighResMIP [6hrPlev_extr]', 'PMIP [PMIP-day]', 'HighResMIP [Amon_sparc]', 'CFMIP [cfSites_new]', 'CFMIP [cf3hr_sim]'],
177	'ocean':['PMIP [PMIP-Omon]', 'C4MIP [C_Ocean_T1]', 'C4MIP [C_Ocean_T2]', 'FAFMIP [fafOyrB]', 'FAFMIP [fafOmon]', 'ISMIP6 [new_Omon]', 'PMIP [PMIP-OIclim]', 'FAFMIP [fafOmonB]', 'PMIP [PMIP-Oclim]'],
178	'land':['LS3MIP [LCmon]', 'C4MIP [C_Hist]', 'C4MIP [L_day]', 'PMIP [PMIP-Lclim]', 'C4MIP [C_Basic]', 'LUMIP [Lmon_Lut]', 'LUMIP [Lyr_Lut]', 'C4MIP [C_LandT1]', 'PMIP [PMIP-Lmon]', 'C4MIP [C_LandT2]'],
179	'seaIce':['SIMIP [seaicemon]', 'SIMIP [seaiceday]' ],
180	'landIce':['ISMIP6 [new_LImon]', 'PMIP [PMIP-LIclim]', 'ISMIP6 [icesheetyear]', 'ISMIP6 [new_fx]', 'ISMIP6 [icesheetmon]'],
181	'__unset__':['LS3MIP [LWday]', 'LS3MIP [LEday]'] }
182
183	realmdefix = {}
184	for k in realmdef.keys():
185	if k != '__all__':
186	for s in realmdef[k]:
187	assert not realmdefix.has_key(s), 'Duplicate targ value in realmdef: %s [%s]' % (s,k)
188	realmdefix[s] = k
189
190	for k in realmdef['__all__']:
191	assert realmdefix.has_key(k), 'Key %s not found in realmdefix' % k
192
193	class cls_s1(object):
194	def __init__(self):
195	self.a = collections.defaultdict( list )
196
197	def mycc(s):
198	s = string.replace( s, '&', '' )
199	s = string.replace( s, '.', '-' )
200	if len(s) > 0 and s[-1] == '+':
201	s = s[:-1] + 'ETC'
202	## camelcase the input string
203	s1 = string.lower( string.replace(string.strip(str( s) ), '-', ' ') )
204	return string.replace( string.capwords( s1 ), ' ', '' )
205
206	empty=re.compile('^$')
207
208	def test( x,m):
209	if not x:
210	print m
211	return x
212
213	class lcm(object):
214	def __init__(self,a,b):
215	self.a = {}
216	self.b = {}
217	for i in a:
218	self.a[string.lower(i)] = i
219	for i in b:
220	self.b[string.lower(i)] = i
221
222	class main(object):
223
224	def __init__(self,src,rq,doRepl=False,run=False,schemaMode='dreq'):
225	self.schemaMode = schemaMode
226	self.requestlinkuid = set()
227	self.src=src
228	self.vgcheck = {}
229	self.rqlPreset = {}
230	fok = [test(os.path.isfile(src),'%s not found' % src), ]
231	assert all( fok), 'Required input file(s) missing'
232	self.fix = WbMods.fix
233	self.spsh = WbMods.ls
234	self.spsh.addTime()
235	self.cmDims = WbMods.cm
236	self.cmvMap002 = dict()
237	self.importWbMods = WbMods
238	self.rq = rq
239	self.deferredLabMap = {}
240	self.struidmapextra = {}
241
242	## groupItems:grps
243	for sh in [rq.expt,rq.exgp,rq.objec,rq.refti, rq.revti, rq.grps, rq.rqli, rq.rqit]:
244	print sh['__info__']
245
246	self.repl = collections.defaultdict( list )
247	self.replItems = {}
248	self.rep = {'cmv':replaceItems()}
249	##self.deferredRequestLinkClone = collections.defaultdict( set )
250
251	self.err0010 = collections.defaultdict( int )
252	self.defaultP = {}
253	self.remo = {}
254	self.upda = {}
255	self.insert = {}
256	self.rqLnks = collections.OrderedDict()
257	self.exptSet = set()
258	self.exptGrpDict = {}
259
260	self.objectives = collections.defaultdict( dict )
261	self.objectiveLinks = collections.defaultdict( cls_s1 )
262	self.experiments = {}
263	self.experimentGrp = {}
264	self.exptPntUid = {}
265	self.exptPnt = collections.defaultdict( list )
266	self.mips = set()
267	self.dsortdd = {}
268
269	if doRepl:
270	self.importRepl()
271	self.importRepl(rfile='CMVreplace.csv',mode='cmv')
272	self.importRemove()
273	self.importUpdate()
274	self.importInsert()
275
276	self.e15_10 = 0
277	self.doc = xml.dom.minidom.parse( self.src )
278	self.vocabs = xml.dom.minidom.parse( 'vocabs.xml' )
279	self.skey = {'experiment':rq.expt, 'exptgroup':rq.exgp, \
280	'objective':rq.objec, 'var':None,'ovar':rq.refti, 'groupitem':rq.grps, \
281	'revisedtabitem_xxx':rq.revti, 'requestlink':rq.rqli, 'requestitem':rq.rqit, \
282	'requestvargroup':rq.rqvg, 'tablesection':rq.rqsect, 'requestvar':rq.revti}
283
284	self.newImport = False
285	self.prepVar()
286	self.writeTimeSlice()
287	self.writeMip()
288	self.writeMcfg()
289	self.writeProcNotes()
290	self.writeCmDim()
291	self.writeRvg()
292	self.prep(rq)
293	self.cmvAudit(0)
294	self.prepRequestItem()
295	self.reviewExpt()
296	if run:
297	self.run(rq)
298	self.writeRelations()
299	self.writeRvx()
300	if self.schemaMode == 'dreq2':
301	self.write2()
302	self.writeMipTables()
303	self.writeCellMethods()
304	self.writeVar()
305	self.writeSn()
306	self.finish()
307	self.finish2()
308	extRefCmv.close()
309
310	def prepRequestItem(self):
311	thissh = self.skey['requestitem']
312	## 'mip', 'tab', 'expt','rlid','ny', 'nexmax', 'nenmax', 'nymax','treset','info'
313	s1 = collections.defaultdict( set )
314	for k in thissh:
315	if k[0] != '_':
316	ll = thissh[k][:]
317	ku = str(ll[2])
318	s1[ll[0]].add(ll[9])
319	if len( ku) == 1:
320	print 'SEVERE.rqi.00088: ',ll
321	##
322	self.exptPnt[ku].append( str(k) )
323
324	thissh = self.skey['experiment']
325	for k in thissh:
326	if k[0] != '_':
327	ll = thissh[k][:]
328	thisl = str( ll[2] )
329	if ll[4] in ['SolarMIP']:
330	print 'ERROR: %s experiment: %s' % (ll[4],str(ll))
331	else:
332	self.mips.add( str( ll[4] ) )
333	self.experiments[thisl] = str( ll[0] )
334	self.exptSet.add( nt_expt._make( [str( ll[0] ), thisl, str( ll[4] ), str( ll[1] ) ] ) )
335
336	thissh = self.skey['exptgroup']
337	for k in thissh:
338	if k[0] != '_':
339	ll = thissh[k][:]
340	thisl = str( ll[1] )
341	self.exptGrpDict[thisl] = str( ll[1] )
342	self.experimentGrp[thisl] = str( ll[0] )
343
344	mm = []
345	for m in self.mips:
346	if m not in self.mipUids:
347	mm.append(m)
348	assert len(mm) == 0, 'Not all mips found: %s: %s' % (str(mm),str(self.mipUids))
349
350	def writeProcNotes(self):
351	xx = self.vocabs.getElementsByTagName( 'processingOptions' )
352	assert len(xx) == 1, 'Expecting one element named "processingOptions", found %s' % len(xx)
353	iDoc = xx[0]
354	## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/>
355	thiss = self.doc.getElementsByTagName( 'tags' )[0]
356	dil = thiss.getElementsByTagName('item')
357	for d in dil:
358	thiss.removeChild(d)
359	for i in iDoc.getElementsByTagName('item'):
360	item = self.doc.createElement( 'item' )
361	thisl = i.getAttribute( 'label' )
362	for k in ['title','description','label']:
363	item.setAttribute( k, i.getAttribute( k ) )
364	uid = 'TAG:%s' % thisl
365	item.setAttribute( 'uid', uid )
366	thiss.appendChild( item )
367
368	def writeRelations(self):
369	thiss = self.doc.getElementsByTagName( 'varRelations' )[0]
370	dil = thiss.getElementsByTagName('item')
371	for d in dil:
372	thiss.removeChild(d)
373	for i in self.fix.relations:
374	item = self.doc.createElement( 'item' )
375	for j in range( len( self.fix.relationinfo ) ):
376	item.setAttribute( self.fix.relationinfo[j], i[j] )
377	thiss.appendChild( item )
378
379	thiss = self.doc.getElementsByTagName( 'varRelLnk' )[0]
380	dil = thiss.getElementsByTagName('item')
381	for d in dil:
382	thiss.removeChild(d)
383	cc = collections.defaultdict(int)
384	for i in self.fix.relationlinks:
385	item = self.doc.createElement( 'item' )
386	tvp = i[1]
387	ix = cc[tvp]
388	cc[tvp] += 1
389	for j in range( len( self.fix.relationlinkinfo ) ):
390	item.setAttribute( self.fix.relationlinkinfo[j], i[0][j] )
391
392	if tvp in self.cmvUidByTVP:
393	vid = self.cmvUidByTVP[ tvp ][ix]
394	item.setAttribute( 'rid', vid )
395	print 'INFO.093.00011: found: %s' % str(tvp)
396	else:
397	t,v,p = tvp
398	if ('CMIP5_%s' % t,v,p) in self.cmvUidByTVP:
399	if ix >= len(self.cmvUidByTVP[ ('CMIP5_%s' % t,v,p) ] ):
400	print 'WARN.093.00011b: not found: %s, %s' % (str(tvp),ix)
401	else:
402	vid = self.cmvUidByTVP[ ('CMIP5_%s' % t,v,p) ][ix]
403	item.setAttribute( 'rid', vid )
404	print 'INFO.093.00011a: found: %s' % str(tvp)
405	elif t[-2:] in ['pt','zm'] and ( t[:-2],v,p) in self.cmvUidByTVP:
406	vid = self.cmvUidByTVP[ ( t[:-2],v,p) ][ix]
407	item.setAttribute( 'rid', vid )
408	print 'INFO.093.00011a: found: %s' % str(tvp)
409	else:
410	print 'WARN.093.00011: not found: %s' % str(tvp)
411	thiss.appendChild( item )
412
413	def writeCellMethods(self):
414	thiss = self.doc.getElementsByTagName( 'cellMethods' )[0]
415	dil = thiss.getElementsByTagName('item')
416	for d in dil:
417	thiss.removeChild(d)
418	ss = set()
419	for k in WbMods.cellm.cminfo.keys():
420	t,l,u,d = WbMods.cellm.cminfo[k]
421	item = self.doc.createElement( 'item' )
422	item.setAttribute( 'label', l )
423	item.setAttribute( 'title', t )
424	item.setAttribute( 'cell_methods', k )
425	item.setAttribute( 'uid', u )
426	if d != None:
427	item.setAttribute( 'description', d )
428	thiss.appendChild( item )
429
430	def writeMipTables(self):
431	self.cfgUids = set()
432	xx = self.vocabs.getElementsByTagName( 'table' )
433	assert len(xx) == 1, 'Expecting one element named "table", found %s' % len(xx)
434	iDoc = xx[0]
435	## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/>
436	thiss = self.doc.getElementsByTagName( 'miptable' )[0]
437	dil = thiss.getElementsByTagName('item')
438	for d in dil:
439	thiss.removeChild(d)
440	ss = set()
441	for i in iDoc.getElementsByTagName('item'):
442	thisl = i.getAttribute( 'label' )
443	thisll = string.replace( thisl, '_', '-' )
444	if thisl in self.tables:
445	item = self.doc.createElement( 'item' )
446	ss.add(thisl)
447	item.setAttribute( 'label', thisll )
448	item.setAttribute( 'uid', 'MIPtable::%s' % thisl )
449	for k in ['title','description','frequency']:
450	item.setAttribute( k, i.getAttribute( k ) )
451	thiss.appendChild( item )
452	else:
453	print 'WARN.mtid.002: ignoring mip table record for %s' % thisl
454	for lab in self.tables:
455	if lab not in ss:
456	thisll = string.replace( lab, '_', '-' )
457	print 'WARN.mtid.001: creating dummy mip table record for %s' % lab
458	item = self.doc.createElement( 'item' )
459	item.setAttribute( 'label', thisll )
460	item.setAttribute( 'title', lab )
461	item.setAttribute( 'frequency', '???' )
462	item.setAttribute( 'uid', 'MIPtable::%s' % lab )
463	thiss.appendChild( item )
464
465	##################################################################
466
467	def writeMcfg(self):
468	self.cfgUids = set()
469	xx = self.vocabs.getElementsByTagName( 'modelConfig' )
470	assert len(xx) == 1, 'Expecting one element named "modelConfig", found %s' % len(xx)
471	iDoc = xx[0]
472	## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/>
473	thiss = self.doc.getElementsByTagName( 'modelConfig' )[0]
474	dil = thiss.getElementsByTagName('item')
475	for d in dil:
476	thiss.removeChild(d)
477	for i in iDoc.getElementsByTagName('item'):
478	item = self.doc.createElement( 'item' )
479	thisl = i.getAttribute( 'label' )
480	for k in ['title','MIPs','usage','range','type']:
481	item.setAttribute( k, i.getAttribute( k ) )
482	uid = str( thisl )
483	item.setAttribute( 'uid', uid )
484	item.setAttribute( 'label', uid )
485	self.cfgUids.add( uid )
486	thiss.appendChild( item )
487
488	def writeCmDim(self):
489	thiss = self.doc.getElementsByTagName( 'grids' )[0]
490	dil = thiss.getElementsByTagName('item')
491	for d in dil:
492	thiss.removeChild(d)
493	kk = ['tables','label','altLabel','description','standardName','title','axis','units','isIndex','coords','bounds','direction','valid_min','valid_max','type','positive','value','boundsValues','requested','boundsRequested','tolRequested','isGrid' ]
494	for j in self.cmDims.ss.keys():
495	item = self.doc.createElement( 'item' )
496	for i in range(len(kk)):
497	if kk[i] in ['valid_min','valid_max']:
498	if str( self.cmDims.ss[j][i] ) != '':
499	item.setAttribute( kk[i], str( self.cmDims.ss[j][i] ) )
500	else:
501	item.setAttribute( kk[i], str( self.cmDims.ss[j][i] ) )
502	uid = 'dim:%s' % str( self.cmDims.ss[j][1] )
503	item.setAttribute( 'uid', uid )
504	thiss.appendChild( item )
505
506	def writeVar(self):
507	thiss = self.doc.getElementsByTagName( 'var' )[0]
508	dil = thiss.getElementsByTagName('item')
509	for d in dil:
510	thiss.removeChild(d)
511	#<item description="dummyAt" id="001.001.001" label="example-01" procComment="dummyAt" procnote="dummyAt" prov="dummyAt" provmip="dummyAt" sn="dummyAt" title="dummy title string" uid="fa349c44-9ccb-11e5-8176-5404a60d96b5" units="dummyAt"/>
512	kk= ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp','replaced_by']
513	self.usedSn = set()
514	for uid in self.importWbMods.vars.d1:
515	item = self.doc.createElement( 'item' )
516	thisr = self.importWbMods.vars.d1[uid]
517	sn = thisr[7]
518	self.usedSn.add( sn )
519	if sn.find( ' ' ) != -1:
520	print 'WARN.sn.00001: space in sn: ',sn
521	thisr[7] = sn.replace( ' ', '_' )
522	if sn.find( '?' ) != -1:
523	print 'WARN.sn.00002: question in sn: ',sn
524	thisr[7] = sn.replace( '?', '' )
525
526	for i in range( len(kk) -2 ):
527	item.setAttribute( kk[i], thisr[i] )
528	if len( self.importWbMods.vars.c1[ thisr[0] ] ) == 1:
529	id = thisr[0]
530	else:
531	id = '%s.%s' % (thisr[0],self.importWbMods.vars.c1[ thisr[0] ].index( uid ) )
532	print 'INFO.090.00050: setting id for non-unique variable name: %s' % id
533	item.setAttribute( 'id', id )
534	thiss.appendChild( item )
535
536	def writeRvg(self):
537	thiss = self.doc.getElementsByTagName( 'requestVarGroup' )[0]
538	dil = thiss.getElementsByTagName('item')
539	for d in dil:
540	thiss.removeChild(d)
541	#<item description="dummyAt" id="001.001.001" label="example-01" procComment="dummyAt" procnote="dummyAt" prov="dummyAt" provmip="dummyAt" sn="dummyAt" title="dummy title string" uid="fa349c44-9ccb-11e5-8176-5404a60d96b5" units="dummyAt"/>
542	kk= ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp','replaced_by']
543	kk= ['uid', 'label', 'title', 'mip', 'ref', 'refNote']
544	for uid in self.importWbMods.vg.vg:
545	item = self.doc.createElement( 'item' )
546	thisr = self.importWbMods.vg.vg[uid]
547	for i in range( len(kk)):
548	item.setAttribute( kk[i], thisr[i] )
549	##item.setAttribute( 'id', id )
550	thiss.appendChild( item )
551
552	def writeRvx(self):
553	thiss = self.doc.getElementsByTagName( 'requestVar' )[0]
554
555	## 'vid', 'title', 'label', 'priority', 'vgid', 'mip'
556	cols = self.rq.rqvarx['__cols__']
557	nn = 0
558	for k in self.rq.rqvarx.keys():
559	if k[0] != '_':
560	rec = self.rq.rqvarx[k]
561	print 'INFO.writeRvx.01001: ',k,rec
562	item = self.doc.createElement( 'item' )
563	item.setAttribute( 'uid', k )
564	for i in range(6):
565	item.setAttribute( cols[i], rec[i] )
566	vid = item.getAttribute( 'vid' )
567
568	if vid in self.cmvMap002:
569	vid = self.cmvMap002[vid]
570	item.setAttribute( 'vid', vid )
571	if vid in extRefCmvMapped:
572	vid = extRefCmvMapped[vid]
573	item.setAttribute( 'vid', vid )
574
575	if vid not in self.cmvByUid:
576	print 'ERROR.fixcmv.0005: attempt to use invalid vid (writeRvx): ',vid,k,rec
577	thiss.appendChild( item )
578	nn += 1
579	print 'INFO.writeRvx.00001: %s records written' % nn
580
581	def writeSn(self):
582	import loadcf
583	cf = loadcf.cf()
584	thiss = self.doc.getElementsByTagName( 'standardname' )[0]
585	dil = thiss.getElementsByTagName('item')
586	for d in dil:
587	thiss.removeChild(d)
588
589	for i in sorted( cf.names.keys() ):
590	d,u = cf.names[i]
591	self.appendSnItem(thiss,i,d,u)
592
593	for i in sorted( cf.alias.keys() ):
594	if i in self.usedSn:
595	print 'WARN.aliassn.00001: standard name alias: %s' % i
596	sn = cf.alias[i]
597	d,u = cf.names[sn]
598	self.appendSnItem(thiss,i,d,u)
599
600	def appendSnItem(self,thiss,sn,desc,units):
601	item = self.doc.createElement( 'item' )
602	uid = str(sn)
603	if uid.find( ' ' ) != -1:
604	print 'SEVERE.sn.00001: BAD SN: %s' % uid
605	uid = uid.replace( ' ', '' )
606	title = string.capwords( string.replace( uid, '_', ' ' ) )
607	label = string.replace( title, ' ', '' )
608	for w in ['Of','In','On','By']:
609	ww = ' %s ' % w
610	title = title.replace( ww, string.lower( ww ) )
611	item.setAttribute( 'uid', uid )
612	item.setAttribute( 'label', label )
613	item.setAttribute( 'title', title )
614	item.setAttribute( 'units', str(units) )
615	item.setAttribute( 'description', uniCleanFunc( desc ) )
616	thiss.appendChild( item )
617
618	def writeTimeSlice(self):
619	xx = self.vocabs.getElementsByTagName( 'timeSlice' )
620	assert len(xx) == 1, 'Expecting one element named "activity", found %s' % len(xx)
621	iDoc = xx[0]
622	thiss = self.doc.getElementsByTagName( 'timeSlice' )[0]
623	dil = thiss.getElementsByTagName('item')
624	for d in dil:
625	thiss.removeChild(d)
626
627	for i in iDoc.getElementsByTagName('item'):
628	item = self.doc.createElement( 'item' )
629	label = i.getAttribute( 'label' )
630	for k in ['label','title','type']:
631	item.setAttribute( k, i.getAttribute( k ) )
632	for k in ['child']:
633	if i.getAttribute( k ) != '':
634	item.setAttribute( k, i.getAttribute( k ) )
635	for k in ['start', 'end', 'step', 'sliceLen', 'nyears']:
636	if i.getAttribute( k ) != '':
637	v = str( int( float( i.getAttribute( k ) ) ) )
638	item.setAttribute( k, v )
639	item.setAttribute( 'uid', '_slice_%s' % label )
640	thiss.appendChild( item )
641
642	def writeMip(self):
643	self.mipUids = set()
644	xx = self.vocabs.getElementsByTagName( 'activity' )
645	assert len(xx) == 1, 'Expecting one element named "activity", found %s' % len(xx)
646	mipDoc = xx[0]
647	## <item id="tmpid.0003" label="C4MIP" status="" title="Coupled Climate Carbon Cycle Model Intercomparison Project" url="http://c4mip.lsce.ipsl.fr/"/>
648	thiss = self.doc.getElementsByTagName( 'mip' )[0]
649	dil = thiss.getElementsByTagName('item')
650	for d in dil:
651	thiss.removeChild(d)
652	for i in mipDoc.getElementsByTagName('item'):
653	item = self.doc.createElement( 'item' )
654	mip = i.getAttribute( 'label' )
655	for k in ['title','url']:
656	item.setAttribute( k, i.getAttribute( k ) )
657	uid = string.replace( mip, ' ', '' )
658	item.setAttribute( 'uid', uid )
659	item.setAttribute( 'label', uid )
660	self.mipUids.add( uid )
661	thiss.appendChild( item )
662
663	if "DECK" not in self.mipUids:
664	item = self.doc.createElement( 'item' )
665	item.setAttribute( 'uid', 'DECK' )
666	item.setAttribute( 'url', '' )
667	item.setAttribute( 'label', 'DECK' )
668	item.setAttribute( 'title', 'DECK - set of standard CMIP runs' )
669	self.mipUids.add( "DECK" )
670	thiss.appendChild( item )
671
672	def prepVar(self):
673	thissh = self.skey['var']
674	thatsh = self.skey['ovar']
675	self.varsxx = None
676	self.varsxxx = None
677	dothis = False
678	if self.newImport:
679	#['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov', 'priority0', 'rlm0']
680	self.vars = collections.defaultdict( list )
681	for k in thissh:
682	if k[0] != '_':
683	ll = thissh[k][:]
684	thisl = str( ll[2] )
685	self.vars[thisl].append( k )
686	else:
687	if dothis:
688	self.varsxx = collections.defaultdict( list )
689	for k in thissh:
690	if k[0] != '_':
691	ll = thissh[k][:]
692	self.varsxx[str( ll[0] )].append( k )
693	else:
694	self.varsxxx = collections.defaultdict( list )
695	for k in thatsh:
696	if k[0] != '_':
697	ll = thatsh[k][:]
698	thisl = str( ll[21] )
699	self.varsxxx[thisl].append( k )
700
701	def prep(self,rq):
702	## keys here match section element names in schema
703	#for k in [ 'var','ovar','groupitem', 'revisedtabitem']:
704	## lists for 3 new sections
705	### uuid, spid, tmid, cell_methods, cell_measures, description, procNote, prov
706	structure = []
707	self.structInfo = {}
708	### uuid, label, dimensions, shape, levels, levelFlag
709	spatialshp = []
710	### uuid, label, dimension, tstyle
711	temporalshp = []
712	### cmor variables
713	self.cmv = []
714	self.rqvList = []
715	self.rqvgList = []
716	self.cmvUidByFL = collections.defaultdict( cls_s1 )
717	self.cmvUidByTab = collections.defaultdict( cls_s1 )
718	self.cmvUidByTVP = collections.defaultdict( list )
719	self.cmvByLab = collections.defaultdict( dict )
720	self.cmvs = set()
721	self.cmvinfo = ('uid','stid','label','title','description','vid', \
722	'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', \
723	'deflate', 'valid_min', 'flag_values', 'modeling_realm', 'valid_max', 'positive', 'mipTable', \
724	'prov', 'provNote', 'frequency', 'rowIndex', 'coordinates','defaultPriority','mipTableSection','processing')
725	self.cmvinfo2 = ('uid','stid','label','title','description','vid', \
726	'deflate_level', 'shuffle', 'type', \
727	'deflate', 'modeling_realm', 'positive', 'mipTable', 'mipTableSection', \
728	'prov', 'provNote', 'frequency', 'rowIndex', 'defaultPriority','processing')
729	self.rqvinfo = ('uid','label','vid','gpid','priority','mip','table' )
730
731
732	thissh = self.skey['var']
733	self.varUidByName = self.importWbMods.vars.c1
734	##self.varUidByName = collections.defaultdict( list )
735	if self.newImport:
736	oo = open('refDefaultP.txt', 'w' )
737	for k in thissh:
738	if k[0] != '_':
739	##['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov', 'priority0']
740	thisuuid = str(k)
741	self.defaultP[thisuuid] = thissh[k][-1]
742	oo.write( '%s\t%s\t%s\t%s\n' % (k, thissh[k][0], thissh[k][2], thissh[k][-1] ) )
743	##self.varUidByName[thissh[k][0]].append( thisuuid )
744	oo.close()
745	else:
746	## ['label', 'title', 'description', 'procComment', 'procnote', 'prov', 'provmip', 'sn', 'units', 'uid','defaultp']
747	self.vars = collections.defaultdict( list )
748	self.vars1 = collections.defaultdict( list )
749	for uid in self.importWbMods.vars.d1:
750	self.defaultP[uid] = self.importWbMods.vars.d1[uid][-1]
751	## collection indexed by short_name
752	self.vars1[self.importWbMods.vars.d1[uid][0]].append( uid )
753	## collection indexed by standard_name
754	self.vars[self.importWbMods.vars.d1[uid][7]].append( uid )
755
756	### request variables
757	self.rqv = []
758	self.realmByVar = collections.defaultdict( list )
759	for ksect in [ 'ovar','groupitem']:
760	thissh = self.skey[ksect]
761	if ksect == 'ovar':
762	setu1 = set()
763	kk = ['uid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid', 'processing', 'gpid','rowIndex','ssect', 'priority']
764	id = kk.index('dimensions')
765	ivid = kk.index('vid')
766	thisnt = collections.namedtuple( 'ovar', kk )
767	ov = []
768	dset = set()
769	ds2 = set()
770	ovuu = collections.defaultdict( set )
771	ovll = collections.defaultdict( list )
772	for k in thissh:
773	if k[0] != '_':
774	rr = thissh[k][:]
775	rr[-3] = int( rr[-3] )
776	if rr[id] == 's\|c\|a\|l\|a\|r':
777	print 'ERROR.060.0001: bad dimension string detected and corrected: %s' % rr[id]
778	rr[id] = ''
779	if string.find( rr[23], 'plevs' ) != -1:
780	print 'INFO.qqqq01: ',rr[21:24]
781	rr[23] = string.replace(rr[23], 'plevs', 'plev19')
782	thisdim = rr[id]
783	if thisdim.find( ' ' ) != -1:
784	thisdim = '\|'.join( [x.strip() for x in thisdim.split() ] )
785	rr[id] = thisdim
786
787	tt = thisnt._make( rr )
788	if tt.uid in setu1:
789	print 'ERROR.099.0002: duplicate uuid: %s: %s' % (uuid,str(rr))
790	if str(tt.vid) == '__var_not_found_012__':
791	print 'ERROR.099.00102: bad vid string: %s [%s]' % (tt.vid,tt.out_name)
792
793	if tt.out_name == 'clisccp':
794	print 'INFO.clisccp.00002: ',rr,k
795	if tt.out_name == 'intppcalc':
796	print 'INFO.intppcalc.00002: ',rr,k
797	if tt.out_name == 'ch4':
798	print 'INFO.ch4.00002: ',rr,k
799
800	if str(tt.vid) not in self.importWbMods.vars.d1:
801	if tt.out_name in self.importWbMods.vars.c1:
802	if len( self.importWbMods.vars.c1[ tt.out_name ] ) == 1:
803	rr[ivid] = self.importWbMods.vars.c1[ tt.out_name ][0]
804	tt = thisnt._make( rr )
805	print 'INFO.099.00103: bad vid string replaced: %s [%s]' % (tt.vid,tt.out_name)
806	else:
807	print 'ERROR.099.00104: ambiguous bad vid string: %s [%s]' % (tt.vid,tt.out_name)
808	else:
809	print 'ERROR.099.00105: bad vid string: %s [%s]' % (tt.vid,tt.out_name)
810	setu1.add(tt.uid)
811	ov.append( tt )
812
813	dset.add( thisdim )
814	if tt.out_name in ['al','albs']:
815	print 'INFO.albs.00001: ',thisdim, rr
816	if tt.dimensions == 'location\|time1':
817	print 'INFO.dimensions.00001',rr
818	if tt.out_name == 'msftmyz':
819	print 'INFO.msftmyz.00002: ',self.cmv[-1]
820
821	ds2.add( (tt.dimensions,tt.cell_methods,tt.cell_measures,tt.flag_values, tt.flag_meanings) )
822	ovuu[tt.vid].add( tt.modeling_realm )
823	ovll[tt.vid].append( tt )
824	if rr[22][-3:] in ['mon','day','3hr']:
825	fr = rr[22][-3:]
826	elif rr[22][-2:] in [ 'yr','fx']:
827	fr = rr[22][-2:]
828	else:
829	fr = tab2freq[rr[22]]
830
831	prov = '%s ((isd.003))' % tt.mipTable
832	if tt.mipTable[:4] == 'OMIP':
833	prov = 'CMIP6 [OMIP] ((isd.001))'
834	elif string.find( tt.mipTable, '_' ) != -1:
835	prov = '%s [%s] ((isd.002))' % tuple( string.split( tt.mipTable, '_' ) )
836
837	if tt.var[:5] == 'xx..ccmi_':
838	pass
839	assert False, 'blind pass disabled .....'
840	else:
841	lab = tt.var
842	if tt.out_name != tt.var:
843	print 'WARN.outname.00001: out name issue.... ',tt.out_name,tt.var
844	if lab in {'*','include Oyr 3D tracers'}:
845	lab = '-copy-'
846	print 'ERROR.copy.00090: ',tt
847	##
848	## insert conditional on redundant tables here ...
849	##
850	if tt.mipTable in redundantTables:
851	if tt.mipTable not in reportedRedundantTables:
852	print 'INFO.000.11111: redundant table spotted: ',tt.mipTable
853	reportedRedundantTables.add( tt.mipTable )
854	else:
855	if lab != '-copy-' and string.find( lab, '-' ) != -1:
856	print 'ERROR.207.00010: bad character in label: %s' % lab
857	lab = string.replace( lab, '-', '' )
858	if tt.ssect != '':
859	ssect,sectid = tt.ssect
860	else:
861	ssect = ''
862	sectid = None
863
864	uid = chkRefCmv( tt.mipTable, lab, tt.uid, 'cmv.01009' )
865	if uid in self.cmvs:
866	print 'ERROR.090.09010: duplicate cmv uid %s : %s' % (uid, str(tt.__dict__) )
867	self.cmvs.add( uid )
868	self.cmv.append( (uid, (tt.dimensions,tt.cell_methods,tt.cell_measures,tt.flag_values,tt.flag_meanings), \
869	lab, tt.title, tt.comment, tt.vid, \
870	tt.deflate_level, tt.shuffle, tt.ok_max_mean_abs, tt.flag_meanings, tt.type, tt.ok_min_mean_abs, \
871	tt.deflate, tt.valid_min, tt.flag_values, tt.modeling_realm, tt.valid_max, tt.positive, tt.mipTable, \
872	prov, ssect, fr, tt.rowIndex, '',int(tt.priority), ssect, tt.processing) )
873	if tt.ok_max_mean_abs != '':
874	try:
875	xx = float( tt.ok_max_mean_abs )
876	except:
877	print 'FAILED to take float of ok_max_mean_abs -.....'
878	print tt.__dict__
879	raise
880	if tt.out_name == 'intppcalc':
881	print 'INFO.intppcalc.00003: ',self.cmv[-1]
882	if tt.out_name == 'msftmyz':
883	print 'INFO.msftmyz.00003: ',self.cmv[-1]
884	if tt.processing != '':
885	print 'INFO.processing.0001: ',lab,tt.title,tt.processing
886
887	self.realmByVar[ lab ].append( tt.modeling_realm )
888	vgid = tt.gpid
889	self.rqv.append( (str( uuid.uuid1() ),lab, uid, vgid, tt.priority,'CMIP5',tt.mipTable) )
890	if sectid not in (None, '_na_'):
891	self.rqv.append( (str( uuid.uuid1() ),lab, uid, sectid, tt.priority,'CMIP5',tt.mipTable) )
892	if lab == 'somint':
893	print 'INFO.somint.0002:', self.rqv[-1]
894	if lab == 'sisnconc':
895	print 'INFO.sisnconc.0002:', self.rqv[-1]
896	thisd = {}
897	for k9 in range(len(self.rqvinfo)):
898	thisd[self.rqvinfo[k9]] = self.rqv[-1][k9]
899	self.rqvList.append( (thisd, 'prep:thisd' ) )
900
901	self.dsort( dset )
902	print self.dsortdd[ksect][0].keys()
903
904	elif ksect == 'groupitem':
905	kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'priority', 'mip', 'uid', 'oldv', 'rowIndex', 'cmorv', 'new', 'gpid', 'vkey', 'vid']
906	thisnt = collections.namedtuple( 'groupitem', kk )
907	il = kk.index( 'levels' )
908	iu = kk.index( 'uid' )
909	idx = kk.index( 'descriptionEx' )
910	ixvid = kk.index( 'vid' )
911	ixvar = kk.index( 'var' )
912	ixcmv = kk.index( 'cmorv' )
913	ixp = kk.index( 'priority' )
914	dset = set()
915	dsgpi = set()
916	gi = []
917	s1 = set()
918	sm = set()
919	##
920	nov = 0
921	setu = set()
922	for k in thissh.keys():
923	if k[0] != '_':
924	rr = thissh[k][:]
925	if trace.GROUPITEM_ta and rr[ixvar] == 'ta':
926	print 'INFO.045.00004: groupitem: ',rr
927
928	if rr[ixvar] == 'snc':
929	print 'INFO.snc.0010:',rr
930	##
931	## convert floats back to integer
932	##
933	if type(rr[ixp]) == type(1.):
934	rr[ixp] = int(rr[ixp])
935	##
936	if rr[ixcmv] == '':
937	rr[ixcmv] = rr[1]
938	tt = thisnt._make( rr )
939
940	if tt.var == 'hus':
941	print 'INFO.hus.0001: ',tt.__dict__
942	if str(tt.mip) != 'OMIP':
943	gi.append( tt )
944	sm.add( tt.mask )
945
946	smdd = {}
947	dsn = {'floating ice shelf':u'floating ice sheet', u'ocean':u'sea', 'yes':'seaIce' }
948	d1 = {u'where land over land':(False,'area: mean where land over land'), \
949	u'land':(False,'area: mean where land'), \
950	u'hourly max':(False,'area: mean time: mean within hours time: maximum over hours'), \
951	u'grounded ice sheet':(True,'area: mean where grounded_ice_sheet'), \
952	u'ice sheet':(True,'area: mean where ice_sheet'), \
953	u'floating ice sheet':(True,'area: mean where floating_ice_shelf'), \
954	u'floating land ice':(True,'area: mean where floating_ice_shelf'), \
955	u'where snow on sea ice':(True,'area: mean where snow over sea_ice'), \
956	u'where melt ponds':(True,'area: mean where melt_pond over sea_ice'), \
957	u'where ridge':(True,'area: mean where sea_ice_ridge over sea_ice'), \
958	u'where sea ice':(True,'area: mean where sea_ice'), \
959	u'floating ice shelf':(True,'area: mean where floating_ice_sheet'), \
960	u'seaIce':(True,'area: mean where sea_ice over sea'), \
961	u'where sea ice over sea':(True,'area: mean where sea_ice over sea'), \
962	u'sea':(False,'area: mean where sea') }
963
964	for s in sm:
965	if s in ['', 'no', u'none',u'Southern hemisphere', u'Northern hemisphere']:
966	smdd[s] = (False,'')
967	else:
968	s1 = string.strip(s)
969	try:
970	smdd[s] = d1[ dsn.get( s1,s1 ) ]
971	except:
972	print s, dsn.get( s1,s )
973	raise
974
975	eeu = dict()
976
977	ccchk = ivg.checkCmv(gi)
978	nrep = 0
979
980	for tt in gi:
981	tabsect = ''
982	gr = 'a'
983	if tt.mip == 'OMIP' \
984	or any( [string.find(tt.table,i) != -1 for i in ['Omon','Oyr','Oclim','OImon','Ocean'] ] ) \
985	or any( [string.find(tt.group,i) != -1 for i in ['Omon','Oyr','Oclim','OImon','Ocean'] ] ) \
986	or tt.cmorv in oceanVars:
987	gr = 'o'
988	elif tt.cmorv in siSurfFluxes:
989	gr = "M"
990	print 'INFO.simip.00010: ', tt.cmorv
991
992	if self.varsxx != None:
993	if tt.cmorv in self.varsxx:
994	sss = set()
995	for k in self.varsxx[tt.cmorv]:
996	sss.add(self.skey['var'][k][-1] )
997	sss = list(sss)
998	if len(sss) == 1:
999	if sss[0] == 'ocean':
1000	gr = 'o'
1001	elif sss[0] == 'atmos':
1002	gr = 'a'
1003	print 'INFO:lllw:', tt.cmorv, sss, gr
1004	elif self.varsxxx != None:
1005	if tt.cmorv in self.varsxxx:
1006	sss = set()
1007	for k in self.varsxxx[tt.cmorv]:
1008	sss.add(self.skey['ovar'][k][21] )
1009	sss = list(sss)
1010	if len(sss) == 1:
1011	if sss[0] == 'ocean':
1012	gr = 'o'
1013	elif sss[0] == 'atmos':
1014	gr = 'a'
1015	print 'INFO:lllw:', tt.cmorv, sss, gr
1016
1017
1018	if tt.cmorv in ['talknat', 'sltbasin','clisccp','fCLandToOcean']:
1019	print 'INFO.llll: ',tt,gr
1020	##
1021	## adding label for mip here messes up referencing later ...
1022	##
1023	if tt.mip in [ 'SIMIP','LUMIP']:
1024	grprov = tt.mip
1025	else:
1026	grprov = ""
1027
1028	if tt.cmorv[-3:] in ['500','700','200','300','850','000']:
1029	print 'INFO.022.0002: Setting p... coordinate for ',tt
1030	if tt.cmorv[-5:] in [ 't2000']:
1031	thisc = 'depth2000m'
1032	elif tt.cmorv[-4:] in [ 't300', 't700']:
1033	thisc = 'depth%sm' % tt.cmorv[-3:]
1034	elif tt.cmorv[-3:] == '000':
1035	assert tt.cmorv[-4:] == '1000',str(tt)
1036	thisc = 'p%s' % tt.cmorv[-4:]
1037	else:
1038	thisc = 'p%s' % tt.cmorv[-3:]
1039	thissh = '%s\|%s' % (tt.shape, thisc)
1040	else:
1041	thissh = tt.shape
1042
1043	if thissh.find( ':' ) != -1:
1044	thissh, realmsuggest = thissh.split( ':' )
1045	elif tt.cmorv in varRealmMapsInv:
1046	realmsuggest = varRealmMapsInv[tt.cmorv]
1047	else:
1048	realmsuggest = None
1049
1050	### need to check where variables are, perhaps, refering to "ovar"
1051	##kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'misc', 'mip', 'uid', 'rowIndex', 'new', 'gpid', 'vkey', 'vid']
1052	prov = '%s [%s]' % (tt.mip,tt.group)
1053	provNote = ''
1054	lab = tt.cmorv
1055	if lab in {'*','include Oyr 3D tracers'}:
1056	lab = '-copy-'
1057	elif lab == 'wap (omega)':
1058	lab = 'wap'
1059	provNote = 'wap (omega)'
1060	lab = string.replace( string.strip(lab), '_', '-' )
1061	fr = freqmap.get( tt.freq, tt.freq )
1062	assert tt.group not in ['icesheetmon', 'icesheetyear','icesheetfx'], 'Unexpected tt.group: %s' % str(tt)
1063	if tt.group in ['LImonant','LImongre','LIyrant','LIyrgre','LIfxant','LIfxgre']:
1064	gr = 'g'
1065	ntab = tt.group
1066	elif tt.group in ['ImonAnt','ImonGre','IyrAnt','IyrGre','IfxAnt','IfxGre']:
1067	gr = 'g'
1068	ntab = tt.group
1069	elif tt.group in ['icesheetmonant','icesheetmongre','icesheetfxant','icesheetfxgre']:
1070	gr = 'g'
1071	ntab = 'LI' + tt.group[8:]
1072	elif tt.group in ['icesheetyearant','icesheetyeargre']:
1073	gr = 'g'
1074	ntab = 'LIyr' + tt.group[-3:]
1075	elif tt.table in ['Omon','Oyr','day'] and tt.shape == '=':
1076	ntab = tt.table
1077	print 'INFO.table.02002: new request var for existing CMOR var: %s, %s' % (tt.table,lab)
1078	else:
1079	ntab = 'em%s' % string.capwords( fr )
1080
1081	if fr == '6hr' and tt.levels in ['7h',4.0, '4']:
1082	ntab = '6hrPlev'
1083	print 'INFO.033.00001: adding to 6hrPlev ....',lab, tt.__dict__
1084
1085	elif tt.gpid in self.importWbMods.vg.vg and self.importWbMods.vg.vg[tt.gpid][1] == 'ISMIP6-LImon-01':
1086	##kk= ['uid', 'label', 'title', 'mip', 'ref', 'refNote']
1087
1088	ntab = 'LImon'
1089	tabsect = 'new'
1090	print 'INFO.limon.0001: Adding %s to LImon/new' % lab
1091
1092	if thissh == 'YZT' and ntab[-1] != 'Z':
1093	ntab += 'Z'
1094
1095	dset.add( (thissh, tt.levels,tt.tstyle,gr) )
1096	if 'area: mean where ice_sheet' == smdd[tt.mask][1] and gr == 'o':
1097	print 'SEVERE.structure.30001: ',tt.__dict__,gr
1098	dsgpi.add( (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask], 'None', 'None', grprov) )
1099	#
1100	# picking out re-purposed CMIP5 variables
1101	#
1102	if ovuu.has_key(tt.vid) and len( ovuu[tt.vid] ) == 1:
1103	print 'INFO.ovuu.00001: lookup started %s, %s' % (tt.cmorv,tt.group)
1104	if len( ovuu[tt.vid] ) > 1:
1105	print 'WARN.ovuu.00002: multiple realms: %s, %s: %s' % (tt.cmorv,tt.group, str( ovuu[tt.vid] ) )
1106	tt1 = ovll[tt.vid][0]
1107	thisrealm = tt1.modeling_realm
1108	ttl = ovll[tt.vid][0].title
1109	p1 = ovll[tt.vid][0].priority
1110	#tt1 = ovuu[tt.vid]
1111	if thisrealm == 'seaIce':
1112	ntab = 'OImon'
1113	if tt.priority in [1,2,3]:
1114	p = tt.priority
1115	else:
1116	p = int(p1)
1117
1118	if gr == 'o' and thisrealm == 'atmos':
1119	thisrealm = 'ocean'
1120
1121	if string.find( tt.descriptionEx, '//' ) != -1:
1122	ttl += ' ' + string.split( tt.descriptionEx, '/' )[0]
1123
1124	uid = chkRefCmv( ntab, lab, tt.uid, 'cmv.01001' )
1125	if uid != tt.uid:
1126	self.cmvMap002[tt.uid] = uid
1127
1128	thiscmv = (uid, (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask],tt1.flag_values,tt1.flag_meanings),
1129	lab, ttl, tt.descriptionEx, tt1.vid, \
1130	tt1.deflate_level, tt1.shuffle, tt1.ok_max_mean_abs, tt1.flag_meanings, tt1.type, tt1.ok_min_mean_abs, \
1131	tt1.deflate, tt1.valid_min, tt1.flag_values, thisrealm, tt1.valid_max, tt1.positive, ntab, \
1132	prov, provNote, tt.freq, tt.rowIndex, '', int(p),tabsect, '')
1133
1134	if type(tt1.valid_max) == type(1.) and tt1.valid_max <= tt1.valid_min:
1135	print 'ERROR.valmx.00001: valid_max below/equal valid_min: ',tt1
1136	self.realmByVar[ lab ].append( thisrealm )
1137	thisrqv = [str( uuid.uuid1() ),lab, uid, tt.gpid, p, tt.mip, tt.group]
1138	if lab in [ 'sisnconc','mrro','snc']:
1139	print 'INFO.%s.0001a:' % lab, thisrqv
1140	nov += 1
1141	else:
1142	print 'INFO.ovuu.00003: lookup started %s, %s' % (tt.cmorv,tt.group)
1143	thisrealm = ''
1144	if realmsuggest != None:
1145	thisrealm = realmsuggest
1146	elif realmdefix.has_key( prov ):
1147	thisrealm = realmdefix[prov]
1148	else:
1149	print 'WARN:070.00001: realm not found for %s,%s' % (lab,prov)
1150	if gr == 'o' and thisrealm == 'atmos':
1151	thisrealm = 'ocean'
1152
1153	if thisrealm == 'seaIce':
1154	if fr == 'mon':
1155	ntab = 'OImon'
1156	elif fr == 'day':
1157	ntab = 'SIday'
1158	else:
1159	raise
1160
1161	if tt.priority in [1,2,3]:
1162	p = tt.priority
1163	ptr = 1
1164	elif self.defaultP.get( tt.vid, 102) in [1,2,3]:
1165	p = self.defaultP.get( tt.vid, 102)
1166	ptr = 2
1167	else:
1168	p = 1
1169	print 'WARN:070.00002: priority defaulting to 1 for %s,%s' % (lab,prov)
1170	ptr = 3
1171
1172	if lab != '-copy-' and string.find( lab, '-' ) != -1:
1173	print 'ERROR.207.00020: bad character in label: %s' % lab
1174	lab = string.replace( lab, '-', '' )
1175	ttl = '__from %s' % tt.table
1176	if lab != '-copy-':
1177	if tt.vid in self.importWbMods.vars.d1:
1178	ttl = self.importWbMods.vars.d1[tt.vid][1]
1179	if string.find( tt.descriptionEx, '//' ) != -1:
1180	ttl += ' ' + string.split( tt.descriptionEx, '/' )[0]
1181
1182	if ntab in refCmv.refUid[ntab] and lab in refCmv.refUid[ntab]:
1183	uid = refCmv.refUid[ntab][lab]
1184	else:
1185	kk = str( '%s.%s' % (ntab,lab) )
1186	if kk in extRefCmv:
1187	uid = extRefCmv[kk]
1188	print 'INFO.cmv.01003: uid found in extRedCmv: %s, %s (%s)' % (ntab,lab,uid)
1189	else:
1190	if tt.uid in extRefCmvInv:
1191	print 'SEVERE.cmv.09909: uid issue %s [%s] %s' % (kk,tt.uid,extRefCmvInv[tt.uid])
1192	if extRefCmvMode != 'r':
1193	extRefCmv[kk] = tt.uid
1194	print 'ERROR.cmv.01003: uid not found: %s, %s (%s)' % (ntab,lab,tt.uid)
1195	uid = tt.uid
1196
1197	if tt.levels in ['7h',]:
1198	if lab.find( tt.levels) == -1:
1199	print 'WARN.cmv.00880: extending label %s --> %s (%s)' % (lab,lab+tt.levels, str( tt.__dict__))
1200	lab += tt.levels
1201	uid = chkRefCmv( ntab, lab, tt.uid, 'cmv.01011' )
1202	if uid in eeu:
1203	print 'WARN.cmv.90001: duplicate cmv uid: ',uid,tt.__dict__,eeu[uid]
1204	thiscmv = (uid, (thissh,tt.levels,tt.tstyle,gr,smdd[tt.mask], 'None', 'None'), lab, ttl, tt.descriptionEx, tt.vid, \
1205	0, 0, None, 0, '', None, \
1206	0,None,0, thisrealm, None, '', ntab, \
1207	prov, provNote, tt.freq, tt.rowIndex, '', str(p), tabsect, '')
1208	self.realmByVar[ lab ].append( thisrealm )
1209
1210	thisrqv = [str( uuid.uuid1() ),lab, uid, tt.gpid, p, tt.mip, tt.group]
1211	if lab in ['mrro', 'sisnconc','snc']:
1212	print 'INFO.%s.0001b:' % lab, thisrqv, ptr, thiscmv
1213	if trace.GROUPITEM_ta and lab == 'ta':
1214	print 'INFO.045.00005: groupitem: ',thisrqv
1215
1216	if ccchk.uidReplace.has_key(thiscmv[0]):
1217	nrep += 1
1218	print 'ERROR.cmv.09005: duplicate cmv uuid: %s: %s' % (uid, str(tt.__dict__) )
1219	thisrqv[2] = ccchk.uidReplace[thiscmv[0]]
1220	else:
1221	if uid in setu1:
1222	print 'ERROR.cmv.09001: duplicate cmv uuid: %s: %s ' % (uid,str(tt.__dict__))
1223	elif uid not in eeu:
1224	eeu[uid] = tt.__dict__
1225	assert len(thiscmv) == len( self.cmvinfo ), 'BAD CMV specification length: %s' % str(cmv)
1226	self.cmv.append( thiscmv )
1227	print 'INFO.cmv.x0001: append cmv: ',tt.__dict__
1228	self.rqv.append( thisrqv )
1229	if thisrqv[1] == 'somint':
1230	print 'INFO.somint.0003',thisrqv
1231	## Variables marked with shape '?' from PMIP are actually defined by OMIP ....
1232	if tt.shape == '?':
1233	print 'ERROR.801.00001: shape not set: %s, %s, %s' % ( tt.shape,tt.mip,tt.var)
1234
1235	##['', u'none', u'land', u'grounded ice sheet', u'floating ice sheet', u'floating ice shelf', u'ocean', u'ice sheet', u'sea', u'Southern hemisphere', u'Northern hemisphere']
1236	self.dsort( dset,mode=ksect )
1237	print 'INFO.088.00001: nrep=%s' % nrep
1238	print 'INFO.088.00002: nov=%s [%s]' % (nov,len(gi))
1239
1240	nrep = []
1241	for c in self.cmv:
1242	u,v,f,t,p,pn = (c[0],c[2],c[21],c[18],c[19],c[20])
1243	self.cmvUidByFL[f].a[v].append( u )
1244	self.cmvUidByTab[t].a[v].append( u )
1245	if v[0] != '-':
1246	if (t,v,p) in self.cmvUidByTVP:
1247	nrep.append( (t,v,p) )
1248	self.cmvUidByTVP[ (t,v,p) ].append( u )
1249	self.cmvByLab[v][ u ] = (f,t)
1250	if len( nrep) > 0:
1251	print 'WARN.108.00088: unresolved .... %s' % len(nrep)
1252	for n in nrep:
1253	print n
1254
1255	#######################################################################################
1256	#################
1257	std = set()
1258	ssd = set()
1259	esd = {}
1260
1261	for ksect in [ 'ovar','groupitem']:
1262	#### create structure list for spatial and temporal dimensions
1263	for t in self.dsortdd[ksect][0].keys():
1264	tdim, tt = self.dsort_u1( self.dsortdd[ksect][0][t], self.dsortdd[ksect][1] )
1265
1266	std.add( tdim )
1267	ssd.add( tt )
1268
1269	etd = self.spsh.etd
1270	for s in std:
1271	assert s in etd, 'Temporal dimension record not found for %s (see importWbMods.loadSpatial)' % s
1272
1273	kk = 0
1274	self.spidInfo = {}
1275	for s in ssd:
1276	d,lfl,nl = s
1277	if d == 'longitude\|latitude\|plev7':
1278	print 'INFO.plev7.0002: resetting plev7 to plev7c'
1279	d = 'longitude\|latitude\|plev7c'
1280	if d == 'longitude\|latitude\|':
1281	print 'INFO.plev7.0003: removing trailing \|'
1282	d = 'longitude\|latitude'
1283	if d not in WbMods.ls.ss:
1284	print 'SEVERE.spatial.0001: dimensions not found in WbMods.ls: %s' % d
1285	assert False, 'Spatial shapes must be specified in spatialShape_ref.xls: %s' % d
1286	u = str( uuid.uuid1() )
1287	lab = 'ssd-%3.3i' % kk
1288	description = 'ssd.%3.3i' % kk
1289	else:
1290	u = WbMods.ls.ss[d][4]
1291	lab = WbMods.ls.ss[d][0]
1292	description = WbMods.ls.ss[d][1]
1293	print 'INFO.spatial.0002: setting spatial shape uid: ',u,lab,description
1294
1295	kk+=1
1296	esd[s] = (u,lab,description,d,lfl,nl)
1297	self.spidInfo[u] = (u,lab,description,d,lfl,nl)
1298
1299	## clumsy here .. uid copied from spatialShape_ref.xls
1300	esd['Y-na'] = ('6dc25718-a24a-11e5-a29d-5404a60d96b5','extra','extra','latitude','false',1)
1301	self.dimsTime = etd
1302	self.dimsTimeInfo = ['uid','label','title','dimensions']
1303	self.dimsSpace = esd
1304	self.dimsSpaceInfo = ['uid','label','title','dimensions','levelFlag','levels', 'dimids']
1305
1306	sshp = set()
1307	eshp = set()
1308	self.lls = ivg.structureList()
1309	nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tmid','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] )
1310	kk = 0
1311	strdd = {}
1312	ee3 = {}
1313	for dims,cmet,cmea,flgv, flgm in ds2:
1314	tdim, tt = self.dsort_u1( self.dsortdd['ovar'][0][dims], self.dsortdd['ovar'][1] )
1315	d1,odim,d2,crd = self.dsortdd['ovar'][0][dims]
1316	sdim,lfl,nl = tt
1317	u = str( uuid.uuid1() )
1318	spid = esd[tt][0]
1319	splab = esd[tt][1]
1320	tmid = etd[tdim][0]
1321	tlab = etd[tdim][1]
1322	kk += 1
1323	label = 'str-a%3.3i' % kk
1324	proc = ''
1325	prov = 'CMIP5/OMIP'
1326	desc = ''
1327	##print 'INFO.zzz01: ',dims, crd
1328	t0 = (dims,cmet,cmea,flgv,flgm)
1329	if cmet in ['None',None]:
1330	if tlab == 'time-mean':
1331	cmet = 'time: mean'
1332	elif tlab == 'time-point':
1333	cmet = 'time: point'
1334	else:
1335	cmet = ''
1336	if cmea in ['None',None]:
1337	cmea = ''
1338
1339	if flgv in ['None',None,u'None']:
1340	flgv = ''
1341	if flgm in ['None',None,u'None']:
1342	flgm = ''
1343	t1 = (dims,cmet,cmea,flgv,flgm)
1344	##errc,u,label = self.lls.add( [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] )
1345	errc,u,label = self.lls.add2( [u,label,splab,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] )
1346	print 'INFO.structure.00011:',t0,t1,u,label,spid,splab
1347	self.structInfo[u] = (label,cmet,spid,tmid)
1348	strdd[t0] = u
1349	strdd[t1] = u
1350
1351
1352	ttt = [ {u'Instantaneous (end of year)', 'instantaneous', 'point', 'Synoptic', u'synoptic', u'time: point'},
1353	{'Mean', u'time: mean', u'Time mean', u'daily mean', u'mean', u'time mean', u'time: day', u'time: mean', u'weighted time mean', 'Cumulative annual fraction'}, set(), {''}, 'Daily max', 'Daily min']
1354	cmetl = ['time: point', 'time: mean', 'time: mean within years time: mean over years ', '', 'time: maximum', 'time: minimum']
1355	##cmetlsi = ['time: point', ('time: mean area: mean where sea_ice over sea','time: mean where sea_ice area: mean where sea_ice'), 'time: mean within years time: mean over years ', '']
1356	ee1 = {}
1357	for shp,lvls,sty,gr,tmsk, flgv, flgm, prov in dsgpi:
1358	t = (shp,lvls,sty,gr)
1359	ttx = (shp,lvls,sty,gr,tmsk, flgv, flgm, prov)
1360	if "t2000" in shp:
1361	print 'CCCC',shp,lvls,sty,gr,tmsk, flgv, flgm, prov
1362	tdim, tt = self.dsort_u1( self.dsortdd['groupitem'][0][t], self.dsortdd['groupitem'][1] )
1363	d1,odim,d2,crd = self.dsortdd['groupitem'][0][t]
1364	tflm, msk = tmsk
1365	sdim,lfl,nl = tt
1366	u = str( uuid.uuid1() )
1367	spid = esd[tt][0]
1368	splab = WbMods.ls.labByUid[ spid ]
1369	tmid = etd[tdim][0]
1370	tlab = etd[tdim][1]
1371	print 'INFO.spatial.6666: %s, %s, %s' % (splab,spid,str(tt))
1372
1373	if "XY" in splab:
1374	if gr == "M":
1375	cmea = "--MODEL"
1376	else:
1377	cmea = 'area: areacell%s' % gr
1378	else:
1379	cmea = ''
1380
1381	cmet = ''
1382	for i in range(4):
1383	if sty in ttt[i]:
1384	if i == 1 and prov == 'SIMIP':
1385	if sty == u'weighted time mean':
1386	cmet = 'area: time: mean where sea_ice'
1387	elif sty == u'partial mean over sea':
1388	cmet = 'area: mean where sea_ice over sea time: mean'
1389	else:
1390	cmet = 'area: time: mean'
1391	elif prov == 'LUMIP' and shp == 'XYLT':
1392	print 'INFO.lumip.00505: ',shp,lvls,sty,gr,tmsk, flgv, flgm, prov
1393	if sty == 'Time mean':
1394	cmet = 'area: time: mean where landuse'
1395	elif sty == 'Cumulative annual fraction':
1396	cmet = 'area: mean time: sum'
1397	elif sty == 'Instantaneous (end of year)':
1398	cmet = 'area: mean where landUse time: point'
1399	else:
1400	cmet = cmetl[i]
1401	if cmet == '':
1402	cmet = msk
1403	elif msk != '':
1404	if "time:" in msk:
1405	print 'INFO.cmet.05005: using time: info in msk: ',msk,cmet
1406	cmet = msk
1407	else:
1408	cmet = msk + ' ' + cmet
1409	kk += 1
1410	label = 'str-b%3.3i' % kk
1411	proc = ''
1412	if prov == '':
1413	prov = 'CMIP5, endorsed MIPs'
1414	desc = ''
1415	dims = tt[0]
1416	if tdim != '':
1417	dims += '\|' + tdim
1418	if odim == 'spectband':
1419	print 'INFO.spectband.00002: ',tt, tdim, cmet, cmea, label, u
1420
1421	if strdd.has_key( (dims,cmet,cmea,'','') ) and crd == '' and odim == '':
1422	u = strdd[ (dims,cmet,cmea,'','') ]
1423	##print 'INFO.zzzy03: ',tt, tdim, cmet, cmea, label, u
1424	elif ee1.has_key( (tmid, spid, odim, cmet, cmea, crd) ):
1425	u = ee1[(tmid, spid, odim, cmet, cmea, crd)]
1426	##print 'INFO.zzzy04: ', tt, tdim, cmet, label, u
1427	else:
1428	errc,u,label = self.lls.add2( [u,label,splab,tlab, odim, crd, cmet, cmea, '', '', desc, proc, prov] )
1429	print 'INFO.zzzy02: ',tt, tdim, cmet, label, u, ttx
1430
1431	ee1[(tmid, spid, odim, cmet, cmea, crd)] = u
1432
1433	if sty == u'Synoptic':
1434	print 'yyyy: ',(shp,lvls,sty,gr,tmsk,flgv,flgm)
1435	strdd[(shp,lvls,sty,gr,tmsk,flgv,flgm)] = u
1436	if flgv == 'None' and flgm == 'None':
1437	strdd[(shp,lvls,sty,gr,tmsk,'','')] = u
1438	self.structInfo[u] = (label,cmet,spid,tmid)
1439
1440	##nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tmid','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] )
1441	##self.lls.close()
1442	self.strdd = strdd
1443
1444	def cmvAudit(self,nn):
1445	nrep = []
1446	self.cmvByUid = {}
1447	for c in self.cmv:
1448	u,v,f,t,p,pn = (c[0],c[2],c[21],c[18],c[19],c[20])
1449	self.cmvUidByFL[f].a[v].append( u )
1450	self.cmvUidByTab[t].a[v].append( u )
1451	if v[0] != '-':
1452	if (t,v,p) in self.cmvUidByTVP:
1453	nrep.append( (t,v,p) )
1454	self.cmvUidByTVP[ (t,v,p) ].append( u )
1455	self.cmvByLab[v][ u ] = (f,t)
1456	self.cmvByUid[u] = c
1457	if len( nrep) > 0:
1458	print 'WARN.108.00088: unresolved .... %s' % len(nrep)
1459	for n in nrep:
1460	print n
1461	topop = []
1462	cc1 = collections.defaultdict( int )
1463	cc2 = collections.defaultdict( int )
1464	for k in self.fix.fixcmv:
1465	vid = str( self.fix.fixcmv[k][4] )
1466	fn = str( self.fix.fixcmv[k][6] )
1467	cc1[fn] += 1
1468	if vid not in self.cmvByUid:
1469	print 'WARN.fixcmv.0001: invalid vid %s [%s]' % ( vid, str( self.fix.fixcmv[k] ) )
1470	topop.append( k )
1471	cc2[fn] += 1
1472	if len(topop) > 0:
1473	print 'WARN.fixcmv.0009: %s invalid vid value popped from fixcmv [%s]' % ( len( topop ), len( self.fix.fixcmv ) )
1474	for k in topop:
1475	self.fix.fixcmv.pop( k )
1476	for f in sorted( cc2.keys() ):
1477	print 'WARN.fixcmv.0010: %s invalid vid value popped from fixcmv:%s [%s]' % (cc2[f],f,cc1[f])
1478
1479	def dsort_u1(self,eed,sde):
1480	tdim = eed[2]
1481	sdim = eed[0]
1482	nl = sde[sdim]
1483	lfl = 'true'
1484	if nl == -1:
1485	lfl = 'false'
1486	nl = 0
1487	elif nl == -2:
1488	nl = 1
1489	pass
1490	return (tdim, (sdim,lfl,nl) )
1491
1492
1493	def dsort( self, dset, mode='ovar' ):
1494	"""Sort the dimensions string into spatial, temporal and other sections,
1495	returns a dictionary of tuples: spatial dimensions, other, temporal, coords and a 2nd, keyyed on spatial dimensions with a level count indicator. """
1496	vdims = {'plev7':7, 'plev':4, 'plevs':19, 'plev3':3,'plev8':8,'alt40':40,'alev1':1, 'alevhalf':-1,'olevel':-1,'rho':-1,'sdepth':-1,'alevel':-1}
1497	odims = {'sza5','vegtype','scatratio','dbze','typepdec','vgidx','tau','effectRad'}
1498	dl = sorted( list(dset) )
1499	if mode == 'ovar':
1500	eed = {}
1501	sss = [set(),set(),set(),set()]
1502	for s in dl:
1503	if s[:4] == 'time':
1504	bits = ['',s[4:]]
1505	else:
1506	bits = string.split(s, '\|time', maxsplit=1 )
1507	if len(bits) == 1:
1508	ts = ''
1509	else:
1510	ts = 'time'
1511
1512	b0 = bits[0]
1513	b1 = ''
1514	bb = string.split(bits[0],'\|')
1515	if bb[-1] in odims:
1516	if bb[-1] == 'tauxxx':
1517	b0 = bits[0][:-4]
1518	ts = 'tau\|time'
1519	else:
1520	b0 = string.join( bb[:-1], '\|' )
1521	b1 = bb[-1]
1522
1523	if len(bits) == 1:
1524	tup = (b0,b1,'','')
1525	elif len(bits) == 2:
1526	b2 = string.split( bits[1], '\|', maxsplit=1 )
1527	if len(b2) == 2:
1528	tup = (b0,b1, ts + b2[0], b2[1])
1529	else:
1530	tup = (b0,b1, ts + bits[1], '')
1531	eed[s] = tup
1532	for k in range(4):
1533	sss[k].add(tup[k])
1534
1535	lcount = {}
1536	for s in sss[0]:
1537	bits = string.split( s, '\|' )
1538	nl = -2
1539	thisb = None
1540	for b in bits:
1541	if vdims.has_key(b):
1542	assert nl == -2, 'Duplicate vertical level count in %s' % s
1543	nl = vdims[b]
1544	thisb = b
1545	lcount[s] = nl
1546	if nl == -1 and s == 'latitude\|basin':
1547	print 'SEVERE: error in level deduction'
1548	print s, bits, nl, thisb
1549	raise
1550	else:
1551	## input is a set of tuples, shape, levels, tstyle, o/a
1552	## aim is to extract spatial, other, temporal dimensions + coords.
1553	sss = [set(),set(),set()]
1554	ttt = [ ('time1', u'Instantaneous (end of year)', 'instantaneous', 'point', 'Synoptic', u'synoptic', u'time: point'),
1555	('Daily max', 'Daily min', 'time', 'Mean', u'time: mean', u'Time mean', u'daily mean', u'mean', u'time mean', u'time: day', u'time: mean', u'weighted time mean', 'Cumulative annual fraction','partial mean over sea'),
1556	(),
1557	('diurnal','time3'), ('',)]
1558	ttdd = {}
1559	kkk = ['time1','time','time2','time3','']
1560	for i in range(5):
1561	for k in ttt[i]:
1562	ttdd[k] = kkk[i]
1563
1564	##lll = {[u'1.0', u'14.0', u'16.0', u'17.0', u'27.0', u'3.0', u'36.0', u'4.0', u'40.0', u'7.0', ]:'flt', [u'17 (or 23 )', u'17 (or 23)', u'17 (or23)']:'obs', [u'Model levels or 27Plevs', u'all', u'all model levels', u'all model levels above 400hPa', u'all*', u'integrated over depth'], [u'single level',], [u'soil levels',], [u'surface',], [[u'100 m',] }
1565	dds = {'2D':'XYT', 'XY':'XYT', '':'?', '2D ':'XYT', 'XYZ':'XYZT', }
1566	## 'K'?? 'XYK'
1567	##
1568	##
1569	## spatial dimensions, other, time, coordinates, wild card in spatial dimensions.
1570	##
1571	dd = {'XKT':('latitude\|basin','','time','',False), \
1572	'XYZKT':('longitude\|latitude\|%s','tau','time','',True), \
1573	'XYZKmT':('longitude\|latitude\|%s','misrBands','time','',True), \
1574	'XYZKKT':('longitude\|latitude\|%s','effectRadXX\|tau','time','',True), \
1575	'XYZKKlT':('longitude\|latitude\|%s','effectRadLi\|tau','time','',True), \
1576	'XYZKKiT':('longitude\|latitude\|%s','effectRadIc\|tau','time','',True), \
1577	'XYZKdbT':('longitude\|latitude\|%s','dbze','time','',True), \
1578	'XYZKscT':('longitude\|latitude\|%s','scatratio','time','',True), \
1579	'XYZT':('longitude\|latitude\|%s','','time','',True), \
1580	'XYST':('longitude\|latitude\|sdepth','','time','',False), \
1581	'XYSnT':('longitude\|latitude\|snowdepth','','time','',False), \
1582	'XYS1T':('longitude\|latitude','sdepth1','time','',False), \
1583	'XYH1T':('longitude\|latitude','height100','time','',False), \
1584	'XYZT*':('longitude\|latitude\|%s','','time','',True), \
1585	'KZT':('alevel','spectband','time','',False), \
1586	'Ksp':('','spectband','','',False), \
1587	'XYKspT':('longitude\|latitude','spectband','time','',False), \
1588	'XYZKspT':('longitude\|latitude\|%s','spectband','time','',True), \
1589	'XYT':('longitude\|latitude','','time','',False), \
1590	'2DV':('longitude\|latitude','','time','',False), \
1591	'XYKT':('longitude\|latitude','sza5','time','',False), \
1592	'XYLT':('longitude\|latitude','landUse','time','',False), \
1593	'XYant':('xant\|yant','','time','',False), \
1594	'XYgre':('xgre\|ygre','','time','',False), \
1595	'XYKszT':('longitude\|latitude','sza5','time','',False), \
1596	'YZT':('latitude\|%s','','time','',True), \
1597	## snowband was an error ... all the variables should have been on ice bands ... easier to correct here.
1598	'XYKsn':('longitude\|latitude','iceband','time','',False), \
1599	'XYKic':('longitude\|latitude','iceband','time','',False), \
1600	'K4':('siline','','time','',False), \
1601	'scalar':('','','time','',False), \
1602	'':('','','time','',False), \
1603	'sheet':('icesheet','','time','',False), \
1604	'ZST1':('alevel','site','time','',False), \
1605	'BasinYT':('latitude\|basin','','time','',False), \
1606	'?':( '?', '?','?','?',False)
1607	}
1608
1609	for d in dl:
1610	for k in range(3):
1611	sss[k].add( d[k] )
1612
1613	ls0 = set()
1614	ls1 = set()
1615	##
1616	## sort shapes
1617	##
1618	for s0 in sss[0]:
1619	if string.find( s0, '\|' ) != -1:
1620	s,crd0 = string.split( s0, '\|' )
1621	else:
1622	s,crd0 = s0,None
1623
1624	if not dd.has_key( dds.get(s,s) ):
1625	print 'ERROR.shape.0001: NO SOLUTION FOUND FOR SHAPE: "%s"' % s
1626	else:
1627	tt = dd[dds.get(s,s)]
1628	print 'INFO.structure.00077: mapping %s to %s' % (s,str(tt))
1629	if tt[-1]:
1630	ls0.add( s0 )
1631	##
1632	## look at shapes which require additional information about vertical coord
1633	##
1634	for d in dl:
1635	if d[0] in ls0:
1636	ls1.add( d[1] )
1637
1638	lset = {'soil levels':'sdepth', 'all model levels above 400hPa':'alevel'}
1639	lsdd = {}
1640	print 'ls1....',ls1
1641	for l in ls1:
1642	ii = 0
1643	if l in ['all','all*','all model levels']:
1644	ii = -1
1645	this = '%slevel'
1646	elif lset.has_key(l):
1647	ii = -1
1648	this = lset[l]
1649	elif l == '':
1650	ii = -2
1651	this = l
1652	elif l == '7h':
1653	ii = 7
1654	this = 'plev7h'
1655	elif l == '3h':
1656	ii = 3
1657	this = 'plev3h'
1658	else:
1659	ii = int( float(l) )
1660	if ii in [40,16]:
1661	this = 'alt%s' % ii
1662	elif ii > 1.5:
1663	if ii > 35:
1664	ii = 39
1665	##
1666	## merge 14 and 17 level specifications to a 19 level coordinate
1667	##
1668	if ii in [14,17]:
1669	ii = 19
1670
1671	this = 'plev%s' % ii
1672	if ii == 7:
1673	this += 'c'
1674
1675	else:
1676	this = ''
1677	lsdd[l] = (ii,this)
1678	##['', 1.0, u'all', 3.0, 4.0, u'soil levels', 7.0, 40.0, u'all model levels', u'17 (or 23)', u'all*', 14.0, u'Model levels or 27Plevs', 16.0, 17.0, u'all model levels above 400hPa', u'17 (or23)', 36.0, u'17 (or 23 )', 27.0]
1679
1680	lcount = {}
1681	eed = {}
1682	for d in dl:
1683	s0 = d[0]
1684	if string.find( s0, '\|' ) != -1:
1685	s,crd0 = string.split( s0, '\|' )
1686	else:
1687	s,crd0 = s0,None
1688	if not dd.has_key( dds.get(s,s) ):
1689	print 'NO SOLUTION FOUND FOR SHAPE: "%s"' % s
1690	else:
1691	tt = dd[dds.get(s,s)]
1692	if tt[-1]:
1693	n,zz = lsdd[d[1]]
1694	if len(zz) > 0 and zz[0] == '%':
1695	zz = zz % d[3]
1696	sc = tt[0] % zz
1697	if sc[-1] == "\|":
1698	print 'WARN.080.05000: Bad dimension found %s [%s] %s %s' % (sc,str(tt), str((n,zz)), str(d) )
1699	else:
1700	sc = tt[0]
1701	if (string.find( sc, 'alevel' ) != -1) or (string.find( sc, 'olevel' ) != -1):
1702	n = -1
1703	else:
1704	n = -2
1705
1706	lcount[sc] = n
1707	if n == -1 and sc == 'latitude\|basin':
1708	print 'SEVERE [2]: error in level deduction'
1709	print sc, tt[0], tt[-1], n
1710	raise
1711	if crd0 != None:
1712	tt3 = tt[3] + crd0
1713	else:
1714	tt3 = tt[3]
1715
1716	if d[2] == '' and tt[2] not in ['','?']:
1717	d2 = tt[2]
1718	else:
1719	d2 = d[2]
1720	print 'INFO:qqq05: ',d, sc, tt, ttdd.get( d2, '%s' % d2), tt3
1721	if d2 not in ttdd:
1722	print 'ERROR.time.00800: ',d2,ttdd.keys()
1723	eed[d] = (sc,tt[1],ttdd.get( d2, '%s' % d2),tt3)
1724	print 'INFO.ddddd.00002:',d,eed[d]
1725	self.dsortdd[mode] = (eed,lcount)
1726
1727	def run(self,rq):
1728
1729	xx = self.doc.getElementsByTagName( 'main' )
1730	assert len(xx) == 1, 'Expecting one element named "main", found %s' % len(xx)
1731	main = xx[0]
1732	## main = self.doc.childNodes[0]
1733	xsn = []
1734	ff = {}
1735	for c in main.childNodes:
1736	tag = c.nodeName
1737	if tag != '#text':
1738	xsn.append(tag)
1739	il = []
1740	for i in c.childNodes:
1741	if i.nodeName == 'item':
1742	il.append(i)
1743	xx = dict( il[0].attributes.items() )
1744	ff[string.lower(tag)] = (c,xx)
1745
1746	print xsn
1747	print ff.keys()
1748	self.ff = ff
1749
1750	## keys here match section element names in schema
1751	## dec 2015 -- removing 'var' from this list
1752	assert self.schemaMode == 'dreq2', 'Old schema mode not supported here'
1753	sectlist = [ 'experiment','exptgroup','objective','requestlink', 'requestitem','requestvargroup','tablesection','requestvar' ]
1754	sectlist = [ 'experiment','exptgroup','objective','requestlink', 'requestitem','tablesection','requestvar' ]
1755
1756	for k in sectlist:
1757	thissh = self.skey[k]
1758	## set labmod False for section in which label coercion is not wanted
1759	labmod = True
1760	lll = []
1761	for i in thissh.keys():
1762	if i[0] != '_':
1763	lll.append(i)
1764	idk = 0
1765	if len(lll) > 0:
1766	##
1767	## remove example items, xcept where this has been done in previous function
1768	##
1769	if k not in ['xxxx']:
1770	dil = ff[k][0].getElementsByTagName('item')
1771	for d in dil:
1772	ff[k][0].removeChild(d)
1773
1774	vare = {}
1775	nfr = set()
1776	for i in lll:
1777	ll = map( uniCleanFunc, thissh[i] )
1778	assert len(ll) == len( thissh[i] ), 'Lost list elements after uniClean'
1779	idk += 1
1780	#################################################################
1781	######### NEED TO CLEAN THIS IF ITEMS FROM OTHER SECTIONS ARE FILTERED
1782	#######################################################################
1783	dothis = True
1784	if k not in ['var','experiment']:
1785	item = self.doc.createElement( 'item' )
1786
1787	if k == 'objective':
1788	## - mip; tab; expt; rlid; ny
1789	labmod = False
1790	kk = ['mip', 'label', 'title', 'description']
1791	##thisl = string.replace(string.strip(str( ll[1] ) ), '-', ' ')
1792	## camelcase this label.
1793	##thisl = string.replace( string.capwords( thisl ), ' ', '' )
1794	thisl = mycc( ll[1] )
1795	mip = str( ll[0] )
1796	self.objectives[mip][thisl] = i
1797	for j in range(len(kk)):
1798	thisv = str( ll[j] )
1799	item.setAttribute( kk[j], thisv )
1800	item.setAttribute( 'uid', str(i) )
1801	elif k == 'experiment':
1802	labmod = False
1803	kk = ['uid','egid','label','description','mip','mcfg','tier','nstart','starty','endy','yps','ensz','ntot','comment']
1804	thisl = str( ll[2] )
1805	## self.mips.add( str( ll[4] ) )
1806	if ll[4] in ['SolarMIP']:
1807	dothis = False
1808	else:
1809	item = self.doc.createElement( 'item' )
1810	for j in range(len(kk)):
1811	if kk[j] in ['tier','ensz']:
1812	if type(ll[j]) == type( 1 ):
1813	thisv = str( ll[j] )
1814	else:
1815	thisv = string.join( [str(x) for x in ll[j]] )
1816	else:
1817	thisv = str( uniCleanFunc(ll[j]) )
1818	item.setAttribute( kk[j], thisv )
1819	if kk[j] == 'yps':
1820	if type(ll[j]) == type([]):
1821	print 'ERROR.yps.00001: ', ll
1822	item.setAttribute( 'yps', '25' )
1823	##################################################################################################
1824	elif k == 'exptgroup':
1825	labmod = False
1826	kk = ['uid','label','tierMin','ntot']
1827	thisl = str( ll[1] )
1828	for j in range(len(kk)):
1829	thisv = str( ll[j] )
1830	item.setAttribute( kk[j], thisv )
1831
1832	##################################################################################################
1833	elif k == 'groupitem':
1834	kk = ['group', 'var', 'table', 'freq', 'descriptionEx', 'shape', 'levels', 'tstyle', 'mask', 'misc', 'mip', 'uid', 'new', 'gpid', 'vkey', 'vid']
1835	assert len(kk) == len(ll), 'length mismatch, %s %s' % (len(kk),len(ll))
1836	# - group; table; freq; descriptionEx; shape; levels; tstyle; mask; mip; mip2; uuid; new; vid
1837	ix_gpid = kk.index( 'gpid' )
1838	thisl = string.strip(str( ll[1] ) )
1839	for j in range(len(kk)):
1840	if j != 1:
1841	##item.setAttribute( kk[j], str( ll[j] ) )
1842	if kk[j] == 'misc':
1843	item.setAttribute( 'priority', str( ll[j] ) )
1844	else:
1845	item.setAttribute( kk[j], str( ll[j] ) )
1846	item.setAttribute( 'title', thisl )
1847	if string.find(thisl, ' ') != -1:
1848	thisl = string.split(thisl)[0]
1849	thisuuid = ll[11]
1850
1851	gpid = str(ll[ix_gpid])
1852	if not self.importWbMods.vg.vg.has_key(gpid):
1853	print 'ERROR.015.0010: gpid not found %s' % str(ll)
1854	self.e15_10 += 1
1855	if self.repl.has_key( thisuuid ):
1856	for tag,old,new in self.repl[thisuuid]:
1857	thisold = item.getAttribute( tag )
1858	assert thisold == old, 'Attempt to replace value which is not present'
1859	item.setAttribute( tag, new )
1860
1861	#################################### from revised CMIP5 tables ### request variables referencing core tables
1862	elif k in ['requestvar']:
1863	#### need to fill gaps in variable groups --- or do better job upstream ########## !!!!!!!!!!!!!!!!
1864	## - table; mip; uuid; priority
1865	##
1866	## variable group ids pulled through via mapping above .. very messy
1867	## still missing some OMIP matches .... OMIP revised tables are not explicitly requested anywhere.
1868	##
1869	kk = ['var', 'table', 'mip', 'vid', 'priority','vgid']
1870	##vgmaps = {'Oyr':'OMIP-Oyr','OImon':'SIMIP-seaicemon'}
1871
1872	vgid = str( ll[5] )
1873	doThis = True
1874	if vgid == '__unset__01__':
1875	dothis = False
1876	else:
1877	thisl = string.strip(str( ll[0] ) )
1878	if len( thisl ) > 6 and thisl[-6:] == 'Adjust':
1879	dothis = False
1880
1881	if dothis:
1882	thisuuid = str( uuid.uuid1() )
1883	thisd = {}
1884
1885	if ll[2] not in refMips:
1886	print 'SEVERE.mip.000001: bad mip label in request item: %s --- %s' % (ll[2],str(ll))
1887
1888	for j in range(len(kk)):
1889	if j != 0 and kk[j] != 'table':
1890	item.setAttribute( kk[j], str( ll[j] ) )
1891	thisd[kk[j]] = str( ll[j] )
1892	vid = str( ll[3] )
1893	if vid in extRefCmvMapped:
1894	print 'INFO.isd005.0002: mapping vid: ',vid,extRefCmvMapped[vid]
1895	vid = extRefCmvMapped[vid]
1896
1897	print 'INFO.isd005.0001: ',thisl, uid, vgid
1898	item.setAttribute( 'title', thisl + ' ((isd.005))' )
1899	item.setAttribute( 'uid', thisuuid )
1900	item.setAttribute( 'vgid', vgid )
1901	item.setAttribute( 'vid', vid )
1902
1903	thisd['title'] = thisl + ' ((isd.005))'
1904	thisd['uid'] = thisuuid
1905	if vid in self.cmvMap002:
1906	print 'INFO.vid.00001: resetting requestvar vid: %s ---> %s' % (vid,self.cmvMap002[vid])
1907	vid = self.cmvMap002[vid]
1908	thisd['vid'] = vid
1909	thisd['vgid'] = vgid
1910
1911	##################################
1912	elif k == 'requestlink':
1913	## - uuid; mip; tab; objective; grid; gridreq; comment
1914	kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'ref', 'refNote', 'refid']
1915
1916	uid = str( ll[0] )
1917	mip = string.strip(str( ll[1] ) )
1918	obj = string.strip(str( ll[3] ) )
1919	if mip == "GMMIP":
1920	bits = [ mycc(x) for x in string.split( obj, ' ' ) ]
1921	else:
1922	bits = [ mycc(x) for x in string.split( obj, ',' ) ]
1923
1924	##
1925	## look for objectives specified in experiment column ....
1926	##
1927
1928	for b in bits:
1929	self.objectiveLinks[mip].a[b].append( str( ll[0] ) )
1930	self.requestlinkuid.add( str(ll[0] ) )
1931	self.rqlPreset[str(ll[0])] = ll[9]
1932	ttl = string.strip(str( ll[2] ) )
1933	self.rqLnks[str(ll[0])] = ll[:]
1934	dothis = False
1935
1936	##################################
1937	elif k == 'requestitem':
1938	## - mip; tab; expt; rlid; ny
1939	## defer so that cross-check of "expt" can be done .. or scan this earlier ....
1940	kk = ['mip', 'tab', 'expt', 'rlid', 'ny', 'nexmax', 'nenmax', 'nymax','treset','info']
1941	finishedClones = False
1942	ku = str(ll[2])
1943	if self.exptPntUid[ku][1] == 'list' and len( self.exptPntUid[ku][2] ) > 0:
1944	cloning = True
1945	clones = set( self.exptPntUid[ku][2] )
1946	kcl = 0
1947	else:
1948	cloning = False
1949
1950	while not finishedClones:
1951	thisl = mycc( '%s %s' % (ll[0],ll[1]) )
1952	if str( ll[3] ) not in self.rqlPreset:
1953	print 'ERROR.090.02000: requestlink not found .. ',ll
1954	preset = '-99'
1955	else:
1956	preset = str( self.rqlPreset[ str( ll[3] ) ] )
1957	if str( ll[3] ) not in self.requestlinkuid:
1958	print 'SEVERE.005.00005: rlid not found: %s' % str(ll)
1959	for j in range(len(kk)-1):
1960	if kk[j] == 'ny':
1961	if ll[j] == '':
1962	thisv = 0
1963	else:
1964	try:
1965	thisv = str( int(ll[j]) )
1966	except:
1967	print 'SEVERE: failed to set "ny": %s' % str(ll)
1968	thisv = 999
1969	else:
1970	thisv = str( ll[j] )
1971	if not (kk[j] == 'treset' and thisv in ['None','']):
1972	item.setAttribute( kk[j], thisv )
1973	item.setAttribute( 'title', '%s, %s, %s' % (ll[0],ll[1],ll[2]) )
1974	item.setAttribute( 'uid', str(i) )
1975	item.setAttribute( 'preset', preset )
1976	if cloning:
1977	ku = clones.pop()
1978	if kcl != 0:
1979	item.setAttribute( 'uid', str(uuid.uuid1()) )
1980	kcl += 1
1981	##print 'INFO.clone.00001: ',ku, kcl
1982	if ku in self.experiments:
1983	esid = self.experiments[ku]
1984	esidmsg = 'Experiment %s' % ku
1985	elif ku in self.exptPntUid:
1986	esid = self.exptPntUid[ku][0]
1987	esidmsg = str(self.exptPntUid[ku])
1988	else:
1989	print 'ERROR.esid.00001:', ku, ' Not Found'
1990	esidmsg = 'ERROR: Experiment %s Not Found' % ku
1991	esid = str(uuid.uuid1())
1992
1993	item.setAttribute( 'esid', esid )
1994	item.setAttribute( 'esidComment', esidmsg )
1995	if type(ll[9]) in [type(''), type(u'')]:
1996	kkk = ll[9].strip( )
1997	if kkk[:3] == '_sl':
1998	item.setAttribute( 'tslice', kkk )
1999	elif kkk not in ['','all']:
2000	print 'ERROR.tslice.0007: unrecognised slice: %s' % kkk
2001
2002	finishedClones = (not cloning) or ( len( clones) == 0 )
2003	if not finishedClones:
2004	ff[k][0].appendChild( item )
2005	item = self.doc.createElement( 'item' )
2006	##################################
2007	elif k in ['requestvargroup','tablesection']:
2008	if k == 'requestvargroup':
2009	kk = ['uid', 'mip', 'label', 'title', 'ref', 'refNote']
2010	else:
2011	kk = ['uid', 'gpid', 'mip', 'label', 'title', 'ref', 'refNote']
2012	thisl = string.replace( str( ll[2] ), '.', '-' )
2013	omitrvg = {'C4MIP-Omon','C4MIP-Oclim','HighResMIP-OImon','HighResMIP-Oclim'}
2014	if thisl in omitrvg:
2015	dothis = False
2016	else:
2017	thisd = {}
2018	for j in range(len(kk)):
2019	item.setAttribute( kk[j], str( ll[j] ) )
2020	thisd[kk[j]] = str( ll[j] )
2021	####
2022	if dothis:
2023	if labmod:
2024	if string.find( thisl, '_' ) != -1:
2025	thisl = string.replace( thisl, '_', '-' )
2026	if string.find( thisl, '*' ) != -1:
2027	thisl = string.replace( thisl, '*', '-' )
2028	if string.find( thisl, '!' ) != -1:
2029	thisl = string.replace( thisl, '!', '-' )
2030	if string.find( thisl, ' ' ) != -1:
2031	thisl = string.replace( thisl, ' ', '-' )
2032	if string.find( thisl, ':' ) != -1:
2033	thisl = string.replace( thisl, ':', '-' )
2034	item.setAttribute( 'label', thisl )
2035	ff[k][0].appendChild( item )
2036	if k in ['requestvargroup','tablesection']:
2037	thisd['label'] = thisl
2038	self.rqvgList.append( thisd )
2039	if len(nfr) > 0:
2040	print 'ERROR:nfr: ',nfr
2041
2042	def fixfixcmv(self):
2043	"""deal with changing cmv ids relative to fixcmv"""
2044	##self.cmvinfo = ('uid','stid','label','title','description','vid', \
2045	##'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', \
2046	##'deflate', 'valid_min', 'flag_values', 'modeling_realm', 'valid_max', 'positive', 'mipTable', \
2047	##'prov', 'provNote', 'frequency', 'rowIndex', 'coordinates','defaultPriority','mipTableSection')
2048	ff = {}
2049	for v in self.cmv:
2050	kkk = '%s.%s.%s' % (v[2],v[18],v[19])
2051	if kkk in self.fix.fixcmvx:
2052	for k in self.fix.fixcmvx[kkk]:
2053	if k != v[0]:
2054	print 'WARN.cmv.00040: changed uid:', k, v
2055	ff[k] = v[0]
2056	## ['label','uid','table','mode','uid2','prov']
2057
2058	for vid in self.fix.fixcmv.keys():
2059	r = list( self.fix.fixcmv[vid] )
2060	print r
2061	if r[1] in ff or r[4] in ff:
2062	if r[1] in ff:
2063	r[1] = ff[r[1]]
2064	if r[4] in ff:
2065	r[4] = ff[r[4]]
2066	self.fix.fixcmv[r[1]] = r
2067
2068
2069	def write2(self):
2070	"""write sections for cmor variable, request variable, structure, spatial and temporal dimensions"""
2071	spsResets = collections.defaultdict(list)
2072	doneStructures = set()
2073	doneMessages = set()
2074	self.fixfixcmv()
2075	self.spLab2Uid = {}
2076	self.tmLab2Uid = {}
2077	self.tmUid2Lab = {}
2078	self.tmsp2str = {}
2079	self.tables = set()
2080	for ksect in [ u'spatialshape', u'temporalshape', u'structure', u'cmorvar', u'requestvar', u'requestlink']:
2081	##for ksect in [ u'cmorvar', u'spatialshape', u'temporalshape', u'structure']:
2082	dil = self.ff[ksect][0].getElementsByTagName('item')
2083	if ksect == 'requestvar':
2084	for item in dil:
2085	vid = item.getAttribute( 'vid' )
2086	if vid in self.fix.fixcmv:
2087	if self.fix.fixcmv[vid][3] == u'replace':
2088	vid1 = str( self.fix.fixcmv[vid][4] )
2089	if vid1 not in self.cmvByUid:
2090	print 'ERROR.fixcmv.0002: attempt to use invalid vid: ',item.getAttribute( 'label' ),vid1,vid in self.cmvByUid
2091	else:
2092	item.setAttribute( 'vid', vid1 )
2093	print 'INFO.fixcmv.00020: replacing link in requestVar %s: %s' % (item.getAttribute( 'uid' ),vid1)
2094	else:
2095	for d in dil:
2096	self.ff[ksect][0].removeChild(d)
2097
2098	if ksect == u'cmorvar':
2099	ktable = self.cmvinfo.index('mipTable')
2100	kfreq = self.cmvinfo.index('frequency')
2101	klabel = self.cmvinfo.index('label')
2102
2103	s1 = dict()
2104	stab = set()
2105	tmidpt = self.tmLab2Uid['time-point']
2106	self._cmvLabByUid = {}
2107	for r in self.cmv:
2108	if r[0] in self.rep['cmv'].items:
2109	print 'INFO.090.09099: detected cmv replace request: ',r
2110	elif r[0] in s1:
2111	print 'ERROR.090.09005: duplicate key at ',r,s1[r[0]]
2112	else:
2113	s1[ r[0] ] = (r[ktable],r[kfreq],r[klabel])
2114	stab.add( r[ktable] )
2115	u0 = r[0]
2116	l0 = r[2]
2117	#
2118	# identify structures for AerChemMIP tables
2119	#
2120	if len( r[1] ) == 2:
2121	spid = self.spLab2Uid[r[1][0]]
2122	if r[1][1] == 'mean':
2123	tmid = self.tmLab2Uid['time-mean']
2124	tmode = 'mean'
2125	else:
2126	tmid = self.tmLab2Uid['time-point']
2127	tmode = 'point'
2128
2129	if (tmid,spid,tmode) in self.lls.tmsp2str:
2130	stid = self.lls.tmsp2str[ (tmid,spid,tmode) ]
2131	else:
2132	print 'SEVERE.stid.0001: no structure record for ',tmid,spid,tmode,r[1][0]
2133	stid = '__stid_not_found__'
2134	print 'INFO.klklkl: AerChemMIP structure .... ',r[1], spid
2135	else:
2136	if r[1] in self.strdd:
2137	stid = self.strdd[r[1]]
2138	else:
2139	for k in self.strdd.keys():
2140	if k[0] == r[1][0]:
2141	print '*** ',k
2142	else:
2143	print '+++ ',k
2144	print 'SEVERE.999.00099: strdd key not found: %s' % str(r[1])
2145	print r
2146	stid = '__struct_not_found_001__'
2147
2148	stinfo = self.structInfo.get( stid, ['','','',''] )
2149	spinfo = self.spidInfo.get( stinfo[2] , ['','','','',''] )
2150	tmid = stinfo[3]
2151	if "plev7h" in spinfo[3]:
2152	lsfx = '7h'
2153	elif "plev7c" in spinfo[3]:
2154	lsfx = ''
2155	elif "plev4" in spinfo[3]:
2156	lsfx = '4'
2157	elif "plev23" in spinfo[3] and r[2] == 'tntlw' :
2158	##print 'plev7h found',r,spinfo
2159	lsfx = '23'
2160	else:
2161	lsfx = ''
2162
2163	if r[2] == 'clisccp':
2164	print 'INFO.clisccp.00001:',r
2165
2166	if r[0] in self.fix.fixcmv:
2167	## print 'INFO.fixcmv.00010: omitting record %s, %s, %s' % (str( self.fix.fixcmv[r[0]] ), r[0],r[2] )
2168	pass
2169	else:
2170	item = self.doc.createElement( 'item' )
2171	for k in range(len(self.cmvinfo)):
2172	n = self.cmvinfo[k]
2173	if n in self.cmvinfo2:
2174	if n == 'mipTable' and str(r[k]) == 'CMIP5_Oyr':
2175	print 'ERROR.047.0002: ',r
2176	if n == "vid":
2177	if str(r[k]) == '__var_not_found_012__':
2178	print 'ERROR.099.00202: bad vid string: %s [%s]' % (r[k],str(r[:3]))
2179	vid = str(r[k])
2180	if vid in self.importWbMods.vars.vmap:
2181	ov = vid
2182	vid = self.importWbMods.vars.vmap[vid]
2183	print 'INFO.040.04444: mapping',ov,vid
2184
2185	if n == "rowIndex":
2186	ix = str(r[k])
2187	elif n == "defaultPriority":
2188	priority = str(r[k])
2189	elif n == "title":
2190	title = str(r[k])
2191
2192	if n == "uid":
2193	if r[k].find( 'ea313ee8-a0de-11e6-bc63-ac72891c3257' ) != -1:
2194	print 'INFO.uid.9999: ',r
2195	item.setAttribute( n, r[k].strip() )
2196	elif n == "stid":
2197	item.setAttribute( n, stid )
2198	elif n in ["mipTableSection","processing"]:
2199	if str(r[k]) != '':
2200	item.setAttribute( n, str(r[k]) )
2201	elif n == "label":
2202	thisl = string.replace( r[k], '_', '' )
2203	if len(lsfx) > 0:
2204	if thisl[ - len( lsfx): ] != lsfx:
2205	thisl += lsfx
2206	item.setAttribute( n, thisl )
2207	elif n in ["valid_min","valid_max","ok_max_mean_abs","ok_min_mean_abs"]:
2208	if str(r[k]) != "" and r[k] not in [None, 'None']:
2209	item.setAttribute( n, str(r[k]) )
2210	elif n == "frequency":
2211	fr = freqmap.get( str(r[k]), str(r[k]) )
2212	item.setAttribute( n, fr )
2213	elif n == "mipTable":
2214	tn = tableNameMap.get( str(r[k]), str(r[k]) )
2215	if tn == 'em6hr':
2216	if 'alev' in spinfo[3]:
2217	tn = '6hrLev'
2218	else:
2219	if tmid == tmidpt:
2220	tn = '6hrPlevpt'
2221	else:
2222	tn = '6hrPlev'
2223	print 'INFO.6hrPlev.0001:', tmid, tmidpt, tmid == tmidpt, r,tn, self.tmUid2Lab[tmid]
2224	item.setAttribute( n, tn )
2225	else:
2226	item.setAttribute( n, str(r[k]) )
2227
2228	### reset table name for OImon ... correcting for frequency as well
2229	###
2230	omitThis = False
2231	mtable = item.getAttribute( 'mipTable' )
2232	prov = item.getAttribute( 'prov' )
2233	realm = item.getAttribute( 'modeling_realm' )
2234	if realm in ['','__unset__']:
2235	if prov in knownRealmA:
2236	item.setAttribute( 'modeling_realm', 'atmos' )
2237	elif prov in knownRealmO:
2238	item.setAttribute( 'modeling_realm', 'ocean' )
2239
2240	mtable0 = mtable
2241	if mtable0 == 'OImon':
2242	if r[kfreq] == 'mon':
2243	mtable = 'SImon'
2244	else:
2245	mtable = 'SIday'
2246	elif mtable0 in ['em6hr','em3hr','emDay','emMon']:
2247	if stid in self.structInfo:
2248	if string.find( self.structInfo[stid][1], 'time: point' ) != -1 or tmid == tmidpt:
2249	mtable += 'pt'
2250	if string.find( self.spatialDims[ self.structInfo[stid][2] ], 'longitude' ) == -1 and mtable[-1] != 'Z':
2251	mtable += 'Z'
2252	else:
2253	print 'WARN.090.09999: structure uid not found',stid, r
2254	elif mtable0 == 'Amon' and thisl[-6:] == 'Adjust':
2255	print 'INFO.adjust.0001: %s moved to AmonAdj .. and omitted' % thisl
2256	mtable = 'AmonAdj'
2257	omitThis = True
2258	if mtable != r[ktable]:
2259	item.setAttribute( 'mipTable', mtable )
2260	item.setAttribute( 'mtid', 'MIPtable::%s' % mtable )
2261	self.tables.add( mtable )
2262
2263	thisuid = r[0]
2264	if tableUtils.getSubGroup(spinfo[3],mtable,thisl,ix,title,thisuid,priority):
2265	item.setAttribute( 'subGroup', tableUtils.subGroup )
2266
2267	if str(vid) not in self.importWbMods.vars.d1:
2268
2269	if thisl in self.importWbMods.vars.c1 or (thisl[-2:] in ['17','19','23','27','7c','7h'] and thisl[:-2] in self.importWbMods.vars.c1):
2270	if thisl in self.importWbMods.vars.c1:
2271	thisn = thisl
2272	else:
2273	thisn = thisl[:-2]
2274
2275	if len( self.importWbMods.vars.c1[ thisn ] ) == 1:
2276	newvid = self.importWbMods.vars.c1[ thisn ][0]
2277	item.setAttribute( 'vid', newvid )
2278	print 'INFO.099.00110: bad vid string replaced: %s [%s]' % (newvid,thisl)
2279	else:
2280	print 'ERROR.099.00111: ambiguous bad vid string: %s [%s]' % (vid,thisl)
2281	else:
2282	print 'ERROR.099.00112: bad vid string: %s [%s]' % (vid,thisl)
2283	elif thisl == 'H2p':
2284	print 'WARN.h2p.00001: H2p vid recording as OK here .... %s' % vid
2285	if self.repl.has_key( thisuid ):
2286	for tag,old,new in self.repl[thisuid]:
2287	thisold = item.getAttribute( tag )
2288	if thisold != old:
2289	##print 'WARN.yyy: Attempt to replace value which is not present'
2290	print 'WARN.050.00051: Failed attempt to replace values in cmv: %s old:%s new:%s' % (tag,old,new)
2291	else:
2292	if tag == 'vid' and new not in self.importWbMods.vars.d1:
2293	print 'ERROR.050.00052: Attempt to replave values in cmv with invalid vid: %s old:%s new:%s' % (tag,old,new)
2294	else:
2295	item.setAttribute( tag, new )
2296	print 'INFO.050.00050: replacing values in cmv: %s old:%s new:%s' % (tag,old,new)
2297
2298	if thisuid in self.deferredLabMap:
2299	lab0,lab1 = self.deferredLabMap[thisuid]
2300	labold = item.getAttribute( 'label' )
2301	if labold != lab0:
2302	print 'ERROR.labmap.0001: attempt to change wrong label: ',lab0,lab1,labold
2303	else:
2304	item.setAttribute( 'label', lab1 )
2305	print 'INFO.labmap.0001: attempt to change label: ',lab0,lab1
2306
2307	### final check on structure .. but won't work if tuple not found ... but why is structure not there??
2308	stid = item.getAttribute( 'stid' )
2309	if stid in self.struidmapextra:
2310	mid = ('INFO.structure2',stid)
2311	if mid not in doneMessages:
2312	print 'INFO.structure: mapping [2]: %s to %s' % (stid,self.struidmapextra[stid])
2313	doneMessages.add(mid)
2314	item.setAttribute( 'stid', self.struidmapextra[stid] )
2315	stid = self.struidmapextra[stid]
2316	if stid in WbMods.str.uidMap:
2317	mid = ('INFO.structure',stid)
2318	if mid not in doneMessages:
2319	print 'INFO.structure: mapping: %s to %s' % (stid,WbMods.str.uidMap[stid])
2320	doneMessages.add(mid)
2321	if stid in self.struidmapextra:
2322	assert self.struidmapextra[stid] == WbMods.str.uidMap[stid], 'CONFLICT in stid mappings: %s, %s, %s' % (stid,self.struidmapextra[stid], WbMods.str.uidMap[stid])
2323	item.setAttribute( 'stid', WbMods.str.uidMap[stid] )
2324
2325	## append CMORvar record
2326	u1 = item.getAttribute('uid')
2327	if u1 in extRefCmvMapped:
2328	print 'WARN.uidmap.00004: mapping CMORvar uid %s --> %s' % (u1,extRefCmvMapped[u1] )
2329	item.setAttribute('uid',extRefCmvMapped[u1])
2330	l1 = item.getAttribute( 'label')
2331	#up: rlus rlut rlutcs hfls hfss mcu rsus rsut rsutcs down:rlds rsds
2332	if l1 in varPositive:
2333	item.setAttribute( 'positive', varPositive[l1] )
2334	print 'INFO.positive.00001: setting positive [%s]: %s' % (l1,varPositive[l1])
2335	if l1 != l0:
2336	print 'WARN.cmvlab.00030: changed cmv label: %s %s %s %s' % (l0,l1,u0,u1)
2337	self._cmvLabByUid[item.getAttribute('uid')] = item.getAttribute( 'label')
2338	if not omitThis:
2339	self.ff[ksect][0].appendChild( item )
2340	print 'INFO.tables.0001: ',stab
2341
2342	elif ksect == u'structure':
2343	ers = {}
2344	for kk in spsResets:
2345	if len( spsResets[kk] ) > 0:
2346	utarg = spsResets[kk][0][0]
2347	for r in spsResets[kk][1:]:
2348	ers[r[0]] = utarg
2349	##flds = self.structList[0][1]._fields
2350	flds = self.lls.structAttr
2351	for u,nt in self.lls.ll:
2352	smode = 'new'
2353	if smode == 'new':
2354	d1 = self.lls.asDict( u )
2355	tlab = d1['time label']
2356	##print 'INFO.stid.00072: %s, %s' % (u,str( d1 ))
2357	else:
2358	##print 'INFO.stid.00071: %s, %s' % (u,str( nt.__dict__ ))
2359	d1 = {}
2360	for f in flds:
2361	if f == 'spid' and nt.__dict__[f] in ers:
2362	d1[f] = ers[ nt.__dict__[f] ]
2363	print 'INFO.801.00011: updating target spid .... ', d1[f]
2364	elif f == 'coords' and 't2000' in nt.__dict__[f]:
2365	d1[f] = str( nt.__dict__[f] )
2366	elif f == 'coords' and nt.__dict__[f][:4] in ['plev']:
2367	cnew = 'p' + nt.__dict__[f][4:]
2368	d1[f] = str( cnew )
2369
2370	print 'INFO.801.00013: updating target coords .... %s --> %s ' % ( nt.__dict__[f], cnew )
2371	elif f == 'uid':
2372	d1[f] = u
2373	elif f == 'tlab':
2374	tlab = nt.tlab
2375	tmid = self.tmLab2Uid[tlab]
2376	d1['tmid'] = tmid
2377
2378	else:
2379	d1[f] = str( nt.__dict__[f] )
2380	if d1.get( 'cell_measures', '' ) == '@OPT':
2381	print 'WARN.structure.00077: changing cell_measures @OPT to --OPT',str(d1 )
2382	d1['cell_measures'] = '--OPT'
2383
2384	if d1['spid'] in WbMods.ls.labByUid:
2385	slab = WbMods.ls.labByUid[ d1['spid'] ]
2386	print 'INFO.801.00061: spid found in structure record ... %s\n%s' % (d1['spid'],slab)
2387	else:
2388	slab = '?'
2389	print 'SEVERE.801.00060: bad spid found in structure record ... %s\n%s' % (d1['spid'],str(d1))
2390	cm = d1['cell_methods']
2391	if cm == 'null':
2392	if tlab == 'time-point':
2393	cm = 'time: point'
2394	d1['cell_methods'] = cm
2395	elif tlab == 'time-mean':
2396	cm = 'time: mean'
2397	d1['cell_methods'] = cm
2398	elif cm in WbMods.cellm.cmmap:
2399	cm0 = cm
2400	cm = WbMods.cellm.cmmap[cm]
2401	print 'INFO.cmmap.00001: ',cm0,cm
2402	d1['cell_methods'] = cm
2403
2404	cc = d1['coords']
2405	oo = d1['odims']
2406	if cc not in ['','?']:
2407	print 'INFO.coords.00001: >%s<' % cc
2408	d1['cids'] = 'dim:%s' % cc
2409	if oo not in ['','?']:
2410	ooc = string.join( ['dim:%s' % x for x in oo.split( '\|' )] )
2411	print 'INFO.odims.00001: >%s<' % ooc
2412	d1['cids'] = ooc
2413	t = (tlab,slab,d1['odims'],d1['coords'],d1['cell_methods'],d1['cell_measures'],d1['flag_meanings'])
2414
2415	dothis = True
2416	if t not in WbMods.str.lookup:
2417	print 'WARN.structure.0040: tuple not in lookup: ',t,d1['uid'],d1['spid'],d1
2418	else:
2419	print 'WARN.structure.0041: tuple in lookup ignored: ',t,d1['uid'],d1['spid'],d1
2420	lab = d1['label']
2421	if lab in WbMods.str.labMap:
2422	print 'INFO.structure.0050: omitting structure record ....',t,lab, d1['uid']
2423	lab2 = WbMods.str.labMap[lab]
2424	dothis = False
2425	self.struidmapextra[d1['uid']] = WbMods.str.uidByLab[lab2]
2426	print 'INFO.structure.0062: setting stid mapping ....',t,d1['uid'],WbMods.str.uidByLab[lab2], d1['label'], lab2
2427
2428	lab = d1['label']
2429	if lab in WbMods.str.mods:
2430	for k in WbMods.str.mods[lab]:
2431	print 'INFO.structure.0055: modifying attributes: %s: %s: %s' % (lab,k,WbMods.str.mods[lab][k])
2432	d1[k] = WbMods.str.mods[lab][k]
2433	if k == 'cell_methods':
2434	cm = WbMods.str.mods[lab][k]
2435	if cm in WbMods.cellm.cminfo:
2436	d1['cmid'] = WbMods.cellm.cminfo[cm][2]
2437	else:
2438	print 'SEVERE.structure.0055: cmid for new cell method not found: %s' % (cm)
2439
2440	if dothis and d1['uid'] in doneStructures:
2441	print 'WARN.structure.05021: repeat uid in structure definitions ....%s' % d1['uid']
2442	if dothis and d1['uid'] not in doneStructures and d1['uid'] not in WbMods.str.uidMap:
2443	if cm not in WbMods.cellm.cminfo:
2444	print 'WARN.cellm.0001: Cell methods not found: ',cm, d1['uid']
2445	else:
2446	d1['cmid'] = WbMods.cellm.cminfo[cm][2]
2447
2448	doneStructures.add( d1['uid'] )
2449	item = self.doc.createElement( 'item' )
2450	for k in d1:
2451	if k not in ['time_label','spatial_label', 'time label','spatial label']:
2452	item.setAttribute( k.replace( ' ','_'), d1[k] )
2453	self.ff[ksect][0].appendChild( item )
2454	else:
2455	if not dothis:
2456	print 'INFO.structure.09001: omitting %s' % d1['label']
2457	elif d1['uid'] in doneStructures:
2458	print 'INFO.structure.09002: omitting %s' % d1['label']
2459	elif d1['uid'] in WbMods.str.uidMap:
2460	print 'INFO.structure.09003: omitting %s' % d1['label']
2461	###
2462	elif ksect == u'spatialshape':
2463
2464	## self.dimsSpaceInfo = ['uid','label','title','dimensions','levelFlag','levels']
2465	self.spatialDims = {}
2466	for k in self.dimsSpace.keys():
2467	rr = list( self.dimsSpace[k] )
2468	kk = rr[3]
2469	if kk == 'scalar':
2470	kk = ''
2471	rr = list(rr)
2472	rr[3] = ''
2473	elif kk == 'longitude\|latitude\|plev':
2474	kk = 'longitude\|latitude\|plev4'
2475	rr = list(rr)
2476	rr[3] = kk
2477	elif kk == 'longitude\|latitude\|plev7':
2478	kk = 'longitude\|latitude\|plev7c'
2479	print 'INFO.plev7.00001: plev7 --> plev7c'
2480	rr = list(rr)
2481	rr[3] = kk
2482	self.spatialDims[rr[0]] = rr[3]
2483	bits = [x for x in rr[3].split( '\|' ) if x not in ['','?']]
2484	rr.append( string.join( ['dim:%s' % x for x in bits] ) )
2485	if kk not in spsResets:
2486	item = self.doc.createElement( 'item' )
2487	for i in range(len(self.dimsSpaceInfo)):
2488	item.setAttribute( self.dimsSpaceInfo[i], str( rr[i] ) )
2489	##assert rr[3] in self.spsh.ss, 'FATAL: spatial shape record not recognised: %s' % str(rr)
2490	if rr[3] in self.spsh.ss:
2491	lab,ttl,lf,li,u = self.spsh.ss[rr[3]]
2492	if trace.SPATIALSHAPE_01:
2493	print 'INFO.035.0001: new spatial shape label: %s, %s' % (lab, ttl)
2494	if ttl[:4] != 'ssd.':
2495	item.setAttribute( 'title', ttl )
2496	if len(lab) < 4 or lab[:4] != 'ssd-':
2497	item.setAttribute( 'label', lab )
2498	item.setAttribute( 'levels', li )
2499	item.setAttribute( 'uid', u )
2500	item.setAttribute( 'levelFlag', string.lower(lf) )
2501	else:
2502	print 'ERROR.055.00001: spatial shape record not recognised: %s' % str(rr)
2503	print 'INFO.801.00120: new shape for %s' % str(rr)
2504	thisl = item.getAttribute( 'label' )
2505	assert thisl not in self.spLab2Uid.keys(), 'Duplicate spatial shape label: %s' % thisl
2506	self.spLab2Uid[thisl] = item.getAttribute( 'uid' )
2507	self.ff[ksect][0].appendChild( item )
2508	else:
2509	print 'INFO.801.00121: duplicate shape for %s' % str(rr)
2510	spsResets[kk].append( rr )
2511
2512	elif ksect == u'temporalshape':
2513	for k in self.dimsTime.keys():
2514	rr = self.dimsTime[k]
2515	item = self.doc.createElement( 'item' )
2516	for i in range(len(self.dimsTimeInfo)):
2517	item.setAttribute( self.dimsTimeInfo[i], str( rr[i] ) )
2518	if self.dimsTimeInfo[i] == 'dimensions':
2519	item.setAttribute( 'dimid', 'dim:%s' % rr[i] )
2520	self.ff[ksect][0].appendChild( item )
2521	self.tmLab2Uid[ item.getAttribute( 'label' ) ] = item.getAttribute( 'uid' )
2522	self.tmUid2Lab[ item.getAttribute( 'uid' ) ] = item.getAttribute( 'label' )
2523	elif ksect == u'requestvar':
2524	##kk = ['var', 'table', 'mip', 'vid', 'priority']
2525	kk = ['label', 'table', 'mip', 'vid', 'priority','vgid','title','uid']
2526	kktr = {'vgid':'gpid', 'title':'label'}
2527	kkf = 0
2528	for t in self.rqv:
2529	if t[5][:6] == 'SPECS_':
2530	print 'SEVERE.specs.00001: mis-assigned mip: ',t
2531	if t[1][-6:] == 'Adjust':
2532	doThis = False
2533	else:
2534	item = self.doc.createElement( 'item' )
2535
2536	for i in range(len(kk)):
2537	k = kktr.get( kk[i], kk[i] )
2538	v = t[ self.rqvinfo.index(k) ]
2539	if k == 'priority':
2540	item.setAttribute( kk[i], str( int(v) ) )
2541	elif kk[i] == 'label':
2542	thisl = string.replace( v, '_', '' )
2543	item.setAttribute( kk[i], thisl )
2544	elif kk[i] == 'title':
2545	item.setAttribute( kk[i], str( v ) + ' ((isd.006))' )
2546	elif kk[i] == 'table':
2547	pass
2548	else:
2549	item.setAttribute( kk[i], str( v ) )
2550
2551	vid = item.getAttribute( 'vid' )
2552	if vid in self.fix.fixcmv:
2553	if self.fix.fixcmv[vid][3] == u'replace':
2554	vidn = str( self.fix.fixcmv[vid][4] )
2555	if vidn not in self._cmvLabByUid:
2556	print 'SEVERE.fixcmv.00040: attempt to redirect request to non-existant variable: %s,%s,%s::%s' % (vid,vidn,thisl,str(self.fix.fixcmv[vid]) )
2557	else:
2558	labn = self.cmvByUid[vidn][2]
2559	if vid in self.cmvByUid:
2560	lab = self.cmvByUid[vid][2]
2561	if lab != labn:
2562	print 'WARN.fixcmv.00001: label change: %s -- %s (%s --> %s)' % (lab,labn,vid,vidn)
2563	elif labn != thisl:
2564	print 'WARN.fixcmv.00002: label change [2]: %s -- %s (%s --> %s)' % (thisl,labn,vid,vidn)
2565	item.setAttribute( 'vid', vidn )
2566	vid = vidn
2567	print 'INFO.fixcmv.00050: replacing link in requestVar %s %s %s' % (str(t),vidn,labn)
2568
2569	newd = None
2570	if vid in self.importWbMods.mmsi.ss:
2571	print 'INFO.046.0002: mip map found: ', self.importWbMods.mmsi.ss[vid]
2572	mm = self.importWbMods.mmsi.ss[vid]
2573	##newd = self._editRequestVar01( item, mm, t, {} )
2574
2575	vid = item.getAttribute( 'vid' )
2576	if vid in self.cmvMap002:
2577	vid = self.cmvMap002[vid]
2578	item.setAttribute( 'vid', vid )
2579
2580	if vid in extRefCmvMapped:
2581	vid = extRefCmvMapped[vid]
2582	item.setAttribute( 'vid', vid )
2583
2584	if vid not in self.cmvByUid:
2585	print 'ERROR.reqvar.000001: vid not found: %s' % str(t)
2586	else:
2587	lab0 = self.cmvByUid[vid][2]
2588	if vid not in self._cmvLabByUid:
2589	print 'WARN.cmvlab.00020: no _cmv.. record ... %s %s: %s' % (vid,lab0,str( self.cmvByUid[vid] ) )
2590	else:
2591	lab = self._cmvLabByUid[vid]
2592	if lab != lab0:
2593	print 'WARN.cmvlab.00010: difference in records ... %s %s: %s' % (vid,lab,str( self.cmvByUid[vid] ) )
2594	if lab != thisl:
2595	##print 'INFO.reqvar.00001: inconsistent requestvar label: %s --> %s (%s)' % (thisl, lab, vid)
2596	print 'INFO.reqvar.00001: changing requestvar label: %s --> %s' % (thisl, lab)
2597	item.setAttribute( 'label', lab )
2598
2599	tm = item.getAttribute( 'mip' )
2600
2601	did = '%s__%s' % (t[2],t[3])
2602	if self.vgcheck.has_key(did):
2603	print 'ERROR.090.09002: Duplicate request ....: ',t,self.vgcheck[did]
2604	else:
2605	self.vgcheck[did] = t
2606	self.ff[ksect][0].appendChild( item )
2607	##
2608	######################################################################################
2609	##
2610	elif ksect == u'requestlink':
2611	## - uuid; mip; tab; objective; grid; gridreq; comment
2612	kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'ref', 'refNote', 'refid']
2613	kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt', 'opar', 'preset', 'refid']
2614	##kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt','opar','ref', 'refNote', 'refid']
2615
2616	for k,ll in self.rqLnks.items():
2617	if k[0] != '_':
2618	assert len(ll) == len(kk), 'Bad record length in requestLink'
2619	thisl = string.strip(str( ll[2] ) )
2620	uid = str( ll[0] ).strip()
2621	item = self.doc.createElement( 'item' )
2622	for j in range(len(kk)):
2623	thiskk = kk[j]
2624	## preset moved to request item .... so it can also be used on a per-experiment basis
2625	if thiskk != 'preset':
2626	if kk[j] == 'comment':
2627	v = utils_wb.uniCleanFunc( ll[j] )
2628	else:
2629	v = str( ll[j] )
2630
2631	### redirect to new variable group if specified in mappings
2632	##
2633	item.setAttribute( thiskk, v )
2634
2635	item.setAttribute( 'label', thisl.replace( '.', '-' ) )
2636	item.setAttribute( 'title', thisl )
2637	self.ff[ksect][0].appendChild( item )
2638	##################################
2639
2640	def _editRequestVar01( self, item, mm, t, thisd, vid=None ):
2641	if vid == None:
2642	vid = mm[0].vid2
2643	item.setAttribute( 'vid', vid )
2644	item.setAttribute( 'label', mm[0].lab2 )
2645	thisd['vid'] = vid
2646	thisd['label'] = mm[0].lab2
2647	print 'INFO.fffff.00001:', mm[0].lab2, mm[0].vid2
2648	othd = None
2649	if len( mm ) > 1:
2650	othd = thisd.copy()
2651	assert len( mm ) < 3, 'Unable to deal with triple clone'
2652	cln = item.cloneNode(True)
2653	uid = str( uuid.uuid1() )
2654	cln.setAttribute( 'uid', uid )
2655	cln.setAttribute( 'label', mm[1].lab2 )
2656	othd['uid'] = uid
2657	othd['label'] = mm[1].lab2
2658	self.ff['requestvar'][0].appendChild( cln )
2659
2660	print 'INFO.046.0003: request variable split:', t[1]
2661
2662	def finishExptGrp(self):
2663	"""finish exptGroup section and create exptgrouplink section"""
2664	this = self.doc.getElementsByTagName('exptGroup')[0]
2665	### kk = ['uid','label','tierMin','ntot']
2666	for k in self.exptPntUid.keys():
2667	u,typ = self.exptPntUid[k]
2668	## if not isg:
2669	##item = self.doc.createElement( 'item' )
2670	##item.setAttribute( uid, u )
2671	##this = self.doc.getElementsByTagName('exptGroupLink')[0]
2672	##dil = this.getElementsByTagName('item')
2673	##for d in dil:
2674	##this.removeChild(d)
2675	##this.appendChild( item )
2676
2677	def reviewExpt(self):
2678	"""Prepare cross-linking information to populate requestitem, exptgroup and exptgrouplink"""
2679	sh = shelve.open( '../framework/sh_links/exptMappings', 'r' )
2680	extraMappings={ 'LS3MIP-1, LS3MIP-2':['LmipH', 'LmipF'], \
2681	'LULCC_hist, CPL_1pctDF, LND_DF, ATM_DF, CPL_DF, CPL_noLULCC_hist_conc (LUMIP)':['LULCC_hist', 'CPL_1pctDF', 'LND_DF', 'ATM_DF', 'CPL_DF', 'CPL_noLULCC_hist_conc'] \
2682	}
2683
2684	##
2685	## extra fix for HighResMIP .... after encoding in template made consistent with other MIPs
2686	##
2687	shm = {}
2688	for k in sh.keys():
2689	if string.find( k, ' ' ) != -1 and k[:4] == 'High':
2690	k1 = string.split( k)[0]
2691	assert k1 not in shm, 'Duplicate abbreviated key: %s [%s and %s]' % (k1,k,shm[k1])
2692	shm[k1] = k
2693	ks = sorted( self.exptPnt.keys() )
2694	self.exptPntMap = collections.defaultdict( list )
2695	self.exptPntUid = {}
2696	ner = 0
2697	ngr = 0
2698	nmr = 0
2699	nuns = 0
2700	nls = 0
2701	for k0 in ks:
2702	k = string.replace( string.replace( k0, '\n', ' ' ), ' ', ' ')
2703	thisl = None
2704	if sh.has_key(k):
2705	thisl = sh[k]
2706	elif shm.has_key(k):
2707	thisl = sh[shm[k]]
2708	elif extraMappings.has_key(k):
2709	thisl = extraMappings[k]
2710	elif self.experiments.has_key(k):
2711	thisl = [k,]
2712	else:
2713	thisl = [k,]
2714	print 'WARN.080.00010: no mapping founds for experiment %s' % k
2715
2716
2717	if thisl != None:
2718	thisu = None
2719	thisls = set( thisl )
2720	linkType = None
2721	for k2 in thisls:
2722	ok = True
2723	if self.experiments.has_key(k2):
2724	ner += 1
2725	linkType = 'expt'
2726	thisu = self.experiments[k2]
2727	elif self.experimentGrp.has_key(k2):
2728	ngr += 1
2729	thisu = self.experimentGrp[k2]
2730	linkType = 'grp'
2731	elif k2 in self.mips:
2732	nmr += 1
2733	thisu = k2
2734	linkType = 'mip'
2735	else:
2736	k2c = dreq_utils.labcoerce(k2)
2737	if self.experimentGrp.has_key(k2c):
2738	ngr += 1
2739	thisu = self.experimentGrp[k2c]
2740	linkType = 'grp'
2741	else:
2742	ok = False
2743	print 'ERROR.080.00999: target experiment not found: %s [%s]' % (k2,k)
2744	if not ok:
2745	self.exptPntMap[k].append( k2 )
2746	##
2747	## store uid of expt group to be pointed at.
2748	##
2749	if len(thisls) == 1 and thisu != None:
2750	self.exptPntUid[k0] = (thisu, linkType, thisls)
2751	else:
2752	self.exptPntUid[k0] = (str( uuid.uuid1() ), 'list', tuple(thisls) )
2753	print 'INFO.expt.00001: ',k0, thisl, thisls, self.exptPntUid[k0]
2754	nls += 1
2755	else:
2756	nuns += 1
2757	self.exptPntUid[k0] = (None,None)
2758
2759	sh.close()
2760	print 'INFO.080.00010: Expt refs: %s, Expt group refs: %s, MIP expt set refs: %s, unset: %s, lists: %s' % (ner,ngr,nmr, nuns, nls)
2761
2762	def reviewObj(self):
2763	## review objectives
2764	##
2765	ol = []
2766	fnd1 = set()
2767	for m in self.objectives.keys():
2768	not_used = []
2769	used = set()
2770	not_found = []
2771	found = []
2772	alist = ['All','All?','all']
2773	aaa = [self.objectiveLinks[m].a.has_key(x) for x in ['All','All?','all'] ]
2774	us1 = set()
2775	for k in self.objectiveLinks[m].a:
2776	x = None
2777	if k == 'AllExceptPaleorcms':
2778	x = self.objectives[m].keys()
2779	x.pop( x.index( 'Paleorcms' ) )
2780	elif k == 'AllExceptNaturalVariability':
2781	x = self.objectives[m].keys()
2782	x.pop( x.index( 'NaturalVariability' ) )
2783
2784	if x != None:
2785	fnd1.add(k)
2786	for i in x:
2787	used.add(i)
2788	us1.add(i)
2789	for u in self.objectiveLinks[m].a[k]:
2790	ol.append( (m,self.objectives[m][i], u ) )
2791
2792	allused = (m != 'GMMIP') and any( aaa )
2793	for o in self.objectives[m].keys():
2794	if allused or self.objectiveLinks[m].a.has_key(o):
2795	used.add(o)
2796	for u in self.objectiveLinks[m].a[o]:
2797	ol.append( (m,self.objectives[m][o], u ) )
2798	for k in alist:
2799	if self.objectiveLinks[m].a.has_key(k):
2800	for u in self.objectiveLinks[m].a[k]:
2801	ol.append( (m,self.objectives[m][o], u ) )
2802	elif o not in us1:
2803	not_used.append(o)
2804
2805	for o in self.objectiveLinks[m].a.keys():
2806	if (o in {'All','All?'} and len(self.objectives[m].keys()) > 0 ) or self.objectives[m].has_key(o) or o in fnd1:
2807	found.append(o)
2808	else:
2809	not_found.append(o)
2810	if len(not_used) > 0 or len(not_found) > 0:
2811	print 'WARN.080.00002: %s: OK %s: unused: %s: not found:%s' % (m,str(used),str(not_used),str(not_found))
2812	else:
2813	print 'INFO.080.00002: %s: all objectives matched' % m
2814	self.objList = ol
2815
2816
2817	def finish(self):
2818	## varChoice section
2819	self.choices()
2820	##
2821	self.reviewObj()
2822	##
2823	return
2824	#d1 = {}
2825	#for i in self.rqvgList:
2826	#d1[i['uid']] = i
2827	#this = self.doc.getElementsByTagName('requestVarGroup')[0]
2828	#dil = this.getElementsByTagName('item')
2829	kkk = 0
2830	kk0 = []
2831	ss = {}
2832	#for item in dil:
2833	#uid = item.getAttribute( 'uid' )
2834	#refn = item.getAttribute( 'refNote' )
2835	#bits = string.split( refn, '_' )
2836	#if len(bits) == 2:
2837	#if string.find(bits[0], '.' ) != -1:
2838	#bits[0] = string.split( bits[0], '.')[-1]
2839	#ss[ tuple( bits ) ] = uid
2840	#if uid not in d1:
2841	#kk0.append( 'Record for uid %s not found' % uid )
2842	#else:
2843	#lab = item.getAttribute( 'label' )
2844	#if d1[uid]['label'] == lab:
2845	#kkk += 1
2846	#else:
2847	#kk0.append( 'Mismatch in label for %s' % uid )
2848	#print 'INFO.rvg.00001: %s records checked OK; %s failed' % (kkk,len(kk0))
2849	notNeeded = ( ('Oyr','3dtr'), ('Amon','2d'), ('cf3hr','grid'))
2850	##nnf = 0
2851	dt1 = {}
2852	for s in tableUtils.sgset:
2853	a,b = string.split( s, '_' )
2854	if (a,b) not in notNeeded:
2855	if (a,b) in ss:
2856	##
2857	## create lookup --- dt1[<sub-group id>] = <requestVarGroup uid>
2858	##
2859	dt1[s] = ss[(a,b)]
2860	else:
2861	print 'WARN.rvg.00002: %s %s not found' % (a,b)
2862	nnf += 1
2863
2864	assert nnf == 0, 'Not all sub-groups found'
2865	###
2866	### read through request vars ... and look to see how many refer to subgroups identified in dt1:
2867	### how many of these are identified in tableUtils, how many in tableUtils are missing?
2868	###
2869	dt1vs = {}
2870	for s in dt1:
2871	dt1vs[ dt1[s] ] = s
2872
2873	##print 'INFO.rvg.00009: dt1vs: %s' % str(dt1vs)
2874	this = self.doc.getElementsByTagName('requestVar')[0]
2875	dil = this.getElementsByTagName('item')
2876	nf0 = 0
2877	nf1 = 0
2878	nf2 = 0
2879	ss0 = set()
2880	sn0 = collections.defaultdict( int )
2881	sn1 = collections.defaultdict( int )
2882	sn2 = collections.defaultdict( int )
2883	for item in dil:
2884	vgid = item.getAttribute( 'vgid' )
2885	if vgid in dt1vs:
2886	s = dt1vs[vgid]
2887	nf0 += 1
2888	sn0[s] += 1
2889	vid = item.getAttribute( 'vid' )
2890	ss0.add( vid )
2891	if vid in tableUtils.vdict:
2892	nf1 += 1
2893	sn1[s] += 1
2894	else:
2895	this.removeChild(item)
2896
2897	for vid in tableUtils.vdict:
2898	if vid not in ss0:
2899	var,ttl,mip,tab,sg,priority = tableUtils.vdict[vid]
2900	a,b = string.split( sg, '_' )
2901	if (a,b) not in notNeeded:
2902	nf2 += 1
2903	sn2[ tableUtils.vdict[vid][4] ] += 1
2904	assert sg in dt1, 'Cannot locate requestVarGroupd for %s: %s' % (sg,str(dt1.keys()))
2905	vgid = dt1[sg]
2906	item = self.doc.createElement( 'item' )
2907	item.setAttribute( 'uid', str(uuid.uuid1()) )
2908	item.setAttribute( 'vid', vid )
2909	item.setAttribute( 'vgid', vgid )
2910	item.setAttribute( 'title', '%s ((isd.009))' % var )
2911	item.setAttribute( 'label', var )
2912	item.setAttribute( 'mip', mip )
2913	item.setAttribute( 'priority', priority )
2914	this.appendChild( item )
2915
2916	print 'INFO.rvg.00010: variables found in sg rvgs: %s' % nf0
2917	print 'INFO.rvg.00011: of these, number in tableUtils: %s' % nf1
2918	print 'INFO.rvg.00012: variables in tableUtils not in sg rvgs: %s [from %s]' % (nf2,len(tableUtils.vdict.keys()))
2919	for k in sorted( sn0.keys() ):
2920	print 'INFO.rvg.00020: # in tableUtils [%s] %s [from %s]' % (k,sn0[k],sn1[k])
2921	for k in sorted( sn2.keys() ):
2922	print 'INFO.rvg.00021: # in tableUtils [%s] not found %s' % (k,sn2[k])
2923
2924
2925	def finish2(self):
2926	##
2927	## objectiveLink
2928	##
2929	this = self.doc.getElementsByTagName('objectiveLink')[0]
2930	dil = this.getElementsByTagName('item')
2931	for d in dil:
2932	this.removeChild(d)
2933	for t in self.objList:
2934	item = self.doc.createElement( 'item' )
2935	item.setAttribute( 'label', t[0] )
2936	item.setAttribute( 'oid', t[1] )
2937	item.setAttribute( 'rid', t[2] )
2938	thisuuid = str( uuid.uuid1() )
2939	item.setAttribute( 'uid', thisuuid )
2940	this.appendChild( item )
2941	##
2942	## remarks
2943	##
2944	this = self.doc.getElementsByTagName('remarks')[0]
2945	dil = this.getElementsByTagName('item')
2946	for d in dil:
2947	this.removeChild(d)
2948
2949	##mn = self.doc.getElementsByTagName('main')[0]
2950	for item in self.doc.getElementsByTagName('item'):
2951	for k,v in item.attributes.items():
2952	if type(v) not in ( type( u'x' ), type( 'x' ) ):
2953	print 'TYPE ERROR %s, %s: %s' % (item.parentNode.tagName, k,v)
2954	txt = self.doc.toprettyxml(indent='\t', newl='\n', encoding=None)
2955	if self.schemaMode == 'dreq2':
2956	ofn = 'trial2_20150831.xml'
2957	else:
2958	ofn = 'trial_20150831.xml'
2959	oo = open( ofn, 'w' )
2960	lines = string.split( txt, '\n' )
2961	for line in lines:
2962	l = utils_wb.uniCleanFunc( string.strip(line) )
2963	if empty.match(l):
2964	continue
2965	else:
2966	oo.write(l + '\n')
2967	oo.close()
2968
2969	def choices(self):
2970	"""Populate the 'varChoice' section, using the information in the 'choices' dictionary in 'choicecfg'"""
2971
2972	ccfg = choiceCfg.choices( self )
2973	self.ccfg = ccfg
2974	##
2975	## choices
2976	##
2977	this = self.doc.getElementsByTagName('varChoice')[0]
2978	dil = this.getElementsByTagName('item')
2979	for d in dil:
2980	this.removeChild(d)
2981	for k in ccfg.choices.keys():
2982	item = self.doc.createElement( 'item' )
2983	cc = ccfg.choices[k]
2984	for kk in cc.keys():
2985	item.setAttribute( kk, cc[kk] )
2986	this.appendChild( item )
2987	##print 'INFO.zzz.00008: Appending ',cc[kk]
2988
2989	this = self.doc.getElementsByTagName('varChoiceLinkR')[0]
2990	dil = this.getElementsByTagName('item')
2991	for d in dil:
2992	this.removeChild(d)
2993	for cc in ccfg.varChoiceLnk:
2994	item = self.doc.createElement( 'item' )
2995	##print 'INFO.zzz', cc
2996	for i in range( len( ccfg.varChoiceLnkInfo ) ):
2997	item.setAttribute( ccfg.varChoiceLnkInfo[i], cc[i] )
2998	this.appendChild( item )
2999
3000	this = self.doc.getElementsByTagName('varChoiceLinkC')[0]
3001	dil = this.getElementsByTagName('item')
3002	for d in dil:
3003	this.removeChild(d)
3004	for cc in ccfg.varChoiceLnkCfg:
3005	item = self.doc.createElement( 'item' )
3006	for i in range( len( ccfg.varChoiceLnkCfgInfo ) ):
3007	item.setAttribute( ccfg.varChoiceLnkCfgInfo[i], cc[i] )
3008	this.appendChild( item )
3009
3010	##self.varChoiceLnkCfgInfo = [ 'uid', 'label', 'title', 'vid', 'cid', 'cfgid', 'cfg'
3011
3012	def importRepl(self,rfile='uuidreplace.csv',mode='var'):
3013	for l in open(rfile).readlines():
3014	bits = string.split( string.strip(l), '\t' )
3015	assert len(bits) == 7, 'Could not parse item replacement file'
3016	if mode == 'var':
3017	self.repl[ bits[3] ].append( (bits[2],bits[0],bits[1] ) )
3018	self.replItems[bits[0]] = bits[1]
3019	elif mode == 'cmv':
3020	self.rep['cmv'].v[ bits[3] ].append( (bits[2],bits[0],bits[1] ) )
3021	self.rep['cmv'].items[bits[0]] = bits[1]
3022
3023
3024	def importRemove(self,rfile='uuidremove.csv'):
3025	for l in open(rfile).readlines():
3026	self.remo[ string.strip(l) ] = 1
3027
3028	def importUpdate(self,rfile='uuidupdate.csv'):
3029	for l in open(rfile).readlines():
3030	bits = string.split( string.strip(l), '\t' )
3031	tgs = string.strip( bits[2] )
3032	if string.find( tgs, ' ' ) == -1:
3033	tags = [tgs,]
3034	else:
3035	tags = string.split( tgs )
3036	self.upda[ bits[0] ] = {'comment':bits[1], 'tags':tags, 'label':bits[3], 'title':bits[4] }
3037
3038	def importInsert(self,rfile='uuidinsert.csv'):
3039	for l in open(rfile).readlines():
3040	bits = string.split( string.strip(l), '\t' )
3041	if bits[0] == 'unique':
3042	self.insert[bits[1]] = bits[3]
3043
3044	mode = 'dreq2'
3045	sampleXml = '../framework/out/%sSample.xml' % mode
3046	from scansh02 import rq
3047
3048	m = main(sampleXml, rq,run=True, schemaMode=mode, doRepl=False)
3049

Note: See TracBrowser for help on using the repository browser.

Download in other formats: