import collections, xlsxwriter, uuid, json, sys import docAppend from dreqPy import dreq from dreqPy import scope from dreqPy import volsum from dreqPy.extensions import collect ### ### need to check that Omon-3d and Omon-oth are complete ..... ### extend revocean3d method ..... class uuidManage(object): ##uuidlist = ['x%5.5i_%s' % (i,str( uuid.uuid1() ) ) for i in range(5000)] uuidlist = [x.strip() for x in open( 'uuid_july6.txt' ).readlines() ] def __init__(self): self.l = list() self.used = set() def pop(self): u = self.uuidlist.pop() self.used.add( u ) return u uuidlist = uuidManage() newUid = collections.defaultdict( set ) class rev01(object): def __init__(self): """Identify variables which are commonly used in the historical and piControl experiments and append them to the ScenarioMIP experiment request; Add ESMval low frequency request to all experiments.""" self.dq = dreq.loadDreq(manifest='dreqManifest_review.txt') collect.add( self.dq ) self.l00 = tuple([len( self.dq.coll[x].items ) for x in ['requestVarGroup','requestLink','requestItem','requestVar'] ] ) self.exptix = dict() self.varix = dict() self.cmvix = dict() for i in self.dq.coll['experiment'].items: self.exptix[i.label] = i.uid for i in self.dq.coll['var'].items: self.varix[i.label] = i.uid for i in self.dq.coll['CMORvar'].items: self.cmvix[(i.mipTable,i.label)] = i.uid def revocean3d(self): o3d = self.dq.inx.uid['efc154f6-5629-11e6-9079-ac72891c3257'] ooth = self.dq.inx.uid['efc143a8-5629-11e6-9079-ac72891c3257'] omon_table = self.dq.inx.uid['MIPtable::Omon'] s3d = set() soth = set() for i in self.dq.inx.iref_by_sect[omon_table.uid].a['CMORvar']: cmv = self.dq.inx.uid[i] st = self.dq.inx.uid[cmv.stid] sp = self.dq.inx.uid[st.spid] if sp.dimensions.find( 'olevel' ) != -1 and sp.dimensions.find( 'latitude' ) != -1 and sp.dimensions.find( 'longitude' ) != -1: s3d.add( i ) else: soth.add( i ) cc = collections.defaultdict( set ) s3dx = set() for i in self.dq.inx.iref_by_sect[o3d.uid].a['requestVar']: rv = self.dq.inx.uid[i] cmv = self.dq.inx.uid[rv.vid] s3dx.add(rv.vid) st = self.dq.inx.uid[cmv.stid] sp = self.dq.inx.uid[st.spid] if sp.dimensions.find( 'olevel' ) == -1: cc[st.uid].add( rv.uid ) sothx = set() for i in self.dq.inx.iref_by_sect[ooth.uid].a['requestVar']: rv = self.dq.inx.uid[i] cmv = self.dq.inx.uid[rv.vid] sothx.add( rv.vid ) st = self.dq.inx.uid[cmv.stid] sp = self.dq.inx.uid[st.spid] if sp.dimensions.find( 'olevel' ) != -1: cc[st.uid].add( rv.uid ) rqv1 = {'priority':1, 'vgid':'efc143a8-5629-11e6-9079-ac72891c3257', 'mip':'OMIP'} thisrqv = self.dq.coll['requestVar'].items[0] for u in soth: if u not in sothx: cmv = self.dq.inx.uid[u] this = rqv1.copy() this['vid'] = u this['priority'] = cmv.defaultPriority this['uid'] = uuidlist.pop() this['label'] = 'OMIP-%s' % cmv.label this['title'] = 'OMIP monthly, excluding 3d: %s' % cmv.label thisrqv._append( this ) rqv1 = {'priority':1, 'vgid':'efc154f6-5629-11e6-9079-ac72891c3257', 'mip':'OMIP'} for u in s3d: if u not in s3dx: cmv = self.dq.inx.uid[u] this = rqv1.copy() this['vid'] = u this['priority'] = cmv.defaultPriority this['uid'] = uuidlist.pop() this['label'] = 'OMIP-%s' % cmv.label this['title'] = 'OMIP monthly, 3d: %s' % cmv.label thisrqv._append( this ) print 'Found in Omon-oth group:' for s in cc: print '%s: %s' % (s,len(cc[s])) self.rmu = cc['f7e4000e-562c-11e6-a2a4-ac72891c3257'] def reva(self): # expand experiment label list to set of uid's of experiments + mips + experiment groups which, if referenced by esid attribute, imply a request from these experiments. eset = set( ['historical','piControl'] ) esetx = set() for lab in eset: u = self.exptix[lab] esetx.add(u) i = self.dq.inx.uid[u] esetx.add(i.mip) esetx.add(i.egid) ## find set of ScenarioMIP experiments and experiment groups esetr = set(['ScenarioMIP',]) i = self.dq.inx.uid['ScenarioMIP'] for u in self.dq.inx.iref_by_sect[i.uid].a['experiment']: esetr.add(u) esetr.add(self.dq.inx.uid[u].egid) ## find requestLink items which link to an experiment in eset. rql01 = self._eset_to_rqlset(esetx) rqlScen = self._eset_to_rqlset(esetr) ## for each variable, collect names of mips requesting it cc = self._rqlset_to_varmipdict(rql01) self.ccScen = self._rqlset_to_varmipdict(rqlScen) ## rank variables ee = collections.defaultdict( set ) for i in self.dq.coll['CMORvar'].items: t = (i.mipTable,i.label) ee[len(cc[t])].add( t ) ks = sorted( ee.keys() ) ks.reverse() kac = 0 ll = len( self.dq.coll['CMORvar'].items ) fac = 100./ll for k in ks: kac += len( ee[k] ) print '%2s: %4s (%6.2f%%)' % (k, len(ee[k]), kac*fac) for k in ks: vv = collections.defaultdict( set ) print '%s:: Request Rank %s' % (k,k) for t in ee[k]: vv[t[0]].add(t[1]) for t in sorted( vv.keys() ): print '%s:: %s' % (t, ','.join( sorted( vv[t] ) ) ) self.rankedList = ee self.dumpXlsx( 'test' ) eset = self._eset_per_mip_by_request() depcomp = self._mip_dep_comp(eset) def _rev_thetao(self): self.vgid_with_thetao_only = dict() for tab in ['Omon','Odec']: thetao = self.cmvix[(tab,'thetao')] bigthetao = self.cmvix[(tab,'bigthetao')] s1 = set() s2 = set() for i in self.dq.inx.iref_by_sect[thetao].a['requestVar']: vgid = self.dq.inx.uid[i].vgid s1.add( vgid ) for i in self.dq.inx.iref_by_sect[bigthetao].a['requestVar']: vgid = self.dq.inx.uid[i].vgid s2.add( vgid ) ss = {x for x in s1 if x not in s2} self.vgid_with_thetao_only[tab] = (ss,bigthetao) print 'THETAO: %s: variable groups to edit: %s' % (tab, len(ss) ) def _add_bigthetao(self): ## add in requestVars ##[u'uid', u'vid', u'title', u'label', u'priority', u'vgid', u'mip'] rqv1 = {'priority':1, 'mip':'OMIP'} thisrqv = self.dq.coll['requestVar'].items[0] for tab in ['Omon','Odec']: ss, bigthetao = self.vgid_with_thetao_only[tab] for vgid in ss: this = rqv1.copy() this['vid'] = bigthetao this['vgid'] = vgid this['uid'] = uuidlist.pop() this['label'] = 'OMIP-bigthetao' this['title'] = 'bigthetao -- alternative to thetao use in some models' thisrqv._append( this ) def dumpXlsx(self,fn): wb = xlsxwriter.Workbook('%s.xlsx' % fn) s = wb.add_worksheet(name='Info') s.write( 0,0, 'Review of Data Request: Summary' ) s.write( 1,0, 'Rank' ) s.write( 1,1, 'Number of MIPs requesting variable for either historical or piControl' ) s.write( 1,0, "Count" ) s.write( 1,1, "Number of variables" ) s.write( 2,0, "Cumulative %" ) s.write( 2,1, "Cumulative number of variables, as a percentage" ) s.write( 3,0, "Count*" ) s.write( 3,1, "Number of variables which in this set which are already requested for ScenarioMIP experiments by VIACSAB" ) s.write( 4,0, "Rank" ) s.write( 4,1, "Count" ) s.write( 4,2, "Cumulative" ) s.write( 4,3, "Cumulative %" ) s.write( 4,4, "Count*" ) s.write( 4,5, "Cumulative*" ) ks = sorted( self.rankedList.keys() ) ks.reverse() kac = 0 kvac = 0 ll = len( self.dq.coll['CMORvar'].items ) fac = 100./ll ir = 4 for k in ks: kac += len( self.rankedList[k] ) kv = len( [t for t in self.rankedList[k] if 'VIACSAB' in self.ccScen[t] ] ) kv2 = len( [t for t in self.rankedList[k] if len( self.ccScen[t] ) > 0 ] ) kvac += kv ir += 1 s.write( ir,0, k ) s.write( ir,1, len( self.rankedList[k] ) ) s.write( ir,2, kac ) s.write( ir,3, (kac*fac) ) s.write( ir,4, kv ) s.write( ir,5, kvac ) s.write( ir,6, kv2 ) for k in ks: s = wb.add_worksheet(name='Rank %s' % k) s.write( 0,0, "Table" ) s.write( 0,1, "Variable" ) ##vv = collections.defaultdict( set ) tt = sorted( list( self.rankedList[k] ) ) ir = 0 for t in tt: ir += 1 s.write( ir,0, t[0] ) s.write( ir,1, t[1] ) s.write( ir,2, 'VIACSAB' in self.ccScen[t] ) s.write( ir,3, self.dq.inx.uid[ self.cmvix[t] ].title ) wb.close() def _review_table_groups(self): ss = set() for i in self.dq.coll['requestVarGroup'].items: if i.refNote.find( 'MIP Table: ' ) != -1: tab = i.refNote.split( )[-1] u = 'MIPtable::%s' % tab if u in self.dq.inx.uid: ss.add((i,u)) else: print i.title, '%s not found' % u oo = open( 'groupVsTable.txt','w') for i,u in ss: ## CMORvar items in table s1 = self.dq.inx.iref_by_sect[u].a['CMORvar'] ## CMORvar items in group s2 = [self.dq.inx.uid[k].vid for k in self.dq.inx.iref_by_sect[i.uid].a['requestVar']] s1x = [k for k in s1 if k not in s2] s2x = [k for k in s2 if k not in s1] if len(s1x) == 0 and len(s2x) == 0: print 'Table %s OK [%s/%s]' % (i.title, len(s1), len(s2)) else: print 'ERROR: Table %s: %s;; %s' % (i.title, len(s1x), len(s2x)) for u in s1x: item = self.dq.inx.uid[u] oo.write( 'Table only\t%s\t%s\t%s\n' % (i.label, item.label, item.uid) ) for u in s2x: item = self.dq.inx.uid[u] oo.write( 'Group only\t%s\t%s\t%s\n' % (i.label, item.label, item.uid) ) oo.close() def _mip_dep_comp(self,eset): esdoc = json.load( open( 'ingest/mipExptDep.json', 'r' ) ) es = esdoc['mips'] esex = set() drex = set() for k in es: for e in es[k]['expts']: esex.add(e) for k in eset: for e in eset[k]: drex.add(e) d1 = esex.difference( drex ) if len( d1 ) > 0: print 'EXPERIMENTS ONLY IN es-doc:', d1 else: print 'experiments not different, %s -- %s' % (len(esex), len(drex)) d1 = drex.difference( esex ) if len( d1 ) > 0: print 'EXPERIMENTS ONLY IN dreq:', d1 for k in es: if k not in eset: print 'SEVERE.eset.0001: esdoc key not in request dict: %s' % k for k in eset: if k not in es: print 'SEVERE.eset.0001: request dict key not in esdoc dict: %s' % k res = dict() consistent = set() for k in es: if k in eset: esdoc_dependency_only = set() dreq_requests_only = set() for e in es[k]['expts']: if e not in eset[k]: esdoc_dependency_only.add(e) for e in eset[k]: if e not in es[k]['expts']: dreq_requests_only.add(e) if len( dreq_requests_only ) == 0 and len( esdoc_dependency_only ) == 0: consistent.add( k ) else: res[k] = {'esdoc_only':sorted(list(esdoc_dependency_only)), 'dreq_only':sorted(list(dreq_requests_only))} ee = {'consistent':sorted(list(consistent)), 'differences':res} oo = open( 'mipExptDepComp.json', 'w' ) json.dump( {'info':{'title':'Analysis of dependencies of MIPS on expts'}, 'analysis':ee}, oo, indent=4, sort_keys=True ) oo.close() oo = open( 'esdoc_only.csv', 'w' ) for k in sorted(list(res.keys())): if len( res[k]['esdoc_only'] ) > 0: oo.write( '\t'.join( [k,] + res[k]['esdoc_only'] ) + '\n' ) oo.close() oo = open( 'dreq_only.csv', 'w' ) for k in sorted(list(res.keys())): if len( res[k]['dreq_only'] ) > 0: oo.write( '\t'.join( [k,] + res[k]['dreq_only'] ) + '\n' ) oo.close() return ee def _eset_per_mip_by_request(self): """Find set of experiments from which a MIP requests data""" eset = collections.defaultdict(set) for i in self.dq.coll['requestLink'].items: for u in self.dq.inx.iref_by_sect[i.uid].a['requestItem']: this = self.dq.inx.uid[u] ex = self.dq.inx.uid[this.esid] if ex._h.label == 'experiment': eset[i.mip].add(ex.label) else: for uuu in self.dq.inx.iref_by_sect[this.esid].a['experiment']: eset[i.mip].add(self.dq.inx.uid[uuu].label ) res = dict() for k in eset: res[k] = eset[k] if 'DECK' in res: print 'DECK???????? :',res[k] else: print 'DECK not here ....' return res def _eset_to_rqlset(self,eset): """Find requestLink items which link to an experiment in eset. Input: eset: set of uid's of experiments, experiment groups and mips; Output: rqlset: set of uid's of requestLink records. """ rqlset = set() for i in self.dq.coll['requestLink'].items: for u in self.dq.inx.iref_by_sect[i.uid].a['requestItem']: this = self.dq.inx.uid[u] if this.esid in eset: rqlset.add( i.uid ) return rqlset def _rqlset_to_varmipdict(self,rqlset): """For each variable, collect names of mips requesting it: Input: rqlset: set of uid's of requestLink records; Output: cc: dictionary (collections.defaultdict instance): keys: tuple of (mipTable, label), values: set of MIPS. """ cc = collections.defaultdict( set ) for i in rqlset: rvg = self.dq.inx.uid[i].refid mip = self.dq.inx.uid[i].mip for u in self.dq.inx.iref_by_sect[rvg].a['requestVar']: this = self.dq.inx.uid[u] cmv = self.dq.inx.uid[this.vid] cc[(cmv.mipTable,cmv.label)].add( mip ) return cc def _add_areatype(self): s1 = set() for i in self.dq.coll['grids'].items: if i.standardName == 'area_type': for u in self.dq.inx.iref_by_sect[i.uid].a['structure']: s1.add(u) s2 = set() for u in s1: for k in self.dq.inx.iref_by_sect[u].a['CMORvar']: s2.add(k) ss = set() for k in s2: i = self.dq.inx.uid[k] v = self.dq.inx.uid[i.vid] if v.sn.find( 'area_fraction' ) != -1 and i.mipTable not in ['IyrGre', 'IyrAnt', 'SIDay']: ss.add(i) # for u in ['MIPtable::Ofx','MIPtable::fx']: for k in self.dq.inx.iref_by_sect[u].a['CMORvar']: i = self.dq.inx.uid[k] ss.add(i) rvg1 = {u'refNote':'rvg1', u'uid':uuidlist.pop(), u'title':'Baseline set of variables for all experiments (fixed and masks)', u'label':'maskBaseline', u'mip':'CMIP', u'ref':'auto'} self.dq.coll['requestVarGroup'].items[0]._append( rvg1 ) rql1 = {u'comment':'Baseline set of variables for all experiments (fixed and masks)', u'refNote':'added by requestReview script', u'uid':uuidlist.pop(), u'opar':'', u'title':'Masks baseline', u'opt':'', u'label':'maskBaseline', u'grid':'', u'tab':'', u'objective':'ModelIntercomparison1', u'mip':'CMIP', u'ref':'', u'refid':rvg1['uid'], u'gridreq':''} objl1 = {u'uid':uuidlist.pop(), u'title':'', u'label':'ModelIntercomparison1', 'oid':self.modelIntObjUid, 'rid':rql1['uid']} rqi1 = {u'nexmax':1, u'uid':None, u'title':'', u'nenmax':1, u'nymax':999, 'preset':-1, u'expt':'', u'ny':10, u'rlid':rql1['uid'], u'tab':'', u'esid':'', u'mip':'CMIP', u'label':'', u'esidComment':''} self.dq.coll['objectiveLink'].items[0]._append( objl1 ) self.dq.coll['requestLink'].items[0]._append( rql1 ) thisrqi = self.dq.coll['requestItem'].items[0] for exg in self.dq.coll['exptgroup'].items: this = rqi1.copy() this['uid'] = uuidlist.pop() this['esid'] = exg.uid this['expt'] = exg.label this['label'] = 'CMIP-masks-%s' % exg.label this['title'] = 'Masks baseline: Link to %s' % exg.label thisrqi._append( this ) ## add in requestVars ##[u'uid', u'vid', u'title', u'label', u'priority', u'vgid', u'mip'] rqv1 = {'priority':1, 'vgid':rvg1['uid'], 'mip':'CMIP'} thisrqv = self.dq.coll['requestVar'].items[0] for cmv in ss: u = cmv.uid this = rqv1.copy() this['vid'] = u this['uid'] = uuidlist.pop() this['label'] = 'CMIP-%s' % cmv.label this['title'] = 'Masks baseline .. %s' % cmv.label thisrqv._append( this ) def _add_esmvalBaseline(self): ## add request for esmval low freq. groupd (b7d445ce-c16a-11e6-bb6a-ac72891c3257) to every expt. esmvalBase = "b7d445ce-c16a-11e6-bb6a-ac72891c3257" rvg1 = {u'refNote':'rvg1', u'uid':uuidlist.pop(), u'title':'Baseline set of esmval variables for all experiments', u'label':'validationBaseline', u'mip':'CMIP', u'ref':'auto'} self.dq.coll['requestVarGroup'].items[0]._append( rvg1 ) rql1 = {u'comment':'Baseline set of variables for all experiments', u'refNote':'added by requestReview script', u'uid':uuidlist.pop(), u'opar':'', u'title':'ESMval baseline', u'opt':'', u'label':'esmvalBaseline', u'grid':'', u'tab':'', u'objective':'ModelIntercomparison1', u'mip':'CMIP', u'ref':'', u'refid':rvg1['uid'], u'gridreq':''} thisobj = self.dq.coll['objective'].items obj1 = {u'uid':uuidlist.pop(), u'title':'Diagnostics for model intercomparison', 'desription':'Diagnostics for model intercomparison', u'label':'ModelIntercomparison1', 'mip':'CMIP'} self.modelIntObjUid = obj1['uid'] objl1 = {u'uid':uuidlist.pop(), u'title':'', u'label':'ModelIntercomparison1', 'oid':obj1['uid'], 'rid':rql1['uid']} thisobj.append( thisobj[0].__class__( idict=obj1, id='auto' ) ) thisobjl = self.dq.coll['objectiveLink'].items thisobjl.append( thisobjl[0].__class__( idict=objl1, id='auto' ) ) thislist = self.dq.coll['requestLink'].items thislist.append( thislist[0].__class__( idict=rql1, id='auto' ) ) thisrqi = self.dq.coll['requestItem'].items rqi1 = {u'nexmax':1, u'uid':None, u'title':'', u'nenmax':1, u'nymax':999, 'preset':-1, u'expt':'', u'ny':10, u'rlid':rql1['uid'], u'tab':'', u'esid':'', u'mip':'CMIP', u'label':'', u'esidComment':''} for exg in self.dq.coll['exptgroup'].items: this = rqi1.copy() this['uid'] = uuidlist.pop() this['esid'] = exg.uid this['expt'] = exg.label this['label'] = 'CMIP-esmval-%s' % exg.label this['title'] = 'ESMVal baseline: Link to %s' % exg.label thisrqi.append( thisrqi[0].__class__( idict=this, id='auto' ) ) ss = set() for i in self.dq.inx.iref_by_sect[esmvalBase].a['requestVar']: rqv = self.dq.inx.uid[i] cmv = self.dq.inx.uid[rqv.vid] st = self.dq.inx.uid[cmv.stid] sp = self.dq.inx.uid[st.spid] if not (cmv.frequency == 'day' and (sp.dimensions.find( 'plev8' ) != -1)): ss.add(cmv) ## add in requestVars ##[u'uid', u'vid', u'title', u'label', u'priority', u'vgid', u'mip'] rqv1 = {'priority':1, 'vgid':rvg1['uid'], 'mip':'CMIP'} thisrqv = self.dq.coll['requestVar'].items[0] for cmv in ss: u = cmv.uid this = rqv1.copy() this['vid'] = u this['uid'] = uuidlist.pop() this['label'] = 'CMIP-%s' % cmv.label this['title'] = 'Validation baseline .. %s' % cmv.label thisrqv._append( this ) def _add_rql(self): ##[u'refNote', u'uid', u'title', u'label', u'mip', u'ref'] rvg1 = {u'refNote':'rvg1', u'uid':uuidlist.pop(), u'title':'Baseline set of variables for ScenarioMIP experiments', u'label':'scenarioMipBaseline', u'mip':'CMIP', u'ref':'auto'} ##[u'comment', u'refNote', u'uid', u'opar', u'title', u'opt', u'label', u'grid', u'tab', u'objective', u'mip', u'ref', u'refid', u'gridreq'] thisrvg = self.dq.coll['requestVarGroup'].items thislist = self.dq.coll['requestLink'].items thisrqi = self.dq.coll['requestItem'].items thisrqv = self.dq.coll['requestVar'].items rql1 = {u'comment':'Baseline set of variables for ScenarioMIP experiments', u'refNote':'added by requestReview script', u'uid':uuidlist.pop(), u'opar':'', u'title':'ScenarioMIP baseline', u'opt':'', u'label':'scenarioMipBaseline', u'grid':'', u'tab':'', u'objective':'ModelIntercomparison2', u'mip':'CMIP', u'ref':'', u'refid':rvg1['uid'], u'gridreq':''} thislist.append( thislist[0].__class__( idict=rql1, id='auto' ) ) thisrvg.append( thisrvg[0].__class__( idict=rvg1, id='auto' ) ) thisobj = self.dq.coll['objective'].items thisobjl = self.dq.coll['objectiveLink'].items obj1 = {u'uid':uuidlist.pop(), u'title':'Diagnostics for model intercomparison', 'desription':'Diagnostics for model intercomparison from ScenarioMIP experiments', u'label':'ModelIntercomparison2', 'mip':'CMIP'} objl1 = {u'uid':uuidlist.pop(), u'title':'', u'label':'ModelIntercomparison2', 'oid':obj1['uid'], 'rid':rql1['uid']} thisobj.append( thisobj[0].__class__( idict=obj1, id='auto' ) ) thisobjl.append( thisobjl[0].__class__( idict=objl1, id='auto' ) ) ## add in requestItems, linking to scenarioMip experiments ##[u'treset', u'nexmax', u'uid', u'title', u'nenmax', u'nymax', u'expt', u'ny', u'preset', u'tslice', u'rlid', u'tab', u'esid', u'mip', u'label', u'esidComment'] rqi1 = {u'nexmax':1, u'uid':None, u'title':'', u'nenmax':1, u'nymax':999, 'preset':-1, u'expt':'', u'ny':1, u'rlid':rql1['uid'], u'tab':'', u'esid':'', u'mip':'CMIP', u'label':'', u'esidComment':''} i = self.dq.inx.uid['ScenarioMIP'] for u in self.dq.inx.iref_by_sect[i.uid].a['experiment']: ex = self.dq.inx.uid[u] this = rqi1.copy() this['uid'] = uuidlist.pop() this['esid'] = u this['expt'] = ex.label this['label'] = 'CMIP-scenario-%s' % ex.label this['title'] = 'Link to %s' % ex.label thisrqi.append( thisrqi[0].__class__( idict=this, id='auto' ) ) ## add in requestVars ##[u'uid', u'vid', u'title', u'label', u'priority', u'vgid', u'mip'] rqv1 = {'priority':1, 'vgid':rvg1['uid'], 'mip':'CMIP'} for k in self.rankedList.keys(): if k > 6: for t in self.rankedList[k]: u = self.cmvix[t] cmv = self.dq.inx.uid[u] this = rqv1.copy() this['vid'] = u this['uid'] = uuidlist.pop() this['label'] = 'CMIP-%s' % cmv.label this['title'] = 'ScenMipBase .. %s' % cmv.label self.dq.coll['requestVar'].items.append( thisrqv[0].__class__( idict=this, id='auto' ) ) def _finish(self,go=False,delt=True): self.dq.inx = dreq.index( self.dq.coll ) l11 = tuple([len( self.dq.coll[x].items ) for x in ['requestVarGroup','requestLink','requestItem','requestVar'] ] ) if delt==True: assert self.l00 != l11, 'Failed to append items ....' print self.l00, l11 if go: x = scope.dreqUI(['-m','CMIP:ModelIntercomparison2.ModelIntercomparison1','-p','3','-t','3']) x.run(dq=self.dq) sc = scope.dreqQuery( dq=self.dq ) sc.setTierMax( 3 ) sc.gridPolicyDefaultNative = False pmax = 3 vs = volsum.vsum( sc, scope.odsz, scope.npy, odir='revXls', tabByFreqRealm=False ) vs.analAll(pmax) r = rev01() r.reva() r._rev_thetao() r._add_rql() r._add_bigthetao() r._add_esmvalBaseline() r._add_areatype() r._review_table_groups() r.revocean3d() r._finish(go=False,delt=False) ifn = 'test.xml' if len(sys.argv) > 1: ifn = sys.argv[1] ifile = 'out/annotated_20150731_chk.xml' ifile = 'ing02/%s' % ifn ofile = 'ing02/test_01.xml' da = docAppend.dreqAppend(ifile, r.dq, uuidlist.used, rmu=r.rmu ) da.write(ofile=ofile)