Changeset 711
- Timestamp:
- 02/08/16 10:17:34 (5 years ago)
- Location:
- CMIP6dreqbuild/trunk/src/framework/dreqPy
- Files:
-
- 1 added
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
CMIP6dreqbuild/trunk/src/framework/dreqPy/dreq.py
r705 r711 16 16 if sys.version_info[0] == 3: 17 17 python2 = False 18 pythonPre27 = False 18 19 elif sys.version_info[0] == 2: 19 20 pythonPre27 = sys.version_info[1] < 7 -
CMIP6dreqbuild/trunk/src/framework/dreqPy/makeTables.py
r701 r711 1 1 2 import dreq3 2 import collections, string, os, sys 4 import vrev 3 4 try: 5 import dreq 6 import vrev 7 import misc_utils 8 except: 9 import dreqPy.dreq as dreq 10 import dreqPy.vrev as vrev 11 import dreqPy.misc_utils as misc_utils 12 5 13 python2 = True 6 14 if sys.version_info[0] == 3: … … 19 27 else: 20 28 oldpython = True 21 22 import scope_utils23 29 24 30 try: … … 594 600 mlab = setMlab( m ) 595 601 596 cc0 = scope_utils.getExptSum( self.dq, mlab, l1 )602 cc0 = misc_utils.getExptSum( self.dq, mlab, l1 ) 597 603 ks = sorted( list( cc0.keys() ) ) 598 604 if self.verbose: -
CMIP6dreqbuild/trunk/src/framework/dreqPy/packageConfig.py
r702 r711 15 15 16 16 __versionComment__ = "Updates to content for several MIPs; uniqueness of CMOR variable name per table" 17 __version__ = "01.beta.3 2"17 __version__ = "01.beta.33" 18 18 __title__ = "dreqPy" 19 19 __description__ = "CMIP6 Data Request Python API" -
CMIP6dreqbuild/trunk/src/framework/dreqPy/scope.py
r680 r711 6 6 import dreq 7 7 from utilities import cmvFilter 8 import makeTables 8 9 except: 9 import dreqPy.dreq 10 import dreqPy.dreq as dreq 10 11 from dreqPy.utilities import cmvFilter 12 import dreqPy.makeTables as makeTables 11 13 12 14 import collections, string, operator 13 import makeTables14 15 import sys, os 15 16 … … 510 511 for v in vars: 511 512 if 'requestVar' not in inx.iref_by_sect[v].a: 512 print 'Variable with no request ....: %s, %s' % (inx.uid[v].label, inx.uid[v].mipTable)513 print ( 'Variable with no request ....: %s, %s' % (inx.uid[v].label, inx.uid[v].mipTable) ) 513 514 szv[v] = self.sz[inx.uid[v].stid]*npy[inx.uid[v].frequency] 514 515 ov.append( self.dq.inx.uid[v] ) … … 909 910 assert len(ll) == 7, 'Length of model configuration argument must be 7 comma separated integers: %s' % self.adict['mcfg'] 910 911 lli = [ int(x) for x in ll] 911 print lli912 912 913 913 self.sc = dreqQuery( dq=self.dq ) … … 944 944 self.sc.checkDir( odir, 'xls files' ) 945 945 946 ##print mips, eid947 946 if 'txtOpts' in self.adict: 948 947 if self.adict['txtOpts'][0] == 'v': -
CMIP6dreqbuild/trunk/src/framework/dreqPy/scope_utils.py
r682 r711 1 import makeTables2 1 import collections, string 3 4 def rankCMORvars(dq): 5 cc = collections.defaultdict( set ) 6 ee = {} 7 kd = 0 8 ff = {} 9 for ic in dq.coll['CMORvar'].items: 10 s = set() 11 r = set() 12 i = dq.inx.uid[ ic.vid ] 13 if i._h.label != 'remarks': 14 kk = '%s.%s' % (ic.mipTable, ic.label) 15 if i.title != ic.title: 16 print ( '%s: %s, %s' % (kk, ic.title, i.title) ) 17 kd += 1 18 if string.find( ic.modeling_realm, ' ' ) != -1: 19 for x in string.split( ic.modeling_realm ): 20 r.add( string.strip( x ) ) 21 elif ic.modeling_realm not in ['__unset__','']: 22 r.add( ic.modeling_realm ) 23 if 'requestVar' in dq.inx.iref_by_sect[ic.uid].a: 24 for x in dq.inx.iref_by_sect[ic.uid].a['requestVar']: 25 s.add(x) 26 27 if len(s) > 0: 28 ee[kk] = r 29 ff[kk] = i 30 ss = sorted( [dq.inx.uid[x].priority for x in s] ) 31 if len(ss) > 1: 32 kk = '%s-%s' % (ss[0],ss[1]) 33 sn = dq.inx.uid[i.sn] 34 if sn._h.label == 'remarks': 35 kk += 'x' 36 cc[kk].add( i.label ) 37 else: 38 print ( '%s not used' % i.label ) 39 print ( kd ) 40 return (cc,ee,ff) 41 42 def rankVars(dq): 43 cc = collections.defaultdict( set ) 44 ee = {} 45 ff = {} 46 for i in dq.coll['var'].items: 47 s = set() 48 r = set() 49 if 'CMORvar' in dq.inx.iref_by_sect[i.uid].a: 50 for cmv in dq.inx.iref_by_sect[i.uid].a['CMORvar']: 51 ic = dq.inx.uid[cmv] 52 if string.find( ic.modeling_realm, ' ' ) != -1: 53 for x in string.split( ic.modeling_realm ): 54 r.add( string.strip( x ) ) 55 elif ic.modeling_realm not in ['__unset__','']: 56 r.add( ic.modeling_realm ) 57 if 'requestVar' in dq.inx.iref_by_sect[cmv].a: 58 for x in dq.inx.iref_by_sect[cmv].a['requestVar']: 59 s.add(x) 60 61 if len(s) > 0: 62 ee[i.label] = r 63 ff[i.label] = i 64 ss = sorted( [dq.inx.uid[x].priority for x in s] ) 65 if len(ss) > 0: 66 kk = '%s' % (ss[0]) 67 sn = dq.inx.uid[i.sn] 68 if sn._h.label == 'remarks': 69 kk += 'x' 70 cc[kk].add( i.label ) 71 else: 72 print ( '%s not used' % i.label ) 73 return (cc,ee,ff) 74 75 def getExptSum(dq,mip,rqi): 76 """Return a dictionary of experiment uids keyed on MIPs, from list of request items""" 77 cc = collections.defaultdict( set ) 78 for i in rqi: 79 es = dq.inx.uid[i.esid] 80 if es._h.label == 'experiment': 81 cc[es.mip].add(es.uid) 82 elif es._h.label in ['exptgroup','mip']: 83 if 'experiment' in dq.inx.iref_by_sect[i.esid].a: 84 for id in dq.inx.iref_by_sect[i.esid].a['experiment']: 85 ex = dq.inx.uid[id] 86 cc[ex.mip].add(id) 87 88 return cc 89 ##ks = sorted( list( cc.keys() ) ) 90 ##xx = string.join( ['%s: %s' % (k,len(cc[k])) for k in ks], ', ' ) 91 92 class rqiSet(object): 93 npy = {'1hrClimMon':24*12, 'daily':365, u'Annual':1, u'fx':0.01, u'1hr':24*365, u'3hr':8*365, 94 u'monClim':12, u'Timestep':100, u'6hr':4*365, u'day':365, u'1day':365, u'mon':12, u'yr':1, 95 u'1mon':12, 'month':12, 'year':1, 'monthly':12, 'hr':24*365, 'other':24*365, 96 'subhr':24*365, 'Day':365, '6h':4*365, '3 hourly':8*365, '':1 } 97 def __init__(self,dq,rqi=None,byMip=None): 98 self.dq = dq 99 if rqi != None: 100 assert byMip == None, 'ERROR.rqiSet.001: Cannot have rqi and byMip both assigned' 101 self.rqi = rqi 102 elif byMip != None: 103 self.rqi = [i for i in dq.coll['requestItem'].items if i.mip == byMip] 104 else: 105 self.rqi = dq.coll['requestItem'].items 106 107 self.verbose = False 108 if self.verbose: 109 print ( 'INFO.rqiSet.00001: initialised, len(rqi) = %s' % len(self.rqi) ) 110 111 def run(self,vsz,rqi=None,pmax=1,tiermax=1,plist=False): 112 self.exptVarSum(pmax=pmax,plist=plist,tiermax=tiermax) 113 self.exptVarVol(vsz,plist=plist,tiermax=tiermax) 114 115 def getVarList(self,rqi,pmax=1): 116 cc = collections.defaultdict( list ) 117 for i in rqi: 118 rl = self.dq.inx.uid[i.rlid] 119 if 'requestVar' in self.dq.inx.iref_by_sect[rl.refid].a: 120 for id in self.dq.inx.iref_by_sect[rl.refid].a['requestVar']: 121 rq = self.dq.inx.uid[id] 122 if rq.priority <= pmax: 123 cc[rq.vid].append( (i.ny, i.nymax, i.nenmax,rl.grid,i.uid) ) 124 ee = {} 125 for vid in cc.keys(): 126 if len( cc[vid] ) == 1: 127 ee[vid] = cc[vid][0] 128 else: 129 ll = [x[0] for x in cc[vid] if x[0] > 0] 130 if len(ll) == 0: 131 ny = -1 132 else: 133 ny = max(ll) 134 ll = [x[1] for x in cc[vid] if x[1] > 0] 135 if len(ll) == 0: 136 nymax = -1 137 else: 138 nymax = max(ll) 139 ll = [x[2] for x in cc[vid] if x[2] > 0] 140 if len(ll) == 0: 141 nenmax = -1 142 else: 143 nenmax = max(ll) 144 ss = set( [x[3] for x in cc[vid]] ) 145 rqil = [x[4] for x in cc[vid] ] 146 ee[vid] = (ny,nymax,nenmax,list(ss),rqil ) 147 148 return ee 149 150 def exptVarSum(self,exptsOk=False,pmax=1,plist=True,tiermax=1): 151 if not exptsOk: 152 self.exptByMip(tiermax=tiermax) 153 154 self.exvars = {} 155 for m in sorted( self.expts.keys() ): 156 for i in self.expts[m]: 157 rqi = [self.dq.inx.uid[x] for x in self.exrqi[i] ] 158 159 ## obtain dictionary, keyed om CMORvar uid, of variables requested 160 ee = self.getVarList( rqi, pmax=pmax ) 161 ex = self.dq.inx.uid[i] 162 if plist: 163 print ( 'exptVarSum: %s, %s, %s (%s)' % (m,ex.label,len( ee.keys() ), len( rqi)) ) 164 self.exvars[i] = ee 165 166 def exptVarVol(self,vsz,plist=True,tiermax=1): 167 nttt = 0 168 ## 169 ## exvarvol is a dictionary of dictionaries. key 1: experiment uid. 170 ## key 2: cmor variable uid 171 ## content: 5-tuple: ntot: floats requested 172 ## s: floats per time instant 173 ## npy: number of outputs per year 174 ## ny: number of years of output 175 ## nen: number of ensembles 176 #################################################################################### 177 self.exvarvol = {} 178 for m in sorted( self.expts.keys() ): 179 for i in self.expts[m]: 180 ee = self.exvars[i] 181 ex = self.dq.inx.uid[i] 182 ## 183 ## experiment has list of ensemble size (ensz) against tier (tier) 184 ## max ensz st. tier <= tiermax 185 ## 186 l = [x for x in ex.tier if x <= tiermax] 187 exensz = ex.ensz[len(l)-1] 188 189 cmvd = {} 190 nn = 0 191 nerr = 0 192 for k in ee: 193 cmv = self.dq.inx.uid[k] 194 if cmv._h.label == 'CMORvar': 195 s = vsz[cmv.stid] 196 npy = self.npy[cmv.frequency] 197 nyi = ee[k][0] 198 if ex.yps < 0: 199 ny = nyi 200 else: 201 ny = min( [ex.yps,nyi] ) 202 ne = ee[k][2] 203 if ne < 0: 204 nen = exensz 205 else: 206 nen = min( [ne,exensz] ) 207 ntot = s*npy*ny*nen 208 ## 209 ## need to do more on various options here 210 ## 211 cmvd[k] = (ntot,s,npy,ny,nen) 212 nn += ntot 213 else: 214 nerr += 1 215 if plist: 216 print ( 'exptVarVol: %s, %s, %s[%s]: %9.4fTb' % (m,ex.label,len( ee.keys() ), nerr, nn*2.*1.e-12) ) 217 nttt += nn 218 self.exvarvol[i] = cmvd 219 220 if plist: 221 print ( 'TOTAL: %9.3fTb' % (nttt*2*1.e-12) ) 222 223 def exptByMip(self,tiermax=1): 224 cc = collections.defaultdict( list ) 225 for i in self.rqi: 226 cc[i.mip].append( i ) 227 228 ks = sorted( list( cc.keys() ) ) 229 for k in ks: 230 self.getExptByThisMip(k,cc[k],tiermax=tiermax) 231 232 def getExptByThisMip(self,mip,rqi,tiermax=1): 233 self.expts = collections.defaultdict( set ) 234 self.exrqi = collections.defaultdict( set ) 235 for i in rqi: 236 es = self.dq.inx.uid[i.esid] 237 238 ## check to see if "treset" override is present and below tiermax 239 tover = False 240 if 'treset' in i.__dict__ and i.treset != '__unset__': 241 tover = i.treset <= tiermax 242 243 if es._h.label == 'experiment': 244 if es.tier[0] <= tiermax or tover: 245 self.expts[es.mip].add(es.uid) 246 self.exrqi[es.uid].add( i.uid ) 247 elif es._h.label in ['exptgroup','mip']: 248 if 'experiment' in self.dq.inx.iref_by_sect[i.esid].a: 249 for id in self.dq.inx.iref_by_sect[i.esid].a['experiment']: 250 ex = self.dq.inx.uid[id] 251 if ex.tier[0] <= tiermax or tover: 252 self.expts[ex.mip].add(id) 253 self.exrqi[id].add( i.uid ) 254 ks = sorted( list( self.expts.keys() ) ) 255 xx = string.join( ['%s: %s' % (k,len(self.expts[k])) for k in ks], ', ' ) 256 print ( '%s:: %s' % (mip,xx) ) 257 258 class c1(object): 259 def __init__(self): 260 self.a = collections.defaultdict( int ) 2 try: 3 import makeTables 4 except: 5 import dreqPy.makeTables as makeTables 261 6 262 7 ##NT_txtopts = collections.namedtuple( 'txtopts', ['mode'] ) 263 8 264 9 class xlsTabs(object): 10 """used in scope.py; uses makeTables.py""" 265 11 def __init__(self,sc,tiermax=1,pmax=1,xls=True, txt=False, txtOpts=None, odir='xls'): 266 12 self.pmax=pmax -
CMIP6dreqbuild/trunk/src/framework/dreqPy/vrev.py
r691 r711 1 1 """This module has a class which will analyse the usage of variables in the data request""" 2 2 import operator 3 import dreq4 3 import collections 5 4 … … 124 123 else: 125 124 ei = dq.inx.uid[e] 126 print 'ERROR.exptgroup.00001: empty experiment group: %s: %s' % (ei.label, ei.title)125 print ( 'ERROR.exptgroup.00001: empty experiment group: %s: %s' % (ei.label, ei.title) ) 127 126 if r._h.label in [ 'remarks','exptgroup']: 128 127 ##print 'WARNING: link to remarks encountered' … … 139 138 140 139 if __name__ == '__main__': 140 try: 141 import dreq 142 except: 143 import dreqPy.dreq as dreq 141 144 dq = dreq.loadDreq() 142 145 c = checkVar(dq)
Note: See TracChangeset
for help on using the changeset viewer.