1 | """Date Request Scoping module |
---|
2 | ------------------------------ |
---|
3 | The scope.py module contains the dreqQuery class and a set of ancilliary functions. The dreqQuery class contains methods for analysing the data request. |
---|
4 | """ |
---|
5 | |
---|
6 | class exYr(object): |
---|
7 | def __init__(self): |
---|
8 | pass |
---|
9 | |
---|
10 | try: |
---|
11 | import dreq |
---|
12 | imm=1 |
---|
13 | except: |
---|
14 | import dreqPy.dreq as dreq |
---|
15 | imm=2 |
---|
16 | |
---|
17 | if imm == 1: |
---|
18 | from utilities import cmvFilter, gridOptionSort |
---|
19 | import misc_utils |
---|
20 | import fgrid |
---|
21 | import volsum |
---|
22 | else: |
---|
23 | import dreqPy.volsum as volsum |
---|
24 | import dreqPy.fgrid as fgrid |
---|
25 | from dreqPy.utilities import cmvFilter, gridOptionSort |
---|
26 | import dreqPy.misc_utils as misc_utils |
---|
27 | |
---|
28 | import collections, string, operator |
---|
29 | import sys, os |
---|
30 | |
---|
31 | def intdict(): |
---|
32 | return collections.defaultdict( int ) |
---|
33 | |
---|
34 | if sys.version_info >= (2,7): |
---|
35 | oldpython = False |
---|
36 | else: |
---|
37 | oldpython = True |
---|
38 | |
---|
39 | gridSorter = gridOptionSort( oldpython ) |
---|
40 | |
---|
41 | def sortTimeSlice( tsl ): |
---|
42 | |
---|
43 | s = set() |
---|
44 | for ts in tsl: |
---|
45 | if ts[0] == None: |
---|
46 | return (1,ts,'Taking unsliced option') |
---|
47 | s.add( ts[0][1] ) |
---|
48 | if len(s) > 1: |
---|
49 | return (-1,None,'Multiple slice types') |
---|
50 | tst = s.pop() |
---|
51 | if not ( tst == 'simpleRange' or (len(tst) > 13 and tst[:13] == 'branchedYears') ): |
---|
52 | return (-2,None,'slice type aggregation not supported') |
---|
53 | if len(tsl) == 2: |
---|
54 | tsll = list( tsl ) |
---|
55 | sa,ea = tsll[0][0][2:] |
---|
56 | sb,eb = tsll[1][0][2:] |
---|
57 | if sa <= sb and ea >= eb: |
---|
58 | return (1,tsll[0], 'Taking largest slice') |
---|
59 | if sb <= sa and eb >= ea: |
---|
60 | return (1,tsll[1], 'Taking largest slice') |
---|
61 | if ea < sb or eb < sa: |
---|
62 | return (2,tsll, 'Slices are disjoint') |
---|
63 | return (-3,None, 'Overlapping slices') |
---|
64 | else: |
---|
65 | tsll = sorted( list(tsl), key=lambda x: x[0][3] ) |
---|
66 | if min( [x[0][2] for x in tsll] ) == tsll[-1][0][2]: |
---|
67 | return (1,tsll[-1], 'Taking largest slice') |
---|
68 | return (-4,None, 'Cannot sort slices') |
---|
69 | |
---|
70 | odsz = {'landUse':(5,'free'), 'tau':7, 'scatratio':15, 'effectRadLi|tau':(28,'query pending'), 'vegtype':(8,'free'), 'sza5':5, 'site':(119,'73 for aquaplanet .. '), 'iceband':(5,'free'), 'dbze':15, 'spectband':(10,'free'), 'misrBands':(7,'query pending'), 'effectRadIc|tau':(28,'query pending')} |
---|
71 | |
---|
72 | python2 = True |
---|
73 | if sys.version_info[0] == 3: |
---|
74 | python2 = False |
---|
75 | from functools import reduce |
---|
76 | try: |
---|
77 | from utilP3 import mlog3 |
---|
78 | except: |
---|
79 | from dreqPy.utilP3 import mlog3 |
---|
80 | mlg = mlog3() |
---|
81 | else: |
---|
82 | from utilP2 import util |
---|
83 | mlg = util.mlog() |
---|
84 | |
---|
85 | class c1(object): |
---|
86 | def __init__(self): |
---|
87 | self.a = collections.defaultdict( int ) |
---|
88 | class c1s(object): |
---|
89 | def __init__(self): |
---|
90 | self.a = collections.defaultdict( set ) |
---|
91 | |
---|
92 | NT_txtopts = collections.namedtuple( 'txtopts', ['mode'] ) |
---|
93 | |
---|
94 | def vfmt(ss): |
---|
95 | stb = ss*1.e-12 |
---|
96 | if stb < .099: |
---|
97 | return '%7.2fGb' % (stb*100) |
---|
98 | else: |
---|
99 | return '%7.2fTb' % stb |
---|
100 | |
---|
101 | class baseException(Exception): |
---|
102 | """Basic exception for general use in code.""" |
---|
103 | |
---|
104 | def __init__(self,msg): |
---|
105 | self.msg = 'scope:: %s' % msg |
---|
106 | |
---|
107 | def __str__(self): |
---|
108 | return repr( self.msg ) |
---|
109 | |
---|
110 | def __repr__(self): |
---|
111 | return self.msg |
---|
112 | |
---|
113 | nt_mcfg = collections.namedtuple( 'mcfg', ['nho','nlo','nha','nla','nlas','nls','nh1'] ) |
---|
114 | class cmpd(object): |
---|
115 | def __init__(self,dct): |
---|
116 | self.d = dct |
---|
117 | def cmp(self,x,y,): |
---|
118 | return cmp( self.d[x], self.d[y] ) |
---|
119 | |
---|
120 | |
---|
121 | def filter1( a, b ): |
---|
122 | if b < 0: |
---|
123 | return a |
---|
124 | else: |
---|
125 | return min( [a,b] ) |
---|
126 | |
---|
127 | def filter2( a, b, tt, tm ): |
---|
128 | ## largest tier less than or equal to tm |
---|
129 | ll = [t for t in tt if t <= tm] |
---|
130 | if len( ll ) > 0: |
---|
131 | t1 = [t for t in tt if t <= tm][-1] |
---|
132 | it1 = tt.index(t1) |
---|
133 | aa = a[it1] |
---|
134 | if b < 0: |
---|
135 | return aa |
---|
136 | else: |
---|
137 | return min( [aa,b] ) |
---|
138 | else: |
---|
139 | return 0 |
---|
140 | |
---|
141 | npy = {'1hrClimMon':24*12, 'daily':365, u'Annual':1, u'fx':0.01, u'1hr':24*365, u'3hr':8*365, |
---|
142 | u'monClim':12, u'Timestep':100, u'6hr':4*365, u'day':365, u'1day':365, u'mon':12, u'yr':1, |
---|
143 | u'1mon':12, 'month':12, 'year':1, 'monthly':12, 'hr':24*365, 'other':24*365, |
---|
144 | 'subhr':24*365, 'Day':365, '6h':4*365, '3 hourly':8*365, '':1, 'dec':0.1 } |
---|
145 | |
---|
146 | ## There are 4 cmor variables with blank frequency .... |
---|
147 | |
---|
148 | def vol01( sz, v, npy, freq, inx ): |
---|
149 | n1 = npy[freq] |
---|
150 | s = sz[inx.uid[v].stid] |
---|
151 | assert type(s) == type(1), 'Non-integer size found for %s' % v |
---|
152 | assert type(n1) in (type(1),type(0.)), 'Non-number "npy" found for %s, [%s]' % (v,freq) |
---|
153 | return s*n1 |
---|
154 | |
---|
155 | class col_list(object): |
---|
156 | def __init__(self): |
---|
157 | self.a = collections.defaultdict(list) |
---|
158 | |
---|
159 | class col_count(object): |
---|
160 | def __init__(self): |
---|
161 | self.a = collections.defaultdict(int) |
---|
162 | |
---|
163 | class dreqQuery(object): |
---|
164 | __doc__ = """Methods to analyse the data request, including data volume estimates""" |
---|
165 | def __init__(self,dq=None,tierMax=1): |
---|
166 | if dq == None: |
---|
167 | self.dq = dreq.loadDreq() |
---|
168 | else: |
---|
169 | self.dq=dq |
---|
170 | self.rlu = {} |
---|
171 | for i in self.dq.coll['objective'].items: |
---|
172 | k = '%s.%s' % (i.mip,i.label) |
---|
173 | ##assert not k in self.rlu, 'Duplicate label in objectives: %s' % k |
---|
174 | if k in self.rlu: |
---|
175 | print ( 'SEVERE: Duplicate label in objectives: %s' % k ) |
---|
176 | self.rlu[k] = i.uid |
---|
177 | |
---|
178 | self.odsz = odsz |
---|
179 | self.npy = npy |
---|
180 | self.strSz = dict() |
---|
181 | self.cmvFilter = cmvFilter( self ) |
---|
182 | self.tierMax = tierMax |
---|
183 | self.gridPolicyDefaultNative = False |
---|
184 | self.gridOceanStructured = True |
---|
185 | self.gridPolicyForce = None |
---|
186 | self.retainRedundantRank = False |
---|
187 | self.intersection = False |
---|
188 | self.gridPolicyTopOnly = True |
---|
189 | self.exptFilter = None |
---|
190 | self.exptFilterBlack = None |
---|
191 | self.uniqueRequest = False |
---|
192 | |
---|
193 | self.mips = set( [x.label for x in self.dq.coll['mip'].items ] ) |
---|
194 | self.mips = ['CMIP','AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] |
---|
195 | self.mipsp = self.mips[:-4] |
---|
196 | self.cmvGridId, i4 = fgrid.fgrid( self.dq ) |
---|
197 | assert len(i4) == 0 |
---|
198 | |
---|
199 | self.experiments = set( [x.uid for x in self.dq.coll['experiment'].items ] ) |
---|
200 | self.exptByLabel = {} |
---|
201 | for x in self.dq.coll['experiment'].items: |
---|
202 | if x.label in self.exptByLabel: |
---|
203 | print ( 'ERROR: experiment label duplicated: %s' % x.label ) |
---|
204 | self.exptByLabel[x.label] = x.uid |
---|
205 | self.mipls = sorted( list( self.mips ) ) |
---|
206 | |
---|
207 | self.default_mcfg = nt_mcfg._make( [259200,60,64800,40,20,5,100] ) |
---|
208 | self.mcfg = self.default_mcfg._asdict() |
---|
209 | self.mcfgNote = None |
---|
210 | self.szcfg() |
---|
211 | self.requestItemExpAll( ) |
---|
212 | |
---|
213 | def showOpts(self): |
---|
214 | print ( ( self.tierMax, self.gridPolicyDefaultNative, self.gridOceanStructured, self.gridPolicyForce, |
---|
215 | self.retainRedundantRank, self.gridPolicyTopOnly, self.exptFilter, self.exptFilterBlack, |
---|
216 | self.uniqueRequest )) |
---|
217 | |
---|
218 | def setMcfg(self, ll, msg=None ): |
---|
219 | assert len(ll) == 7, 'Model config must be of length 7: %s' % str(ll) |
---|
220 | assert all( [type(x) == type(1) for x in ll] ) |
---|
221 | self.mcfg = nt_mcfg._make( ll )._asdict() |
---|
222 | if msg == None: |
---|
223 | self.mcfgNote = 'User supplied model configuration: %s' % str(ll) |
---|
224 | else: |
---|
225 | self.mcfgNote = msg |
---|
226 | self.szcfg() |
---|
227 | |
---|
228 | def szcfg(self): |
---|
229 | szr = {'100km':64800, '1deg':64800, '2deg':16200 } |
---|
230 | self.szss = {} |
---|
231 | self.sz = {} |
---|
232 | self.szg = collections.defaultdict( dict ) |
---|
233 | self.szgss = collections.defaultdict( dict ) |
---|
234 | self.isLatLon = {} |
---|
235 | self.szSrf = collections.defaultdict( dict ) |
---|
236 | self.szssSrf = collections.defaultdict( dict ) |
---|
237 | for i in self.dq.coll['spatialShape'].items: |
---|
238 | gtype = 'a' |
---|
239 | if i.levelFlag == False: |
---|
240 | ds = i.dimensions.split( '|' ) |
---|
241 | if ds[-1] in ['site', 'basin']: |
---|
242 | vd = ds[-2] |
---|
243 | else: |
---|
244 | vd = ds[-1] |
---|
245 | |
---|
246 | if vd[:4] == 'olev' or vd == 'rho': |
---|
247 | gtype = 'o' |
---|
248 | nz = self.mcfg['nlo'] |
---|
249 | elif vd[:4] == 'alev': |
---|
250 | nz = self.mcfg['nla'] |
---|
251 | elif vd in ['slevel']: |
---|
252 | nz = self.mcfg['nls'] |
---|
253 | elif vd in ['snowdepth','sdepth']: |
---|
254 | nz = 5 |
---|
255 | elif vd == 'aslevel': |
---|
256 | nz = self.mcfg['nlas'] |
---|
257 | else: |
---|
258 | mlg.prnt( 'Failed to parse dimensions %s: %s' % (i.label,i.dimensions) ) |
---|
259 | raise |
---|
260 | else: |
---|
261 | nz = i.levels |
---|
262 | |
---|
263 | dims = set( i.dimensions.split( '|' ) ) |
---|
264 | if 'latitude' in dims and 'longitude' in dims: |
---|
265 | if gtype == 'o': |
---|
266 | nh = self.mcfg['nho'] |
---|
267 | self.isLatLon[i.uid] = 'o' |
---|
268 | else: |
---|
269 | nh = self.mcfg['nha'] |
---|
270 | self.isLatLon[i.uid] = 'a' |
---|
271 | else: |
---|
272 | nh = 10 |
---|
273 | self.isLatLon[i.uid] = False |
---|
274 | |
---|
275 | self.szss[i.uid] = nh*nz |
---|
276 | if self.isLatLon[i.uid] != False and len(dims) == 2: |
---|
277 | self.szssSrf[i.uid] = { 'a':self.mcfg['nha']*nz, 'o':self.mcfg['nho']*nz } |
---|
278 | |
---|
279 | for k in szr: |
---|
280 | if self.isLatLon[i.uid] != False: |
---|
281 | self.szgss[k][i.uid] = szr[k]*nz |
---|
282 | else: |
---|
283 | self.szgss[k][i.uid] = nh*nz |
---|
284 | |
---|
285 | for i in self.dq.coll['structure'].items: |
---|
286 | s = 1 |
---|
287 | knownAtmos = False |
---|
288 | if i.__dict__.get('odims','') != '': |
---|
289 | if i.odims in odsz: |
---|
290 | sf = odsz[i.odims] |
---|
291 | else: |
---|
292 | ## print 'SEVERE.odims.00001: no information on dimension size: %s' % i.odims |
---|
293 | sf = 5 |
---|
294 | if type( sf ) == type( () ): |
---|
295 | sf = sf[0] |
---|
296 | s = s*sf |
---|
297 | if i.odims not in ['iceband']: |
---|
298 | knownAtmos = True |
---|
299 | if i.spid in self.szss: |
---|
300 | self.sz[i.uid] = self.szss[i.spid]*s |
---|
301 | |
---|
302 | if i.uid in self.szssSrf: |
---|
303 | if knownAtmos: |
---|
304 | self.sz[i.uid] = self.szssSrf[i.spid]['a']*s |
---|
305 | else: |
---|
306 | for k in ['a','o']: |
---|
307 | self.szSrf[i.uid][k] = self.szssSrf[i.spid][k]*s |
---|
308 | |
---|
309 | for k in szr: |
---|
310 | self.szg[k][i.uid] = self.szgss[k][i.spid]*s |
---|
311 | else: |
---|
312 | print ('WARNING: spid has no size info: %s [%s]' % (i.spid,i.uid) ) |
---|
313 | self.sz[i.uid] = 0. |
---|
314 | for k in szr: |
---|
315 | self.szg[k][i.uid] = 0. |
---|
316 | |
---|
317 | def getRequestLinkByMip( self, mipSel ): |
---|
318 | """Return the set of request links which are associated with specified MIP""" |
---|
319 | |
---|
320 | if type(mipSel) == type( {} ): |
---|
321 | return self.getRequestLinkByMipObjective(self,mipSel) |
---|
322 | |
---|
323 | if type(mipSel) == type(''): |
---|
324 | t1 = lambda x: x == mipSel |
---|
325 | elif type(mipSel) == type(set()): |
---|
326 | t1 = lambda x: x in mipSel |
---|
327 | |
---|
328 | s = set() |
---|
329 | for i in self.dq.coll['requestLink'].items: |
---|
330 | if t1(i.mip): |
---|
331 | if 'requestItem' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
332 | if any( [ self.rqiExp[x][3] > 0 for x in self.dq.inx.iref_by_sect[i.uid].a['requestItem'] if x in self.rqiExp ] ): |
---|
333 | s.add( i ) |
---|
334 | |
---|
335 | self.rqs = list( s ) |
---|
336 | return self.rqs |
---|
337 | |
---|
338 | def getRequestLinkByMipObjective( self, mipSel ): |
---|
339 | """Return the set of request links which are associated with specified MIP and its objectives""" |
---|
340 | |
---|
341 | assert type(mipSel) == type( {} ),'Argument must be a dictionary, listing objectives for each MIP' |
---|
342 | |
---|
343 | s = set() |
---|
344 | for i in self.dq.coll['requestLink'].items: |
---|
345 | if i.mip in mipSel: |
---|
346 | if len(mipSel[i.mip]) == 0: |
---|
347 | s.add( i ) |
---|
348 | elif 'objectiveLink' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
349 | ss = set( [self.dq.inx.uid[k].label for k in self.dq.inx.iref_by_sect[i.uid].a['objectiveLink'] ] ) |
---|
350 | if any( [x in mipSel[i.mip] for x in ss] ): |
---|
351 | s.add( i ) |
---|
352 | ## |
---|
353 | ## filter requestLinks by tierMax: check to see whether they link to experiments with tier below or equal to tiermax. |
---|
354 | ## |
---|
355 | s1 = set() |
---|
356 | for i in s: |
---|
357 | if 'requestItem' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
358 | if any( [ self.rqiExp[x][-1] > 0 for x in self.dq.inx.iref_by_sect[i.uid].a['requestItem'] if x in self.rqiExp ] ): |
---|
359 | s1.add( i ) |
---|
360 | |
---|
361 | self.rqs = list( s1 ) |
---|
362 | return self.rqs |
---|
363 | |
---|
364 | def varGroupXexpt(self, rqList ): |
---|
365 | """For a list of request links, return a list of variable group IDs for each experiment""" |
---|
366 | self.cc = collections.defaultdict( list ) |
---|
367 | ## dummy = {self.cc[i.expt].append(i.rlid) for i in self.dq.coll['requestItem'].items if i.rlid in {j.uid for j in rqList} } |
---|
368 | return self.cc |
---|
369 | |
---|
370 | def yearsInRequest(self, rql ): |
---|
371 | self.ntot = sum( [i.ny for i in self.dq.coll['requestItem'].items if i.rlid == rql.uid] ) |
---|
372 | return self.ntot |
---|
373 | |
---|
374 | def rqlByExpt( self, l1, ex, pmax=2, expFullEx=False ): |
---|
375 | """rqlByExpt: return a set of request links for an experiment""" |
---|
376 | ## |
---|
377 | inx = self.dq.inx |
---|
378 | |
---|
379 | if ex != None: |
---|
380 | |
---|
381 | exi = self.dq.inx.uid[ex] |
---|
382 | if exi._h.label == 'experiment': |
---|
383 | exset = set( [ex,exi.egid,exi.mip] ) |
---|
384 | else: |
---|
385 | exset = set( self.esid_to_exptList(ex,deref=False,full=expFullEx) ) |
---|
386 | ## |
---|
387 | ## rql is the set of all request links which are associated with a request item for this experiment set |
---|
388 | ## |
---|
389 | |
---|
390 | l1p = set() |
---|
391 | for i in l1: |
---|
392 | if i.preset < 0 or i.preset <= pmax: |
---|
393 | if i.esid in exset: |
---|
394 | l1p.add(i) |
---|
395 | else: |
---|
396 | exset = None |
---|
397 | l1p = l1 |
---|
398 | |
---|
399 | rql0 = set() |
---|
400 | for i in l1p: |
---|
401 | rql0.add(i.rlid) |
---|
402 | |
---|
403 | rqlInv = set() |
---|
404 | for u in rql0: |
---|
405 | if inx.uid[u]._h.label == 'remarks': |
---|
406 | rqlInv.add( u ) |
---|
407 | if len(rqlInv) != 0: |
---|
408 | mlg.prnt ( 'WARNING.001.00002: %s invalid request links from request items ...' % len(rqlInv) ) |
---|
409 | rql = set() |
---|
410 | for u in rql0: |
---|
411 | if inx.uid[u]._h.label != 'remarks': |
---|
412 | rql.add( u ) |
---|
413 | |
---|
414 | return rql, l1p, exset |
---|
415 | |
---|
416 | def varsByRql( self, rql, pmax=2, intersection=False, asDict=False): |
---|
417 | """The complete set of variables associated with a set of request links.""" |
---|
418 | inx = self.dq.inx |
---|
419 | cc1 = collections.defaultdict( set ) |
---|
420 | for i in rql: |
---|
421 | o = inx.uid[i] |
---|
422 | if o.opt == 'priority': |
---|
423 | p = int( float( o.opar ) ) |
---|
424 | assert p in [1,2,3], 'Priority incorrectly set .. %s, %s, %s' % (o.label,o.title, o.uid) |
---|
425 | cc1[inx.uid[i].mip].add( (inx.uid[i].refid,p) ) |
---|
426 | else: |
---|
427 | cc1[inx.uid[i].mip].add( inx.uid[i].refid ) |
---|
428 | |
---|
429 | if intersection: |
---|
430 | ccv = {} |
---|
431 | # |
---|
432 | # set of request variables for each MIP |
---|
433 | # |
---|
434 | for k in cc1: |
---|
435 | thisc = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in cc1[k] ] ) |
---|
436 | rqvgs = collections.defaultdict( set ) |
---|
437 | for x in cc1[k]: |
---|
438 | if type(x) == type( () ): |
---|
439 | rqvgs[x[0]].add( x[1] ) |
---|
440 | else: |
---|
441 | rqvgs[x].add( 3 ) |
---|
442 | |
---|
443 | s = set() |
---|
444 | for vg in rqvgs: |
---|
445 | for l in inx.iref_by_sect[vg].a['requestVar']: |
---|
446 | if inx.uid[l].priority <= min(pmax,max(rqvgs[vg])): |
---|
447 | s.add( inx.uid[l].vid ) |
---|
448 | ccv[k] = s |
---|
449 | |
---|
450 | if len( ccv.keys() ) < len( list(imips) ): |
---|
451 | vars = set() |
---|
452 | else: |
---|
453 | vars = reduce( operator.and_, [ccv[k] for k in ccv] ) |
---|
454 | else: |
---|
455 | rqvgs = collections.defaultdict( set ) |
---|
456 | for k in cc1: |
---|
457 | for x in cc1[k]: |
---|
458 | if type(x) == type( () ): |
---|
459 | rqvgs[x[0]].add( x[1] ) |
---|
460 | else: |
---|
461 | rqvgs[x].add( 3 ) |
---|
462 | |
---|
463 | ###To obtain a set of variables associated with this collection of variable groups: |
---|
464 | |
---|
465 | if asDict: |
---|
466 | vars = collections.defaultdict( list ) |
---|
467 | else: |
---|
468 | vars = set() |
---|
469 | for vg in rqvgs: |
---|
470 | for l in inx.iref_by_sect[vg].a['requestVar']: |
---|
471 | if inx.uid[l].priority <= min(pmax,max(rqvgs[vg])): |
---|
472 | if asDict: |
---|
473 | vars[inx.uid[l].vid].append( vg ) |
---|
474 | else: |
---|
475 | vars.add(inx.uid[l].vid) |
---|
476 | |
---|
477 | ##col1 = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in rqvg ] ) |
---|
478 | ### filter out cases where the request does not point to a CMOR variable. |
---|
479 | ##vars = {vid for vid in vars if inx.uid[vid][0] == u'CMORvar'} |
---|
480 | |
---|
481 | if asDict: |
---|
482 | thisvars = {} |
---|
483 | for vid in vars: |
---|
484 | if inx.uid[vid]._h.label == u'CMORvar': |
---|
485 | thisvars[vid] = vars[vid] |
---|
486 | else: |
---|
487 | thisvars = set() |
---|
488 | for vid in vars: |
---|
489 | if inx.uid[vid]._h.label == u'CMORvar': |
---|
490 | thisvars.add(vid) |
---|
491 | |
---|
492 | return thisvars |
---|
493 | |
---|
494 | def exptYears( self, rqll, ex=None, exBlack=None): |
---|
495 | """Parse a set of request links, and get years requested for each (varGroup, expt, grid) tuple """ |
---|
496 | |
---|
497 | self.tsliceDict = collections.defaultdict( dict ) |
---|
498 | ccts = collections.defaultdict( dict ) |
---|
499 | cc = collections.defaultdict( set ) |
---|
500 | for rl in rqll: |
---|
501 | if 'requestItem' not in self.dq.inx.iref_by_sect[rl.uid].a: |
---|
502 | print ( 'WARN.001.00001: no request items for: %s, %s' % (rl.uid, rl.title) ) |
---|
503 | else: |
---|
504 | |
---|
505 | ##print rl.uid, rl.title, rl.grid, rl.gridreq |
---|
506 | if self.gridPolicyForce != None: |
---|
507 | grd = self.gridPolicyForce |
---|
508 | elif rl.grid in ['1deg','2deg','100km']: |
---|
509 | if rl.grid == '100km': |
---|
510 | grd = '1deg' |
---|
511 | else: |
---|
512 | grd = rl.grid |
---|
513 | else: |
---|
514 | ## note that naming of "gridreq" is unfortunate ... "No" means that native grid is required |
---|
515 | if rl.gridreq in ['No', 'no']: |
---|
516 | #or self.gridPolicyDefaultNative: |
---|
517 | grd = 'native' |
---|
518 | elif rl.gridreq in ['no*1']: |
---|
519 | #or self.gridPolicyDefaultNative: |
---|
520 | grd = 'native:01' |
---|
521 | else: |
---|
522 | ##print ( 'INFO.grd.00001: defaulting to grid ..%s, %s, %s' % (rl.label,rl.title, rl.uid) ) |
---|
523 | grd = 'DEF' |
---|
524 | |
---|
525 | for iu in self.dq.inx.iref_by_sect[rl.uid].a['requestItem']: |
---|
526 | i = self.dq.inx.uid[iu] |
---|
527 | |
---|
528 | ## |
---|
529 | ## apply "treset" filter to request items linked to this group. |
---|
530 | ## |
---|
531 | if self.tierMax < 0 or 'treset' not in i.__dict__ or i.treset <= self.tierMax: |
---|
532 | if iu in self.rqiExp: |
---|
533 | for e in self.rqiExp[iu][1]: |
---|
534 | if (ex == None or e in ex) and (exBlack == None or e not in exBlack): |
---|
535 | this = self.rqiExp[iu][1][e] |
---|
536 | if this != None: |
---|
537 | thisns = this[-3] |
---|
538 | thisny = this[-2] |
---|
539 | thisne = this[-1] |
---|
540 | ##cc[ (rl.refid,e,grd) ].add( filter1( thisns*thisny*thisne, i.nymax) ) |
---|
541 | cc[ (rl.refid,e,grd) ].add( thisns*thisny*thisne ) |
---|
542 | if self.rqiExp[iu][4] != None: |
---|
543 | ccts[(rl.refid,e)][thisns*thisny*thisne] = self.rqiExp[iu][4] |
---|
544 | |
---|
545 | ee = collections.defaultdict( dict ) |
---|
546 | |
---|
547 | revertToLast = True |
---|
548 | ey = exYr() |
---|
549 | if revertToLast: |
---|
550 | for g,e,grd in cc: |
---|
551 | ee[g][(e,grd)] = max( cc[( g,e,grd) ] ) |
---|
552 | if (g,e) in ccts and ee[g][(e,grd)] in ccts[(g,e)]: |
---|
553 | self.tsliceDict[g][e] = ccts[(g,e)][ ee[g][(e,grd)] ] |
---|
554 | ey.exptYears = ee |
---|
555 | return ey |
---|
556 | ff = collections.defaultdict( dict ) |
---|
557 | ## |
---|
558 | ## this needs to be done separately for ocean and atmosphere, because of the default logic workflow ... |
---|
559 | for g,e,grd in cc: |
---|
560 | ee[(g,e)][grd] = max( cc[( g,e,grd) ] ) |
---|
561 | |
---|
562 | xx = collections.defaultdict( dict ) |
---|
563 | for g,e in ee: |
---|
564 | ddef = ee[(g,e)].get( 'DEF', 0 ) |
---|
565 | for grd in ee[(g,e)]: |
---|
566 | if grd != 'DEF': |
---|
567 | xx[(g,'a')][(e, grd)] = ee[(g,e)][grd] |
---|
568 | xx[(g,'o')][(e, grd)] = ee[(g,e)][grd] |
---|
569 | xx[(g,'')][(e, grd)] = ee[(g,e)][grd] |
---|
570 | if grd == 'native' and ddef != 0: |
---|
571 | xx[(g,'a')][(e, 'native')] = max( [xx[(g,'a')][(e, 'native')],ddef] ) |
---|
572 | xx[(g,'')][(e, 'native')] = max( [xx[(g,'')][(e, 'native')],ddef] ) |
---|
573 | if grd == '1deg' and ddef != 0: |
---|
574 | xx[(g,'o')][(e, '1deg')] = max( [xx[(g,'o')][(e, '1deg')],ddef] ) |
---|
575 | |
---|
576 | for grp,flg in xx: |
---|
577 | ff[grp][flg] = xx[(grp,flg)] |
---|
578 | |
---|
579 | ## return dict[<variable group>]{dict[<experiment><grid>]{<years>}} |
---|
580 | ## return dict[<variable group>][grid flag]{dict[<experiment>,<grid>]{<years>}} |
---|
581 | return ff |
---|
582 | |
---|
583 | def volByExpt( self, l1, ex, pmax=1, cc=None, intersection=False,expFullEx=False, adsCount=False ): |
---|
584 | """volByExpt: calculates the total data volume associated with an experiment/experiment group and a list of request items. |
---|
585 | The calculation has some approximations concerning the number of years in each experiment group. |
---|
586 | cc: an optional collector, to accumulate indexed volumes. """ |
---|
587 | ## |
---|
588 | inx = self.dq.inx |
---|
589 | imips = set() |
---|
590 | for i in l1: |
---|
591 | imips.add(i.mip) |
---|
592 | |
---|
593 | rql, l1p, exset = self.rqlByExpt( l1, ex, pmax=pmax, expFullEx=expFullEx ) |
---|
594 | verbose = False |
---|
595 | if verbose: |
---|
596 | for i in rql: |
---|
597 | r = inx.uid[i] |
---|
598 | print ( '%s, %s, %s' % (r.label, r.title, r.uid) ) |
---|
599 | |
---|
600 | if ex != None: |
---|
601 | |
---|
602 | exi = self.dq.inx.uid[ex] |
---|
603 | if exi._h.label == 'experiment': |
---|
604 | exset = set( [ex,exi.egid,exi.mip] ) |
---|
605 | ##### |
---|
606 | if len( rql ) == 0: |
---|
607 | self.vars = set() |
---|
608 | return (0,{},{} ) |
---|
609 | |
---|
610 | ## The complete set of variables associated with these requests: |
---|
611 | vars = self.varsByRql( rql, pmax=pmax, intersection=intersection, asDict=True) |
---|
612 | ## |
---|
613 | ## filter by configuration option and rank |
---|
614 | ## |
---|
615 | if not self.retainRedundantRank: |
---|
616 | len1 = len(vars.keys()) |
---|
617 | cmv = self.cmvFilter.filterByChoiceRank(cmv=vars.keys()) |
---|
618 | vars = cmv |
---|
619 | |
---|
620 | self.vars = vars |
---|
621 | |
---|
622 | e = {} |
---|
623 | for u in rql: |
---|
624 | ### for request variables which reference the variable group attached to the link, add the associate CMOR variables, subject to priority |
---|
625 | i = inx.uid[u] |
---|
626 | e[i.uid] = set() |
---|
627 | si = collections.defaultdict( list ) |
---|
628 | for x in inx.iref_by_sect[i.refid].a['requestVar']: |
---|
629 | if inx.uid[x].priority <= pmax: |
---|
630 | e[i.uid].add( inx.uid[x].vid ) |
---|
631 | |
---|
632 | if verbose: |
---|
633 | cmv = inx.uid[inx.uid[x].vid] |
---|
634 | if cmv._h.label == 'CMORvar': |
---|
635 | si[ cmv.mipTable ].append( inx.uid[x].label ) |
---|
636 | # |
---|
637 | # for each variable, calculate the maximum number of years across all the request links which reference that variable. |
---|
638 | ## |
---|
639 | ## for each request item we have nymax, nenmax, nexmax. |
---|
640 | ## |
---|
641 | nymg = collections.defaultdict( dict ) |
---|
642 | ## |
---|
643 | ## if dataset count rather than volume is wanted, use item 3 from rqiExp tuple. |
---|
644 | if adsCount: |
---|
645 | irqi = 3 |
---|
646 | else: |
---|
647 | irqi = 2 |
---|
648 | |
---|
649 | sgg = set() |
---|
650 | for v in vars: |
---|
651 | s = set() |
---|
652 | sg = collections.defaultdict( set ) |
---|
653 | cc2 = collections.defaultdict( set ) |
---|
654 | cc2s = collections.defaultdict( c1s ) |
---|
655 | for i in l1p: |
---|
656 | ################## |
---|
657 | if (exset == None or i.esid in exset) and v in e[i.rlid]: |
---|
658 | ix = inx.uid[i.esid] |
---|
659 | rl = inx.uid[i.rlid] |
---|
660 | sgg.add( rl.grid ) |
---|
661 | if rl.grid in ['100km','1deg','2deg']: |
---|
662 | grd = rl.grid |
---|
663 | else: |
---|
664 | grd = 'native' |
---|
665 | |
---|
666 | this = None |
---|
667 | if exset == None: |
---|
668 | thisz = 100 |
---|
669 | ## |
---|
670 | ## for a single experiment, look up n years, and n ensemble. |
---|
671 | ## should have nstart???? |
---|
672 | ## |
---|
673 | elif exi._h.label == 'experiment' or ix._h.label == 'experiment': |
---|
674 | this = None |
---|
675 | if ex in self.rqiExp[i.uid][1]: |
---|
676 | this = self.rqiExp[i.uid][1][ex] |
---|
677 | elif ix.uid in self.rqiExp[i.uid][1]: |
---|
678 | this = self.rqiExp[i.uid][1][ix.uid] |
---|
679 | if this != None: |
---|
680 | thisns = this[-3] |
---|
681 | thisny = this[-2] |
---|
682 | thisne = this[-1] |
---|
683 | cc2s[grd].a[u].add( filter1( thisns*thisny*thisne, i.nymax) ) |
---|
684 | else: |
---|
685 | thisz = None |
---|
686 | if 'experiment' in inx.iref_by_sect[i.esid].a: |
---|
687 | for u in inx.iref_by_sect[i.esid].a['experiment']: |
---|
688 | if u in self.rqiExp[i.uid][1]: |
---|
689 | this = self.rqiExp[i.uid][1][u] |
---|
690 | thisns = this[-3] |
---|
691 | thisny = this[-2] |
---|
692 | thisne = this[-1] |
---|
693 | ## |
---|
694 | ### aggregate year count for each experiment and output grid |
---|
695 | ## clarify definition and usage of nymax -- should be redundant ... could be replaced by inward references from "timeSlice" |
---|
696 | cc2s[grd].a[u].add( filter1( thisns*thisny*thisne, i.nymax) ) |
---|
697 | |
---|
698 | if exset != None: |
---|
699 | sg[grd].add( self.rqiExp[i.uid][irqi] ) |
---|
700 | |
---|
701 | ### |
---|
702 | ### sum over experiments of maximum within each experiment |
---|
703 | ### |
---|
704 | for g in sg: |
---|
705 | nymg[v][g] = sum( [max( cc2s[g].a[k] ) for k in cc2s[g].a] ) |
---|
706 | |
---|
707 | szv = {} |
---|
708 | ov = [] |
---|
709 | for v in vars: |
---|
710 | if 'requestVar' not in inx.iref_by_sect[v].a: |
---|
711 | print ( 'Variable with no request ....: %s, %s' % (inx.uid[v].label, inx.uid[v].mipTable) ) |
---|
712 | try: |
---|
713 | szv[v] = self.sz[inx.uid[v].stid]*npy[inx.uid[v].frequency] |
---|
714 | except: |
---|
715 | if inx.uid[v].stid not in self.sz: |
---|
716 | print ('ERROR: size not found for stid %s (v=%s, %s)' % (inx.uid[v].stid,v,inx.uid[v].label) ) |
---|
717 | if inx.uid[v].frequency not in npy: |
---|
718 | print ('ERROR: npy not found for frequency %s (v=%s, %s)' % (inx.uid[v].frequency,v,inx.uid[v].label) ) |
---|
719 | szv[v] = 0 |
---|
720 | ov.append( self.dq.inx.uid[v] ) |
---|
721 | |
---|
722 | ff = {} |
---|
723 | for v in vars: |
---|
724 | if adsCount: |
---|
725 | ff[v] = 1 |
---|
726 | else: |
---|
727 | if 'native' in nymg[v]: |
---|
728 | ff[v] = szv[v] |
---|
729 | ny = nymg[v]['native'] |
---|
730 | else: |
---|
731 | ks0 = nymg[v].keys() |
---|
732 | if len(ks0) == 0: |
---|
733 | ff[v] = 0. |
---|
734 | ny = 0. |
---|
735 | else: |
---|
736 | ks = gridSorter.sort( nymg[v].keys() )[0] |
---|
737 | ##ks = list( nymg[v].keys() )[0] |
---|
738 | ny = nymg[v][ks] |
---|
739 | if inx.uid[v].stid in self.szg[ks]: |
---|
740 | ff[v] = self.szg[ks][ inx.uid[v].stid ] * npy[inx.uid[v].frequency] |
---|
741 | else: |
---|
742 | ff[v] = 0. |
---|
743 | |
---|
744 | if inx.uid[v].frequency != 'monClim': |
---|
745 | ff[v] = ff[v]*ny |
---|
746 | |
---|
747 | ee = self.listIndexDual( ov, 'mipTable', 'label', acount=None, alist=None, cdict=ff, cc=cc ) |
---|
748 | self.ngptot = sum( [ ff[v] for v in vars] ) |
---|
749 | return (self.ngptot, ee, ff ) |
---|
750 | |
---|
751 | def esid_to_exptList(self,esid,deref=False,full=False): |
---|
752 | if not esid in self.dq.inx.uid: |
---|
753 | mlg.prnt ( 'Attempt to dereferece invalid uid: %s' % esid ) |
---|
754 | raise |
---|
755 | |
---|
756 | if self.dq.inx.uid[esid]._h.label == 'experiment': |
---|
757 | expts = [esid,] |
---|
758 | elif self.dq.inx.uid[esid]._h.label != 'remarks': |
---|
759 | if esid in self.dq.inx.iref_by_sect and 'experiment' in self.dq.inx.iref_by_sect[esid].a: |
---|
760 | expts = list( self.dq.inx.iref_by_sect[esid].a['experiment'][:] ) |
---|
761 | else: |
---|
762 | expts = [] |
---|
763 | |
---|
764 | ## add in groups and mips for completeness |
---|
765 | ## |
---|
766 | if full: |
---|
767 | if self.dq.inx.uid[esid]._h.label == 'mip': |
---|
768 | s = set() |
---|
769 | for e in expts: |
---|
770 | if self.dq.inx.uid[e]._h.label != 'experiment': |
---|
771 | mlg.prnt ( 'ERROR: %s, %s, %s ' % (esid,e, self.dq.inx.uid[e].title ) ) |
---|
772 | s.add( self.dq.inx.uid[e].egid ) |
---|
773 | for i in s: |
---|
774 | expts.append( i ) |
---|
775 | expts.append( esid ) |
---|
776 | else: |
---|
777 | return None |
---|
778 | |
---|
779 | if self.tierMax > 0: |
---|
780 | expts1 = [] |
---|
781 | for i in expts: |
---|
782 | if self.dq.inx.uid[i]._h.label == 'experiment': |
---|
783 | if self.dq.inx.uid[i].tier[0] <= self.tierMax: |
---|
784 | expts1.append( i ) |
---|
785 | elif self.dq.inx.uid[i]._h.label == 'exptgroup': |
---|
786 | if self.dq.inx.uid[i].tierMin <= self.tierMax: |
---|
787 | expts1.append( i ) |
---|
788 | else: |
---|
789 | expts1.append( i ) |
---|
790 | else: |
---|
791 | expts1 = expts |
---|
792 | |
---|
793 | if deref: |
---|
794 | return [self.dq.inx.uid[e] for e in expts1] |
---|
795 | else: |
---|
796 | return expts1 |
---|
797 | ## |
---|
798 | ## need to call this on load |
---|
799 | ## then use instead of i.ny etc below |
---|
800 | ## |
---|
801 | def requestItemExpAll( self ): |
---|
802 | self.rqiExp = {} |
---|
803 | for rqi in self.dq.coll['requestItem'].items: |
---|
804 | a,b,c,d,e = self.requestItemExp( rqi ) |
---|
805 | if a != None: |
---|
806 | self.rqiExp[rqi.uid] = (a,b,c,d,e) |
---|
807 | |
---|
808 | def requestItemExp( self, rqi ): |
---|
809 | assert rqi._h.label == "requestItem", 'Argument to requestItemExp must be a requestItem' |
---|
810 | tsl = None |
---|
811 | if 'tslice' in rqi.__dict__: |
---|
812 | ts = self.dq.inx.uid[ rqi.tslice ] |
---|
813 | if ts._h.label == 'timeSlice': |
---|
814 | if ts.type == 'simpleRange': |
---|
815 | tsl = (ts.label,'simpleRange', ts.start,ts.end) |
---|
816 | elif ts.type == 'branchedYears': |
---|
817 | tsl = (ts.label,'%s:%s' % (ts.type,ts.child), ts.start,ts.end) |
---|
818 | else: |
---|
819 | tsl = (ts.label, ts.type, None, None ) |
---|
820 | |
---|
821 | |
---|
822 | u = rqi.esid |
---|
823 | if self.dq.inx.uid[u]._h.label == 'experiment': |
---|
824 | expts = [u,] |
---|
825 | elif self.dq.inx.uid[u]._h.label != 'remarks': |
---|
826 | if u in self.dq.inx.iref_by_sect and 'experiment' in self.dq.inx.iref_by_sect[u].a: |
---|
827 | expts = self.dq.inx.iref_by_sect[u].a['experiment'] |
---|
828 | else: |
---|
829 | expts = [] |
---|
830 | else: |
---|
831 | return (None, None, None, None,None) |
---|
832 | |
---|
833 | if self.tierMax > 0: |
---|
834 | expts = [i for i in expts if self.dq.inx.uid[i].tier[0] <= self.tierMax] |
---|
835 | |
---|
836 | self.multiTierOnly = False |
---|
837 | if self.multiTierOnly: |
---|
838 | expts = [i for i in expts if len(self.dq.inx.uid[i].tier) > 1] |
---|
839 | print ('Len expts: %s' % len(expts) ) |
---|
840 | |
---|
841 | if len(expts) > 0: |
---|
842 | e = [self.dq.inx.uid[i] for i in expts] |
---|
843 | for i in e: |
---|
844 | if i._h.label != 'experiment': |
---|
845 | mlg.prnt ( 'ERROR: %s, %s, %s ' % ( u,i._h.label, i.label, i.title ) ) |
---|
846 | dat2 = {} |
---|
847 | for i in e: |
---|
848 | dat2[i.uid] = (i.ntot, i.yps, i.ensz, i.tier, i.nstart, filter1(i.yps,rqi.nymax), filter2(i.ensz,rqi.nenmax,i.tier,self.tierMax) ) |
---|
849 | |
---|
850 | nytot = sum( [dat2[x][-2]*dat2[x][-3] for x in dat2 ] ) |
---|
851 | netot = sum( [dat2[x][-1] for x in dat2 ] ) |
---|
852 | else: |
---|
853 | dat2 = {} |
---|
854 | nytot = 0 |
---|
855 | netot = 0 |
---|
856 | |
---|
857 | ## |
---|
858 | ## to get list of years per expt for each requestLink ... expts is union of all dat2 keys, |
---|
859 | ## and want max of dat2[x][0] for each experiment x. |
---|
860 | ## |
---|
861 | return (expts, dat2, nytot, netot, tsl ) |
---|
862 | |
---|
863 | def setTierMax( self, tierMax ): |
---|
864 | """Set the maxium tier and recompute request sizes""" |
---|
865 | if tierMax != self.tierMax: |
---|
866 | self.tierMax = tierMax |
---|
867 | self.requestItemExpAll( ) |
---|
868 | |
---|
869 | def summaryByMip( self, pmax=1 ): |
---|
870 | bytesPerFloat = 2. |
---|
871 | for m in self.mipls: |
---|
872 | v = self.volByMip( m, pmax=pmax ) |
---|
873 | mlg.prnt ( '%12.12s: %6.2fTb' % (m,v*bytesPerFloat*1.e-12) ) |
---|
874 | |
---|
875 | def rqlByMip( self, mip): |
---|
876 | if mip == 'TOTAL': |
---|
877 | mip = self.mips |
---|
878 | |
---|
879 | if type(mip) in [type( '' ),type( u'') ]: |
---|
880 | if mip not in self.mips: |
---|
881 | mlg.prnt ( self.mips ) |
---|
882 | raise baseException( 'rqiByMip: Name of mip not recognised: %s' % mip ) |
---|
883 | l1 = [i for i in self.dq.coll['requestLink'].items if i.mip == mip] |
---|
884 | elif type(mip) in [ type( set()), type( [] ) ]: |
---|
885 | nf = [ m for m in mip if m not in self.mips] |
---|
886 | if len(nf) > 0: |
---|
887 | raise baseException( 'rqlByMip: Name of mip(s) not recognised: %s' % str(nf) ) |
---|
888 | l1 = [i for i in self.dq.coll['requestLink'].items if i.mip in mip] |
---|
889 | elif type(mip) == type( dict()): |
---|
890 | nf = [ m for m in mip if m not in self.mips] |
---|
891 | if len(nf) > 0: |
---|
892 | raise baseException( 'rqlByMip: Name of mip(s) not recognised: %s' % str(nf) ) |
---|
893 | l1 = [] |
---|
894 | for i in self.dq.coll['requestLink'].items: |
---|
895 | if i.mip in mip: |
---|
896 | ok = False |
---|
897 | if len( mip[i.mip] ) == 0: |
---|
898 | ok = True |
---|
899 | else: |
---|
900 | for ol in self.dq.inx.iref_by_sect[i.uid].a['objectiveLink']: |
---|
901 | o = self.dq.inx.uid[ol] |
---|
902 | if self.dq.inx.uid[o.oid].label in mip[i.mip]: |
---|
903 | ok = True |
---|
904 | if ok: |
---|
905 | l1.append( i ) |
---|
906 | else: |
---|
907 | raise baseException( 'rqiByMip: "mip" (1st explicit argument) should be type string or set: %s -- %s' % (mip, type(mip)) ) |
---|
908 | |
---|
909 | return l1 |
---|
910 | |
---|
911 | def rqiByMip( self, mip): |
---|
912 | l1 = self.rqlByMip( mip ) |
---|
913 | if len(l1) == 0: |
---|
914 | return [] |
---|
915 | l2 = [] |
---|
916 | for i in l1: |
---|
917 | if 'requestItem' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
918 | for u in self.dq.inx.iref_by_sect[i.uid].a['requestItem']: |
---|
919 | l2.append( self.dq.inx.uid[u] ) |
---|
920 | |
---|
921 | l20 = self.rqiByMip0( mip ) |
---|
922 | for i in l20: |
---|
923 | assert i in l2 |
---|
924 | return l2 |
---|
925 | |
---|
926 | |
---|
927 | def rqiByMip0( self, mip): |
---|
928 | |
---|
929 | if mip == 'TOTAL': |
---|
930 | mip = self.mips |
---|
931 | if type(mip) in [type( '' ),type( u'') ]: |
---|
932 | if mip not in self.mips: |
---|
933 | mlg.prnt ( self.mips ) |
---|
934 | raise baseException( 'rqiByMip: Name of mip not recognised: %s' % mip ) |
---|
935 | l1 = [i for i in self.dq.coll['requestItem'].items if i.mip == mip] |
---|
936 | elif type(mip) in [ type( set()), type( [] ) ]: |
---|
937 | nf = [ m for m in mip if m not in self.mips] |
---|
938 | if len(nf) > 0: |
---|
939 | raise baseException( 'rqiByMip: Name of mip(s) not recognised: %s' % str(nf) ) |
---|
940 | l1 = [i for i in self.dq.coll['requestItem'].items if i.mip in mip] |
---|
941 | elif type(mip) == type( dict()): |
---|
942 | nf = [ m for m in mip if m not in self.mips] |
---|
943 | if len(nf) > 0: |
---|
944 | raise baseException( 'rqiByMip: Name of mip(s) not recognised: %s' % str(nf) ) |
---|
945 | l1 = [] |
---|
946 | for i in self.dq.coll['requestLink'].items: |
---|
947 | if i.mip in mip: |
---|
948 | ok = False |
---|
949 | if len( mip[i.mip] ) == 0: |
---|
950 | ok = True |
---|
951 | else: |
---|
952 | for ol in self.dq.inx.iref_by_sect[i.uid].a['objectiveLink']: |
---|
953 | o = self.dq.inx.uid[ol] |
---|
954 | if self.dq.inx.uid[o.oid].label in mip[i.mip]: |
---|
955 | ok = True |
---|
956 | if ok: |
---|
957 | if 'requestItem' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
958 | for u in self.dq.inx.iref_by_sect[i.uid].a['requestItem']: |
---|
959 | l1.append( self.dq.inx.uid[u] ) |
---|
960 | else: |
---|
961 | raise baseException( 'rqiByMip: "mip" (1st explicit argument) should be type string or set: %s -- %s' % (mip, type(mip)) ) |
---|
962 | |
---|
963 | return l1 |
---|
964 | |
---|
965 | def checkDir(self,odir,msg): |
---|
966 | if not os.path.isdir( odir ): |
---|
967 | try: |
---|
968 | os.mkdir( odir ) |
---|
969 | except: |
---|
970 | print ('\n\nFailed to make directory "%s" for: %s: make necessary subdirectories or run where you have write access' % (odir,msg) ) |
---|
971 | print ( '\n\n' ) |
---|
972 | raise |
---|
973 | print ('Created directory %s for: %s' % (odir,msg) ) |
---|
974 | |
---|
975 | def xlsByMipExpt(self,m,ex,pmax,odir='xls',xls=True,txt=False,txtOpts=None): |
---|
976 | import scope_utils |
---|
977 | mxls = scope_utils.xlsTabs(self,tiermax=self.tierMax,pmax=pmax,xls=xls, txt=txt, txtOpts=txtOpts,odir=odir) |
---|
978 | mlab = misc_utils.setMlab( m ) |
---|
979 | mxls.run( m, mlab=mlab ) |
---|
980 | |
---|
981 | def cmvByInvMip( self, mip,pmax=1,includeYears=False, exptFilter=None,exptFilterBlack=None ): |
---|
982 | mips = set( self.mips[:] ) |
---|
983 | if type(mip) == type( '' ): |
---|
984 | mips.discard( mip ) |
---|
985 | else: |
---|
986 | for m in mip: |
---|
987 | mips.discard( m ) |
---|
988 | |
---|
989 | return self.cmvByMip( mips,pmax=pmax,includeYears=includeYears, exptFilter=exptFilter, exptFilterBlack=exptFilterBlack ) |
---|
990 | |
---|
991 | def cmvByMip( self, mip,pmax=1,includeYears=False, exptFilter=None, exptFilterBlack=None ): |
---|
992 | if exptFilter != None: |
---|
993 | assert type(exptFilter) == type( set() ), 'Argument exptFilter must be None or a set: %s' % str(exptFilter) |
---|
994 | if exptFilterBlack != None: |
---|
995 | assert type(exptFilterBlack) == type( set() ), 'Argument exptFilterBlack must be None or a set: %s' % str(exptFilterBlack) |
---|
996 | if exptFilter != None: |
---|
997 | assert len( exptFilter.difference( exptFilterBlack ) ) > 0, 'If exptFilter and exptFilterBlack are both set, exptFilter must have non-black listed elements' |
---|
998 | |
---|
999 | l1,ee = self.rvgByMip( mip, includePreset=True, returnLinks=True ) |
---|
1000 | if includeYears: |
---|
1001 | expys = self.exptYears( l1, ex=exptFilter, exBlack=exptFilterBlack ) |
---|
1002 | cc = collections.defaultdict( set ) |
---|
1003 | ccts = collections.defaultdict( set ) |
---|
1004 | |
---|
1005 | mipsByVar = collections.defaultdict( set ) |
---|
1006 | ss = set() |
---|
1007 | for pr in ee: |
---|
1008 | ### loop over request var groups. |
---|
1009 | for i in ee[pr]: |
---|
1010 | if 'requestVar' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
1011 | # |
---|
1012 | # loop over request vars in group |
---|
1013 | # |
---|
1014 | for x in self.dq.inx.iref_by_sect[i.uid].a['requestVar']: |
---|
1015 | i1 = self.dq.inx.uid[x] |
---|
1016 | |
---|
1017 | thisp = i1.priority |
---|
1018 | if pr != -1: |
---|
1019 | thisp = pr |
---|
1020 | |
---|
1021 | if thisp <= pmax: |
---|
1022 | if includeYears and i1.vid in self.cmvGridId: |
---|
1023 | ##assert i.uid in expys, 'No experiment info found for requestVarGroup: %s' % i.uid |
---|
1024 | ## may have no entry as a consequence of tierMin being set in the requestLink(s). |
---|
1025 | assert i1.vid in self.cmvGridId, 'No grid identification lookup found for %s: %s' % (i1.label,i1.vid) |
---|
1026 | assert self.cmvGridId[i1.vid] in ['a','o','si','li'], 'Unexpected grid id: %s: %s:: %s' % (i1.label,i1.vid, self.cmvGridId[i1.vid]) |
---|
1027 | gflg = {'si':'','li':''}.get( self.cmvGridId[i1.vid], self.cmvGridId[i1.vid] ) |
---|
1028 | rtl = True |
---|
1029 | |
---|
1030 | if i.uid in expys.exptYears: |
---|
1031 | mipsByVar[i1.vid].add( i.mip ) |
---|
1032 | if rtl: |
---|
1033 | for e,grd in expys.exptYears[i.uid]: |
---|
1034 | if exptFilter == None or e in exptFilter: |
---|
1035 | if grd == 'DEF': |
---|
1036 | if gflg == 'o' and not self.gridPolicyDefaultNative: |
---|
1037 | ##if gflg == 'o': |
---|
1038 | grd1 = '1deg' |
---|
1039 | else: |
---|
1040 | grd1 = 'native' |
---|
1041 | else: |
---|
1042 | grd1 = grd |
---|
1043 | cc[(i1.vid,e,grd1)].add( expys.exptYears[i.uid][e,grd] ) |
---|
1044 | if i.uid in self.tsliceDict and e in self.tsliceDict[i.uid]: |
---|
1045 | ccts[(i1.vid,e)].add( (self.tsliceDict[i.uid][e],thisp) ) |
---|
1046 | else: |
---|
1047 | ccts[(i1.vid,e)].add( (None,thisp) ) |
---|
1048 | |
---|
1049 | |
---|
1050 | else: |
---|
1051 | |
---|
1052 | for gf in expys.exptYears[i.uid]: |
---|
1053 | for e,grd in expys.exptYears[i.uid][gf]: |
---|
1054 | if grd in ["1deg",'2deg'] or gf == gflg: |
---|
1055 | if exptFilter == None or e in exptFilter: |
---|
1056 | cc[(i1.vid,e,grd)].add( expys.exptYears[i.uid][gf][e,grd] ) |
---|
1057 | else: |
---|
1058 | print ( 'SKIPPING %s: %s' % (i1.label,i1.vid) ) |
---|
1059 | ss.add( i1.vid ) |
---|
1060 | |
---|
1061 | if self.intersection and type(mip) == type( set() ) and len(mip) > 1: |
---|
1062 | sint = set( [k for k in mipsByVar if len( mipsByVar[k] ) == len(mip)] ) |
---|
1063 | print ( 'INTERSECTION: %s out of %s variables [%s]' % (len(sint),len(mipsByVar.keys()),str(mip)) ) |
---|
1064 | xxx = [t for t in cc if t[0] not in sint] |
---|
1065 | for t in xxx: |
---|
1066 | del cc[t] |
---|
1067 | if includeYears: |
---|
1068 | l2 = collections.defaultdict( dict ) |
---|
1069 | l2x = collections.defaultdict( dict ) |
---|
1070 | ## |
---|
1071 | ## this removes lower ranked grids .... but for some groups want different grids for different variable categories |
---|
1072 | ## |
---|
1073 | if self.gridPolicyTopOnly: |
---|
1074 | for v,e,g in cc: |
---|
1075 | l2x[(v,e)][g] = max( list( cc[(v,e,g)] ) ) |
---|
1076 | for v,e in l2x: |
---|
1077 | if len( l2x[(v,e)].keys() ) == 1: |
---|
1078 | g,val = list( l2x[(v,e)].items() )[0] |
---|
1079 | else: |
---|
1080 | kk = gridSorter.sort( l2x[(v,e)].keys() ) |
---|
1081 | gflg = {'si':'','li':''}.get( self.cmvGridId[v], self.cmvGridId[v] ) |
---|
1082 | g = kk[0] |
---|
1083 | if g not in l2x[(v,e)]: |
---|
1084 | print ( '%s not found in %s (%s):' % (g,str(l2x[(v,e)].keys()),str(kk)) ) |
---|
1085 | val = l2x[(v,e)][g] |
---|
1086 | |
---|
1087 | l2[v][(e,g)] = val |
---|
1088 | else: |
---|
1089 | for v,e,g in cc: |
---|
1090 | l2[v][(e,g)] = max( list( cc[(v,e,g)] ) ) |
---|
1091 | |
---|
1092 | l2ts = collections.defaultdict( dict ) |
---|
1093 | for v in l2: |
---|
1094 | for e,g in l2[v]: |
---|
1095 | if (v,e) in ccts: |
---|
1096 | if len( ccts[(v,e)] ) > 1: |
---|
1097 | rc, ts, msg = sortTimeSlice( ccts[(v,e)] ) |
---|
1098 | if rc == 1: |
---|
1099 | l2ts[v][e] = tuple( list(ts) + [g,] ) |
---|
1100 | elif rc == 2: |
---|
1101 | try: |
---|
1102 | ##(('abrupt5', 'simpleRange', 0, 5), 1), (('abrupt30', 'simpleRange', 121, 150), 1)] |
---|
1103 | yl = list( range( ts[0][0][2], ts[0][0][3] + 1) ) + list( range( ts[1][0][2], ts[1][0][3] + 1) ) |
---|
1104 | except: |
---|
1105 | print ( 'FAILED TO GENERATE YEARLIST' ) |
---|
1106 | print ( str((v,e) ) ) |
---|
1107 | print ( 'range( ts[0][0][2], ts[0][0][3] + 1) + range( ts[1][0][2], ts[1][0][3] + 1)' ) |
---|
1108 | print ( str(ts) ) |
---|
1109 | raise |
---|
1110 | l2ts[v][e] = ('_union', 'YEARLIST', len(yl), str(yl), ts[1], g ) |
---|
1111 | else: |
---|
1112 | print ('TIME SLICE MULTIPLE OPTIONS FOR : %s, %s, %s, %s' % (v,e,str(ccts[(v,e)]), msg ) ) |
---|
1113 | else: |
---|
1114 | a,b = ccts[(v,e)].pop() |
---|
1115 | if type(a) == type( [] ): |
---|
1116 | l2ts[v][e] = a + [b,g,] |
---|
1117 | elif type(a) == type( () ): |
---|
1118 | l2ts[v][e] = list(a) + [b,g,] |
---|
1119 | elif a == None: |
---|
1120 | l2ts[v][e] = [None,b,g] |
---|
1121 | else: |
---|
1122 | assert False, 'Bad type for ccts record: %s' % type( a) |
---|
1123 | return l2, l2ts |
---|
1124 | else: |
---|
1125 | l2 = sorted( [i for i in [self.dq.inx.uid[i] for i in ss] if i._h.label != 'remarks'], key=lambda x: x.label ) |
---|
1126 | return l2 |
---|
1127 | |
---|
1128 | def exptFilterList(self,val,option,ret='uid'): |
---|
1129 | if type( val ) not in [[],()]: |
---|
1130 | val = [val,] |
---|
1131 | |
---|
1132 | if option == 'lab': |
---|
1133 | v0 = val[:] |
---|
1134 | val = [] |
---|
1135 | mm = [] |
---|
1136 | for v in v0: |
---|
1137 | if v not in self.exptByLabel: |
---|
1138 | mm.append( v ) |
---|
1139 | else: |
---|
1140 | val.append( self.exptByLabel[v] ) |
---|
1141 | |
---|
1142 | assert len(mm) == 0, 'Experiment names not all recognised: %s' % str(mm) |
---|
1143 | |
---|
1144 | oo = set() |
---|
1145 | for v in val: |
---|
1146 | i = self.dq.inx.uid[v] |
---|
1147 | if i._h.label in ['exptgroup','mip']: |
---|
1148 | if 'experiment' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
1149 | for u in self.dq.inx.iref_by_sect[i.uid].a['experiment']: |
---|
1150 | oo.add( u ) |
---|
1151 | elif i._h.label == 'experiment': |
---|
1152 | oo.add( i.uid ) |
---|
1153 | else: |
---|
1154 | print ('WARNING .. skipping request for experiment which links to record of type %s' % i._h.label ) |
---|
1155 | return oo |
---|
1156 | |
---|
1157 | def getFreqStrSummary(self,mip,pmax=1): |
---|
1158 | ## |
---|
1159 | ## get a dictionary keyed on CMORvar uid, containing dictionary keyed on (experiment, grid) with value as number of years. |
---|
1160 | ## |
---|
1161 | if not self.uniqueRequest: |
---|
1162 | cmv, self.cmvts = self.cmvByMip(mip,pmax=pmax,includeYears=True,exptFilter=self.exptFilter,exptFilterBlack=self.exptFilterBlack) |
---|
1163 | else: |
---|
1164 | cmv1, cmvts1 = self.cmvByInvMip(mip,pmax=pmax,includeYears=True,exptFilter=self.exptFilter,exptFilterBlack=self.exptFilterBlack) |
---|
1165 | cmv2, cmvts2 = self.cmvByMip('TOTAL',pmax=pmax,includeYears=True,exptFilter=self.exptFilter,exptFilterBlack=self.exptFilterBlack) |
---|
1166 | cmv = self.differenceSelectedCmvDict( cmv1, cmv2 ) |
---|
1167 | |
---|
1168 | if not self.retainRedundantRank: |
---|
1169 | len1 = len(cmv) |
---|
1170 | self.cmvFilter.filterByChoiceRank(cmv=cmv,asDict=True) |
---|
1171 | len2 = len(cmv) |
---|
1172 | ##print 'INFO.redundant.0001: length %s --> %s' % (len1,len2) |
---|
1173 | |
---|
1174 | self.selectedCmv = cmv |
---|
1175 | return self.cmvByFreqStr( cmv ) |
---|
1176 | |
---|
1177 | def differenceSelectedCmvDict( self, cmv1, cmv2 ): |
---|
1178 | """Return the diffence between two dictionaries of cmor variables returned by self.cmvByMip. |
---|
1179 | The dictionaries contain dictionaries of values. Differences may be subdictionaries not present, |
---|
1180 | elements of sub-dictionaries not present, or elements of sub-dictionaries present with different values. |
---|
1181 | A one sided difference is returned.""" |
---|
1182 | |
---|
1183 | cmv = {} |
---|
1184 | for i in cmv2: |
---|
1185 | if i not in cmv1: |
---|
1186 | cmv[i] = cmv2[i] |
---|
1187 | else: |
---|
1188 | eei = {} |
---|
1189 | for t in cmv2[i]: |
---|
1190 | if t not in cmv1[i]: |
---|
1191 | eei[t] = cmv2[i][t] |
---|
1192 | else: |
---|
1193 | if cmv2[i][t] > cmv1[i][t]: |
---|
1194 | eei[t] = cmv2[i][t] - cmv1[i][t] |
---|
1195 | if len( eei.keys() ) != 0: |
---|
1196 | cmv[i] = eei |
---|
1197 | return cmv |
---|
1198 | |
---|
1199 | def cmvByFreqStr(self,cmv,asDict=True,exptFilter=None,exptFilterBlack=None): |
---|
1200 | if exptFilter != None: |
---|
1201 | assert type(exptFilter) == type( set() ), 'Argument exptFilter must be None or a set: %s' % str(exptFilter) |
---|
1202 | if exptFilterBlack != None: |
---|
1203 | assert type(exptFilterBlack) == type( set() ), 'Argument exptFilterBlack must be None or a set: %s' % str(exptFilterBlack) |
---|
1204 | if exptFilter != None: |
---|
1205 | assert len( exptFilter.difference( exptFilterBlack ) ) > 0, 'If exptFilter and exptFilterBlack are both set, exptFilter must have non-black listed elements' |
---|
1206 | |
---|
1207 | cc = collections.defaultdict( list ) |
---|
1208 | for i in cmv: |
---|
1209 | if asDict: |
---|
1210 | ii = self.dq.inx.uid[i] |
---|
1211 | if ii._h.label != 'remarks': |
---|
1212 | st = self.dq.inx.uid[ ii.stid ] |
---|
1213 | if st._h.label != 'remarks': |
---|
1214 | cc0 = collections.defaultdict( float ) |
---|
1215 | cc1 = collections.defaultdict( int ) |
---|
1216 | se = collections.defaultdict( set ) |
---|
1217 | for e,g in cmv[i]: |
---|
1218 | cc0[g] += cmv[i][(e,g)] |
---|
1219 | cc1[g] += 1 |
---|
1220 | se[g].add(e) |
---|
1221 | for g in cc0: |
---|
1222 | g1 = g |
---|
1223 | if self.isLatLon[st.spid] != False: |
---|
1224 | g1 = g |
---|
1225 | if g1 == 'DEF' and self.isLatLon[st.spid] == 'o': |
---|
1226 | if self.gridPolicyDefaultNative: |
---|
1227 | g1 = 'native' |
---|
1228 | else: |
---|
1229 | g1 = '1deg' |
---|
1230 | elif g == 'native:01': |
---|
1231 | gflg = {'si':'','li':''}.get( self.cmvGridId[i], self.cmvGridId[i] ) |
---|
1232 | if gflg == 'o' and not self.gridOceanStructured: |
---|
1233 | g1 = '1deg' |
---|
1234 | else: |
---|
1235 | g1 = 'native' |
---|
1236 | elif g1 in ['1deg','2deg','native']: |
---|
1237 | pass |
---|
1238 | else: |
---|
1239 | print ( 'WARNING --- blind default to native: %s' % g ) |
---|
1240 | g1 = 'native' |
---|
1241 | elif g == 'native:01': |
---|
1242 | g1 = 'native' |
---|
1243 | |
---|
1244 | cc[ (st.spid,st.__dict__.get('odims',''),ii.frequency,g1) ].append( (i,cc0[g],cc1[g],se[g]) ) |
---|
1245 | |
---|
1246 | else: |
---|
1247 | st = self.dq.inx.uid[ i.stid ] |
---|
1248 | cc[ (st.spid,st.__dict__.get('odims',''),i.frequency) ].append( i.label ) |
---|
1249 | |
---|
1250 | self.thiscmvset = set() |
---|
1251 | c2 = collections.defaultdict( dict ) |
---|
1252 | sf = set() |
---|
1253 | if asDict: |
---|
1254 | for s,o,f,g in cc.keys(): |
---|
1255 | c2[(s,o,g)][f] = cc[ (s,o,f,g) ] |
---|
1256 | sf.add( f ) |
---|
1257 | else: |
---|
1258 | for s,o,f in cc.keys(): |
---|
1259 | c2[(s,o)][f] = cc[ (s,o,f) ] |
---|
1260 | sf.add( f ) |
---|
1261 | lf = sorted( list(sf) ) |
---|
1262 | c3 = collections.defaultdict( dict ) |
---|
1263 | |
---|
1264 | for tt in sorted( c2.keys() ): |
---|
1265 | if asDict: |
---|
1266 | s,o,g = tt |
---|
1267 | else: |
---|
1268 | s,o = tt |
---|
1269 | g = 'native' |
---|
1270 | i = self.dq.inx.uid[ s ] |
---|
1271 | |
---|
1272 | if asDict: |
---|
1273 | for f in c2[tt]: |
---|
1274 | isClim = f.lower().find( 'clim' ) != -1 |
---|
1275 | ny = 0 |
---|
1276 | expts = set() |
---|
1277 | labs = [] |
---|
1278 | labs = collections.defaultdict( int ) |
---|
1279 | ccx = collections.defaultdict( list ) |
---|
1280 | for cmvi, ny1, ne, eset in c2[tt][f]: |
---|
1281 | ccx[cmvi].append( (ny1, ne, eset) ) |
---|
1282 | net = 0 |
---|
1283 | for cmvi in ccx: |
---|
1284 | if len( ccx[cmvi] ) == 1: |
---|
1285 | ny1, ne, eset = ccx[cmvi][0] |
---|
1286 | else: |
---|
1287 | ny1, ne, eset = ( 0,0,set() ) |
---|
1288 | for a,b,s in ccx[cmvi]: |
---|
1289 | ny1 += a |
---|
1290 | ne += b |
---|
1291 | eset = eset.union( s ) |
---|
1292 | |
---|
1293 | net += ne |
---|
1294 | if len(eset) != ne: |
---|
1295 | print ( 'WARNING: inconsistency in volume estimate ... possible duplication for %s,%s' % (cmvi,f) ) |
---|
1296 | for e in eset: |
---|
1297 | elab = self.dq.inx.uid[e].label |
---|
1298 | expts.add(elab) |
---|
1299 | |
---|
1300 | if exptFilter != None: |
---|
1301 | expts = exptFilter.intersection( expts ) |
---|
1302 | if exptFilterBlack != None: |
---|
1303 | expts = expts.difference( exptFilterBlack ) |
---|
1304 | |
---|
1305 | if len(expts) > 0: |
---|
1306 | lab = self.dq.inx.uid[cmvi].label |
---|
1307 | self.thiscmvset.add( cmvi ) |
---|
1308 | ny += ny1 |
---|
1309 | labs[cmvi] += ny1 |
---|
1310 | ne = len( expts ) |
---|
1311 | nn = len( labs.keys() ) |
---|
1312 | |
---|
1313 | if isClim: |
---|
1314 | ny = net/float(nn) |
---|
1315 | else: |
---|
1316 | ny = ny/float(nn) |
---|
1317 | assert tt[2] in ['native','1deg','2deg','native:01'], 'BAD grid identifier: %s' % str(tt) |
---|
1318 | c3[tt][f] = (nn,ny,ne, labs,expts) |
---|
1319 | return (sf,c3) |
---|
1320 | |
---|
1321 | def getStrSz( self, g, stid=None, s=None, o=None, tt=False, cmv=None ): |
---|
1322 | assert stid == None or (s==None and o==None), 'Specify either stid or s and o' |
---|
1323 | assert stid != None or (s!=None and o!=None), 'Specify either stid or s and o' |
---|
1324 | |
---|
1325 | if stid != None: |
---|
1326 | st = self.dq.inx.uid[stid] |
---|
1327 | if st._h.label != 'remarks': |
---|
1328 | s = st.spid |
---|
1329 | o = st.__dict__.get( 'odims', '' ) |
---|
1330 | else: |
---|
1331 | self.strSz[ (stid,g) ] = (False,0) |
---|
1332 | if tt: |
---|
1333 | return (self.strSz[ (stid,g) ], None) |
---|
1334 | else: |
---|
1335 | return self.strSz[ (stid,g) ] |
---|
1336 | |
---|
1337 | g1 = g |
---|
1338 | if g1 == 'DEF': |
---|
1339 | if self.isLatLon[s] == 'o': |
---|
1340 | g1 = '1deg' |
---|
1341 | else: |
---|
1342 | g1 = 'native' |
---|
1343 | elif g1 == 'native:01': |
---|
1344 | assert cmv != None, 'Need a valid cmor variable id .... ' |
---|
1345 | gflg = {'si':'','li':''}.get( self.cmvGridId[cmv], self.cmvGridId[cmv] ) |
---|
1346 | if gflg == 'o' and not self.gridOceanStructured: |
---|
1347 | g1 = '1deg' |
---|
1348 | else: |
---|
1349 | g1 = 'native' |
---|
1350 | if (s,o,g) not in self.strSz: |
---|
1351 | |
---|
1352 | if o == '': |
---|
1353 | sf = 1 |
---|
1354 | elif o in self.odsz: |
---|
1355 | sf = self.odsz[o] |
---|
1356 | else: |
---|
1357 | sf = 5 |
---|
1358 | |
---|
1359 | if type( sf ) == type( () ): |
---|
1360 | sf = sf[0] |
---|
1361 | |
---|
1362 | try: |
---|
1363 | if g1 != 'native' and self.isLatLon[s] != False: |
---|
1364 | szg = self.szgss[g1][s] |
---|
1365 | else: |
---|
1366 | szg = self.szss[s] |
---|
1367 | except: |
---|
1368 | print ( 'Failed to get size for: %s, %s, %s' % (g,g1,s ) ) |
---|
1369 | raise |
---|
1370 | |
---|
1371 | szg = szg * sf |
---|
1372 | self.strSz[ (s,o,g) ] = (True,szg) |
---|
1373 | |
---|
1374 | if tt: |
---|
1375 | return (self.strSz[ (s,o,g) ], (s,o,g1) ) |
---|
1376 | else: |
---|
1377 | return self.strSz[ (s,o,g) ] |
---|
1378 | |
---|
1379 | def rvgByMip( self, mip, years=False, includePreset=False, returnLinks=False ): |
---|
1380 | l1 = self.rqlByMip( mip ) |
---|
1381 | if includePreset: |
---|
1382 | cc = collections.defaultdict( set ) |
---|
1383 | ss = set() |
---|
1384 | for i in l1: |
---|
1385 | if 'requestItem' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
1386 | prs = set() |
---|
1387 | for x in self.dq.inx.iref_by_sect[i.uid].a['requestItem']: |
---|
1388 | prs.add(self.dq.inx.uid[x].preset) |
---|
1389 | |
---|
1390 | for p in prs: |
---|
1391 | assert p in [-1,1,2,3], 'Bad preset value' |
---|
1392 | cc[p].add( i.refid ) |
---|
1393 | ee = {} |
---|
1394 | for p in cc: |
---|
1395 | l2 = sorted( [self.dq.inx.uid[i] for i in cc[p]], key=lambda x: x.label ) |
---|
1396 | ee[p] = l2 |
---|
1397 | if returnLinks: |
---|
1398 | return (l1,ee) |
---|
1399 | else: |
---|
1400 | return ee |
---|
1401 | else: |
---|
1402 | ss = set( [i.refid for i in l1] ) |
---|
1403 | l2 = sorted( [self.dq.inx.uid[i] for i in ss], key=lambda x: x.label ) |
---|
1404 | if returnLinks: |
---|
1405 | return (l1,l2) |
---|
1406 | else: |
---|
1407 | return l2 |
---|
1408 | |
---|
1409 | def volByMip2( self, mip, pmax=2, intersection=False, adsCount=False, exptid=None,makeTabs=False, odir='xls'): |
---|
1410 | vs = volsum.vsum( self, odsz, npy ) |
---|
1411 | rqf = 'dummy' |
---|
1412 | vsmode='short' |
---|
1413 | if makeTabs: |
---|
1414 | mlab = misc_utils.setMlab( mip ) |
---|
1415 | rqf = '%s/requestVol_%s_%s_%s' % (odir,mlab,self.tierMax,pmax) |
---|
1416 | vsmode='full' |
---|
1417 | vs.run( mip, rqf, pmax=pmax, doxlsx=makeTabs ) |
---|
1418 | vs.anal(olab='dummy', doUnique=False, mode=vsmode, makeTabs=makeTabs) |
---|
1419 | self.vf = vs.res['vf'].copy() |
---|
1420 | for f in sorted( vs.res['vf'].keys() ): |
---|
1421 | mlg.prnt ( 'Frequency: %s: %s' % (f, vs.res['vf'][f]*2.*1.e-12 ) ) |
---|
1422 | ttl = sum( [x for k,x in vs.res['vu'].items()] ) |
---|
1423 | self.res = vs.res |
---|
1424 | self.indexedVol = collections.defaultdict( dict ) |
---|
1425 | for u in vs.res['vu']: |
---|
1426 | cmv = self.dq.inx.uid[u] |
---|
1427 | self.indexedVol[cmv.frequency]['%s.%s' % (cmv.mipTable,cmv.label)] = vs.res['vu'][u] |
---|
1428 | return ttl |
---|
1429 | |
---|
1430 | def volByMip( self, mip, pmax=2, intersection=False, adsCount=False, exptid=None): |
---|
1431 | |
---|
1432 | l1 = self.rqiByMip( mip ) |
---|
1433 | |
---|
1434 | #### The set of experiments/experiment groups: |
---|
1435 | if exptid == None: |
---|
1436 | exps = self.experiments |
---|
1437 | elif type( exptid ) == type(''): |
---|
1438 | exps = set( [exptid,] ) |
---|
1439 | else: |
---|
1440 | assert type( exptid ) == type( set() ),'exptid arg to volByMip must be None, string or set: %s' % type( exptid ) |
---|
1441 | exps = exptid |
---|
1442 | |
---|
1443 | self.volByE = {} |
---|
1444 | vtot = 0 |
---|
1445 | cc = collections.defaultdict( col_count ) |
---|
1446 | self.allVars = set() |
---|
1447 | for e in exps: |
---|
1448 | expts = self.esid_to_exptList(e,deref=True,full=False) |
---|
1449 | if expts not in [None,[]]: |
---|
1450 | for ei in expts: |
---|
1451 | self.volByE[ei.label] = self.volByExpt( l1, ei.uid, pmax=pmax, cc=cc, intersection=intersection, adsCount=adsCount ) |
---|
1452 | vtot += self.volByE[ei.label][0] |
---|
1453 | self.allVars = self.allVars.union( self.vars ) |
---|
1454 | self.indexedVol = cc |
---|
1455 | |
---|
1456 | return vtot |
---|
1457 | |
---|
1458 | def listIndexDual(self, ll, a1, a2, acount=None, alist=None, cdict=None, cc=None ): |
---|
1459 | do_count = acount != None |
---|
1460 | do_list = alist != None |
---|
1461 | assert not (do_count and do_list), 'It is an error to request both list and count' |
---|
1462 | if not (do_count or do_list): |
---|
1463 | acount = '__number__' |
---|
1464 | do_count = True |
---|
1465 | |
---|
1466 | if cc == None: |
---|
1467 | if do_count: |
---|
1468 | cc = collections.defaultdict( col_count ) |
---|
1469 | elif do_list: |
---|
1470 | cc = collections.defaultdict( col_list ) |
---|
1471 | |
---|
1472 | if do_count: |
---|
1473 | for l in ll: |
---|
1474 | if cdict != None: |
---|
1475 | v = cdict[l.uid] |
---|
1476 | elif acount == '__number__': |
---|
1477 | v = 1 |
---|
1478 | else: |
---|
1479 | v = l.__dict__[acount] |
---|
1480 | |
---|
1481 | cc[ l.__dict__[a1] ].a[ l.__dict__[a2] ] += v |
---|
1482 | elif do_list: |
---|
1483 | for l in ll: |
---|
1484 | if cdict != None: |
---|
1485 | v = cdict[l.uid] |
---|
1486 | elif alist == '__item__': |
---|
1487 | v = l |
---|
1488 | else: |
---|
1489 | v = l.__dict__[alist] |
---|
1490 | cc[ l.__dict__[a1] ].a[ l.__dict__[a2] ].append( v ) |
---|
1491 | |
---|
1492 | od = {} |
---|
1493 | for k in cc.keys(): |
---|
1494 | d2 = {} |
---|
1495 | for k2 in cc[k].a.keys(): |
---|
1496 | d2[k2] = cc[k].a[k2] |
---|
1497 | od[k] = d2 |
---|
1498 | return od |
---|
1499 | |
---|
1500 | class dreqUI(object): |
---|
1501 | """Data Request Command line. |
---|
1502 | ------------------------- |
---|
1503 | -v : print version and exit; |
---|
1504 | --unitTest : run some simple tests; |
---|
1505 | -m <mip>: MIP of list of MIPs (comma separated; for objective selection see note [1] below); |
---|
1506 | -l <options>: List for options: |
---|
1507 | o: objectives |
---|
1508 | e: experiments |
---|
1509 | -q <options>: List information about the schema: |
---|
1510 | s: sections |
---|
1511 | <section>: attributes for a section |
---|
1512 | <section:attribute>: definition of an attribute. |
---|
1513 | -h : help: print help text; |
---|
1514 | -e <expt>: experiment; |
---|
1515 | -t <tier> maxmum tier; |
---|
1516 | -p <priority> maximum priority; |
---|
1517 | --xls : Create Excel file with requested variables; |
---|
1518 | --sf : Print summary of variable count by structure and frequency [default]; |
---|
1519 | --legacy : Use legacy approach to volume estimation (deprecated); |
---|
1520 | --xfr : Output variable lists in sheets organised by frequency and realm instead of by MIP table; |
---|
1521 | --SF : Print summary of variable count by structure and frequency for all MIPs; |
---|
1522 | --grdpol <native|1deg> : policy for default grid, if MIPs have not expressed a preference; |
---|
1523 | --grdforce <native|1deg> : force a specific grid option, independent of individual preferences; |
---|
1524 | --ogrdunstr : provide volume estimates for unstructured ocean grid (interpolation requirements of OMIP data are different in this case); |
---|
1525 | --allgrd : When a variable is requested on multiple grids, archive all grids requested (default: only the finest resolution); |
---|
1526 | --unique : List only variables which are requested uniquely by this MIP, for at least one experiment; |
---|
1527 | --esm : include ESM experiments (default is to omit esm-hist etc from volume estimates); |
---|
1528 | --txt : Create text file with requested variables; |
---|
1529 | --mcfg : Model configuration: 7 integers, comma separated, 'nho','nlo','nha','nla','nlas','nls','nh1' |
---|
1530 | default: 259200,60,64800,40,20,5,100 |
---|
1531 | --txtOpts : options for content of text file: (v|c)[(+|-)att1[,att2[...]]] |
---|
1532 | --xlsDir <directory> : Directory in which to place variable listing [xls]; |
---|
1533 | --printLinesMax <n> : Maximum number of lines to be printed (default 20) |
---|
1534 | --printVars : If present, a summary of the variables (see --printLinesMax) fitting the selection options will be printed |
---|
1535 | --intersection : Analyse the intersection of requests rather than union. |
---|
1536 | |
---|
1537 | NOTES |
---|
1538 | ----- |
---|
1539 | [1] A set of objectives within a MIP can be specified in the command line. The extended syntax of the "-m" argument is: |
---|
1540 | -m <mip>[:objective[.obj2[.obj3 ...]]][,<mip2]...] |
---|
1541 | |
---|
1542 | e.g. |
---|
1543 | drq -m HighResMIP:Ocean.DiurnalCycle |
---|
1544 | """ |
---|
1545 | def __init__(self,args): |
---|
1546 | self.adict = {} |
---|
1547 | self.knownargs = {'-m':('m',True), '-p':('p',True), '-e':('e',True), '-t':('t',True), \ |
---|
1548 | '-h':('h',False), '--printLinesMax':('plm',True), \ |
---|
1549 | '-l':('l',True), |
---|
1550 | '-q':('q',True), |
---|
1551 | '--printVars':('vars',False), '--intersection':('intersection',False), \ |
---|
1552 | '--count':('count',False), \ |
---|
1553 | '--txt':('txt',False), \ |
---|
1554 | '--sf':('sf',False), \ |
---|
1555 | '--legacy':('legacy',False), \ |
---|
1556 | '--xfr':('xfr',False), \ |
---|
1557 | '--SF':('SF',False), \ |
---|
1558 | '--esm':('esm',False), \ |
---|
1559 | '--grdpol':('grdpol',True), \ |
---|
1560 | '--ogrdunstr':('ogrdunstr',False), \ |
---|
1561 | '--grdforce':('grdforce',True), \ |
---|
1562 | '--omitCmip':('omitcmip',False), \ |
---|
1563 | '--allgrd':('allgrd',False), \ |
---|
1564 | '--unique':('unique',False), \ |
---|
1565 | '--mcfg':('mcfg',True), \ |
---|
1566 | '--txtOpts':('txtOpts',True), \ |
---|
1567 | '--xlsDir':('xlsdir',True), '--xls':('xls',False) \ |
---|
1568 | } |
---|
1569 | aa = args[:] |
---|
1570 | notKnownArgs = [] |
---|
1571 | while len(aa) > 0: |
---|
1572 | a = aa.pop(0) |
---|
1573 | if a in self.knownargs: |
---|
1574 | b = self.knownargs[a][0] |
---|
1575 | if self.knownargs[a][1]: |
---|
1576 | v = aa.pop(0) |
---|
1577 | self.adict[b] = v |
---|
1578 | else: |
---|
1579 | self.adict[b] = True |
---|
1580 | else: |
---|
1581 | notKnownArgs.append(a) |
---|
1582 | |
---|
1583 | assert self.checkArgs( notKnownArgs ), 'FATAL ERROR 001: Arguments not recognised: %s' % (str(notKnownArgs) ) |
---|
1584 | |
---|
1585 | if 'm' in self.adict: |
---|
1586 | if self.adict['m'] == '_all_': |
---|
1587 | pass |
---|
1588 | elif self.adict['m'].find( ':' ) != -1: |
---|
1589 | ee = {} |
---|
1590 | for i in self.adict['m'].split(','): |
---|
1591 | bits = i.split( ':' ) |
---|
1592 | if len( bits ) == 1: |
---|
1593 | ee[bits[0]] = [] |
---|
1594 | else: |
---|
1595 | assert len(bits) == 2, 'Cannot parse %s' % self.adict['m'] |
---|
1596 | ee[bits[0]] = bits[1].split( '.' ) |
---|
1597 | self.adict['m'] = ee |
---|
1598 | else: |
---|
1599 | self.adict['m'] = set(self.adict['m'].split(',') ) |
---|
1600 | if 'omitcmip' not in self.adict and 'CMIP' not in self.adict['m']: |
---|
1601 | self.adict['m'].add( 'CMIP' ) |
---|
1602 | |
---|
1603 | if 'grdpol' in self.adict: |
---|
1604 | assert self.adict['grdpol'] in ['native','1deg'], 'Grid policy argument --grdpol must be native or 1deg : %s' % self.adict['grdpol'] |
---|
1605 | |
---|
1606 | if 'grdforce' in self.adict: |
---|
1607 | assert self.adict['grdforce'] in ['native','1deg'], 'Grid policy argument --grdforce must be native or 1deg : %s' % self.adict['grdforce'] |
---|
1608 | |
---|
1609 | integerArgs = set( ['p','t','plm'] ) |
---|
1610 | for i in integerArgs.intersection( self.adict ): |
---|
1611 | self.adict[i] = int( self.adict[i] ) |
---|
1612 | |
---|
1613 | self.intersection = self.adict.get( 'intersection', False ) |
---|
1614 | |
---|
1615 | |
---|
1616 | def checkArgs( self, notKnownArgs ): |
---|
1617 | if len( notKnownArgs ) == 0: |
---|
1618 | return True |
---|
1619 | print ('--------------------------------------') |
---|
1620 | print ('------------ %s Arguments Not Recognised ------------' % len(notKnownArgs) ) |
---|
1621 | k = 0 |
---|
1622 | for x in notKnownArgs: |
---|
1623 | k += 1 |
---|
1624 | if x[1:] in self.knownargs: |
---|
1625 | print ( '%s PERHAPS %s instead of %s' % (k, x[1:],x) ) |
---|
1626 | elif '-%s' % x in self.knownargs: |
---|
1627 | print ( '%s PERHAPS -%s instead of %s' % (k, x,x) ) |
---|
1628 | elif x[0] == '\xe2': |
---|
1629 | print ( '%s POSSIBLY -- (double hyphen) instead of long dash in %s' % (k, x) ) |
---|
1630 | print ('--------------------------------------') |
---|
1631 | |
---|
1632 | return len( notKnownArgs ) == 0 |
---|
1633 | |
---|
1634 | def run(self, dq=None): |
---|
1635 | if 'h' in self.adict: |
---|
1636 | mlg.prnt ( self.__doc__ ) |
---|
1637 | return |
---|
1638 | |
---|
1639 | if 'q' in self.adict: |
---|
1640 | if dq == None: |
---|
1641 | dq = dreq.loadDreq(configOnly=True) |
---|
1642 | s = self.adict['q'] |
---|
1643 | if self.adict['q'] == 's': |
---|
1644 | ss = sorted( [(i.title,i.label) for i in dq.coll['__sect__'].items] ) |
---|
1645 | for s in ss: |
---|
1646 | mlg.prnt( '%16s:: %s' % (s[1],s[0]) ) |
---|
1647 | else: |
---|
1648 | ss = [i.label for i in dq.coll['__sect__'].items] |
---|
1649 | if s.find( ':' ) != -1: |
---|
1650 | s,a = s.split( ':' ) |
---|
1651 | else: |
---|
1652 | a = None |
---|
1653 | if s not in ss: |
---|
1654 | mlg.prnt( 'ERROR: option must be a section; use "-q s" to list sections' ) |
---|
1655 | elif a == None: |
---|
1656 | x = [i for i in dq.coll['__sect__'].items if i.label == s] |
---|
1657 | s1 = [i for i in dq.coll['__main__'].items if 'ATTRIBUTE::%s' % s in i.uid] |
---|
1658 | mlg.prnt( x[0].title ) |
---|
1659 | mlg.prnt( ' '.join( sorted ([i.label for i in s1] ) )) |
---|
1660 | else: |
---|
1661 | x = [i for i in dq.coll['__main__'].items if i.uid == 'ATTRIBUTE::%s.%s' % (s,a) ] |
---|
1662 | if len(x) == 0: |
---|
1663 | mlg.prnt( 'ERROR: attribute not found' ) |
---|
1664 | s1 = [i for i in dq.coll['__main__'].items if 'ATTRIBUTE::%s' % s in i.uid] |
---|
1665 | mlg.prnt( 'ATTRIBUTES: ' + ' '.join( sorted ([i.label for i in s1] ) )) |
---|
1666 | else: |
---|
1667 | mlg.prnt( 'Section %s, attribute %s' % (s,a) ) |
---|
1668 | mlg.prnt( x[0].title ) |
---|
1669 | mlg.prnt( x[0].description ) |
---|
1670 | return |
---|
1671 | |
---|
1672 | if not ('m' in self.adict or 'SF' in self.adict): |
---|
1673 | mlg.prnt ( 'Current version requires -m or --SF argument' ) |
---|
1674 | mlg.prnt ( self.__doc__ ) |
---|
1675 | sys.exit(0) |
---|
1676 | |
---|
1677 | if dq == None: |
---|
1678 | self.dq = dreq.loadDreq() |
---|
1679 | else: |
---|
1680 | self.dq = dq |
---|
1681 | |
---|
1682 | if 'l' in self.adict: |
---|
1683 | self.printList() |
---|
1684 | return |
---|
1685 | |
---|
1686 | if 'mcfg' in self.adict: |
---|
1687 | ll = string.split( self.adict['mcfg'], ',' ) |
---|
1688 | assert len(ll) == 7, 'Length of model configuration argument must be 7 comma separated integers: %s' % self.adict['mcfg'] |
---|
1689 | lli = [ int(x) for x in ll] |
---|
1690 | |
---|
1691 | self.sc = dreqQuery( dq=self.dq ) |
---|
1692 | self.sc.intersection = self.intersection |
---|
1693 | |
---|
1694 | if 'grdforce' in self.adict: |
---|
1695 | self.sc.gridPolicyForce = self.adict['grdforce'] |
---|
1696 | if 'grdpol' in self.adict: |
---|
1697 | self.sc.gridPolicyDefaultNative = self.adict['grdpol'] == 'native' |
---|
1698 | print ( 'SETTING grid policy: %s' % self.sc.gridPolicyDefaultNative ) |
---|
1699 | if 'allgrd' in self.adict: |
---|
1700 | self.sc.gridPolicyTopOnly = False |
---|
1701 | print ( 'SETTING grid policy for multiple preferred grids: %s' % self.sc.gridPolicyTopOnly ) |
---|
1702 | if 'unique' in self.adict: |
---|
1703 | self.sc.uniqueRequest = True |
---|
1704 | self.sc.gridOceanStructured = not self.adict.get( 'ogrdunstr', False ) |
---|
1705 | |
---|
1706 | if 'mcfg' in self.adict: |
---|
1707 | self.sc.setMcfg( lli ) |
---|
1708 | |
---|
1709 | tierMax = self.adict.get( 't', 1 ) |
---|
1710 | self.sc.setTierMax( tierMax ) |
---|
1711 | pmax = self.adict.get( 'p', 1 ) |
---|
1712 | |
---|
1713 | makeXls = self.adict.get( 'xls', False ) |
---|
1714 | makeTxt = self.adict.get( 'txt', False ) |
---|
1715 | ##doSf = 'SF' in self.adict or 'sf' in self.adict |
---|
1716 | doSf = 'legacy' not in self.adict |
---|
1717 | if doSf: |
---|
1718 | self.adict['sf'] = True |
---|
1719 | assert not ('legacy' in self.adict and 'sf' in self.adict), "Conflicting command line argument, 'legacy' and 'sf': use only one of these" |
---|
1720 | if makeXls or makeTxt or doSf: |
---|
1721 | xlsOdir = self.adict.get( 'xlsdir', 'xls' ) |
---|
1722 | self.sc.checkDir( xlsOdir, 'xls files' ) |
---|
1723 | |
---|
1724 | tabByFreqRealm = self.adict.get( 'xfr', False ) |
---|
1725 | if 'SF' in self.adict: |
---|
1726 | self.sc.gridPolicyDefaultNative = True |
---|
1727 | vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) |
---|
1728 | vs.analAll(pmax) |
---|
1729 | |
---|
1730 | self.sc.gridPolicyDefaultNative = False |
---|
1731 | vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) |
---|
1732 | vs.analAll(pmax) |
---|
1733 | |
---|
1734 | self.sc.setTierMax( 3 ) |
---|
1735 | vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) |
---|
1736 | vs.analAll(3) |
---|
1737 | return |
---|
1738 | |
---|
1739 | ok = True |
---|
1740 | if self.adict['m'] == '_all_': |
---|
1741 | self.adict['m'] = set(self.sc.mips ) |
---|
1742 | mlab = 'TOTAL' |
---|
1743 | else: |
---|
1744 | for i in self.adict['m']: |
---|
1745 | if i not in self.sc.mips: |
---|
1746 | ok = False |
---|
1747 | tt = misc_utils.mdiff().diff( i,self.sc.mips ) |
---|
1748 | assert not tt[0], 'Bad logic ... unexpected return from misc_utils.mdiff' |
---|
1749 | ##cms = difflib.get_close_matches(i,self.sc.mips ) |
---|
1750 | if tt[1] == 0: |
---|
1751 | mlg.prnt ( 'NOT FOUND: %s' % i ) |
---|
1752 | else: |
---|
1753 | msg = [] |
---|
1754 | for ix in tt[2]: |
---|
1755 | msg.append( '%s [%4.1f]' % (','.join( ix[1] ),ix[0]) ) |
---|
1756 | |
---|
1757 | mlg.prnt( '----------------------------------------' ) |
---|
1758 | if tt[1] == 1 and len(tt[2][0][1]) == 1: |
---|
1759 | mlg.prnt ( 'NOT FOUND: %s: SUGGESTION: %s' % (i,msg[0]) ) |
---|
1760 | else: |
---|
1761 | mlg.prnt ( 'NOT FOUND: %s: SUGGESTIONS: %s' % (i,'; '.join( msg ) ) ) |
---|
1762 | mlg.prnt( '----------------------------------------' ) |
---|
1763 | mlab = misc_utils.setMlab( self.adict['m'] ) |
---|
1764 | assert ok,'Available MIPs: %s' % str(self.sc.mips) |
---|
1765 | |
---|
1766 | eid = None |
---|
1767 | ex = None |
---|
1768 | if 'e' in self.adict: |
---|
1769 | ex = self.adict['e'] |
---|
1770 | if ex in self.sc.mipsp: |
---|
1771 | eid = set( self.dq.inx.iref_by_sect[ex].a['experiment'] ) |
---|
1772 | self.sc.exptFilter = eid |
---|
1773 | elif self.adict['e'] in self.sc.exptByLabel: |
---|
1774 | eid = self.sc.exptByLabel[ self.adict['e'] ] |
---|
1775 | self.sc.exptFilter = set( [eid,] ) |
---|
1776 | else: |
---|
1777 | ns = 0 |
---|
1778 | md = misc_utils.mdiff() |
---|
1779 | ttm = md.diff( self.adict['e'],self.sc.mipsp ) |
---|
1780 | tte = md.diff( self.adict['e'],self.sc.exptByLabel.keys() ) |
---|
1781 | if ttm[1] > 0 and tte[1] == 0 or (ttm[2][0][0] > 0.6*tte[2][0][0]): |
---|
1782 | oo = md.prntprep( self.adict['e'], ttm ) |
---|
1783 | for l in oo: |
---|
1784 | mlg.prnt( l ) |
---|
1785 | if tte[1] > 0 and ttm[1] == 0 or (tte[2][0][0] > 0.6*ttm[2][0][0]): |
---|
1786 | oo = md.prntprep( self.adict['e'], tte ) |
---|
1787 | for l in oo: |
---|
1788 | mlg.prnt( l ) |
---|
1789 | assert False, 'Experiment/MIP %s not found' % self.adict['e'] |
---|
1790 | |
---|
1791 | if not self.adict.get( 'esm', False ): |
---|
1792 | ss = set() |
---|
1793 | for e in ['esm-hist','esm-hist-ext','esm-piControl','piControl-spinup','esm-piControl-spinup']: |
---|
1794 | ss.add( self.sc.exptByLabel[ e ] ) |
---|
1795 | self.sc.exptFilterBlack = ss |
---|
1796 | makeXls = self.adict.get( 'xls', False ) |
---|
1797 | |
---|
1798 | if 'sf' in self.adict: |
---|
1799 | vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) |
---|
1800 | vs.run( self.adict['m'], '%s/requestVol_%s_%s_%s' % (xlsOdir,mlab,tierMax,pmax), pmax=pmax, doxlsx=makeXls ) |
---|
1801 | totalOnly = False |
---|
1802 | if len( self.adict['m'] ) == 1 or totalOnly: |
---|
1803 | if makeXls: |
---|
1804 | vsmode='full' |
---|
1805 | else: |
---|
1806 | vsmode='short' |
---|
1807 | vs.anal(olab=mlab,doUnique=False, mode=vsmode, makeTabs=makeXls) |
---|
1808 | for f in sorted( vs.res['vf'].keys() ): |
---|
1809 | mlg.prnt ( 'Frequency: %s: %s' % (f, vs.res['vf'][f]*2.*1.e-12 ) ) |
---|
1810 | ttl = sum( [x for k,x in vs.res['vu'].items()] )*2.*1.e-12 |
---|
1811 | mlg.prnt( 'TOTAL volume: %8.2fTb' % ttl ) |
---|
1812 | self.printListCc(vs.res['vu']) |
---|
1813 | return |
---|
1814 | |
---|
1815 | mips = self.adict['m'] |
---|
1816 | if type(mips) in [type(set()),type(dict())]: |
---|
1817 | mips = self.adict['m'].copy() |
---|
1818 | if len(mips) > 1: |
---|
1819 | if type(mips) == type(set()): |
---|
1820 | mips.add( '*TOTAL' ) |
---|
1821 | else: |
---|
1822 | mips['*TOTAL'] = '' |
---|
1823 | |
---|
1824 | vs.analAll(pmax,mips=mips,html=False,makeTabs=makeXls) |
---|
1825 | thisd = {} |
---|
1826 | for m in sorted( self.adict['m'] ) + ['*TOTAL',]: |
---|
1827 | for f in sorted( vs.rres[m].keys() ): |
---|
1828 | mlg.prnt ( '%s:: Frequency: %s: %s' % (m,f, vs.rres[m][f]*2.*1.e-12 ) ) |
---|
1829 | for m in sorted( self.adict['m'] ) + ['*TOTAL',]: |
---|
1830 | thisd[m] = sum( [x for k,x in vs.rres[m].items()] ) |
---|
1831 | mlg.prnt( '%s:: TOTAL volume: %8.2fTb' % (m, thisd[m]*2.*1.e-12 ) ) |
---|
1832 | self.printListCc(vs.rresu['*TOTAL']) |
---|
1833 | return |
---|
1834 | |
---|
1835 | adsCount = self.adict.get( 'count', False ) |
---|
1836 | |
---|
1837 | self.getVolByMip(pmax,eid,adsCount) |
---|
1838 | makeTxt = self.adict.get( 'txt', False ) |
---|
1839 | if makeXls or makeTxt: |
---|
1840 | mips = self.adict['m'] |
---|
1841 | |
---|
1842 | if 'txtOpts' in self.adict: |
---|
1843 | if self.adict['txtOpts'][0] == 'v': |
---|
1844 | txtOpts = NT_txtopts( 'var' ) |
---|
1845 | else: |
---|
1846 | txtOpts = NT_txtopts( 'cmv' ) |
---|
1847 | else: |
---|
1848 | txtOpts=None |
---|
1849 | |
---|
1850 | self.sc.xlsByMipExpt(mips,eid,pmax,odir=xlsOdir,xls=makeXls,txt=makeTxt,txtOpts=txtOpts) |
---|
1851 | |
---|
1852 | def printListCc(self,cc): |
---|
1853 | if self.adict.get( 'vars', False ): |
---|
1854 | if python2: |
---|
1855 | vl = sorted( cc.keys(), cmp=cmpd(cc).cmp, reverse=True ) |
---|
1856 | else: |
---|
1857 | vl = sorted( cc.keys(), key=lambda x: cc[x], reverse=True ) |
---|
1858 | printLinesMax = self.adict.get( 'plm', 20 ) |
---|
1859 | if printLinesMax > 0: |
---|
1860 | mx = min( [printLinesMax,len(vl)] ) |
---|
1861 | else: |
---|
1862 | mx = len(vl) |
---|
1863 | |
---|
1864 | for k in vl[:mx]: |
---|
1865 | cmv = self.dq.inx.uid[k] |
---|
1866 | print ('%s.%s:: %sTb' % (cmv.mipTable, cmv.label, cc[k]*2.*1.e-12) ) |
---|
1867 | |
---|
1868 | def printList(self): |
---|
1869 | mips = self.adict['m'] |
---|
1870 | ee = {} |
---|
1871 | for i in self.dq.coll['mip'].items: |
---|
1872 | if i.label in mips: |
---|
1873 | ee[i.label] = i |
---|
1874 | if self.adict['l'] in ['o','e']: |
---|
1875 | targ = {'o':'objective', 'e':'experiment' }[self.adict['l']] |
---|
1876 | for k in sorted( ee.keys() ): |
---|
1877 | if targ in self.dq.inx.iref_by_sect[ee[k].uid].a: |
---|
1878 | for u in self.dq.inx.iref_by_sect[ee[k].uid].a[targ]: |
---|
1879 | print ( '%s: %s' % (ee[k].label, self.dq.inx.uid[u].label) ) |
---|
1880 | else: |
---|
1881 | print ('list objective *%s* not recognised (should be e or o)' % self.adict['l'] ) |
---|
1882 | |
---|
1883 | def getVolByMip(self,pmax,eid,adsCount): |
---|
1884 | |
---|
1885 | v0 = self.sc.volByMip( self.adict['m'], pmax=pmax, intersection=self.intersection, adsCount=adsCount, exptid=eid ) |
---|
1886 | mlg.prnt ( 'getVolByMip: %s [%s]' % (v0,misc_utils.vfmt(v0*2.)) ) |
---|
1887 | cc = collections.defaultdict( int ) |
---|
1888 | for e in self.sc.volByE: |
---|
1889 | for v in self.sc.volByE[e][2]: |
---|
1890 | cc[v] += self.sc.volByE[e][2][v] |
---|
1891 | x = 0 |
---|
1892 | for v in cc: |
---|
1893 | x += cc[v] |
---|
1894 | |
---|
1895 | if python2: |
---|
1896 | vl = sorted( cc.keys(), cmp=cmpd(cc).cmp, reverse=True ) |
---|
1897 | else: |
---|
1898 | vl = sorted( cc.keys(), key=lambda x: cc[x], reverse=True ) |
---|
1899 | if self.adict.get( 'vars', False ): |
---|
1900 | printLinesMax = self.adict.get( 'plm', 20 ) |
---|
1901 | if printLinesMax > 0: |
---|
1902 | mx = min( [printLinesMax,len(vl)] ) |
---|
1903 | else: |
---|
1904 | mx = len(vl) |
---|
1905 | |
---|
1906 | for v in vl[:mx]: |
---|
1907 | mlg.prnt ( '%s.%s: %s' % (self.dq.inx.uid[v].mipTable,self.dq.inx.uid[v].label, misc_utils.vfmt( cc[v]*2. ) ) ) |
---|
1908 | if mx < len(vl): |
---|
1909 | mlg.prnt ( '%s variables not listed (use --printLinesMax to print more)' % (len(vl)-mx) ) |
---|
1910 | |
---|