Changeset 902 for CMIP6dreqbuild
- Timestamp:
- 13/12/16 09:57:21 (4 years ago)
- Location:
- CMIP6dreqbuild/trunk/src/framework
- Files:
-
- 1 added
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
CMIP6dreqbuild/trunk/src/framework/Makefile
r792 r902 66 66 cp dreqPy/utilP2/*.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/utilP2/ 67 67 cp dreqPy/dreqCmdl.py dreqPy/makeTables.py dreqPy/vrev.py dreqPy/fgrid.py dreqPy/overviewTabs.py dreqPy/volsum.py dreqPy/__init__.py dreqPy/simpleCheck.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy 68 cp dreqPy/misc_utils.py dreqPy/ rvgExtraTable.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy68 cp dreqPy/misc_utils.py dreqPy/table_utils.py dreqPy/rvgExtraTable.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy 69 69 cp vocabDemo.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy 70 70 cp compare.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/ -
CMIP6dreqbuild/trunk/src/framework/dreqPy/examples/ex202.py
r736 r902 10 10 sys.exit(0) 11 11 else: 12 print ('Specified directo ty does not exist' )12 print ('Specified directory does not exist' ) 13 13 sys.exit(0) 14 14 else: … … 18 18 sc = scope.dreqQuery() 19 19 p=1 20 sc. xlsByMipExpt('DCPP',None,p,odir='xls')20 sc.volByMip2('DCPP',p, makeTabs=True) -
CMIP6dreqbuild/trunk/src/framework/dreqPy/makeTables.py
r895 r902 8 8 import rvgExtraTable 9 9 import volsum 10 import table_utils 10 11 except: 11 12 import dreqPy.volsum as volsum … … 13 14 import dreqPy.vrev as vrev 14 15 import dreqPy.misc_utils as misc_utils 16 import dreqPy.table_utils as table_utils 15 17 import dreqPy.rvgExtraTable as rvgExtraTable 16 17 18 18 19 def realmFlt( ss ): … … 34 35 return -1 35 36 36 if sys.version_info >= (2,7):37 from functools import cmp_to_key38 oldpython = False39 else:40 oldpython = True41 42 try:43 import xlsxwriter44 except:45 print ('No xlsxwrite: will not make tables ...')46 47 ##NT_txtopts = collections.namedtuple( 'txtopts', ['mode'] )48 49 37 setMlab = misc_utils.setMlab 50 def setMlab_xx( m ):51 if type(m) == type(''):52 if m == '_all_':53 mlab = 'TOTAL'54 else:55 mlab = m56 else:57 ll = sorted( list(m) )58 if len(ll) == 1:59 mlab = list(m)[0]60 else:61 mlab='.'.join( [ x[:2].lower() for x in m ] )62 return mlab63 64 class xlsx(object):65 def __init__(self,fn,xls=True,txt=False,txtOpts=None):66 self.xls=xls67 self.txt=txt68 self.txtOpts = txtOpts69 self.mcfgNote = 'Reference Volume (1 deg. atmosphere, 0.5 deg. ocean)'70 if xls:71 self.wb = xlsxwriter.Workbook('%s.xlsx' % fn)72 self.hdr_cell_format = self.wb.add_format({'text_wrap': True, 'font_size': 14, 'font_color':'#0000ff', 'bold':1, 'fg_color':'#aaaacc'})73 self.hdr_cell_format.set_text_wrap()74 self.sect_cell_format = self.wb.add_format({'text_wrap': True, 'font_size': 14, 'font_color':'#0000ff', 'bold':1, 'fg_color':'#ccccbb'})75 self.sect_cell_format.set_text_wrap()76 self.cell_format = self.wb.add_format({'text_wrap': True, 'font_size': 11})77 self.cell_format.set_text_wrap()78 79 if txt:80 self.oo = open( '%s.csv' % fn, 'w' )81 82 def header(self,tableNotes,collected):83 if self.xls:84 sht = self.newSheet( 'Notes' )85 sht.write( 0,0, '', self.hdr_cell_format )86 sht.write( 0,1, 'Notes on tables', self.hdr_cell_format )87 ri = 088 sht.set_column(0,0,30)89 sht.set_column(1,1,60)90 self.sht = sht91 for t in tableNotes:92 ri += 193 for i in range(2):94 sht.write( ri,i, t[i], self.cell_format )95 96 if collected != None:97 ri += 298 sht.write( ri, 0, 'Table', self.sect_cell_format )99 sht.write( ri, 1, self.mcfgNote, self.sect_cell_format )100 ttl = 0.101 for k in sorted( collected.keys() ):102 ri += 1103 sht.write( ri, 0, k )104 sht.write( ri, 1, vfmt( collected[k]*2. ) )105 ttl += collected[k]106 107 ri += 1108 sht.write( ri, 0, 'TOTAL' )109 sht.write( ri, 1, vfmt( ttl*2. ) )110 111 if self.txt:112 self.oo.write( '\t'.join( ['Notes','','Notes on tables']) + '\n' )113 for t in tableNotes:114 self.oo.write( '\t'.join( ['Notes',] + list(t)) + '\n' )115 116 if collected != None:117 self.oo.write( '\t'.join( ['Notes','Table','Reference Volume (1 deg. atmosphere, 0.5 deg. ocean)']) + '\n')118 for k in sorted( collected.keys() ):119 self.oo.write( '\t'.join( ['Notes',k,vfmt( collected[k]*2. )]) + '\n' )120 121 def cmvtabrec(self,j,t,orec):122 if self.xls:123 for i in range(len(orec)):124 if str( type(orec[i]) ) == "<class 'dreq.dreqItem_CoreAttributes'>":125 self.sht.write( j,i, '', self.cell_format )126 else:127 ##print i, orec[i], type( orec[i] )128 self.sht.write( j,i, orec[i], self.cell_format )129 130 if self.txt:131 self.oo.write( '\t'.join( [t,] + orec) + '\n' )132 133 def varrec(self,j,orec):134 if self.xls:135 for i in range(len(orec)):136 self.sht.write( j,i, orec[i], self.cell_format )137 138 if self.txt:139 self.oo.write( '\t'.join( orec, '\t') + '\n' )140 141 def var(self):142 if self.xls:143 self.sht = self.newSheet( 'var' )144 j = 0145 hrec = ['Long name', 'units', 'description', 'Variable Name', 'CF Standard Name' ]146 if self.xls:147 self.sht.set_column(1,1,40)148 self.sht.set_column(1,2,30)149 self.sht.set_column(1,3,60)150 self.sht.set_column(1,4,40)151 self.sht.set_column(1,5,40)152 153 if self.xls:154 for i in range(len(hrec)):155 self.sht.write( j,i, hrec[i], self.hdr_cell_format )156 157 if self.txt:158 for i in range(len(hrec)):159 self.oo.write( hrec[i] + '\t' )160 self.oo.write( '\n' )161 162 def cmvtab(self,t,addMips,mode='c',tslice=False,byFreqRealm=False):163 if self.xls:164 self.sht = self.newSheet( t )165 j = 0166 ncga = 'NetCDF Global Attribute'167 if mode == 'c':168 hrec = ['Priority','Long name', 'units', 'description', 'comment', 'Variable Name', 'CF Standard Name', 'cell_methods', 'positive', 'type', 'dimensions', 'CMOR Name', 'modeling_realm', 'frequency', 'cell_measures', 'prov', 'provNote','rowIndex','UID','vid','stid','Structure Title','valid_min', 'valid_max', 'ok_min_mean_abs', 'ok_max_mean_abs']169 hcmt = ['Default priority (generally overridden by settings in "requestVar" record)',ncga,'','','Name of variable in file','','','CMOR directive','','','CMOR name, unique within table','','','','','','','','','','CMOR variable identifier','MIP variable identifier','Structure identifier','','','','','']170 if self.xls:171 self.sht.set_column(1,1,40)172 self.sht.set_column(1,3,50)173 self.sht.set_column(1,4,30)174 self.sht.set_column(1,5,50)175 self.sht.set_column(1,6,30)176 self.sht.set_column(1,9,40)177 self.sht.set_column(1,18,40)178 self.sht.set_column(1,19,40)179 else:180 hrec = ['','Long name', 'units', 'description', '', 'Variable Name', 'CF Standard Name', '','', 'cell_methods', 'valid_min', 'valid_max', 'ok_min_mean_abs', 'ok_max_mean_abs', 'positive', 'type', 'dimensions', 'CMOR name', 'modeling_realm', 'frequency', 'cell_measures', 'flag_values', 'flag_meanings', 'prov', 'provNote','rowIndex','UID']181 if addMips:182 hrec.append( 'MIPs (requesting)' )183 hrec.append( 'MIPs (by experiment)' )184 185 if byFreqRealm:186 hrec = ['Table',] + hrec187 hcmt = ['CMOR table',] + hcmt188 if tslice:189 hrec += ['Number of Years','Slice Type','Years']190 hcmt += ['','','']191 192 if self.xls:193 for i in range(len(hrec)):194 self.sht.write( j,i, hrec[i], self.hdr_cell_format )195 if hcmt[i] != '':196 self.sht.write_comment( j,i,hcmt[i])197 198 if self.txt:199 self.oo.write( 'MIP table\t' )200 for i in range(len(hrec)):201 self.oo.write( hrec[i] + '\t' )202 self.oo.write( '\n' )203 self.oo.write( t + '\t' )204 for i in range(len(hrec)):205 if hcmt[i] != '':206 self.oo.write( hcmt[i] + '\t')207 else:208 self.oo.write( '\t')209 self.oo.write( '\n' )210 211 def newSheet(self,name):212 self.worksheet = self.wb.add_worksheet(name=name)213 return self.worksheet214 215 def close(self):216 if self.xls:217 self.wb.close()218 if self.txt:219 self.oo.close()220 221 def vfmt( x ):222 if x < 1.e9:223 s = '%sM' % int( x*1.e-6 )224 elif x < 1.e12:225 s = '%sG' % int( x*1.e-9 )226 elif x < 1.e13:227 s = '%3.1fT' % ( x*1.e-12 )228 elif x < 1.e15:229 s = '%3iT' % int( x*1.e-12 )230 elif x < 1.e18:231 s = '%3iP' % int( x*1.e-15 )232 else:233 s = '{:,.2f}'.format( x*1.e-9 )234 return s235 38 236 39 #priority long name units comment questions & notes output variable name standard name unconfirmed or proposed standard name unformatted units cell_methods valid min valid max mean absolute min mean absolute max positive type CMOR dimensions CMOR variable name realm frequency cell_measures flag_values flag_meanings … … 265 68 266 69 return cmp( 0,0 ) 267 268 def cmpAnnex( x, y ):269 ax = len(x) > 2 and x[:2] == 'em'270 ay = len(y) > 2 and y[:2] == 'em'271 bx = len(x) > 5 and x[:5] in ['CMIP5','CORDE','SPECS']272 by = len(y) > 5 and y[:5] in ['CMIP5','CORDE','SPECS']273 if ax == ay and bx == by:274 return cmp(x,y)275 elif ax:276 if by:277 return cmp(0,1)278 else:279 return cmp(1,0)280 elif ay:281 if bx:282 return cmp(1,0)283 else:284 return cmp(0,1)285 elif bx:286 return cmp(1,0)287 else:288 return cmp(0,1)289 290 if not oldpython:291 kAnnex = cmp_to_key( cmpAnnex )292 kCmpdn = cmp_to_key( cmpdn(['sn','label']).cmp )293 kCmpdnPrl = cmp_to_key( cmpdn(['prov','rowIndex','label']).cmp )294 70 295 71 import re … … 307 83 oo.close() 308 84 309 class makeTab(object):310 def __init__(self, dq, subset=None, mcfgNote=None, dest='tables/test', skipped=set(), collected=None,xls=True,txt=False,txtOpts=None,byFreqRealm=False, tslice=None):311 """txtOpts: gives option to list MIP variables instead of CMOR variables"""312 if subset != None:313 cmv = [x for x in dq.coll['CMORvar'].items if x.uid in subset]314 else:315 cmv = dq.coll['CMORvar'].items316 self.byFreqRealm=byFreqRealm317 318 ixt = collections.defaultdict(list)319 if not byFreqRealm:320 for i in cmv:321 ixt[i.mipTable].append( i.uid )322 else:323 for i in cmv:324 ixt['%s.%s' % (i.frequency,realmFlt( i.modeling_realm) )].append( i.uid )325 326 if oldpython:327 tables = sorted( ixt.keys(), cmp=cmpAnnex )328 else:329 tables = sorted( ixt.keys(), key=kAnnex )330 331 addMips = True332 if addMips:333 c = vrev.checkVar(dq)334 mode = 'c'335 tableNotes = [336 ('Request Version',str(dq.version)),337 ('MIPs (...)','The last two columns in each row list MIPs associated with each variable. The first column in this pair lists the MIPs which are requesting the variable in one or more experiments. The second column lists the MIPs proposing experiments in which this variable is requested. E.g. If a variable is requested in a DECK experiment by HighResMIP, then HighResMIP appears in the first column and DECK in the second')]338 339 wb = xlsx( dest, xls=xls, txt=txt )340 if mcfgNote != None:341 wb.mcfgNote = mcfgNote342 wb.header( tableNotes, collected)343 344 if txtOpts != None and txtOpts.mode == 'var':345 vl = list( set( [v.vid for v in cmv] ) )346 vli = [dq.inx.uid[i] for i in vl]347 if oldpython:348 thisvli = sorted( vli, cmp=cmpdn(['sn','label']).cmp )349 else:350 thisvli = sorted( vli, key=kCmpdn )351 wb.var()352 353 j = 0354 for v in thisvli:355 ###hrec = ['Long name', 'units', 'description', 'Variable Name', 'CF Standard Name' ]356 orec = [v.title, v.units, v.description, v.label, v.sn]357 j += 1358 wb.varrec( j,orec )359 else:360 withoo = False361 for t in tables:362 if withoo:363 oo = open( 'tables/test_%s.csv' % t, 'w' )364 wb.cmvtab(t,addMips,mode='c',tslice=tslice != None,byFreqRealm=byFreqRealm)365 366 j = 0367 if oldpython:368 thiscmv = sorted( [dq.inx.uid[u] for u in ixt[t]], cmp=cmpdn(['prov','rowIndex','label']).cmp )369 else:370 thiscmv = sorted( [dq.inx.uid[u] for u in ixt[t]], key=kCmpdnPrl )371 372 for v in thiscmv:373 cv = dq.inx.uid[ v.vid ]374 strc = dq.inx.uid[ v.stid ]375 if strc._h.label == 'remarks':376 print ( 'ERROR: structure not found for %s: %s .. %s (%s)' % (v.uid,v.label,v.title,v.mipTable) )377 ok = False378 else:379 sshp = dq.inx.uid[ strc.spid ]380 tshp = dq.inx.uid[ strc.tmid ]381 ok = all( [i._h.label != 'remarks' for i in [cv,strc,sshp,tshp]] )382 #[u'shuffle', u'ok_max_mean_abs', u'vid', '_contentInitialised', u'valid_min', u'frequency', u'uid', u'title', u'rowIndex', u'positive', u'stid', u'mipTable', u'label', u'type', u'description', u'deflate_level', u'deflate', u'provNote', u'ok_min_mean_abs', u'modeling_realm', u'prov', u'valid_max']383 384 if not ok:385 if (t,v.label) not in skipped:386 ml = []387 for i in range(4):388 ii = [cv,strc,sshp,tshp][i]389 if ii._h.label == 'remarks':390 ml.append( ['var','struct','time','spatial'][i] )391 print ( 'makeTables: skipping %s %s: %s' % (t,v.label,','.join( ml)) )392 skipped.add( (t,v.label) )393 else:394 dims = []395 dims += sshp.dimensions.split( '|' )396 dims += tshp.dimensions.split( '|' )397 dims += strc.odims.split( '|' )398 dims += strc.coords.split( '|' )399 dims = ' '.join( dims )400 if "qcranges" in dq.inx.iref_by_sect[v.uid].a:401 u = dq.inx.iref_by_sect[v.uid].a['qcranges'][0]402 qc = dq.inx.uid[u]403 ll = []404 for k in ['valid_min', 'valid_max', 'ok_min_mean_abs', 'ok_max_mean_abs']:405 if qc.hasattr(k):406 ll.append( '%s %s' % (qc.__dict__[k],qc.__dict__['%s_status' % k][0]) )407 else:408 ll.append( '' )409 valid_min, valid_max, ok_min_mean_abs, ok_max_mean_abs = tuple( ll )410 else:411 valid_min, valid_max, ok_min_mean_abs, ok_max_mean_abs = ('','','','')412 413 if mode == 'c':414 orec = [str(v.defaultPriority),cv.title, cv.units, cv.description, v.description, cv.label, cv.sn, strc.cell_methods, v.positive, v.type, dims, v.label, v.modeling_realm, v.frequency, strc.cell_measures, v.prov,v.provNote,str(v.rowIndex),v.uid,v.vid,v.stid,strc.title, valid_min, valid_max, ok_min_mean_abs, ok_max_mean_abs]415 else:416 orec = ['',cv.title, cv.units, v.description, '', cv.label, cv.sn, '','', strc.cell_methods, valid_min, valid_max, ok_min_mean_abs, ok_max_mean_abs, v.positive, v.type, dims, v.label, v.modeling_realm, v.frequency, strc.cell_measures, strc.flag_values, strc.flag_meanings,v.prov,v.provNote,str(v.rowIndex),cv.uid]417 418 if byFreqRealm:419 orec = [v.mipTable,] + orec420 421 if addMips:422 thismips = c.chkCmv( v.uid )423 thismips2 = c.chkCmv( v.uid, byExpt=True )424 orec.append( ','.join( sorted( list( thismips) ) ) )425 orec.append( ','.join( sorted( list( thismips2) ) ) )426 427 if tslice != None:428 if v.uid not in tslice:429 orec += ['All', '','']430 elif type( tslice[v.uid] ) == type( 0 ):431 print 'ERROR: unexpected tslice type: %s, %s' % (v.uid, tslice[v.uid] )432 elif len( tslice[v.uid] ) == 2:433 x,priority = tslice[v.uid]434 orec[0] = priority435 else:436 tslab,tsmode,a,b,priority = tslice[v.uid]437 orec[0] = priority438 439 if tsmode[:4] in ['simp','bran']:440 nys = b + 1 - a441 ys = range(a,b+1)442 orec += [str(nys), '',str(ys)]443 elif tsmode[:4] in ['YEAR']:444 nys = a445 ys = b446 orec += [str(nys), '',str(ys)]447 else:448 orec += ['slice', tslab,'']449 450 if withoo:451 oo.write( '\t'.join(orec ) + '\n' )452 j+=1453 wb.cmvtabrec( j,t,orec )454 455 if withoo:456 oo.close()457 wb.close()458 85 459 86 hdr = """ … … 555 182 oo.write( ftr ) 556 183 oo.close() 557 558 559 184 560 185 class styles(object): … … 670 295 ## very messy ... 671 296 ## 672 vs = volsum.vsum( sc, odsz, npy, makeTab, tables )297 vs = volsum.vsum( sc, odsz, npy, table_utils.makeTab, tables ) 673 298 mlab = setMlab( m ) 674 299 vs.run( m, 'requestVol_%s_%s_%s' % (mlab,sc.tierMax,pmax), pmax=pmax ) … … 676 301 677 302 ## collected=summed volumes by table for first page. 678 makeTab( sc.dq, subset=vs.thiscmvset, dest='%s/%s-%s_%s_%s' % (odir,mlab,mlab2,sc.tierMax,pmax), collected=collector[kkc].a,303 table_utils.makeTab( sc.dq, subset=vs.thiscmvset, dest='%s/%s-%s_%s_%s' % (odir,mlab,mlab2,sc.tierMax,pmax), collected=collector[kkc].a, 679 304 mcfgNote=sc.mcfgNote, 680 305 txt=doTxt, xls=doXls, txtOpts=txtOpts ) 681 682 class tables(object):683 def __init__(self,sc, odir='xls',xls=True,txt=False,txtOpts=None):684 self.sc = sc685 self.dq = sc.dq686 ##self.mips = mips687 self.odir = odir688 self.accReset()689 self.doXls = xls690 self.doTxt = txt691 self.txtOpts = txtOpts692 693 def accReset(self):694 self.acc = [0.,collections.defaultdict(int),collections.defaultdict( float ) ]695 696 def accAdd(self,x):697 self.acc[0] += x[0]698 for k in x[2]:699 self.acc[2][k] += x[2][k]700 701 702 def doTable(self,m,l1,m2,pmax,collector,acc=True, mlab=None,exptids=None,cc=None):703 """*acc* allows accumulation of values to be switched off when called in single expt mode"""704 705 self.verbose = False706 if mlab == None:707 mlab = setMlab( m )708 709 cc0 = misc_utils.getExptSum( self.dq, mlab, l1 )710 ks = sorted( list( cc0.keys() ) )711 if self.verbose:712 print ('Experiment summary: %s %s' % (mlab,', '.join( ['%s: %s' % (k,len(cc0[k])) for k in ks] ) ) )713 714 if m2 in [None, 'TOTAL']:715 x = self.acc716 else:717 x = self.sc.volByExpt( l1, m2, pmax=pmax )718 719 ##self.volByExpt( l1, e, pmax=pmax, cc=cc, retainRedundantRank=retainRedundantRank, intersection=intersection, adsCount=adsCount )720 v0 = self.sc.volByMip( m, pmax=pmax, exptid=m2 )721 ####722 if cc==None:723 cc = collections.defaultdict( int )724 for e in self.sc.volByE:725 if self.verbose:726 print ('INFO.mlab.... %s: %s: %s' % ( mlab, e, len( self.sc.volByE[e][2] ) ) )727 for v in self.sc.volByE[e][2]:728 cc[v] += self.sc.volByE[e][2][v]729 xxx = 0730 for v in cc:731 xxx += cc[v]732 ####733 if acc:734 for e in self.sc.volByE:735 self.accAdd(self.sc.volByE[e])736 737 if m2 not in [ None, 'TOTAL']:738 im2 = self.dq.inx.uid[m2]739 ismip = im2._h.label == 'mip'740 mlab2 = im2.label741 742 x0 = 0743 for e in self.sc.volByE:744 if exptids == None or e in exptids:745 x = self.sc.volByE[e]746 if x[0] > 0:747 collector[mlab].a[mlab2] += x[0]748 x0 += x[0]749 else:750 ismip = False751 mlab2 = 'TOTAL'752 x0 = x[0]753 754 if mlab2 == 'TOTAL' and x0 == 0:755 print ( 'no data detected for %s' % mlab )756 757 if x0 > 0:758 #759 # create sum for each table760 #761 xs = 0762 kkc = '_%s_%s' % (mlab,mlab2)763 kkct = '_%s_%s' % (mlab,'TOTAL')764 if m2 in [None, 'TOTAL']:765 x = self.acc766 x2 = set(x[2].keys() )767 for k in x[2].keys():768 i = self.dq.inx.uid[k]769 xxx = x[2][k]770 xs += xxx771 else:772 x2 = set()773 for e in self.sc.volByE:774 if exptids == None or e in exptids:775 x = self.sc.volByE[e]776 x2 = x2.union( set( x[2].keys() ) )777 for k in x[2].keys():778 i = self.dq.inx.uid[k]779 xxx = x[2][k]780 xs += xxx781 if xxx > 0:782 collector[kkc].a[i.mipTable] += xxx783 if ismip:784 collector[kkct].a[i.mipTable] += xxx785 786 ##787 ## One user was getting false error message here, with ('%s' % x0) == ('%s' % xs)788 ##789 if abs(x0 -xs) > 1.e-8*( abs(x0) + abs(xs) ):790 print ( 'ERROR.0088: consistency problem %s %s %s %s' % (m,m2,x0,xs) )791 if x0 == 0:792 print ( 'Zero size: %s, %s' % (m,m2) )793 if len( x[2].keys() ) > 0:794 print ( 'ERROR:zero: %s, %s: %s' % (m,m2,str(x[2].keys()) ) )795 796 if acc and m2 not in [ None, 'TOTAL']:797 collector[mlab].a['TOTAL'] += x0798 799 dd = collections.defaultdict( list )800 lll = set()801 for v in x2:802 vi = self.sc.dq.inx.uid[v]803 if vi._h.label != 'remarks':804 f,t,l,tt,d,u = (vi.frequency,vi.mipTable,vi.label,vi.title,vi.description,vi.uid)805 lll.add(u)806 dd[t].append( (f,t,l,tt,d,u) )807 808 if len( dd.keys() ) > 0:809 collector[mlab].dd[mlab2] = dd810 if m2 not in [ None, 'TOTAL']:811 if im2._h.label == 'experiment':812 dothis = self.sc.tierMax >= min( im2.tier )813 ###814 ### BUT ... there is a treset in the request item .... it may be that some variables are excluded ...815 ### need the variable list itself .....816 ###817 makeTab( self.sc.dq, subset=lll, dest='%s/%s-%s_%s_%s' % (self.odir,mlab,mlab2,self.sc.tierMax,pmax), collected=collector[kkc].a,818 mcfgNote=self.sc.mcfgNote,819 txt=self.doTxt, xls=self.doXls, txtOpts=self.txtOpts )820 306 821 307 styls = styles() … … 849 335 assert os.path.isdir( 'html/u' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below' 850 336 assert os.path.isdir( 'html/index' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below' 851 assert os.path.isdir( 'tables' ), 'Before running this script you need to create a "tables" sub-directory, or edit the makeTab class'337 assert os.path.isdir( 'tables' ), 'Before running this script you need to create a "tables" sub-directory, or edit the table_utils.makeTab class' 852 338 853 339 dq = dreq.loadDreq( htmlStyles=htmlStyle, manifest='out/dreqManifest.txt' ) … … 880 366 dq.makeHtml( annotations={'var':ht.anno}, ttl0='Data Request [%s]' % dreq.version ) 881 367 try: 882 import xlsxwriter 883 mt = makeTab( dq) 368 mt = table_utils.makeTab( dq) 884 369 except: 885 370 print ('Could not make tables ...') -
CMIP6dreqbuild/trunk/src/framework/dreqPy/misc_utils.py
r885 r902 2 2 import logging 3 3 import time 4 5 def vfmt( x ): 6 if x < 1.e9: 7 s = '%sM' % int( x*1.e-6 ) 8 elif x < 1.e12: 9 s = '%sG' % int( x*1.e-9 ) 10 elif x < 1.e13: 11 s = '%3.1fT' % ( x*1.e-12 ) 12 elif x < 1.e15: 13 s = '%3iT' % int( x*1.e-12 ) 14 elif x < 1.e18: 15 s = '%3iP' % int( x*1.e-15 ) 16 else: 17 s = '{:,.2f}'.format( x*1.e-9 ) 18 return s 4 19 5 20 def setMlab( m ): -
CMIP6dreqbuild/trunk/src/framework/dreqPy/scope.py
r895 r902 13 13 if imm == 1: 14 14 from utilities import cmvFilter, gridOptionSort 15 import m akeTables15 import misc_utils 16 16 import fgrid 17 17 import volsum … … 20 20 import dreqPy.fgrid as fgrid 21 21 from dreqPy.utilities import cmvFilter, gridOptionSort 22 import dreqPy.m akeTables as makeTables22 import dreqPy.misc_utils as misc_utils 23 23 24 24 import collections, string, operator … … 965 965 import scope_utils 966 966 mxls = scope_utils.xlsTabs(self,tiermax=self.tierMax,pmax=pmax,xls=xls, txt=txt, txtOpts=txtOpts,odir=odir) 967 mlab = m akeTables.setMlab( m )967 mlab = misc_utils.setMlab( m ) 968 968 mxls.run( m, mlab=mlab ) 969 969 … … 1370 1370 return l2 1371 1371 1372 def volByMip2( self, mip, pmax=2, intersection=False, adsCount=False, exptid=None ):1372 def volByMip2( self, mip, pmax=2, intersection=False, adsCount=False, exptid=None,makeTabs=False, odir='xls'): 1373 1373 vs = volsum.vsum( self, odsz, npy ) 1374 vs.run( mip, 'dummy', pmax=pmax, doxlsx=False ) 1375 vs.anal(olab='dummy', doUnique=False, mode='short', makeTabs=False) 1374 rqf = 'dummy' 1375 vsmode='short' 1376 if makeTabs: 1377 mlab = misc_utils.setMlab( mip ) 1378 rqf = '%s/requestVol_%s_%s_%s' % (odir,mlab,self.tierMax,pmax) 1379 vsmode='full' 1380 vs.run( mip, rqf, pmax=pmax, doxlsx=makeTabs ) 1381 vs.anal(olab='dummy', doUnique=False, mode=vsmode, makeTabs=makeTabs) 1376 1382 self.vf = vs.res['vf'].copy() 1377 1383 for f in sorted( vs.res['vf'].keys() ): … … 1676 1682 if 'SF' in self.adict: 1677 1683 self.sc.gridPolicyDefaultNative = True 1678 vs = volsum.vsum( self.sc, odsz, npy, makeTab=makeTables.makeTab, tables=makeTables.tables,odir=xlsOdir, tabByFreqRealm=tabByFreqRealm )1684 vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) 1679 1685 vs.analAll(pmax) 1680 1686 1681 1687 self.sc.gridPolicyDefaultNative = False 1682 vs = volsum.vsum( self.sc, odsz, npy, makeTab=makeTables.makeTab, tables=makeTables.tables,odir=xlsOdir, tabByFreqRealm=tabByFreqRealm )1688 vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) 1683 1689 vs.analAll(pmax) 1684 1690 1685 1691 self.sc.setTierMax( 3 ) 1686 vs = volsum.vsum( self.sc, odsz, npy, makeTab=makeTables.makeTab, tables=makeTables.tables,odir=xlsOdir, tabByFreqRealm=tabByFreqRealm )1692 vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) 1687 1693 vs.analAll(3) 1688 1694 return … … 1697 1703 ok = False 1698 1704 mlg.prnt ( 'NOT FOUND: %s' % i ) 1699 mlab = m akeTables.setMlab( self.adict['m'] )1705 mlab = misc_utils.setMlab( self.adict['m'] ) 1700 1706 assert ok,'Available MIPs: %s' % str(self.sc.mips) 1701 1707 … … 1722 1728 1723 1729 if 'sf' in self.adict: 1724 vs = volsum.vsum( self.sc, odsz, npy, makeTab=makeTables.makeTab, tables=makeTables.tables,odir=xlsOdir, tabByFreqRealm=tabByFreqRealm )1730 vs = volsum.vsum( self.sc, odsz, npy, odir=xlsOdir, tabByFreqRealm=tabByFreqRealm ) 1725 1731 vs.run( self.adict['m'], '%s/requestVol_%s_%s_%s' % (xlsOdir,mlab,tierMax,pmax), pmax=pmax, doxlsx=makeXls ) 1726 1732 totalOnly = False … … 1792 1798 1793 1799 v0 = self.sc.volByMip( self.adict['m'], pmax=pmax, intersection=self.intersection, adsCount=adsCount, exptid=eid ) 1794 mlg.prnt ( 'getVolByMip: %s [%s]' % (v0,m akeTables.vfmt(v0*2.)) )1800 mlg.prnt ( 'getVolByMip: %s [%s]' % (v0,misc_utils.vfmt(v0*2.)) ) 1795 1801 cc = collections.defaultdict( int ) 1796 1802 for e in self.sc.volByE: … … 1813 1819 1814 1820 for v in vl[:mx]: 1815 mlg.prnt ( '%s.%s: %s' % (self.dq.inx.uid[v].mipTable,self.dq.inx.uid[v].label, m akeTables.vfmt( cc[v]*2. ) ) )1821 mlg.prnt ( '%s.%s: %s' % (self.dq.inx.uid[v].mipTable,self.dq.inx.uid[v].label, misc_utils.vfmt( cc[v]*2. ) ) ) 1816 1822 if mx < len(vl): 1817 1823 mlg.prnt ( '%s variables not listed (use --printLinesMax to print more)' % (len(vl)-mx) ) -
CMIP6dreqbuild/trunk/src/framework/dreqPy/volsum.py
r895 r902 6 6 import dreq 7 7 import misc_utils 8 import table_utils 8 9 import overviewTabs 9 10 except: 10 11 import dreqPy.dreq as dreq 11 12 import dreqPy.misc_utils as misc_utils 13 import dreqPy.table_utils as table_utils 12 14 import dreqPy.overviewTabs as overviewTabs 13 15 … … 44 46 45 47 class vsum(object): 46 def __init__(self,sc,odsz,npy, makeTab=None,tables=None,exptFilter=None, odir='xls', tabByFreqRealm=False):48 def __init__(self,sc,odsz,npy,exptFilter=None, odir='xls', tabByFreqRealm=False): 47 49 self.tabByFreqRealm = tabByFreqRealm 48 self.makeTab = makeTab49 self.mt_tables = tables50 50 idir = dreq.DOC_DIR 51 51 self.sc = sc … … 125 125 self.uniqueCmv = {} 126 126 if html: 127 r1 = overviewTabs.r1( self.sc, self.mt_tables, pmax=pmax, vols=( volsmm, volsme, volsmmt,volsue ) )127 r1 = overviewTabs.r1( self.sc, table_utils.mt_tables, pmax=pmax, vols=( volsmm, volsme, volsmmt,volsue ) ) 128 128 129 129 def _analSelectedCmv(self,cmv): … … 212 212 vm[ee.mip] += vet[(e,t)] 213 213 ve[e] += vet[(e,t)] 214 215 214 ## 216 215 ## makeTab needs: cc[m]: volume summary, by table, lm[m]: list of CMOR variables … … 227 226 ss = ss.union( lm[m] ) 228 227 if makeTabs: 229 self.makeTab(self.sc.dq, subset=lm[m], dest=self.xlsDest('m',olab,m), collected=cc[m])228 table_utils.makeTab(self.sc.dq, subset=lm[m], dest=self.xlsDest('m',olab,m), collected=cc[m]) 230 229 231 230 if olab != None and makeTabs: 232 self.makeTab(self.sc.dq, subset=ss, dest=self.xlsDest('m',olab,'TOTAL'), collected=cct)231 table_utils.makeTab(self.sc.dq, subset=ss, dest=self.xlsDest('m',olab,'TOTAL'), collected=cct) 233 232 if olab != 'TOTAL' and doUnique: 234 self.makeTab(self.sc.dq, subset=s_lm, dest=self.xlsDest('m',olab,'Unique'), collected=s_cc)233 table_utils.makeTab(self.sc.dq, subset=s_lm, dest=self.xlsDest('m',olab,'Unique'), collected=s_cc) 235 234 236 235 cc = collections.defaultdict( dict ) … … 247 246 tslice[v] = self.sc.cmvts[v][e] 248 247 dest = self.xlsDest('e',olab,el) 249 self.makeTab(self.sc.dq, subset=lex[e], dest=self.xlsDest('e',olab,el), collected=cc[e],byFreqRealm=self.tabByFreqRealm, tslice=tslice)248 table_utils.makeTab(self.sc.dq, subset=lex[e], dest=self.xlsDest('e',olab,el), collected=cc[e],byFreqRealm=self.tabByFreqRealm, tslice=tslice) 250 249 ##self.makeTab(self.sc.dq, subset=lex[e], dest=self.xlsDest('e',olab,el), collected=cc[e],byFreqRealm=self.tabByFreqRealm) 251 250 … … 256 255 if olab != None and makeTabs: 257 256 el = self.sc.dq.inx.uid[e].label 258 self.makeTab(self.sc.dq, subset=s_lex[e], dest=self.xlsDest('u',olab,el), collected=ucc[e])257 table_utils.makeTab(self.sc.dq, subset=s_lex[e], dest=self.xlsDest('u',olab,el), collected=ucc[e]) 259 258 260 259 self.res = { 'vmt':vmt, 'vet':vet, 'vm':vm, 'uve':uve, 've':ve, 'lm':lm, 'lex':lex, 'vu':vu, 'cc':cc, 'cct':cct, 'vf':vf} … … 322 321 self.mip = mip 323 322 orecs, crecs = self.csvFreqStrSummary(mip,pmax=pmax) 324 print 'csvFreqStrSummary: %s, %s: %s, %s' % (str(mip),pmax,len(orecs),len(crecs))325 323 if not doxlsx: 326 324 return
Note: See TracChangeset
for help on using the changeset viewer.