import collections, string import dreq import scope import xlsxwriter import makeTables jsh=''' ''' def vfmt( x ): if x < 1.e9: s = '%sM' % int( x*1.e-6 ) elif x < 1.e12: s = '%sG' % int( x*1.e-9 ) elif x < 1.e13: s = '%3.1fT' % ( x*1.e-12 ) elif x < 1.e15: s = '%3iT' % int( x*1.e-12 ) elif x < 1.e18: s = '%3iP' % int( x*1.e-15 ) else: s = '{:,.2f}'.format( x*1.e-9 ) return s class c1(object): def __init__(self): self.a = collections.defaultdict( int ) class c2(object): def __init__(self): self.a = collections.defaultdict( list ) hmap0 = {'CMIP6':'Historical'} class r1(object): def __init__(self,sc,tiermax=1,pmax=1,only=False): self.pmax=pmax self.tiermax=tiermax self.sc = sc sc.setTierMax( tiermax ) self.cc = collections.defaultdict( c1 ) self.dq = sc.dq self.mips = ['AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'SIMIP', 'VIACSAB'] self.mipsp = ['DECK','CMIP6',] + self.mips[:-4] tabs = makeTables.tables( sc, self.mips ) mipsToDo = self.mips + ['TOTAL',] if only != False: mipsToDo = [only,] for m in mipsToDo: if m == 'TOTAL': l1 = sc.rqiByMip( set( self.mips ) ) else: l1 = sc.rqiByMip( m ) self.cc[m].dd = {} self.cc[m].ee = {} tabs.accReset() for m2 in self.mipsp + ['TOTAL',]: xx = [i for i in self.dq.coll['experiment'].items if i.mip == m2] self.cc[m].ee[m2] = xx ## ## need to check this option, and add a template for a view summarising the experiments for each mip-mip combinations ## ## sss=True not yet tested ## sss = True if sss: for i in xx: tabs.doTable(m,l1,i.uid,pmax,self.cc,acc=False) tabs.doTable(m,l1,m2,pmax,self.cc) if only == False: self.makeMMhtml(m,m2) else: tabs.doTable(m,l1,m2,pmax,self.cc) if only == False: self.writeMips(sss) def makeMMhtml(self,m,m2): """Make a html page for data requested by MIP 'm' from MIP 'm2' experiments""" fss = '%s-%s_%s_%s.html' % (m,m2,self.tiermax, self.pmax) kc = '_%s_%s' % (m,m2) if len( self.cc[kc].a.keys() ) == 0: return oo = open( 'tabs03/%s' % fss, 'w' ) ttl = 'Date requested by %s from %s experiments (tier %s, priority %s)' % (m,m2,self.tiermax,self.pmax) jsh = '' pream = '

%s

\n' % ttl pream += '

All variables in one Excel file

\n' % (m,m2,self.tiermax, self.pmax) pream += '' bdy = pream + '\n' bdy += '\n' if m2 == 'TOTAL': labs = sorted( [x for x in self.cc[m].a.keys() if x in self.sc.exptByLabel] ) else: labs = sorted( [i.label for i in self.cc[m].ee[m2] if (i.label in self.cc[m].a and i._h.label == 'experiment')] ) for ilab in labs: x = self.cc[m].a[ilab]*2. if x > 0: s = vfmt( x ) bdy += '\n' % (ilab,m,ilab,self.tiermax, self.pmax,s) bdy += '
ExperimentVolume (and link to variable lists)
%s%s
\n' oo.write( self.dq.pageTmpl % (ttl, jsh, '../', '../index.html', bdy ) ) oo.close() def writeMips(self,sss=False): oo = open( 'tab01_%s_%s.texfrag' % (self.tiermax,self.pmax), 'w' ) mmh = [] mhdr = [ '\\rot{80}{%s}' % hmap0.get(m,m) for m in self.mipsp + ['TOTAL',]] mhdrh = [ '
%s
' % hmap0.get(m,m) for m in self.mipsp + ['TOTAL',]] oo.write( string.join(['',] + mhdr, ' & ' ) + '\\\\ \n\\hline\n' ) mmh.append( '\n' + string.join(['',] + mhdrh ) + '\n' ) htmltmpl_head = '\n' for m in self.mips + ['TOTAL',]: ll = [m,] llh = [m,] ttl = 0. cct = collections.defaultdict( int ) xt = 0. for m2 in self.mipsp + ['TOTAL',]: if m2 in self.cc[m].dd: oo1 = open( 'html/tt/rq-%s-expt-%s.html' % (m,m2), 'w' ) oo1.write( htmltmpl_head ) oo1.write( '''
\n
\n' ) for k in ks: oo1.write( '
\n' % k ) oo1.write( '
' ) for h in ['Frequency','Table','Label','Title','Description','UID' ]: oo1.write( '' % h ) for t in self.cc[m].dd[m2][k]: oo1.write( '\n\n' ) oo1.write( string.join( ['' % x for x in t ] ) + '\n' ) oo1.write( '
%s
%s
\n' ) oo1.write( '' ) if m2 in self.cc[m].dd or m2 == 'TOTAL': if self.cc[m].a[m2] == 0: ll.append( '' ) llh.append( '' ) else: try: if m2 == 'TOTAL': x = xt else: x = self.cc[m].a[m2]*2. xt += x s = vfmt( x ) kc = '_%s_%s' % (m,m2) if m2 == 'TOTAL': sm = string.join( ['%s: %s' % (k,vfmt(cct[k]*2.)) for k in sorted( cct ) ], '; ' ) print m,cct s1 = '%s' % (sm,s) s = '%s' % s else: for k in self.cc[kc].a.keys(): cct[k] += self.cc[kc].a[k] ll.append( s ) fn = '%s-%s_%s_%s.xlsx' % (m,m2,self.tiermax, self.pmax) sm = string.join( ['%s: %s' % (k,vfmt(self.cc[kc].a[k]*2.)) for k in sorted( self.cc[kc].a.keys() ) ], '; ' ) if m2 == 'TOTAL': #llh.append( s1 ) fn = '%s-%s_%s_%s.xlsx' % (m,'all',self.tiermax, self.pmax) if sss: fss = '%s-%s_%s_%s.html' % (m,m2,self.tiermax, self.pmax) llh.append( '%s' % (sm,fss,s) ) else: llh.append( '%s' % (sm,fn,s) ) except: print ( 'Failed to compute element: %s,%s %s' % (m,m2, str(self.cc[m].a[m2]) ) ) raise else: ll.append( '' ) llh.append( '' ) oo.write( string.join(ll, ' & ' ) + '\\\\ \n\\hline\n' ) mmh.append( '' + string.join(['%s' % x for x in llh] ) + '\n' ) mmh.append( '' ) ttl = 'Data volume overview, upto tier %s and priority %s -- provisional' % (self.tiermax, self.pmax) bdy = '''

%s

Data volumes are estimated for nominal model with 1 degree resolution and 40 levels in the atmosphere and 0.5 degrees with 60 levels in the ocean. The "Requesting MIP" (rows) is the MIP specifying the data required to meet their scientific objectives. The "designing MIP" (columns) is the MIP specifying the experimental design. The figures below represent work in progress: there are still omissions and flaws, more details are on the Data Request home page.

''' % ttl bdy += string.join( mmh, '\n' ) ooh = open( 'tab01_%s_%s.html' % (self.tiermax,self.pmax), 'w' ) ooh.write( self.dq.pageTmpl % (ttl, jsh, './', './index.html', bdy ) ) ooh.close() oo.close() if __name__ == "__main__": sc = scope.dreqQuery() r = r1( sc, tiermax=1, pmax=1 ) r = r1( sc, tiermax=3, pmax=3 )