Changeset 554 for CMIP6dreqbuild

Timestamp:

19/01/16 21:19:26 (6 years ago)

Author:

mjuckes

Message:

further debugging on overview tables; per expt view

Location:

CMIP6dreqbuild/trunk/src/framework

Files:

• Makefile (modified) (2 diffs)
• dreqPy/scope.py (modified) (3 diffs)
• makeTables.py (modified) (2 diffs)
• overviewTabs.py (modified) (7 diffs)

CMIP6dreqbuild/trunk/src/framework/Makefile

@@ -74 +74 @@
 
 localHtml:
+        cp -r tab2/* /var/www/f/data/tabs02/
+        cp -r tabs03/* /var/www/f/tabs03/
+        cp tab01_1_1.html tab01_3_3.html /var/www/f
         cp data2.js /var/www/f/data/mipVarsData.js
         cp websiteExtras/dreq.css /var/www/f/css
@@ -81 +84 @@
         cp -r html/index.html /var/www/f/
 
+
 sHtml:
         zip -r html.zip data2.js html 

CMIP6dreqbuild/trunk/src/framework/dreqPy/scope.py

@@ -327 +327 @@
          if inx.uid[u]._h.label != 'remarks':
            rql.add( u ) 
+    else:
+      exi = self.dq.inx.uid[ex]
+      if exi._h.label == 'experiment':
+        exset = set( [ex,exi.egid,exi.mip] )
 
 #####
@@ -442 +446 @@
         if i.esid in exset and v in e[i.rlid]:
           ix = inx.uid[i.esid]
-          if ix._h.label == 'experiment':
+          if exi._h.label == 'experiment':
+            if ex in self.rqiExp[i.uid][1]:
+              this = self.rqiExp[i.uid][1][ex]
+              cc2[i.esid].add( this[-1]*this[-2] )
+          elif ix._h.label == 'experiment':
             cc2[i.esid].add( self.rqiExp[i.uid][irqi] )
           else:
             if 'experiment' in inx.iref_by_sect[i.esid].a:
               for u in inx.iref_by_sect[i.esid].a['experiment']:
-                cc2[u].add( self.rqiExp[i.uid][irqi] )
+                if u in self.rqiExp[i.uid][1]:
+                  this = self.rqiExp[i.uid][1][u]
+                  cc2[u].add( this[-1]*this[-2] )
+                ###cc2[u].add( self.rqiExp[i.uid][irqi] )
             ##else:
               ##print 'WARNING .... empty experiment set'
@@ -563 +574 @@
         if i._h.label != 'experiment':
           mlg.prnt ( 'ERROR: %s, %s, %s ' % ( u,i._h.label, i.label, i.title ) )
-      dat = [ (i.ntot, i.yps, i.ensz, i.tier, i.nstart, filter1(i.yps,rqi.nymax), filter2(i.ensz,rqi.nenmax,i.tier,self.tierMax) ) for i in e]
-      nytot = sum( [x[-2]*x[-1] for x in dat ] )
-      netot = sum( [x[-1] for x in dat ] )
+      ##dat = [ (i.ntot, i.yps, i.ensz, i.tier, i.nstart, filter1(i.yps,rqi.nymax), filter2(i.ensz,rqi.nenmax,i.tier,self.tierMax) ) for i in e]
+      dat2 = {}
+      for i in e:
+        dat2[i.uid] = (i.ntot, i.yps, i.ensz, i.tier, i.nstart, filter1(i.yps,rqi.nymax), filter2(i.ensz,rqi.nenmax,i.tier,self.tierMax) )
+      ### number of 
+      nytot = sum( [dat2[x][-2]*dat2[x][-3] for x in dat2 ] )
+      netot = sum( [dat2[x][-1] for x in dat2 ] )
       ##print 'debug1:: ',dat, nytot, netot
     else:
-      dat = [ (0,0,0,0,0) ]
+      dat2 = {}
       nytot = 0
       netot = 0
     
-    return (expts, dat, nytot, netot )
+    return (expts, dat2, nytot, netot )
 
   def setTierMax( self, tierMax ):

CMIP6dreqbuild/trunk/src/framework/makeTables.py

@@ -89 +89 @@
       
 class makeTab(object):
-  def __init__(self, subset=None, dest='tables/test.xlsx'):
+  def __init__(self, subset=None, dest='tables/test.xlsx', skipped=set()):
     if subset != None:
       cmv = [x for x in dq.coll['CMORvar'].items if x.uid in subset]
@@ -163 +163 @@
 
           if not ok:
-            print 'skipping %s %s' % (t,v.label)
+            if (t,v.label) not in skipped:
+              print 'makeTables: skipping %s %s' % (t,v.label)
+              skipped.add( (t,v.label) )
           else:
             dims = []

CMIP6dreqbuild/trunk/src/framework/overviewTabs.py

@@ -20 +20 @@
             elif x < 1.e15:
               s = '%3iT' % int( x*1.e-12 )
+            elif x < 1.e18:
+              s = '%3iP' % int( x*1.e-15 )
             else:
               s = '{:,.2f}'.format( x*1.e-9 ) 
@@ -57 +59 @@
 ## sss=True not yet tested
 ##
-        sss = False
+        sss = True
         if sss:
           for i in xx:
-            self.doTable(m,l1,i.uid,self.cc)
+            self.doTable(m,l1,i.uid,self.cc,acc=False)
+          self.makeMMhtml(m,m2)
+          self.doTable(m,l1,m2,self.cc)
         else:
           self.doTable(m,l1,m2,self.cc)
 
-    self.writeMips()
-
-  def writeMips(self):
+    self.writeMips(sss)
+
+  def makeMMhtml(self,m,m2):
+    fss = '%s-%s_%s_%s.html' % (m,m2,self.tiermax, self.pmax)
+    oo = open( 'tabs03/%s' % fss, 'w' )
+    ttl = 'Date requested by %s from %s experiments' % (m,m2)
+    jsh = ''
+    bdy = '<p>All variables in one <a href="../data/tabs02/%s-%s_%s_%s.xlsx">Excel file</a></p>\n' % (m,m2,self.tiermax, self.pmax)
+
+    bdy += '<table>\n'
+    for i in self.cc[m].ee[m2]:
+      x = self.cc[m].a[i.uid]*2.
+      s = vfmt( x )
+      bdy += '<tr><td>%s</td><td><a href="../data/tabs02/%s-%s_%s_%s.xlsx">%s</a></td></tr>\n' % (i.label,m,i.uid,self.tiermax, self.pmax,s)
+    bdy += '</table>\n'
+
+    oo.write( self.dq.pageTmpl % (ttl, jsh, '../', '../index.html', bdy ) )
+    oo.close()
+    
+    
+
+  def writeMips(self,sss=False):
 
     oo = open( 'tab01_%s_%s.texfrag' % (self.tiermax,self.pmax), 'w' )
@@ -102 +125 @@
         oo1.write( '</body></html>' )
         
-        
        if m2 in self.cc[m].dd or m2 == 'TOTAL':
         if self.cc[m].a[m2] == 0:
@@ -117 +139 @@
             fn = '%s-%s_%s_%s.xlsx' % (m,m2,self.tiermax, self.pmax)
             kc = '_%s_%s' % (m,m2)
-            sm = string.join( ['%s: %s' % (k,vfmt(self.cc[kc].a[k])) for k in sorted( self.cc[kc].a.keys() ) ], '; ' )
+            sm = string.join( ['%s: %s' % (k,vfmt(self.cc[kc].a[k]*2.)) for k in sorted( self.cc[kc].a.keys() ) ], '; ' )
             if m2 == 'TOTAL':
               llh.append( s1 )
+            elif sss:
+              fss = '%s-%s_%s_%s.html' % (m,m2,self.tiermax, self.pmax)
+              llh.append( '<a title="Excel variable list: %s" href="tabs03/%s">%s</a>' % (sm,fss,s) )
             else:
               llh.append( '<a title="Excel variable list: %s" href="data/tabs02/%s">%s</a>' % (sm,fn,s) )
@@ -128 +153 @@
           ll.append( '' )
           llh.append( '' )
-      print ll
       oo.write( string.join(ll, ' & ' ) + '\\\\ \n\\hline\n' )
       mmh.append( '<tr>' + string.join(['<td>%s</td>' % x for x in llh] ) + '</tr>\n' )
@@ -143 +167 @@
     oo.close()
 
-  def doTable(self,m,l1,m2,collector):
+  def doTable(self,m,l1,m2,collector,acc=True):
+        """acc allows accumulation of values to be switched off when called in single expt mode"""
         
-        x = sc.volByExpt( l1, m2, expFullEx=True,pmax=self.pmax )
+        x = sc.volByExpt( l1, m2, expFullEx=(m2 in self.mips), pmax=self.pmax )
         collector[m].a[m2] += x[0]
 #
 # create sum for each table
 #
+        xs = 0
         for k in x[2].keys():
           i = self.dq.inx.uid[k]
           xxx =  x[2][k]
+          xs += xxx
           if xxx > 0:
             collector['_%s_%s' % (m,m2)].a[i.mipTable] += xxx
+        assert x[0] == xs, 'ERROR.0088: consistency problem %s  %s %s %s' % (m,m2,x[0],xs)
         if m == 'FAFMIP' and m2 == 'DECK':
           print 'FAFMIP [1]:: ',x[1].keys()
           print 'FAFMIP [2]:: ',collector['_FAFMIP_DECK'].a.keys()
 
-        collector[m].a['TOTAL'] += x[0]
+        if acc:
+          collector[m].a['TOTAL'] += x[0]
         dd = collections.defaultdict( list )
         lll = set()
@@ -171 +200 @@
           collector[m].dd[m2] = dd
           makeTables.makeTab( subset=lll, dest='tab2/%s-%s_%s_%s.xlsx' % (m,m2,self.tiermax,self.pmax) )
-         
 
 if __name__ == "__main__":
   sc = scope.dreqQuery()
-  #r = r1( sc, tiermax=1, pmax=1 )
+  r = r1( sc, tiermax=1, pmax=1 )
   r = r1( sc, tiermax=3, pmax=3 )