Changeset 779

Timestamp:

14/10/16 16:42:24 (3 years ago)

Author:

mjuckes

Message:

save

Location:

CMIP6dreqbuild/trunk

Files:

• src/framework/Makefile (modified) (2 diffs)
• src/framework/dreqPy/fgrid.py (modified) (1 diff)
• src/framework/dreqPy/makeTables.py (modified) (10 diffs)
• src/framework/dreqPy/overviewTabs.py (modified) (1 diff)
• src/framework/dreqPy/scope.py (modified) (11 diffs)
• src/framework/dreqPy/volsum.py (modified) (9 diffs)
• src/framework/ingest/expts.py (added)
• srcMisc/sx202.py (added)

CMIP6dreqbuild/trunk/src/framework/Makefile

@@ -65 +65 @@
         cp htmlTemplates.py dreqPy/scope.py dreqPy/scope_utils.py dreqPy/utilities.py dreqPy/utilP3.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy     
         cp dreqPy/utilP2/*.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/utilP2/
-        cp dreqPy/dreqCmdl.py dreqPy/makeTables.py dreqPy/vrev.py dreqPy/__init__.py dreqPy/simpleCheck.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy
+        cp dreqPy/dreqCmdl.py dreqPy/makeTables.py dreqPy/vrev.py dreqPy/fgrid.py dreqPy/overviewTabs.py dreqPy/volsum.py dreqPy/__init__.py dreqPy/simpleCheck.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy
         cp dreqPy/misc_utils.py dreqPy/rvgExtraTable.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy
         cp vocabDemo.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy    
@@ -76 +76 @@
         cp out/supplement_20150731.xml /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/docs/dreqSupp.xml
         cp out/dreqManifest_dist.txt /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/docs/dreqManifest.txt
-        cp out/md5Manifest.txt /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/docs/
+        cp out/md5Manifest.txt out/sfheadings.csv /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/docs/
         cp -r dreqPy/examples /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/
         cp out/BlockSchema.csv /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/docs/

CMIP6dreqbuild/trunk/src/framework/dreqPy/fgrid.py

@@ -48 +48 @@
 
   if len(i4) > 0:
-    print 'SEVERE.cmvgrid.00001: unidentified grids for %s variables' % len(i4)
+    print ( 'SEVERE.cmvgrid.00001: unidentified grids for %s variables' % len(i4) )
     for i in i4:
-      print i.label, i.title, i.prov
+      print ( '%s: %s   %s' % (i.label, i.title, i.prov))
 
   return ee,i4

CMIP6dreqbuild/trunk/src/framework/dreqPy/makeTables.py

@@ -1 +1 @@
 
-import collections, string, os, sys
+import collections, os, sys
 import volsum
 
@@ -51 +51 @@
           mlab = list(m)[0]
         else:
-          mlab=string.join( [ x[:2].lower() for x in m ], '.' )
+          mlab='.'.join( [ x[:2].lower() for x in m ] )
       return mlab
 
@@ -102 +102 @@
 
     if self.txt:
-      self.oo.write( string.join( ['Notes','','Notes on tables'], '\t') + '\n' )
+      self.oo.write( '\t'.join( ['Notes','','Notes on tables']) + '\n' )
       for t in tableNotes:
-        self.oo.write( string.join( ['Notes',] + list(t), '\t') + '\n' )
+        self.oo.write( '\t'.join( ['Notes',] + list(t)) + '\n' )
 
       if collected != None:
-        self.oo.write( string.join( ['Notes','Table','Reference Volume (1 deg. atmosphere, 0.5 deg. ocean)'], '\t') + '\n')
+        self.oo.write( '\t'.join( ['Notes','Table','Reference Volume (1 deg. atmosphere, 0.5 deg. ocean)']) + '\n')
         for k in sorted( collected.keys() ):
-          self.oo.write( string.join( ['Notes',k,vfmt( collected[k]*2. )], '\t') + '\n' )
+          self.oo.write( '\t'.join( ['Notes',k,vfmt( collected[k]*2. )]) + '\n' )
 
   def cmvtabrec(self,j,t,orec):
@@ -117 +117 @@
 
      if self.txt:
-        self.oo.write( string.join( [t,] + orec, '\t') + '\n' )
+        self.oo.write( '\t'.join( [t,] + orec) + '\n' )
 
   def varrec(self,j,orec):
@@ -125 +125 @@
 
      if self.txt:
-        self.oo.write( string.join( orec, '\t') + '\n' )
+        self.oo.write( '\t'.join( orec, '\t') + '\n' )
 
   def var(self):
@@ -355 +355 @@
                  if ii._h.label == 'remarks':
                    ml.append( ['var','struct','time','spatial'][i] )
-              print ( 'makeTables: skipping %s %s: %s' % (t,v.label,string.join( ml, ',')) )
+              print ( 'makeTables: skipping %s %s: %s' % (t,v.label,','.join( ml)) )
               skipped.add( (t,v.label) )
           else:
             dims = []
-            dims += string.split( sshp.dimensions, '|' )
-            dims += string.split( tshp.dimensions, '|' )
-            dims += string.split( strc.odims, '|' )
-            dims += string.split( strc.coords, '|' )
-            dims = string.join( dims )
+            dims +=  sshp.dimensions.split( '|' )
+            dims +=  tshp.dimensions.split( '|' )
+            dims +=  strc.odims.split( '|' )
+            dims +=  strc.coords.split( '|' )
+            dims = ' '.join( dims )
             if mode == 'c':
               orec = [str(v.defaultPriority),cv.title, cv.units, cv.description, v.description, cv.label, cv.sn, strc.cell_methods, v.positive, v.type, dims, v.label, v.modeling_realm, v.frequency, strc.cell_measures, v.prov,v.provNote,str(v.rowIndex),v.uid,v.vid,v.stid,strc.title]
@@ -371 +371 @@
               thismips = c.chkCmv( v.uid )
               thismips2 = c.chkCmv( v.uid, byExpt=True )
-              orec.append( string.join( sorted( list( thismips) ),',') )
-              orec.append( string.join( sorted( list( thismips2) ),',') )
+              orec.append( ','.join( sorted( list( thismips) ) ) )
+              orec.append( ','.join( sorted( list( thismips2) ) ) )
 
             if withoo:
-              oo.write( string.join(orec, '\t' ) + '\n' )
+              oo.write( '\t'.join(orec ) + '\n' )
             j+=1
             wb.cmvtabrec( j,t,orec )
@@ -445 +445 @@
       bdy.append( '</ul></body></html>\n' )
       oo = open( '%s/%s.html' % (self.odir,v.label), 'w' )
-      oo.write( dq.pageTmpl % ( title, '', '../', '../index.html', string.join( bdy, '\n' ) ) )
+      oo.write( dq.pageTmpl % ( title, '', '../', '../index.html', '\n'.join( bdy ) ) )
       oo.close()
       self.anno[v.label] = '<a href="../t/%s.html">Usage</a>' % v.label
@@ -467 +467 @@
         for k in ['sn','ln','u','var','d']:
     
-          if string.find( d[k], '"' ) != -1:
+          if  d[k].fin( '"' ) != -1:
             print ( "WARNING ... quote in %s .. %s [%s]" % (k,var,d[k]) )
-            d[k] = string.replace( d[k], '"', "'" )
+            d[k] =  d[k].replace( '"', "'" )
             print ( d[k] )
         
@@ -631 +631 @@
       ks = sorted( list( cc0.keys() ) )
       if self.verbose:
-        print ('Experiment summary: %s %s' % (mlab,string.join( ['%s: %s' % (k,len(cc0[k])) for k in ks], ', ' ) ) )
+        print ('Experiment summary: %s %s' % (mlab,', '.join( ['%s: %s' % (k,len(cc0[k])) for k in ks] ) ) )
 
       if m2 in [None, 'TOTAL']:

CMIP6dreqbuild/trunk/src/framework/dreqPy/overviewTabs.py

@@ -238 +238 @@
             if m2 == 'TOTAL':
               sm = string.join( ['%s: %s' % (k,vfmt(cct[k]*2.)) for k in sorted( cct ) ], '; ' )
-              print m,cct
+              print ( '%s, %s' % (m,cct) )
               s1 = '<b><span title="%s">%s</span></b>' % (sm,s)
               s = '<b>%s</b>' % s

CMIP6dreqbuild/trunk/src/framework/dreqPy/scope.py

@@ -23 +23 @@
 import sys, os
 
-odsz = {'tau':7, 'scatratio':15, 'effectRadLi|tau':(28,'query pending'), 'vegtype':(8,'free'), 'sza5':5, 'site':(119,'73 for aquaplanet .. '), 'iceband':(5,'free'), 'dbze':15, 'spectband':(10,'free'), 'misrBands':(7,'query pending'), 'effectRadIc|tau':(28,'query pending')}
+odsz = {'landUse':(5,'free'), 'tau':7, 'scatratio':15, 'effectRadLi|tau':(28,'query pending'), 'vegtype':(8,'free'), 'sza5':5, 'site':(119,'73 for aquaplanet .. '), 'iceband':(5,'free'), 'dbze':15, 'spectband':(10,'free'), 'misrBands':(7,'query pending'), 'effectRadIc|tau':(28,'query pending')}
 
 python2 = True
@@ -231 +231 @@
            sf = odsz[i.odims]
         else:
-           print 'SEVERE.odims.00001: no information on dimension size: %s' % i.odims
+           ## print 'SEVERE.odims.00001: no information on dimension size: %s' % i.odims
            sf = 5
         if type( sf ) == type( () ):
@@ -439 +439 @@
     for rl in rqll:
       if 'requestItem' not in self.dq.inx.iref_by_sect[rl.uid].a:
-        print 'WARN.001.00001: no request items for: ',rl.uid, rl.title
+        print ( 'WARN.001.00001: no request items for: %s, %s' % (rl.uid, rl.title) )
       else:
 
@@ -451 +451 @@
             grd = 'native'
           else:
-            print 'INFO.grd.00001: defaulting to grid ..',rl.label,rl.title, rl.uid
+            ## print ( 'INFO.grd.00001: defaulting to grid ..%s, %s, %s' % (rl.label,rl.title, rl.uid) )
             grd = 'DEF'
 
@@ -909 +909 @@
     ss = set()
     for pr in ee:
-      if pr != -1:
-        print 'INFO.preset.00101: ',pr,len(ee[pr])
 ### loop over request  var groups.
       for i in ee[pr]:
@@ -946 +944 @@
                           cc[(i1.vid,e,grd)].add( expys[i.uid][gf][e,grd] )
               else:
-                print 'SKIPPING %s: %s' % (i1.label,i1.vid)
+                print ( 'SKIPPING %s: %s' % (i1.label,i1.vid) )
                 ss.add( i1.vid )
     if includeYears:
@@ -959 +957 @@
         for v,e in l2x:
           if len( l2x[(v,e)].keys() ) == 1:
-             g,val = l2x[(v,e)].items()[0]
+             g,val = list( l2x[(v,e)].items() )[0]
           else:
             if 'native' in l2x[(v,e)].keys():
@@ -1117 +1115 @@
               net += ne
               if len(eset) != ne:
-                print 'WARNING: inconsistency in volume estimate ... possible duplication for %s,%s' % (cmvi,f)
+                print ( 'WARNING: inconsistency in volume estimate ... possible duplication for %s,%s' % (cmvi,f) )
               for e in eset:
                 elab = self.dq.inx.uid[e].label
@@ -1170 +1168 @@
            sf = self.odsz[o]
         else:
-           print 'SEVERE.odims.00006: no information on dimension size: %s' % o
+           # print ( 'SEVERE.odims.00006: no information on dimension size: %s' % o )
            sf = 5
 
@@ -1183 +1181 @@
             szg = self.szss[s]
         except:
-          print 'Failed to get size for: ',g,g1,s
+          print ( 'Failed to get size for: %s, %s, %s' % (g,g1,s ) )
           raise
 
@@ -1512 +1510 @@
     if 'grdpol' in self.adict:
       self.sc.gridPolicyDefaultNative = self.adict['grdpol'] == 'native'
-      print 'SETTING grid policy: ', self.sc.gridPolicyDefaultNative
+      print ( 'SETTING grid policy: %s' % self.sc.gridPolicyDefaultNative )
     if 'allgrd' in self.adict:
       self.sc.gridPolicyTopOnly = False
-      print 'SETTING grid policy for multiple preferred grids: ', self.sc.gridPolicyTopOnly
+      print ( 'SETTING grid policy for multiple preferred grids: %s' % self.sc.gridPolicyTopOnly )
     if 'unique' in self.adict:
       self.sc.uniqueRequest = True

CMIP6dreqbuild/trunk/src/framework/dreqPy/volsum.py

@@ -76 +76 @@
         self.run( m, '%s/requestVol_%s_%s_%s' % (self.odir,m,self.sc.tierMax,pmax), pmax=pmax )
 
-        self.anal(olab=m,doUnique=True)
+        self.anal(olab=m,doUnique=True, makeTabs=True)
         ttl = sum( [x for k,x in self.res['vu'].items()] )*2.*1.e-12
         print ( '%s volume: %8.2fTb' % (m,ttl) )
@@ -121 +121 @@
     return lex, vet, vf, vu, mvol
 
-  def anal(self,olab=None,doUnique=False):
+  def anal(self,olab=None,doUnique=False,makeTabs=False):
     vmt = collections.defaultdict( int )
     vm = collections.defaultdict( int )
@@ -143 +143 @@
       sp = self.sc.dq.inx.uid[k[0]]
       if k not in self.mvol:
-        print '%s missing from mvol: ' % str(k)
+        print ( '%s missing from mvol: ' % str(k) )
       else:
         if checkMvol > 1:
@@ -149 +149 @@
             la = self.sc.dq.inx.uid[u].label
             if self.mvol[k][u] != mvol[k][u]:
-              print 'MISMATCH IN %s (%s): %s:: %s,%s' % (str(k),sp.label,la,mvol[k][u],self.mvol[k][u])
+              print ( 'MISMATCH IN %s (%s): %s:: %s,%s' % (str(k),sp.label,la,mvol[k][u],self.mvol[k][u]) )
           
     for e in lex:
@@ -177 +177 @@
           cct[t] += cc[m][t]
         ss = ss.union( lm[m] )
-        makeTables.makeTab(self.sc.dq, subset=lm[m], dest='%s/cmvmm_%s_%s_%s_%s%s' % (self.odir,olab,m,self.sc.tierMax,self.pmax,self.efnsfx), collected=cc[m])
-
-    if olab != None:
-      makeTables.makeTab(self.sc.dq, subset=ss, dest='%s/cmvmm_%s_%s_%s_%s%s' % (self.odir,olab,'TOTAL',self.sc.tierMax,self.pmax,self.efnsfx), collected=cct)
-      if olab != 'TOTAL' and doUnique:
-        makeTables.makeTab(self.sc.dq, subset=s_lm, dest='%s/cmvmm_%s_%s_%s_%s%s' % (self.odir,olab,'Unique',self.sc.tierMax,self.pmax,self.efnsfx), collected=s_cc)
+        if makeTabs:
+          makeTables.makeTab(self.sc.dq, subset=lm[m], dest='%s/cmvmm_%s_%s_%s_%s%s' % (self.odir,olab,m,self.sc.tierMax,self.pmax,self.efnsfx), collected=cc[m])
+
+    if olab != None and makeTabs:
+        makeTables.makeTab(self.sc.dq, subset=ss, dest='%s/cmvmm_%s_%s_%s_%s%s' % (self.odir,olab,'TOTAL',self.sc.tierMax,self.pmax,self.efnsfx), collected=cct)
+        if olab != 'TOTAL' and doUnique:
+          makeTables.makeTab(self.sc.dq, subset=s_lm, dest='%s/cmvmm_%s_%s_%s_%s%s' % (self.odir,olab,'Unique',self.sc.tierMax,self.pmax,self.efnsfx), collected=s_cc)
 
     cc = collections.defaultdict( dict )
@@ -190 +191 @@
       cc[e][t] = vet[(e,t) ]
     for e in sorted( ve.keys() ):
-      if olab != None:
+      if olab != None and makeTabs:
         el = self.sc.dq.inx.uid[e].label
         makeTables.makeTab(self.sc.dq, subset=lex[e], dest='%s/cmvme_%s_%s_%s_%s%s' % (self.odir,olab,el,self.sc.tierMax,self.pmax,self.efnsfx), collected=cc[e])
@@ -198 +199 @@
         ucc[e][t] = s_vet[(e,t) ]
       for e in sorted( uve.keys() ):
-        if olab != None:
+        if olab != None and makeTabs:
           el = self.sc.dq.inx.uid[e].label
           makeTables.makeTab(self.sc.dq, subset=s_lex[e], dest='%s/cmvume_%s_%s_%s_%s%s' % (self.odir,olab,el,self.sc.tierMax,self.pmax,self.efnsfx), collected=ucc[e])
@@ -207 +208 @@
       cc8[t] += vet[(e,t)]
     for f in sorted( vf.keys() ):
-      print 'Frequency: %s: %s' % (f,vf[f]*2.*1.e-12 )
+      print ( 'Frequency: %s: %s' % (f,vf[f]*2.*1.e-12 ) )
         
   def csvFreqStrSummary(self,mip,pmax=1):
@@ -279 +280 @@
     self.sht.set_column(2,2,4)
     self.sht.set_column(3,3,15)
-    for k in range( (len(oh)-3)/4 ):
+    for k in range( int( (len(oh)-3)/4 ) ):
       self.sht.set_column((k+1)*4,(k+1)*4,4)
       self.sht.set_column((k+1)*4+1,(k+1)*4+1,8)