Changeset 547

Timestamp:

09/01/16 07:55:30 (5 years ago)

Author:

mjuckes

Message:

updates to xlst etc

Location:

CMIP6dreqbuild/trunk

Files:

• docs/dreq2.xls (modified) (previous)
• docs/vocabFrameworkSchema.xsd (modified) (2 diffs)
• docs/xlst_xdsSchemaGen.xml (modified) (2 diffs)
• srcMisc/dreqSX.py (modified) (5 diffs)
• srcMisc/dreq_consol_dreq.py (modified) (6 diffs)
• srcMisc/dreq_utils.py (modified) (7 diffs)
• srcMisc/sx2.py (modified) (5 diffs)

CMIP6dreqbuild/trunk/docs/vocabFrameworkSchema.xsd

@@ -32 +32 @@
   <xs:attribute name="superclass" type="xs:string"/>
   <xs:attribute name="usage" type="xs:string"/>
+  <xs:attribute name="required" type="xs:string"/>
 </xs:complexType>
 
@@ -46 +47 @@
   <xs:attribute name="labUnique" type="st__labUnique_enum"/>
   <xs:attribute name="itemLabelMode" type="st__itemLabelMode_enum"/>
+  <xs:attribute name="required" type="xs:string"/>
 </xs:complexType>
 

CMIP6dreqbuild/trunk/docs/xlst_xdsSchemaGen.xml

@@ -62 +62 @@
   </xs:simpleType><xsl:text>&#xa;</xsl:text>
 
+<xs:simpleType name="st__integerList">
+<xs:restriction base="aa:base__integerList">
+     <xs:minLength value="0"/>
+</xs:restriction>
+</xs:simpleType>
+
+<xs:simpleType name="base__integerList">
+    <xs:list itemType="xs:integer"/>
+</xs:simpleType><xsl:text>&#xa;</xsl:text>
+
+<xs:simpleType name="st__integerListMonInc">
+  <xs:restriction base="aa:st__integerList" />
+</xs:simpleType><xsl:text>&#xa;</xsl:text>
 
 <xs:simpleType name="st__floatList">
@@ -71 +84 @@
 <xs:simpleType name="base__floatList">
     <xs:list itemType="xs:float"/>
-</xs:simpleType>
+</xs:simpleType><xsl:text>&#xa;</xsl:text>
 
   <xs:complexType name="ct__info" mixed="true">

CMIP6dreqbuild/trunk/srcMisc/dreqSX.py

@@ -9 +9 @@
 shvg = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'r' )
 shnv = shelve.open( 'dreq_consol_tables_nv_shelve_v%s' % vdate, 'r' )
+shnvo = shelve.open( 'dreq_consol_tables_nvom_shelve_v%s' % vdate, 'r' )
 ## from vlsc5b --- includes new names from OMIP
 shold = shelve.open( 'dreq_consol_tables_old_shelve_v20150713', 'r' )
@@ -43 +44 @@
 ee0 = {}
 eenv = {}
+eenvo = {}
 ## dictionary to carry short names of old variables.
 eeold = {}
@@ -49 +51 @@
 for k in shnv:
   eenv[k] = shnv[k]
+for k in shnvo:
+  eenvo[k] = shnvo[k]
 ccl = collections.defaultdict( int )
 for k in shvg:
@@ -129 +133 @@
 ## check cmip5 vs. standard output
 s1 = Set()
+cmip5ByVar = collections.defaultdict( list )
 for k in ee_miptables_vars_old.keys():
   if k[:6] == 'CMIP5_':
@@ -152 +157 @@
   if not s in s1:
     print 'ERROR.010.0002: Table %s not in old mip tables' % s
+  for v in cmip5so.so[s].a.keys():
+    cmip5ByVar[v].append(s)
+    
 
 for s in shlist:

CMIP6dreqbuild/trunk/srcMisc/dreq_consol_dreq.py

@@ -361 +361 @@
     return k
 
-  def addItem( self, mip, tab, nn, nexmax, nenmax, nymax, expt=None, rid=None ):
+  def addItem( self, mip, tab, nn, nexmax, nenmax, nymax, expt=None, rid=None, treset=None ):
     if not self.tabindx.has_key( tab ):
       self.tabindx[tab] = len(self.tablist)
@@ -368 +368 @@
     self.tablist[ix][mip] = nn
     self.mips[mip] += 1
-    self.records.append( (mip,tab,expt, rid,nn, nexmax, nenmax, nymax) )
+    self.records.append( (mip,tab,expt, rid,nn, nexmax, nenmax, nymax,treset) )
 
   def show(self,oo):
@@ -514 +514 @@
                 if ok:
                   nbl = True
-          for je in r4i.othix:
-                ok = self.readDeckColPair( j, je, 100, expt=string.strip(r4[je+1]), rid=uid,mode=3 )
+          for jei in range( len(r4i.othix) ):
+                je = r4i.othix[jei]
+                tr = r4i.treset[jei]
+                ok = self.readDeckColPair( j, je, 100, expt=string.strip(r4[je+1]), rid=uid,mode=3,treset=tr )
                 if ok:
                   nbl = True
@@ -524 +526 @@
     return True
 
-  def readDeckColPair( self,j, ix, nydef, expt=None, rid=None, mode=2 ):
+  def readDeckColPair( self,j, ix, nydef, expt=None, rid=None, mode=2, treset=None ):
       assert mode in [2,3], 'Only modes 2,3 supported, not mode=%s' % mode
       this = map( lambda x: x.value, self.wk1.currentSi.row(j)[ix:ix+mode] )
@@ -634 +636 @@
             tab = '%s!!%s' % (self.mip,tab)
 
-        self.parent.addItem( self.mip, tab, ntot, nexmax, nenmax, nymax, expt=expt, rid=rid )
+        self.parent.addItem( self.mip, tab, ntot, nexmax, nenmax, nymax, expt=expt, rid=rid, treset=treset )
         return True
       except:
@@ -747 +749 @@
 sh['__info__'] = { 'label':'requestItems', 'title':'Specification for a single experiment' , \
      'comment':'Currently only has number of years -- need to include more details' }
-sh['__cols__'] = [ 'mip', 'tab', 'expt','rlid','ny', 'nexmax', 'nenmax', 'nymax']
+sh['__cols__'] = [ 'mip', 'tab', 'expt','rlid','ny', 'nexmax', 'nenmax', 'nymax','treset']
 ##('AerChemMIP', u'Omon_3d', u'CMIP6 historical', 'ad74be9a-26ef-11e5-8d9b-ac72891c3257', 0.0)
 for i in rqs.records:

CMIP6dreqbuild/trunk/srcMisc/dreq_utils.py

@@ -18 +18 @@
         comment = ''
         try:
-          tier = int( r[5] )
+          if type(r[5]) == type(1.):
+            tier = int( r[5] )
+          else:
+            tier = [int( x) for x in string.split( r[5] )]
         except:
           print 'ERROR.001.0001: tier not integer: %s, %s' % (mip,r[5])
@@ -29 +32 @@
         for j in [6,11,12,13]:
           try:
-            ll.append( int( r[j] ) )
+            if type(r[j]) == type(1.):
+              v = int( r[j] )
+            else:
+              v = [int( x) for x in string.split( r[j] )]
+            ll.append( v )
           except:
             k = {6:'nstart', 11:'years per sim', 12:'ensemble size', 13:'total number of years'}[j]
@@ -127 +134 @@
 
   def __init__(self,dd=None):
-    self.nt = collections.namedtuple( 'r4info', ['ixcntl','ixh','ix0','ixm','ownix','ownhr','othix','othhr','mode'] )
+    self.nt = collections.namedtuple( 'r4info', ['ixcntl','ixh','ix0','ixm','ownix','ownhr','othix','othhr','mode','treset'] )
 
     if dd != None:
@@ -188 +195 @@
     self.ownex = []
     self.othex = []
+    self.othetrst = []
     for j in range(self.ix0,self.ixm,2):
       if string.strip(r4[j]) != '':
@@ -194 +202 @@
       if string.strip(r4[j+1]) != '':
         self.othex.append(j)
+        if r4[j+2] != '':
+          self.othetrst.append(int(r4[j+2]))
+        else:
+          self.othetrst.append('')
     
     print mip, self.ixh, self.iccntl, self.ix0, self.ixm , self.ownex, self.othex
@@ -199 +211 @@
     othhr =  map(lambda x: r4[x+1], self.othex)
     print ownhr, othhr
-    self.r4info = self.nt( self.iccntl, self.ixh, self.ix0, self.ixm, self.ownex, ownhr, self.othex, othhr,mode )
+    self.r4info = self.nt( self.iccntl, self.ixh, self.ix0, self.ixm, self.ownex, ownhr, self.othex, othhr,mode,self.othetrst )
     ##( 'r4info', ['ixcntl','ixh','ix0','ixm','ownix','ownhr','othix','othhr','mode'] )
 
@@ -206 +218 @@
 class parseShb(object):
   def __init__(self,vdate,sx,cmip5gplk):
+    """Parse **'dreq_consol_tables_reviewed_b_v%s' % vdate** from dreq_consol_tables.py"""
     shb = shelve.open( 'dreq_consol_tables_reviewed_b_v%s' % vdate, 'r' )
     revTabIds = Set( shb.keys()[:] )

CMIP6dreqbuild/trunk/srcMisc/sx2.py

@@ -393 +393 @@
 sh = shelve.open( '%s/cmipVars' % odir, 'n' )
 sh['__info__'] = { 'label':'cmipVars', 'title':'CMIP variables', 'prov':'sx.eenv and sx.eeold' }
-sh['__cols__'] = ['label','title','sn','units','description','procnote','procComment','prov','priority0']
+sh['__cols__'] = ['label','title','sn','units','description','procnote','procComment','prov','priority0','realm0']
 
 ee = {}
@@ -400 +400 @@
   ##sh[k] = { 'label':this.v, 'title':this.l, 'sn':this.s, 'units':this.u, 'description':this.c, 'procnote':'', 'procComment':'','prov':'CMIP6 endorsement' }
   prov = 'CMIP6 endorsement [%s]' % sx.eenv[k][1]
-  sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',prov, sx.eenv[k][0] ]
+  sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',prov, sx.eenv[k][0], sx.eenv[k][8] ]
   ee[this.u] = k
   if this.u == 'atmos':
@@ -412 +412 @@
   this = nt_vrecix._make( map( lambda x: sx.eeold[k][x], vrecix_old ) )
   ##sh[k] = { 'label':this.v, 'title':this.l, 'sn':this.s, 'units':this.u, 'description':this.c, 'procnote':'', 'procComment':'','prov':sx.eeold[k][6]}
-  sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',sx.eeold[k][6], 101 ]
+  if this.v in sx.cmip5ByVar.keys():
+    s = set()
+    for t in sx.cmip5ByVar[this.v]:
+      s.add( sx.cmip5so.so[t].a[this.v][18] )
+    s = list(s)
+    rlm0 = string.join( s )
+  elif this.v in sx.eenvo:
+    rlm0 = sx.eenvo[this.v][0][8]
+    print 'INFO.llll: using eenvo .....',this.v
+  else:
+    rlm0 = '__na__'
+      
+  sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',sx.eeold[k][6], 101, rlm0 ]
   nk += 1
 
@@ -1220 +1232 @@
 
 
- wb = utils_wb.workbook( 'CMIP6DataRequest_ConsolidatedExperiments_20151028.xls' )
+ ##wb = utils_wb.workbook( 'CMIP6DataRequest_ConsolidatedExperiments_20151028.xls' )
+ wb = utils_wb.workbook( 'CMIP6DataRequest_ConsolidatedExperiments_20160110.xls' )
  sht = wb.book.sheet_by_name( 'Experiments' )
  p = dreq_utils.prconsolexpt()
@@ -1245 +1258 @@
      eegg[gp] = str( uuid.uuid1() )
   
-   eetm[gp].add( r[5] )
+   if type( r[5] ) == type( 1 ):
+     eetm[gp].add( r[5] )
+   else:
+     eetm[gp].add( r[5][-1] )
    eent[gp] += r[11]
    label = labcoerce( r[0] )