Changeset 756 for CMIP6dreqbuild

Timestamp:

22/09/16 14:13:08 (3 years ago)

Author:

mjuckes

Message:

update

Location:

CMIP6dreqbuild/trunk/src/workbook

Files:

• importWbMods.py (modified) (5 diffs)
• importWbMods.pyc (modified) (previous)
• inputs/spatialShape_ref.xls (modified) (previous)
• inputs/strUidMap.txt (modified) (1 diff)
• inputs/structRef.xls (modified) (previous)
• ivg.py (modified) (4 diffs)

CMIP6dreqbuild/trunk/src/workbook/importWbMods.py

@@ -4 +4 @@
 
 nt__varGroup = collections.namedtuple( 'varGroup', ['uid','label','title','mip','ref','refNote'] )
+
+def cellMethodCheck(cm):
+      rv = 0
+      if cm.find( '  ' ) != -1:
+        cm = cm.replace( '  ', ' ' )
+        rv = 1
+
+      if cm.find( "area: where" ) != -1:
+        cm = cm.replace( "area: where", "area: mean where" )
+        rv = 1
+
+      if cm.find( "time:mean" ) != -1:
+        cm = cm.replace( "time:mean", "time: mean" )
+        rv = 1
+      elif cm.find( "weighted b " ) != -1:
+        return  ( cm.replace( "weighted b ", "weighted by " ), 1)
+
+      if cm.find( "time: mean" ) == 0 and len(cm) > 10:
+        if cm[10:].strip()[0] == '(':
+          ix = cm.index(')') + 1
+        else:
+          ix = 10
+        cm = cm[ix:].strip( ) + ' ' + cm[:ix]
+        rv = 1
+
+      if cm.find( "time: point" ) == 0 and len(cm) > 11:
+        if cm[11:].strip()[0] == '(':
+          ix = cm.index(')') + 1
+        else:
+          ix = 11
+        cm = cm[ix:].strip( ) + ' ' + cm[:ix]
+        rv = 1
+      return (cm,rv)
 
 class loadVarGroupInfo20(object):
@@ -97 +130 @@
             print 'SEVERE:loadstr.0001: duplicate lookup tuple: ',t
             print r[1], self.lookup[t]
+        print 'INFO.loadstr.00022: ',r[1],t
         self.lookup[t] = r[1]
 ## Label        Title   Spatial shape   Other dims      coords  cell methods    cell measures   flag meanings   flag values     Spatial shape (title)   Spatial dimensions
@@ -126 +160 @@
       v0 = string.strip(r[0])
       if len(v0) == 0 or v0[0] != '#':
+        vv,rv = cellMethodCheck(v0)
         v1 = string.strip(r[3])
         if r[1] in ['redirect','change']:
           self.cmmap[v0] = v1
+          if r[1] == 'redirect' and vv != v0:
+            self.cmmap[vv] = v1
           v0 = v1
         if r[1] != 'redirect':
           t = string.strip(r[5])
           l = string.strip(r[4])
-          self.cminfo[v0] = (t,l,'CellMethods::%s' % l)
+          if vv != v0:
+            self.cminfo[vv] = (t,l,'CellMethods::%s' % l)
+            self.cmmap[v0] = vv
+          else:
+            self.cminfo[v0] = (t,l,'CellMethods::%s' % l)
         
         if r[1] in ['ok','ok?']:
@@ -402 +443 @@
     s1 = wb.book.sheet_by_name(u'ss_ref')
     self.ss = {}
+    self.ssu = {}
     self.labByUid = {}
+    self.uidByLab = {}
     uu = set()
     for i in range(s1.nrows):
@@ -413 +456 @@
         assert u not in uu, 'Duplicate uid in spatialShape_ref.xls: %s' % u
         self.labByUid[u] = rr[0]
+        self.uidByLab[rr[0]] = u
         self.ss[ k ] = (a,b,lf,li,u)
+        self.ssu[ u ] = (a,b,k,lf,li,u)
+
+  def addTime(self):
+    tshp = { 'time':['time-mean','Temporal mean'], 'time1':['time-point','Instantaneous value (i.e. synoptic or time-step value)'], \
+             'time2':['climatoglogy','Climatological mean'], \
+             'time3':['diurnal-cycle','Mean Diurnal Cycle'], \
+             '':['None','No temporal dimensions ... fixed field'] }
+    tuid = ['7a96eb30-8042-11e6-97ee-ac72891c3257', '7a972f78-8042-11e6-97ee-ac72891c3257', '7a976ce0-8042-11e6-97ee-ac72891c3257', '7a97ae58-8042-11e6-97ee-ac72891c3257','cf34c974-80be-11e6-97ee-ac72891c3257']
+    self.etd = {}
+    self.tsu = {}
+    self.tByLab = {}
+
+    k = 0
+    for s in tshp:
+       label, description = tshp[s]
+       u = tuid[k]
+       self.etd[s] = (u,label,description,s)
+       self.tsu[u] = (label,description,s)
+       self.tByLab[label] = (u,description,s)
+       k+=1
   
 class run(object):

CMIP6dreqbuild/trunk/src/workbook/inputs/strUidMap.txt

@@ -190 +190 @@
 str-a02 5edc273a-75e2-11e6-b907-ac72891c3257
 str-x002 new__x002__
+str-x003 new__x003__
+str-x004 new__x004__
+str-x006 new__x006__

CMIP6dreqbuild/trunk/src/workbook/ivg.py

@@ -1 +1 @@
 from scansh import rq
 import string, collections, uuid, shelve
+import importWbMods
+from importWbMods import cellMethodCheck
 
 cmip5GrpLk = collections.defaultdict( dict )
-
-class cellMethodCheck(object):
-  def __init__(self):
-    pass
-  def check(self,cm):
-      if cm.find( "area: where" ) != -1:
-        return  ( cm.replace( "area: where", "area: mean where" ), 1)
-      elif cm.find( "time:mean" ) != -1:
-        return  ( cm.replace( "time:mean", "time: mean" ), 1)
-      elif cm.find( "weighted b " ) != -1:
-        return  ( cm.replace( "weighted b ", "weighted by " ), 1)
-      else:
-        return (cm,0)
 
 class tableUtils(object):
@@ -108 +97 @@
 class structureList(object):
   def __init__(self):
+    self.force = ['str-a02',]
+    self.spsh = importWbMods.loadSpatial()
+    self.spsh.addTime()
+    self.cellm = importWbMods.loadCellm()
     self.ll = []
     self.lldict = dict()
     self.nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tlab','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] )
     self.ee3 = {}
-    self.cmch = cellMethodCheck()
+    #self.cmch = cellMethodCheck()
     sh = shelve.open( '../framework/inSh/structRef', 'r' )
     self.tmap = {}
     self.messageDone = set()
+    cc = collections.defaultdict( dict )
+    dd = dict()
+    self.orecs = set()
+    self.kr = 0
     for k in sh.keys():
       if k[0] != '_':
@@ -121 +118 @@
         print 'INFO.ivg-str.0001: ',sh[k]
         rr = sh[k][:7]
-        rr[0] = sh[k][-1]
+        rr[0] = sh[k][-2]
         t = tuple( rr )
-        assert t not in self.tmap, 'duplicate lookup tuple for structures ....%s' % str(t)
-        self.tmap[t] = sh[k][7:] + [sh[k][0],]
-
-  def add(self,cand):
-    cm, fl = self.cmch.check( cand[6] )
-    if fl == 1:
-      cand[6] = cm
-    nt =  self.nt_dims._make( cand )
-    t1 = ( nt.tlab, nt.spid, nt.odims, nt.coords, nt.cell_methods, nt.cell_measures, nt.flag_meanings )
-    u = nt.uid
-    label = nt.label
-    tt = tuple( [nt.__dict__[k] for k in ['spid','tlab','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','procNote','prov']] )
+        flg = sh[k][-1]
+        if flg:
+           assert t not in cc[True], 'duplicate lookup tuple for structures ....%s:\n %s\n %s' % ( str(t), str( dd[t] ), str( sh[k] ) )
+           dd[t] = sh[k]
+        cc[flg][t] = sh[k][7:-1] + [sh[k][0],]
+
+    for t in cc[True]:
+        self.tmap[t] = cc[True][t]
+    for t in cc[False]:
+      if t not in self.tmap:
+        self.tmap[t] = cc[False][t]
+
+  def close(self):
+    if len(self.orecs) != 0:
+      sh = shelve.open( 'sh/extraStructureRecords', 'n' )
+      for flg,u,r in self.orecs:
+        sh[u] = (flg,r)
+      sh.close()
+     
+  def add(self,cand,t1=None):
+    ## cand = [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov]
+    ## look-up structure based on time label, spatial id, other dimensions, coordinates, cell methods, cell measures, flag meanings
+    err = False
+    if t1 == None:
+      cm, fl = cellMethodCheck( cand[6] )
+      if fl == 1:
+        cand[6] = cm
+      assert len(cm) < 11 or cm[:10] != 'time: mean', 'Should not have time mean first .... %s, %s, %s' % ( str(cand), cm.find( 'time: mean' ), fl )
+      nt =  self.nt_dims._make( cand )
+      cm = self.cellm.cmmap.get( cm,cm)
+      if cm not in self.cellm.cminfo:
+        print 'SEVERE.cellm.0001: Cell method lookup failed: %s' % cm
+        cmttl = cm
+        cmlab = '---'
+      else:
+        cmttl, cmlab = self.cellm.cminfo[cm][:2]
+ 
+      t1 = ( nt.tlab, nt.spid, nt.odims, nt.coords, nt.cell_methods, nt.cell_measures, nt.flag_meanings )
+      u = nt.uid
+      label = nt.label
+      assert nt.spid in self.spsh.ssu, 'Spatial id not found in spsh.ssu: %s' % nt.spid
+      
+      slab, sttl,sdim = self.spsh.ssu[ nt.spid ][:3]
+      tttl = self.spsh.tByLab[nt.tlab][1]
+      title = '%s, %s [%s]' % (tttl, sttl, slab)
+      if string.strip( nt.coords ) != '' or string.strip( nt.odims ) != '':
+          title += ' {%s:%s}' % (nt.odims,nt.coords)
+      if cmlab != '':
+        title += ' [%s]' % cmlab
+      
+    else:
+      assert False, 'Should not be here .....'
+
+    ##tt = tuple( [nt.__dict__[k] for k in ['spid','tlab','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','procNote','prov']] )
     if t1 in self.tmap:
-      u, label, title, description, fv, tlab, tmid = self.tmap[t1]
+## Label        Title   Spatial shape   Other dims      coords  cell methods    cell measures   flag meanings   flag values     Spatial shape (title)   Spatial dimensions
+
+      u, label, title0, description, fv, tlab, tmid = self.tmap[t1]
       if fv != nt.flag_values:
         print 'ERROR.structure.01020: flag_value mismatch for given flag_meanings: ',cand, fv
+        err = True
       if u != nt.uid or label != nt.label:
         cand[0] = u
         cand[1] = label
         nt =  self.nt_dims._make( cand )
+      orec = tuple( cand + [title,sttl,sdim] )
+      self.orecs.add( (1,u,orec) )
     else:
       if t1 not in self.messageDone:
         print 'ERROR.structure.01010: tuple lookup not found: ',cand,t1
+        if t1[1] in self.spsh.labByUid:
+          slab = self.spsh.labByUid[t1[1]]
+    ## cand = [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov]
+## Label        Title   Spatial shape   Other dims      coords  cell methods    cell measures   flag meanings   flag values     Spatial shape (title)   Spatial dimensions
+          self.kr += 1
+          assert self.kr < 100, 'Too many additional structure records required'
+          label = 'str-d%2.2i' % self.kr
+          cand[1] = label
+          orec = tuple( cand + [title,sttl,sdim] )
+          self.orecs.add( (0,u,orec) )
+          
+        else:
+          slab = '**spatial structure not found** .. %s' % t1[1]
+          assert False, 'Spatial structures must be consistent with spatialShape_ref.xls'
+        print 'EINFO.structure.01000: ',label,t1,slab
+        nnmm = 0
+        for tt in self.tmap:
+             if self.tmap[tt][1] == 'str-195':
+               print 'EINFO.structure.01040: ',tt, self.tmap[tt]
+             nm = sum( [tt[k] == t1[k] for k in range(7)] )
+             if nm > 5:
+               nnmm += 1
+               slab = self.spsh.labByUid[tt[1]]
+               lab = self.tmap[tt][1]
+               print 'EINFO.structure.01010: ',lab,tt, slab
+        if nnmm == 0:
+               print 'EINFO.structure.01020: No close matches found'
       self.messageDone.add(t1)
        
-      if self.ee3.has_key( tt ):
-        u = self.ee3[tt]
+      if self.ee3.has_key( t1 ):
+        u = self.ee3[t1]
       else:
         u = nt.uid
-    self.ee3[tt] = u
+    self.ee3[t1] = u
     if u in self.lldict:
       if self.lldict[u] != nt:
@@ -161 +232 @@
       self.lldict[u]  = nt
       self.ll.append( (u,nt) )
-    return (u,label)
+    return (err,u,label)
 
 freqmap = {'daily':'day', 'Annual':'yr', 'Timestep':'subhr',  '1day':'day', '1mon':'mon', 'month':'mon', 'year':'yr', 'monthly':'mon', 'Day':'day', '6h':'6hr', '3 hourly':'3hr', '3 Hourly':'3hr'  }