Changeset 383 for CMIP6dreqbuild/trunk/src

Timestamp:

15/09/15 11:55:28 (7 years ago)

Author:

mjuckes

Message:

update

Location:

CMIP6dreqbuild/trunk/src/workbook

Files:

• importShDreq.py (modified) (12 diffs)
• scansh.py (modified) (1 diff)
• utils_wb.py (modified) (1 diff)

CMIP6dreqbuild/trunk/src/workbook/importShDreq.py

@@ -13 +13 @@
 from utils_wb import uniCleanFunc
 
+class s1(object):
+  def __init__(self):
+    self.a = collections.defaultdict( list )
+
 empty=re.compile('^$')
 
@@ -39 +43 @@
     self.upda = {}
     self.insert = {}
+
+    self.objectives = collections.defaultdict( dict )
+    self.objectiveLinks = collections.defaultdict( dict )
+
     if repl:
       self.importRepl()
@@ -50 +58 @@
     assert all( fok), 'Required input file(s) missing'
 
-    for sh in [rq.refti, rq.revti, rq.vars, rq.grps, rq.rqvg, rq.rqli, rq.rqit, rq.rqsect]:
+    for sh in [rq.objec,rq.refti, rq.revti, rq.vars, rq.grps, rq.rqvg, rq.rqli, rq.rqit, rq.rqsect]:
        print sh['__info__']
 
@@ -103 +111 @@
     assert len( dups ) == 0, 'Duplicate refs: %s, %s' % (str(dups),str(map( lambda x: rq.rqvg[x[1]], dups )) )
         
-    for k in [ 'var','ovar','groupitem', 'revisedtabitem', 'requestlink', 'requestitem','requestvargroup','tablesection' ]:
+    for k in [ 'objective','var','ovar','groupitem', 'revisedtabitem', 'requestlink', 'requestitem','requestvargroup','tablesection' ]:
+## set labmod False for section in which label coercion is not wanted
+      labmod = True
     ##for k in [ 'var']:
-      thissh = {'var':rq.vars,'ovar':rq.refti, 'groupitem':rq.grps, \
+## keys here match section element names in schema
+      thissh = {'objective':rq.objec, 'var':rq.vars,'ovar':rq.refti, 'groupitem':rq.grps, \
          'revisedtabitem':rq.revti, 'requestlink':rq.rqli, 'requestitem':rq.rqit, \
          'requestvargroup':rq.rqvg, 'tablesection':rq.rqsect}[k]
@@ -126 +137 @@
 ######### NEED TO CLEAN THIS IF ITEMS FROM OTHER SECTIONS ARE FILTERED
 #######################################################################
+          dothis = True
           if k != 'var':
             item = doc.createElement( 'item' )
-            item.setAttribute( 'id', 'tmpid.%4.4i' % idk )
-          dothis = True
-          if k == 'var':
+          if k == 'objective':
+ ## - mip; tab; expt; rlid; ny
+            labmod = False
+            kk = ['mip', 'label', 'title', 'description']
+            thisl = string.replace(string.strip(str( ll[1] ) ), '-', ' ')
+## camelcase this label.
+            thisl = string.replace( string.capwords( thisl ), ' ', '' )
+            mip = str( ll[0] )
+            self.objectives[mip][thisl] = i
+            for j in range(len(kk)):
+                thisv = str( ll[j] )
+                item.setAttribute( kk[j], thisv )
+            item.setAttribute( 'uuid', str(i) )
+          elif k == 'var':
 ##- sn; units; description; procnote; procComment; prov
 ##['label', 'title', 'sn', 'units', 'description', 'procnote', 'procComment', 'prov']
@@ -169 +192 @@
        u'PMIP3_Oclim':'monClim', u'SPECS_Amon':'mon', u'SPECS_Lmon':'mon', u'CMIP5_cf3hr':'3hr', u'CORDEX_3h':'3hr', \
        u'CCMI1_annual':'yr', u'CMIP5_Oyr':'yr'}
-            kk = ['uuid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid', 'gpid','rowIndex']
+            kk = ['uuid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid', 'gpid','rowIndex','priority']
 ## uuid; comment; deflate_level; shuffle; ok_max_mean_abs; flag_meanings; type; ok_min_mean_abs; sn; deflate; valid_min; cell_methods; flag_values; cell_measures; out_name; modeling_realm; units; cell_methods_xx; valid_max; positive; var; mipTable; dimensions; vid
+            ix_gpid = kk.index( 'gpid' )
 
             thisl = string.strip( str( ll[21] ) )
             for j in range( min(len(kk),len(ll)) ):
+              if kk[j] == 'priority':
+                ll[j] = int(ll[j])
+                
               if j != 21:
                 item.setAttribute( kk[j], str( ll[j] ) )
+
             if len(ll) < len(kk):
               print 'ERROR.015.0001: record length short %s (%s):: %s' % (len(ll),len(kk),ll)
@@ -197 +225 @@
             assert len(kk) == len(ll), 'length mismatch, %s  %s' % (len(kk),len(ll))
 # - group;  table; freq; descriptionEx; shape; levels; tstyle; mask; mip; mip2; uuid; new; vid
+            ix_gpid = kk.index( 'gpid' )
             thisl = string.strip(str( ll[1] ) )
             for j in range(len(kk)):
@@ -209 +238 @@
               thisl = string.split(thisl)[0]
             thisuuid = ll[11]
-            gpid = str(ll[-3])
+
+            gpid = str(ll[ix_gpid])
             if not rq.rqvg.has_key(gpid):
               print 'ERROR.015.0010: gpid not found %s' % str(ll)
@@ -262 +292 @@
             kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt','opar','ref', 'refNote', 'refid']
             emap = {'uid':'uuid'}
+            mip = string.strip(str( ll[1] ) )
+            obj = string.strip(str( ll[3] ) )
+            if mip == "GMMIP":
+              bits = string.split( obj, ' ' )
+            else:
+              bits = [ string.replace( string.capwords( string.strip(x) ), ' ', '' ) for x in string.split( obj, ',' ) ]
+            for b in bits:
+              self.objectiveLinks[mip][b] = str( ll[0] )
+        
             thisl = string.strip(str( ll[2] ) )
             for j in range(len(kk)):
@@ -279 +318 @@
             for j in range(len(kk)):
                 if kk[j] == 'ny':
-                  thisv = str( int(ll[j]) )
+                  if ll[j] == '':
+                    thisv = 0
+                  else:
+                    try:
+                      thisv = str( int(ll[j]) )
+                    except:
+                      print 'SEVERE: failed to set "ny": %s' % str(ll)
+                      thisv = 999
                 else:
                   thisv = str( ll[j] )
@@ -296 +342 @@
 ####
           if dothis:
-            if string.find( thisl, '_' ) != -1:
+            if labmod:
+              if string.find( thisl, '_' ) != -1:
               #print 'WARNING: underscore in label', ll
-              thisl = string.replace( thisl, '_', '-' )
-            if string.find( thisl, '*' ) != -1:
-              #print 'WARNING: star in label', ll
-              thisl = string.replace( thisl, '*', '-' )
-            if string.find( thisl, '!' ) != -1:
+                thisl = string.replace( thisl, '_', '-' )
+              if string.find( thisl, '*' ) != -1:
+                #print 'WARNING: star in label', ll
+                thisl = string.replace( thisl, '*', '-' )
+              if string.find( thisl, '!' ) != -1:
               #print 'WARNING: exclamation in label', ll
-              thisl = string.replace( thisl, '!', '-' )
-            if string.find( thisl, ' ' ) != -1:
+                thisl = string.replace( thisl, '!', '-' )
+              if string.find( thisl, ' ' ) != -1:
               #print 'WARNING: space in label', ll
-              thisl = string.replace( thisl, ' ', '-' )
-            if string.find( thisl, ':' ) != -1:
+                thisl = string.replace( thisl, ' ', '-' )
+              if string.find( thisl, ':' ) != -1:
               #print 'WARNING: colon in label', ll
-              thisl = string.replace( thisl, ':', '-' )
+                thisl = string.replace( thisl, ':', '-' )
             item.setAttribute( 'label', thisl )
             if k == 'var':
@@ -321 +368 @@
             print '******************************************'
             
-    ##for k in rqvg_extra.keys():
-            ##item = doc.createElement( 'item' )
-            ##ff['...' ][0].appendChild( item )
-
+
+
+
+  def finish(self):
     this = doc.getElementsByTagName('remarks')[0]
     dil = this.getElementsByTagName('item')

CMIP6dreqbuild/trunk/src/workbook/scansh.py

@@ -24 +24 @@
     self.rqvg = sopen( 'requestVarGroup' )
     self.rqsect = sopen( 'requestVarSections' )
+    self.objec = sopen( 'objectives' )
 
     self.cmip5Grps = shelve.open( '%s/cmip5Groups' % odir, 'r' )

CMIP6dreqbuild/trunk/src/workbook/utils_wb.py

@@ -14 +14 @@
       if type(ss) in [type('x'),type(u'x')]:
           ss = string.replace( ss, u'\u2013', '-' )
+          ss = string.replace( ss, u'\u2014', '-u2014-' )
           ss = string.replace( ss, u'\u2010', '-' )
           ss = string.replace( ss, u'\u2018', "'" )