Changeset 141

Timestamp:

31/03/14 16:07:37 (6 years ago)

Author:

mjuckes

Message:

Updated virtual test and correction script generation

Location:

CCCC/trunk

Files:

• README.txt (modified) (2 diffs)
• amap2nco.py (added)
• amap_ex1.txt (added)
• c4.py (modified) (9 diffs)
• config_c4.py (modified) (1 diff)
• utils_c4.py (modified) (2 diffs)

CCCC/trunk/README.txt

@@ -21 +21 @@
   --blfmode <mode>      # set mode for batch log file -- see log file modes
   --flfmode <mode>      # set mode for file-level log file -- see log file modes
+  --aMap                # Read in some attribute mappings and run tests with virtual substitutions, see also map2nco.py
 
 
@@ -71 +72 @@
 "git clone git://uv-cdat.llnl.gov/gitweb/cordex-cmor-tables.git"
 
+VIRTUAL MODE
+------------
+
+The virtual mode can be used to validate substituions before adjusting systems which have been used to generate data, or as the first step of a procedure for repairing some classes of errors.
+
+To use this mode, a mapping file is needed. This can be generated by an initial run of the checker with no virtual substitutions. A file naemd "amapDraft.txt" will be generated. This file should be inspected to ensure that suggested changes make sense.
+
+A typical directive will be of the form:
+@var=rlus;standard_name=surface_upward_longwave_flux_in_air|standard_name=surface_upwelling_longwave_flux_in_air
+
+The meaning is: for variable "rlus", set the attribute "standard_name" to "surface_upwelling_longwave_flux_in_air" where the input file has "surface_upward_longwave_flux_in_air".
+
+If run with the --aMap flag, the checker will test attributes after making virtual substituions. I.e. there are no changes made to the files at this stage, but results of the tests apply as if changes have been made.
+
+After running in virtual mode, c4.py will generate a file named "attributeMappingsLog.txt" which contains a record for every change to every file. If the results of running in virtual mode are positive, this file can be used to create a script to modify the files, by running "amap2nco.py":
+
+python amap2nco.py attributeMappingsLog.txt /tmp/batch1 /tmp/batch1_corrected
+## this will generate a list of NCO commands in "ncoscript.sh", which will apply the changes and create new files in "/tmp/batch1_corrected".
+
+It is recommended that the data values in the corrected files should be checked after running this script.
+
+By default, the amap2nco.py program will generate commands to modify the tracking_id and creation_date global attributes at the same time as making other changed. The "history" attribute is modified by the NCO library. 
+
 EXCEPTIONS
 ----------

CCCC/trunk/c4.py

@@ -24 +24 @@
 class fileMetadata:
 
-  def __init__(self,dummy=False):
+  def __init__(self,dummy=False,attributeMappingsLog=None):
+     
      self.dummy = dummy
+     self.atMapLog = attributeMappingsLog
+     if self.atMapLog == None:
+       self.atMapLog = open( '/tmp/cccc_atMapLog.txt', 'a' )
+
+  def close(self):
+    self.atMapLog.close()
 
   def loadNc(self,fpath):
+    self.fpath = fpath
     self.fn = string.split( fpath, '/' )[-1]
     self.fparts = string.split( self.fn[:-3], '_' )
@@ -99 +107 @@
                 log.info( 'Setting %s to %s' % (targ,m[2][1]) )
               ##print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1])
+              thisval = self.va[m[1][0][1]].get( targ, None )
               self.va[m[1][0][1]][targ] = m[2][1]
+              self.atMapLog.write( '@var:"%s","%s","%s","%s","%s"\n' % (self.fpath, m[1][0][1], targ, thisval, m[2][1] ) )
 
         elif m[1][0][0] == "@ax":
@@ -120 +130 @@
                 log.info( 'Setting %s to %s' % (targ,m[2][1]) )
               ##print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1])
+              thisval = self.va[m[1][0][1]].get( targ, None )
               self.da[m[1][0][1]][targ] = m[2][1]
+              self.atMapLog.write( '@ax:"%s","%s","%s","%s","%s"\n' % (self.fpath, m[1][0][1], targ, thisval, m[2][1]) )
         elif m[1][0][0][0] != "@":
             this = self.ga
@@ -137 +149 @@
                 log.info( 'Setting %s to %s' % (targ,m[2][1]) )
               print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1])
+              thisval = self.ga.get( targ, None )
               self.ga[targ] = m[2][1]
+              self.atMapLog.write( '@:"%s","%s","%s","%s","%s"\n' % (self.fpath, 'ga', targ, thisval, m[2][1]) )
+##
               if targ in globalAttributesInFn:
-                self.fparts[ globalAttributesInFn.index(targ) ] = m[2][1]
+                i = globalAttributesInFn.index(targ)
+                thisval = self.fparts[ i ]
+                self.fparts[ i ] = m[2][1]
                 self.fn = string.join( self.fparts, '_' ) + '.nc'
+                self.atMapLog.write( '@fn:"%s","%s","%s"\n' % (self.fpath, thisval, m[2][1]) )
         else:
           print 'Token %s not recognised' % m[1][0][0]
@@ -421 +439 @@
 
     self.attributeMappings = []
+    self.attributeMappingsLog = None
     if self.attributeMappingFile != None:
       for l in open( self.attributeMappingFile ).readlines():
@@ -431 +450 @@
             cl.append( string.split(b, '=' ) )
           self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
+      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
 
   def getFileLog( self, fn, flf=None ):
@@ -481 +501 @@
        raise
     pcfg = config.projectConfig( c4i.project )
-    ncReader = fileMetadata(dummy=isDummy)
+    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog)
     self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount)
     rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy )
@@ -492 +512 @@
     c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) )
   
+    self.cc.info.amapListDraft = []
     cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor)
     cbv.impt( c4i.flist )
@@ -576 +597 @@
        except:
          ecount = None
+    ncReader.close()
+    if type( self.cc.info.amapListDraft ) == type( [] ) and len(  self.cc.info.amapListDraft ) > 0:
+      ll =  self.cc.info.amapListDraft
+      ll.sort()
+      oo = open( 'amapDraft.txt', 'w' )
+      oo.write( ll[0] + '\n' )
+      for i in range( 1,len(ll) ):
+        if ll[i] != ll[i-1]:
+          oo.write( ll[i] + '\n' )
+      oo.close()
     rec.dumpAll()
     if printInfo:

CCCC/trunk/config_c4.py

@@ -131 +131 @@
                'experiment_id':utils.listControl( 'experiment_id', lrdr.getSimpleList( 'ccmi_experiments.txt', bit=-1 ) ), \
 ## do not preserve or check relation between model and institution.
-               'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'models_insts.txt', bit=-1 ) ), \
+               'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'models_insts.txt', bit=1 ) ), \
                'model_id':utils.listControl( 'model_id', lrdr.getSimpleList( 'models_insts.txt', bit=0 ) ), \
                'modeling_realm':utils.listControl( 'realm', ['atmos', 'ocean', 'land', 'landIce', 'seaIce', 'aerosol', 'atmosChem', 'ocnBgchem'] ), \

CCCC/trunk/utils_c4.py

@@ -387 +387 @@
         if val != None:
           if string.find( targ, val):
-             mm.append(k)
+             mm.append( (k,targ,val) )
       elif targ != val:
-        mm.append( k )
+        mm.append( (k,targ,val) )
 
     ok &= self.test( len(mm)  == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, str(mm)), part=True )
+    if len( mm  ) != 0:
+      if self.parent.amapListDraft == None:
+        self.parent.amapListDraft = []
+      for m in mm:
+          self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % (varName,m[0],m[1],m[0],m[2]) )
+
     if ok:
        self.log_pass()
@@ -558 +564 @@
       
       for k in heightAtDict.keys():
-        ok &= self.test( va['height'].get( k, None ) == heightAtDict[k], \
-                         'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True )
+        val =  va['height'].get( k, "none" )
+        if not self.test( val == heightAtDict[k], \
+                         'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ):
+          self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % ('height',k,val,k,heightAtDict[k]) )
+          ok = False
 
       if ok: