Changeset 141


Ignore:
Timestamp:
31/03/14 16:07:37 (6 years ago)
Author:
mjuckes
Message:

Updated virtual test and correction script generation

Location:
CCCC/trunk
Files:
2 added
4 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/README.txt

    r137 r141  
    2121  --blfmode <mode>      # set mode for batch log file -- see log file modes 
    2222  --flfmode <mode>      # set mode for file-level log file -- see log file modes 
     23  --aMap                # Read in some attribute mappings and run tests with virtual substitutions, see also map2nco.py 
    2324 
    2425 
     
    7172"git clone git://uv-cdat.llnl.gov/gitweb/cordex-cmor-tables.git" 
    7273 
     74VIRTUAL MODE 
     75------------ 
     76 
     77The virtual mode can be used to validate substituions before adjusting systems which have been used to generate data, or as the first step of a procedure for repairing some classes of errors. 
     78 
     79To use this mode, a mapping file is needed. This can be generated by an initial run of the checker with no virtual substitutions. A file naemd "amapDraft.txt" will be generated. This file should be inspected to ensure that suggested changes make sense. 
     80 
     81A typical directive will be of the form: 
     82@var=rlus;standard_name=surface_upward_longwave_flux_in_air|standard_name=surface_upwelling_longwave_flux_in_air 
     83 
     84The meaning is: for variable "rlus", set the attribute "standard_name" to "surface_upwelling_longwave_flux_in_air" where the input file has "surface_upward_longwave_flux_in_air". 
     85 
     86If run with the --aMap flag, the checker will test attributes after making virtual substituions. I.e. there are no changes made to the files at this stage, but results of the tests apply as if changes have been made. 
     87 
     88After running in virtual mode, c4.py will generate a file named "attributeMappingsLog.txt" which contains a record for every change to every file. If the results of running in virtual mode are positive, this file can be used to create a script to modify the files, by running "amap2nco.py": 
     89 
     90python amap2nco.py attributeMappingsLog.txt /tmp/batch1 /tmp/batch1_corrected 
     91## this will generate a list of NCO commands in "ncoscript.sh", which will apply the changes and create new files in "/tmp/batch1_corrected". 
     92 
     93It is recommended that the data values in the corrected files should be checked after running this script. 
     94 
     95By default, the amap2nco.py program will generate commands to modify the tracking_id and creation_date global attributes at the same time as making other changed. The "history" attribute is modified by the NCO library.  
     96 
    7397EXCEPTIONS 
    7498---------- 
  • CCCC/trunk/c4.py

    r137 r141  
    2424class fileMetadata: 
    2525 
    26   def __init__(self,dummy=False): 
     26  def __init__(self,dummy=False,attributeMappingsLog=None): 
     27      
    2728     self.dummy = dummy 
     29     self.atMapLog = attributeMappingsLog 
     30     if self.atMapLog == None: 
     31       self.atMapLog = open( '/tmp/cccc_atMapLog.txt', 'a' ) 
     32 
     33  def close(self): 
     34    self.atMapLog.close() 
    2835 
    2936  def loadNc(self,fpath): 
     37    self.fpath = fpath 
    3038    self.fn = string.split( fpath, '/' )[-1] 
    3139    self.fparts = string.split( self.fn[:-3], '_' ) 
     
    99107                log.info( 'Setting %s to %s' % (targ,m[2][1]) ) 
    100108              ##print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1]) 
     109              thisval = self.va[m[1][0][1]].get( targ, None ) 
    101110              self.va[m[1][0][1]][targ] = m[2][1] 
     111              self.atMapLog.write( '@var:"%s","%s","%s","%s","%s"\n' % (self.fpath, m[1][0][1], targ, thisval, m[2][1] ) ) 
    102112 
    103113        elif m[1][0][0] == "@ax": 
     
    120130                log.info( 'Setting %s to %s' % (targ,m[2][1]) ) 
    121131              ##print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1]) 
     132              thisval = self.va[m[1][0][1]].get( targ, None ) 
    122133              self.da[m[1][0][1]][targ] = m[2][1] 
     134              self.atMapLog.write( '@ax:"%s","%s","%s","%s","%s"\n' % (self.fpath, m[1][0][1], targ, thisval, m[2][1]) ) 
    123135        elif m[1][0][0][0] != "@": 
    124136            this = self.ga 
     
    137149                log.info( 'Setting %s to %s' % (targ,m[2][1]) ) 
    138150              print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1]) 
     151              thisval = self.ga.get( targ, None ) 
    139152              self.ga[targ] = m[2][1] 
     153              self.atMapLog.write( '@:"%s","%s","%s","%s","%s"\n' % (self.fpath, 'ga', targ, thisval, m[2][1]) ) 
     154## 
    140155              if targ in globalAttributesInFn: 
    141                 self.fparts[ globalAttributesInFn.index(targ) ] = m[2][1] 
     156                i = globalAttributesInFn.index(targ) 
     157                thisval = self.fparts[ i ] 
     158                self.fparts[ i ] = m[2][1] 
    142159                self.fn = string.join( self.fparts, '_' ) + '.nc' 
     160                self.atMapLog.write( '@fn:"%s","%s","%s"\n' % (self.fpath, thisval, m[2][1]) ) 
    143161        else: 
    144162          print 'Token %s not recognised' % m[1][0][0] 
     
    421439 
    422440    self.attributeMappings = [] 
     441    self.attributeMappingsLog = None 
    423442    if self.attributeMappingFile != None: 
    424443      for l in open( self.attributeMappingFile ).readlines(): 
     
    431450            cl.append( string.split(b, '=' ) ) 
    432451          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) ) 
     452      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' ) 
    433453 
    434454  def getFileLog( self, fn, flf=None ): 
     
    481501       raise 
    482502    pcfg = config.projectConfig( c4i.project ) 
    483     ncReader = fileMetadata(dummy=isDummy) 
     503    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog) 
    484504    self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount) 
    485505    rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy ) 
     
    492512    c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) ) 
    493513   
     514    self.cc.info.amapListDraft = [] 
    494515    cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor) 
    495516    cbv.impt( c4i.flist ) 
     
    576597       except: 
    577598         ecount = None 
     599    ncReader.close() 
     600    if type( self.cc.info.amapListDraft ) == type( [] ) and len(  self.cc.info.amapListDraft ) > 0: 
     601      ll =  self.cc.info.amapListDraft 
     602      ll.sort() 
     603      oo = open( 'amapDraft.txt', 'w' ) 
     604      oo.write( ll[0] + '\n' ) 
     605      for i in range( 1,len(ll) ): 
     606        if ll[i] != ll[i-1]: 
     607          oo.write( ll[i] + '\n' ) 
     608      oo.close() 
    578609    rec.dumpAll() 
    579610    if printInfo: 
  • CCCC/trunk/config_c4.py

    r139 r141  
    131131               'experiment_id':utils.listControl( 'experiment_id', lrdr.getSimpleList( 'ccmi_experiments.txt', bit=-1 ) ), \ 
    132132## do not preserve or check relation between model and institution. 
    133                'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'models_insts.txt', bit=-1 ) ), \ 
     133               'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'models_insts.txt', bit=1 ) ), \ 
    134134               'model_id':utils.listControl( 'model_id', lrdr.getSimpleList( 'models_insts.txt', bit=0 ) ), \ 
    135135               'modeling_realm':utils.listControl( 'realm', ['atmos', 'ocean', 'land', 'landIce', 'seaIce', 'aerosol', 'atmosChem', 'ocnBgchem'] ), \ 
  • CCCC/trunk/utils_c4.py

    r139 r141  
    387387        if val != None: 
    388388          if string.find( targ, val): 
    389              mm.append(k) 
     389             mm.append( (k,targ,val) ) 
    390390      elif targ != val: 
    391         mm.append( k ) 
     391        mm.append( (k,targ,val) ) 
    392392 
    393393    ok &= self.test( len(mm)  == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, str(mm)), part=True ) 
     394    if len( mm  ) != 0: 
     395      if self.parent.amapListDraft == None: 
     396        self.parent.amapListDraft = [] 
     397      for m in mm: 
     398          self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % (varName,m[0],m[1],m[0],m[2]) ) 
     399 
    394400    if ok: 
    395401       self.log_pass() 
     
    558564       
    559565      for k in heightAtDict.keys(): 
    560         ok &= self.test( va['height'].get( k, None ) == heightAtDict[k], \ 
    561                          'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ) 
     566        val =  va['height'].get( k, "none" ) 
     567        if not self.test( val == heightAtDict[k], \ 
     568                         'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ): 
     569          self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % ('height',k,val,k,heightAtDict[k]) ) 
     570          ok = False 
    562571 
    563572      if ok: 
Note: See TracChangeset for help on using the changeset viewer.