Changeset 141
- Timestamp:
- 31/03/14 16:07:37 (7 years ago)
- Location:
- CCCC/trunk
- Files:
-
- 2 added
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
CCCC/trunk/README.txt
r137 r141 21 21 --blfmode <mode> # set mode for batch log file -- see log file modes 22 22 --flfmode <mode> # set mode for file-level log file -- see log file modes 23 --aMap # Read in some attribute mappings and run tests with virtual substitutions, see also map2nco.py 23 24 24 25 … … 71 72 "git clone git://uv-cdat.llnl.gov/gitweb/cordex-cmor-tables.git" 72 73 74 VIRTUAL MODE 75 ------------ 76 77 The virtual mode can be used to validate substituions before adjusting systems which have been used to generate data, or as the first step of a procedure for repairing some classes of errors. 78 79 To use this mode, a mapping file is needed. This can be generated by an initial run of the checker with no virtual substitutions. A file naemd "amapDraft.txt" will be generated. This file should be inspected to ensure that suggested changes make sense. 80 81 A typical directive will be of the form: 82 @var=rlus;standard_name=surface_upward_longwave_flux_in_air|standard_name=surface_upwelling_longwave_flux_in_air 83 84 The meaning is: for variable "rlus", set the attribute "standard_name" to "surface_upwelling_longwave_flux_in_air" where the input file has "surface_upward_longwave_flux_in_air". 85 86 If run with the --aMap flag, the checker will test attributes after making virtual substituions. I.e. there are no changes made to the files at this stage, but results of the tests apply as if changes have been made. 87 88 After running in virtual mode, c4.py will generate a file named "attributeMappingsLog.txt" which contains a record for every change to every file. If the results of running in virtual mode are positive, this file can be used to create a script to modify the files, by running "amap2nco.py": 89 90 python amap2nco.py attributeMappingsLog.txt /tmp/batch1 /tmp/batch1_corrected 91 ## this will generate a list of NCO commands in "ncoscript.sh", which will apply the changes and create new files in "/tmp/batch1_corrected". 92 93 It is recommended that the data values in the corrected files should be checked after running this script. 94 95 By default, the amap2nco.py program will generate commands to modify the tracking_id and creation_date global attributes at the same time as making other changed. The "history" attribute is modified by the NCO library. 96 73 97 EXCEPTIONS 74 98 ---------- -
CCCC/trunk/c4.py
r137 r141 24 24 class fileMetadata: 25 25 26 def __init__(self,dummy=False): 26 def __init__(self,dummy=False,attributeMappingsLog=None): 27 27 28 self.dummy = dummy 29 self.atMapLog = attributeMappingsLog 30 if self.atMapLog == None: 31 self.atMapLog = open( '/tmp/cccc_atMapLog.txt', 'a' ) 32 33 def close(self): 34 self.atMapLog.close() 28 35 29 36 def loadNc(self,fpath): 37 self.fpath = fpath 30 38 self.fn = string.split( fpath, '/' )[-1] 31 39 self.fparts = string.split( self.fn[:-3], '_' ) … … 99 107 log.info( 'Setting %s to %s' % (targ,m[2][1]) ) 100 108 ##print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1]) 109 thisval = self.va[m[1][0][1]].get( targ, None ) 101 110 self.va[m[1][0][1]][targ] = m[2][1] 111 self.atMapLog.write( '@var:"%s","%s","%s","%s","%s"\n' % (self.fpath, m[1][0][1], targ, thisval, m[2][1] ) ) 102 112 103 113 elif m[1][0][0] == "@ax": … … 120 130 log.info( 'Setting %s to %s' % (targ,m[2][1]) ) 121 131 ##print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1]) 132 thisval = self.va[m[1][0][1]].get( targ, None ) 122 133 self.da[m[1][0][1]][targ] = m[2][1] 134 self.atMapLog.write( '@ax:"%s","%s","%s","%s","%s"\n' % (self.fpath, m[1][0][1], targ, thisval, m[2][1]) ) 123 135 elif m[1][0][0][0] != "@": 124 136 this = self.ga … … 137 149 log.info( 'Setting %s to %s' % (targ,m[2][1]) ) 138 150 print 'Setting %s:%s to %s' % (m[1][0][1],targ,m[2][1]) 151 thisval = self.ga.get( targ, None ) 139 152 self.ga[targ] = m[2][1] 153 self.atMapLog.write( '@:"%s","%s","%s","%s","%s"\n' % (self.fpath, 'ga', targ, thisval, m[2][1]) ) 154 ## 140 155 if targ in globalAttributesInFn: 141 self.fparts[ globalAttributesInFn.index(targ) ] = m[2][1] 156 i = globalAttributesInFn.index(targ) 157 thisval = self.fparts[ i ] 158 self.fparts[ i ] = m[2][1] 142 159 self.fn = string.join( self.fparts, '_' ) + '.nc' 160 self.atMapLog.write( '@fn:"%s","%s","%s"\n' % (self.fpath, thisval, m[2][1]) ) 143 161 else: 144 162 print 'Token %s not recognised' % m[1][0][0] … … 421 439 422 440 self.attributeMappings = [] 441 self.attributeMappingsLog = None 423 442 if self.attributeMappingFile != None: 424 443 for l in open( self.attributeMappingFile ).readlines(): … … 431 450 cl.append( string.split(b, '=' ) ) 432 451 self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) ) 452 self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' ) 433 453 434 454 def getFileLog( self, fn, flf=None ): … … 481 501 raise 482 502 pcfg = config.projectConfig( c4i.project ) 483 ncReader = fileMetadata(dummy=isDummy )503 ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog) 484 504 self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount) 485 505 rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy ) … … 492 512 c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) ) 493 513 514 self.cc.info.amapListDraft = [] 494 515 cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor) 495 516 cbv.impt( c4i.flist ) … … 576 597 except: 577 598 ecount = None 599 ncReader.close() 600 if type( self.cc.info.amapListDraft ) == type( [] ) and len( self.cc.info.amapListDraft ) > 0: 601 ll = self.cc.info.amapListDraft 602 ll.sort() 603 oo = open( 'amapDraft.txt', 'w' ) 604 oo.write( ll[0] + '\n' ) 605 for i in range( 1,len(ll) ): 606 if ll[i] != ll[i-1]: 607 oo.write( ll[i] + '\n' ) 608 oo.close() 578 609 rec.dumpAll() 579 610 if printInfo: -
CCCC/trunk/config_c4.py
r139 r141 131 131 'experiment_id':utils.listControl( 'experiment_id', lrdr.getSimpleList( 'ccmi_experiments.txt', bit=-1 ) ), \ 132 132 ## do not preserve or check relation between model and institution. 133 'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'models_insts.txt', bit= -1 ) ), \133 'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'models_insts.txt', bit=1 ) ), \ 134 134 'model_id':utils.listControl( 'model_id', lrdr.getSimpleList( 'models_insts.txt', bit=0 ) ), \ 135 135 'modeling_realm':utils.listControl( 'realm', ['atmos', 'ocean', 'land', 'landIce', 'seaIce', 'aerosol', 'atmosChem', 'ocnBgchem'] ), \ -
CCCC/trunk/utils_c4.py
r139 r141 387 387 if val != None: 388 388 if string.find( targ, val): 389 mm.append( k)389 mm.append( (k,targ,val) ) 390 390 elif targ != val: 391 mm.append( k)391 mm.append( (k,targ,val) ) 392 392 393 393 ok &= self.test( len(mm) == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, str(mm)), part=True ) 394 if len( mm ) != 0: 395 if self.parent.amapListDraft == None: 396 self.parent.amapListDraft = [] 397 for m in mm: 398 self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % (varName,m[0],m[1],m[0],m[2]) ) 399 394 400 if ok: 395 401 self.log_pass() … … 558 564 559 565 for k in heightAtDict.keys(): 560 ok &= self.test( va['height'].get( k, None ) == heightAtDict[k], \ 561 'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ) 566 val = va['height'].get( k, "none" ) 567 if not self.test( val == heightAtDict[k], \ 568 'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ): 569 self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % ('height',k,val,k,heightAtDict[k]) ) 570 ok = False 562 571 563 572 if ok:
Note: See TracChangeset
for help on using the changeset viewer.