Changeset 101
- Timestamp:
- 07/12/13 00:03:35 (7 years ago)
- Location:
- CCCC/trunk
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
CCCC/trunk/c4.py
r100 r101 213 213 214 214 class checker: 215 def __init__(self, pcfg, cls ):215 def __init__(self, pcfg, cls,reader): 216 216 self.info = dummy() 217 217 self.info.pcfg = pcfg 218 218 self.calendar = 'None' 219 self.ncReader = reader 219 220 self.cfn = utils.checkFileName( parent=self.info,cls=cls) 220 221 self.cga = utils.checkGlobalAttributes( parent=self.info,cls=cls) … … 234 235 235 236 if attributeMappings != []: 236 ncReader.loadNc( fpath )237 ncReader.applyMap( attributeMappings, self.cfn.globalAttributesInFn, log=log )237 self.ncReader.loadNc( fpath ) 238 self.ncReader.applyMap( attributeMappings, self.cfn.globalAttributesInFn, log=log ) 238 239 ncRed = True 239 thisFn = ncReader.fn240 thisFn = self.ncReader.fn 240 241 else: 241 242 ncRed = False … … 253 254 254 255 if not ncRed: 255 ncReader.loadNc( fpath )256 self.ga = ncReader.ga257 self.va = ncReader.va258 self.da = ncReader.da256 self.ncReader.loadNc( fpath ) 257 self.ga = self.ncReader.ga 258 self.va = self.ncReader.va 259 self.da = self.ncReader.da 259 260 260 261 self.cga.check( self.ga, self.va, self.cfn.var, self.cfn.freq, self.vocabs, self.cfn.fnParts ) … … 274 275 return 275 276 276 if self. pcfg.doCheckGrids:277 if self.info.pcfg.doCheckGrids: 277 278 ##self.cgg.rotatedPoleGrids = config.rotatedPoleGrids 278 279 ##self.cgg.interpolatedGrids = config.interpolatedGrids … … 288 289 class c4_init: 289 290 290 def __init__(self ):291 def __init__(self,args=None): 291 292 self.logByFile = True 292 args = sys.argv[1:] 293 if args==None: 294 args = sys.argv[1:] 293 295 nn = 0 294 296 … … 333 335 flist = [] 334 336 ss = 'abcdefgijk' 337 ss = 'abcdefgijklmnopqrstuvwxyz' 335 338 ss = 'abc' 336 ss = 'abcdefgijklmnopqrstuvwxyz'337 339 for i in range(10): 338 340 v = 'v%s' % i … … 404 406 405 407 408 class main: 409 410 def __init__(self,args=None): 411 logDict = {} 412 c4i = c4_init(args=args) 413 isDummy = c4i.project[:2] == '__' 414 if (not withCdms) and isDummy: 415 print withCdms, c4i.project 416 print 'Cannot proceed with non-dummy project without cdms' 417 raise 418 pcfg = config.projectConfig( c4i.project ) 419 ncReader = fileMetadata(dummy=isDummy) 420 cc = checker(pcfg, c4i.project, ncReader) 421 rec = recorder( c4i.recordFile, dummy=isDummy ) 422 monitorFileHandles = False 423 if monitorFileHandles: 424 monitor = utils.sysMonitor() 425 else: 426 monitor = None 427 428 cal = None 429 430 c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) ) 431 432 cbv = utils.checkByVar( parent=cc.info,cls=c4i.project,monitor=monitor) 433 cbv.impt( c4i.flist ) 434 435 for f in c4i.flist: 436 if monitorFileHandles: 437 nofhStart = monitor.get_open_fds() 438 fn = string.split(f,'/')[-1] 439 c4i.logger.info( 'Starting: %s' % fn ) 440 try: 441 ### need to have a unique name, otherwise get mixing of logs despite close statement below. 442 if c4i.logByFile: 443 fLogger = c4i.getFileLog( fn ) 444 logDict[fn] = c4i.fileLogFile 445 c4i.logger.info( 'Log file: %s' % c4i.fileLogFile ) 446 else: 447 fLogger = c4i.logger 448 449 fLogger.info( 'Starting file %s' % fn ) 450 ## default appending to myapp.log; mode='w' forces a new file (deleting old contents). 451 cc.checkFile( f, log=fLogger,attributeMappings=c4i.attributeMappings ) 452 453 if cc.completed: 454 if cal not in (None,'None'): 455 if cal != cc.calendar: 456 c4i.logger.info( 'Error: change in calendar attribute %s --> %s' % (cal, cc.calendar) ) 457 fLogger.info( 'Error: change in calendar attribute %s --> %s' % (cal, cc.calendar) ) 458 cc.errorCount += 1 459 cal = cc.calendar 460 461 if c4i.logByFile: 462 if cc.completed: 463 fLogger.info( 'Done -- error count %s' % cc.errorCount ) 464 else: 465 fLogger.info( 'Done -- checks not completed' ) 466 c4i.closeFileLog( ) 467 468 if cc.completed: 469 c4i.logger.info( 'Done -- error count %s' % cc.errorCount ) 470 if cc.errorCount == 0: 471 rec.add( f, cc.drs ) 472 else: 473 rec.addErr( f, 'ERRORS FOUND | errorCount = %s' % cc.errorCount ) 474 else: 475 c4i.logger.info( 'Done -- testing aborted because of severity of errors' ) 476 rec.addErr( f, 'ERRORS FOUND AND CHECKS ABORTED' ) 477 except: 478 c4i.logger.error("Exception has occured" ,exc_info=1) 479 rec.addErr( f, 'ERROR: Exception' ) 480 if monitorFileHandles: 481 nofhEnd = monitor.get_open_fds() 482 if nofhEnd > nofhStart: 483 print 'Open file handles: %s --- %s' % (nofhStart, nofhEnd) 484 485 cc.info.log = c4i.logger 486 487 if c4i.project != 'SPECS': 488 cbv.c4i = c4i 489 cbv.setLogDict( logDict ) 490 cbv.check( recorder=rec, calendar=cc.calendar) 491 rec.dumpAll() 492 c4i.hdlr.close() 406 493 if __name__ == '__main__': 407 408 logDict = {} 409 c4i = c4_init() 410 isDummy = c4i.project[:2] == '__' 411 if (not withCdms) and isDummy: 412 print withCdms, c4i.project 413 print 'Cannot proceed with non-dummy project without cdms' 414 raise 415 pcfg = config.projectConfig( c4i.project ) 416 cc = checker(pcfg, cls = c4i.project) 417 rec = recorder( c4i.recordFile, dummy=isDummy ) 418 ncReader = fileMetadata(dummy=isDummy) 419 monitorFileHandles = False 420 if monitorFileHandles: 421 monitor = utils.sysMonitor() 422 else: 423 monitor = None 424 425 cal = None 426 427 c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) ) 428 429 cbv = utils.checkByVar( parent=cc.info,cls=c4i.project,monitor=monitor) 430 cbv.impt( c4i.flist ) 431 432 for f in c4i.flist: 433 if monitorFileHandles: 434 nofhStart = monitor.get_open_fds() 435 fn = string.split(f,'/')[-1] 436 c4i.logger.info( 'Starting: %s' % fn ) 437 try: 438 ### need to have a unique name, otherwise get mixing of logs despite close statement below. 439 if c4i.logByFile: 440 fLogger = c4i.getFileLog( fn ) 441 logDict[fn] = c4i.fileLogFile 442 c4i.logger.info( 'Log file: %s' % c4i.fileLogFile ) 443 else: 444 fLogger = c4i.logger 445 446 fLogger.info( 'Starting file %s' % fn ) 447 ## default appending to myapp.log; mode='w' forces a new file (deleting old contents). 448 cc.checkFile( f, log=fLogger,attributeMappings=c4i.attributeMappings ) 449 450 if cc.completed: 451 if cal not in (None,'None'): 452 if cal != cc.calendar: 453 c4i.logger.info( 'Error: change in calendar attribute %s --> %s' % (cal, cc.calendar) ) 454 fLogger.info( 'Error: change in calendar attribute %s --> %s' % (cal, cc.calendar) ) 455 cc.errorCount += 1 456 cal = cc.calendar 457 458 if c4i.logByFile: 459 if cc.completed: 460 fLogger.info( 'Done -- error count %s' % cc.errorCount ) 461 else: 462 fLogger.info( 'Done -- checks not completed' ) 463 c4i.closeFileLog( ) 464 465 if cc.completed: 466 c4i.logger.info( 'Done -- error count %s' % cc.errorCount ) 467 if cc.errorCount == 0: 468 rec.add( f, cc.drs ) 469 else: 470 rec.addErr( f, 'ERRORS FOUND | errorCount = %s' % cc.errorCount ) 471 else: 472 c4i.logger.info( 'Done -- testing aborted because of severity of errors' ) 473 rec.addErr( f, 'ERRORS FOUND AND CHECKS ABORTED' ) 474 except: 475 c4i.logger.error("Exception has occured" ,exc_info=1) 476 rec.addErr( f, 'ERROR: Exception' ) 477 if monitorFileHandles: 478 nofhEnd = monitor.get_open_fds() 479 if nofhEnd > nofhStart: 480 print 'Open file handles: %s --- %s' % (nofhStart, nofhEnd) 481 482 cc.info.log = c4i.logger 483 484 if c4i.project != 'SPECS': 485 cbv.c4i = c4i 486 cbv.setLogDict( logDict ) 487 cbv.check( recorder=rec, calendar=cc.calendar) 488 rec.dumpAll() 489 c4i.hdlr.close() 494 main() 495 496 490 497 ##else: 491 498 ##f1 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/day/clh/clh_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_day_19810101-19851231.nc' -
CCCC/trunk/unitTestsS2.py
r100 r101 3 3 import utils_c4 4 4 import config_c4 as config 5 from c4 import fileMetadata, dummy 5 from c4 import fileMetadata, dummy, main 6 6 7 7 #### set up log file #### … … 38 38 else: 39 39 print 'Failed [%s] %s: valid file name' % (module,fn) 40 41 42 main( args=['-p', '__dummy'] )
Note: See TracChangeset
for help on using the changeset viewer.