Changeset 304
- Timestamp:
- 05/03/15 12:02:51 (6 years ago)
- Location:
- CCCC/trunk/ceda_cc
- Files:
-
- 1 added
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
CCCC/trunk/ceda_cc/c4.py
r297 r304 1 1 2 2 import sys 3 from ccinit import c4_init 3 4 4 5 testmain=False … … 22 23 23 24 # Standard library imports 24 import os, string, time, logging, sys,glob, pkgutil25 import os, string, time, glob, pkgutil 25 26 import shutil 26 27 ## pkgutil is used in file_utils … … 240 241 self.drs['project'] = self.info.pcfg.projectV.id 241 242 self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount 242 243 class c4_init(object):244 245 def __init__(self,args=None):246 self.logByFile = True247 self.policyFileLogfileMode = 'w'248 self.policyBatchLogfileMode = 'np'249 if args==None:250 args = sys.argv[1:]251 nn = 0252 253 self.attributeMappingFile = None254 self.recordFile = 'Rec.txt'255 self.logDir = 'logs_02'256 self.errs = []257 258 # Set default project to "CORDEX"259 self.project = "CORDEX"260 self.holdExceptions = False261 forceLogOrg = None262 argsIn = args[:]263 264 # Show help if no args or help requested265 if len(args) == 0 or args[0] in ('-h', '-help', '--help'):266 print 'Help command not implemented yet'267 raise SystemExit(0)268 # The --copy-config option must be the first argument if it is present.269 elif args[0] == '--copy-config':270 if len(args) < 2:271 self.commandHints( argsIn )272 args.pop(0)273 dest_dir = args.pop(0)274 config.copy_config(dest_dir)275 print 'Configuration directory copied to %s. Set CC_CONFIG_DIR to use this configuration.' % dest_dir276 print277 raise SystemExit(0)278 279 self.summarymode = args[0] == '--sum'280 if self.summarymode:281 return282 283 self.experimental = False284 self.forceNetcdfLib = None285 fltype = None286 argu = []287 while len(args) > 0:288 next = args.pop(0)289 if next == '-f':290 flist = [args.pop(0),]291 self.logByFile = False292 fltype = '-f'293 self.source = flist[0]294 elif next == '--log':295 x = args.pop(0)296 assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)297 if x in ['multi','m']:298 forceLogOrg = 'multi'299 elif x in ['single','s']:300 forceLogOrg = 'single'301 elif next == '--experimental':302 self.experimental = True303 elif next == '--force-ncq':304 self.forceNetcdfLib = 'ncq3'305 elif next == '--force-cdms2':306 self.forceNetcdfLib = 'cdms2'307 elif next == '--force-pync4':308 self.forceNetcdfLib = 'netCDF4'309 elif next == '--force-scientific':310 self.forceNetcdfLib = 'Scientific'311 elif next == '--flfmode':312 lfmk = args.pop(0)313 assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk314 self.policyFileLogfileMode = lfmk315 elif next == '--blfmode':316 lfmk = args.pop(0)317 assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk318 self.policyBatchLogfileMode = lfmk319 elif next == '-d':320 fdir = args.pop(0)321 flist = glob.glob( '%s/*.nc' % fdir )322 self.source = '%s/*.nc' % fdir323 elif next == '-D':324 flist = []325 fdir = args.pop(0)326 for root, dirs, files in os.walk( fdir, followlinks=True ):327 for f in files:328 fpath = '%s/%s' % (root,f)329 if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':330 flist.append( fpath )331 self.source = '%s/.....' % fdir332 elif next == '-R':333 self.recordFile = args.pop(0)334 elif next == '--ld':335 self.logDir = args.pop(0)336 elif next in ['--catchAllExceptions','--cae']:337 self.holdExceptions = True338 elif next == '--aMap':339 self.attributeMappingFile = args.pop(0)340 assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'341 elif next == "-p":342 self.project = args.pop(0)343 else:344 print 'Unused argument: %s' % next345 argu.append( next )346 nn+=1347 if nn != 0:348 print 'Unused arguments: ', argu349 self.commandHints( argsIn )350 351 if self.project == 'CMIP5' and fltype != '-f':352 fl0 = []353 for f in flist:354 if string.find( f, '/latest/' ) != -1:355 fl0.append(f)356 flist = fl0357 358 if forceLogOrg != None:359 if forceLogOrg == 'single':360 self.logByFile = False361 else:362 self.logByFile = True363 364 if self.project[:2] == '__':365 self.source = 'dummy'366 flist = []367 ss = 'abcdefgijk'368 ss = 'abcdefgijklmnopqrstuvwxyz'369 ss = 'abc'370 for i in range(10):371 v = 'v%s' % i372 for a in ss:373 for b in ss:374 flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )375 flist.sort()376 fnl = []377 for f in flist:378 fn = string.split(f, '/')[-1]379 fnl.append(fn)380 nd = 0381 dupl = []382 for k in range(len(fnl)-1):383 if fnl[k] == fnl[k-1]:384 nd += 1385 dupl.append( fnl[k] )386 self.dupDict = {}387 for f in dupl:388 self.dupDict[f] = 0389 if nd != 0:390 self.errs.append( 'Duplicate file names encountered: %s' % nd )391 self.errs.append( dupl )392 self.flist = flist393 self.fnl = fnl394 if not os.path.isdir( self.logDir ):395 os.mkdir( self.logDir )396 397 tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]398 self.batchLogfile = '%s/qcBatchLog_%s.txt' % ( self.logDir,tstring1)399 ## default appending to myapp.log; mode='w' forces a new file (deleting old contents).400 self.logger = logging.getLogger('c4logger')401 if self.policyBatchLogfileMode in ['n','np']:402 assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile403 m = self.policyBatchLogfileMode[0]404 if m == 'n':405 m = 'w'406 if m == 'a':407 self.hdlr = logging.FileHandler(self.batchLogfile)408 else:409 self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)410 formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')411 self.hdlr.setFormatter(formatter)412 self.logger.setLevel(logging.INFO)413 self.logger.addHandler(self.hdlr)414 415 self.attributeMappings = []416 self.attributeMappingsLog = None417 if self.attributeMappingFile != None:418 for l in open( self.attributeMappingFile ).readlines():419 if l[0] != '#':420 bb = string.split( string.strip(l), '|' )421 assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)422 bits = string.split( bb[0], ';' )423 cl = []424 for b in bits:425 cl.append( string.split(b, '=' ) )426 self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )427 self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )428 429 def commandHints(self, args):430 if args[0] in ['-h','--sum']:431 print 'Arguments look OK'432 elif args[0] == '--copy-config':433 print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'434 else:435 if not( '-f' in args or '-d' in args or '-D' in args):436 print 'No file or target directory specified'437 print """USAGE:438 ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]439 440 With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked.441 """442 raise SystemExit(0)443 444 445 def getFileLog( self, fn, flf=None ):446 if flf == None:447 tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]448 if fn in self.dupDict.keys():449 tag = '__%2.2i' % self.dupDict[fn]450 self.dupDict[fn] += 1451 else:452 tag = ''453 self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)454 if self.policyFileLogfileMode in ['n','np']:455 assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile456 m = self.policyFileLogfileMode[0]457 if m == 'n':458 m = 'w'459 else:460 m = 'a'461 self.fileLogfile = flf462 463 self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))464 if m == 'a':465 self.fHdlr = logging.FileHandler(self.fileLogfile)466 else:467 self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)468 fileFormatter = logging.Formatter('%(message)s')469 self.fHdlr.setFormatter(fileFormatter)470 self.fLogger.addHandler(self.fHdlr)471 self.fLogger.setLevel(logging.INFO)472 return self.fLogger473 474 def closeFileLog(self):475 self.fLogger.removeHandler(self.fHdlr)476 self.fHdlr.close()477 if self.policyFileLogfileMode in ['wo','np']:478 os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )479 480 def closeBatchLog(self):481 self.logger.removeHandler(self.hdlr)482 self.hdlr.close()483 if self.policyBatchLogfileMode in ['wo','np']:484 os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )485 486 243 487 244 class main(object): -
CCCC/trunk/ceda_cc/unitTestsS2.py
r287 r304 50 50 except: 51 51 print 'Failed [%s]: dummy run triggered exception' % testId 52 raise 52 53 raise baseException( 'Failed [%s]: dummy run triggered exception' % testId ) 53 54 -
CCCC/trunk/ceda_cc/utils_c4.py
r302 r304 1 """A set of classes running checks and providing utilities to support checks""" 1 2 import string, re, os, sys, traceback, ctypes 2 3 … … 69 70 70 71 class checkBase(object): 72 """Base class for checks, containing a set of standard methods for managing operation of checks and logging of results""" 71 73 72 74 def __init__(self,cls="CORDEX",reportPass=True,parent=None,monitor=None): 75 """Creat class instance: set defaults, link arguments to instance, create a range of compiled regular expressions""" 73 76 self.cls = cls 74 77 self.project = cls 75 78 self.abortMessageCount = parent.abortMessageCount 76 79 self.monitor = monitor 77 ## check done earlier78 ## assert cls in ['CORDEX','SPECS'],'This version of the checker only supports CORDEX, SPECS'79 80 self.re_isInt = re.compile( '[0-9]+' ) 80 81 self.errorCount = 0 … … 103 104 104 105 def isInt(self,x): 106 """Check that a string is a representation of an integer""" 105 107 return self.re_isInt.match( x ) != None 106 108 107 109 def logMessage(self, msg, error=False ): 110 """Log messages and count messages""" 108 111 self.messageCount += 1 109 112 assert self.abortMessageCount < 0 or self.abortMessageCount > self.messageCount, 'Raising error [TESTX01], perhaps for testing' … … 132 135 133 136 def log_exception( self, msg): 137 """Logging of exceptions -- putting trace information in log files""" 134 138 if self.parent != None and self.parent.log != None: 135 139 self.parent.log.error("Exception has occured" ,exc_info=1) … … 138 142 139 143 def log_error( self, msg ): 144 """Create an error log message and call logMessage; count errors;""" 140 145 self.lastError = msg 141 146 self.errorCount += 1 … … 143 148 144 149 def log_pass( self ): 150 """Create a pass log message and call logMessage; count passes;""" 145 151 self.passCount = True 146 152 if self.reportPass: … … 165 171 166 172 def test(self,res,msg,abort=False,part=False,appendLogfile=(None,None)): 173 """Handle test results. 174 :param res: [True/False] result of test; 175 :param msg: Message describing the test; 176 :param abort: {optional} Set True if checks should be aborted when test fails; 177 :param part: {optional} Set True if this is a component of a test (logging of pass suppressed); 178 :param appendLogfile: {optional} Allows results to be appended to pre-existing log file; 179 """ 167 180 self.appendLogfile = appendLogfile 168 181 if res: … … 176 189 177 190 def runChecks(self): 191 """Run all the checks registered in this instance (in self.checks) and handle exceptions""" 178 192 179 193 try: … … 189 203 190 204 class checkFileName(checkBase): 205 """Check basic syntax of file names (i.e. checks properties of the text string, it does not attempt to access the file). 206 Inherits :class:`checkBase` class. Checks are run by the :meth:`check` method.""" 191 207 192 208 def init(self): … … 201 217 202 218 def check(self,fn): 219 """Initiate checks: manage arguments and then call *runChecks* (inherited from checkBase class). 220 Arguments: fn: file name: the file name to be checked.""" 203 221 self.errorCount = 0 204 222 assert type(fn) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(fn) … … 210 228 ### 211 229 def do_check_fn(self): 230 """Basic file name checks: 231 (1) Check suffix; 232 (1b) [for ESA-CCI files] check presence of "ESACCI" and identify file naming convention; 233 (2) Split file name into components and check number of such components; 234 (3) Additional specialist checks for ESA-CCI, CORDEX, CMIP-type (for the time range). 235 """ 212 236 fn = self.fn 213 237 self.errorCount = 0 … … 343 367 344 368 def do_check_fnextra(self): 369 """Check whether file name components match constraints -- but only if those constraints are not implicitly verified through comparison with global attributes in later checks""" 345 370 self.checkId = ('004','file_name_extra' ) 346 371 vocabs = self.pcfg.vocabs … … 353 378 except: 354 379 print 'failed trying to check file name component %s' % a 355 raise 356 ##raise baseException1( 'failed trying to check file name component %s' % a ) 380 raise baseException( 'failed trying to check file name component %s' % a ) 357 381 358 382 self.test( len(m) == 0, 'File name components do not match constraints: %s' % str(m) ) … … 360 384 361 385 class checkGlobalAttributes(checkBase): 386 """Check global and variable attributes, using tables of valid values""" 362 387 363 388 def init(self): … … 609 634 610 635 class checkStandardDims(checkBase): 636 """Check the dimensions which are defined in the specifications""" 611 637 612 638 def init(self): … … 1049 1075 1050 1076 class checkByVar(checkBase): 1077 """Run some checks on groups of files with a common variable. Checks for continuity of time in group""" 1051 1078 1052 1079 def init(self,fileNameSeparator='_'): … … 1061 1088 1062 1089 def impt(self,flist): 1090 """Scan a list of files and identify groups which a common variable and extract time ranges into a dictionary of lists, keyed on group identifiers. 1091 :param flist: List of file names. 1092 1093 This routine has rather obscure nested logical tests used to identify the group to which a file belongs. The complexity arises from the fact that the identification of the files that should form a continuous time series from the file names alone is not a standardised feature of the file names.""" 1063 1094 ee = {} 1064 1095 elist = [] … … 1073 1104 freq = None 1074 1105 1075 ### isFixed = freq in ['fx','fixed']1076 1106 group = fnParts[ self.pcfg.groupIndex ] 1077 1107 1078 1108 if self.parent.fileIsFixed: 1079 ## if isFixed:1080 1109 trange = None 1081 1110 else: … … 1136 1165 1137 1166 def checkTrange(self): 1167 """Manage time range checks: loop over groups of files identified by :meth:`impt`""" 1138 1168 keys = self.ee.keys() 1139 1169 keys.sort() … … 1146 1176 1147 1177 def checkThisTrange( self, tt, group): 1178 """Check consistency across a list of time ranges""" 1148 1179 1149 1180 if group in ['3hr','6hr']: -
CCCC/trunk/ceda_cc/xceptions.py
r262 r304 1 1 """Some exceptions used in the code""" 2 2 3 3 class abortChecks(Exception): 4 """Raised when checks are aborted following failure of a critical test (e.g. file name cannot be parsed).""" 4 5 pass 5 6 class loggedException(Exception): 6 pass 7 class baseException1(Exception): 7 """Raised after an exception has been caught and logged in a checking class, allowing execution to fall back to the loop over files.""" 8 8 pass 9 9 10 10 class baseException(Exception): 11 """Basic exception for general use in code.""" 11 12 12 13 def __init__(self,msg):
Note: See TracChangeset
for help on using the changeset viewer.