Changeset 304


Ignore:
Timestamp:
05/03/15 12:02:51 (6 years ago)
Author:
mjuckes
Message:

added doc strings, and split intialisation class into ccinit module

Location:
CCCC/trunk/ceda_cc
Files:
1 added
4 edited

Legend:

Unmodified
Added
Removed

  • CCCC/trunk/ceda_cc/c4.py

    r297 r304  
    11 
    22import sys 
     3from ccinit import c4_init 
    34 
    45testmain=False 
    … …  
    2223 
    2324# Standard library imports 
    24 import os, string, time, logging, sys, glob, pkgutil 
     25import os, string, time, glob, pkgutil 
    2526import shutil 
    2627## pkgutil is used in file_utils 
    … …  
    240241    self.drs['project'] = self.info.pcfg.projectV.id 
    241242    self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount 
    242  
    243 class c4_init(object): 
    244  
    245   def __init__(self,args=None): 
    246     self.logByFile = True 
    247     self.policyFileLogfileMode = 'w' 
    248     self.policyBatchLogfileMode = 'np' 
    249     if args==None: 
    250        args = sys.argv[1:] 
    251     nn = 0 
    252  
    253     self.attributeMappingFile = None 
    254     self.recordFile = 'Rec.txt' 
    255     self.logDir = 'logs_02' 
    256     self.errs = [] 
    257      
    258     # Set default project to "CORDEX" 
    259     self.project = "CORDEX" 
    260     self.holdExceptions = False 
    261     forceLogOrg = None 
    262     argsIn = args[:] 
    263     
    264     # Show help if no args or help requested 
    265     if len(args) == 0 or args[0] in ('-h', '-help', '--help'): 
    266        print 'Help command not implemented yet' 
    267        raise SystemExit(0)       
    268     # The --copy-config option must be the first argument if it is present. 
    269     elif args[0] == '--copy-config': 
    270        if len(args) < 2: 
    271          self.commandHints( argsIn ) 
    272        args.pop(0) 
    273        dest_dir = args.pop(0) 
    274        config.copy_config(dest_dir) 
    275        print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir 
    276        print 
    277        raise SystemExit(0) 
    278  
    279     self.summarymode = args[0] == '--sum' 
    280     if self.summarymode: 
    281       return 
    282  
    283     self.experimental = False 
    284     self.forceNetcdfLib = None 
    285     fltype = None 
    286     argu = [] 
    287     while len(args) > 0: 
    288       next = args.pop(0) 
    289       if next == '-f': 
    290         flist = [args.pop(0),] 
    291         self.logByFile = False 
    292         fltype = '-f' 
    293         self.source = flist[0] 
    294       elif next == '--log': 
    295         x = args.pop(0) 
    296         assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x) 
    297         if x in ['multi','m']: 
    298            forceLogOrg = 'multi' 
    299         elif x in ['single','s']: 
    300            forceLogOrg = 'single' 
    301       elif next == '--experimental': 
    302         self.experimental = True 
    303       elif next == '--force-ncq': 
    304         self.forceNetcdfLib = 'ncq3' 
    305       elif next == '--force-cdms2': 
    306         self.forceNetcdfLib = 'cdms2' 
    307       elif next == '--force-pync4': 
    308         self.forceNetcdfLib = 'netCDF4' 
    309       elif next == '--force-scientific': 
    310         self.forceNetcdfLib = 'Scientific' 
    311       elif next == '--flfmode': 
    312         lfmk = args.pop(0) 
    313         assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk 
    314         self.policyFileLogfileMode = lfmk 
    315       elif next == '--blfmode': 
    316         lfmk = args.pop(0) 
    317         assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk 
    318         self.policyBatchLogfileMode = lfmk 
    319       elif next == '-d': 
    320         fdir = args.pop(0) 
    321         flist = glob.glob( '%s/*.nc' % fdir  ) 
    322         self.source = '%s/*.nc' % fdir 
    323       elif next == '-D': 
    324         flist  = [] 
    325         fdir = args.pop(0) 
    326         for root, dirs, files in os.walk( fdir, followlinks=True ): 
    327           for f in files: 
    328             fpath = '%s/%s' % (root,f) 
    329             if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc': 
    330               flist.append( fpath ) 
    331         self.source = '%s/.....' % fdir 
    332       elif next == '-R': 
    333         self.recordFile = args.pop(0) 
    334       elif next == '--ld': 
    335         self.logDir = args.pop(0) 
    336       elif next in ['--catchAllExceptions','--cae']: 
    337         self.holdExceptions = True 
    338       elif next == '--aMap': 
    339         self.attributeMappingFile = args.pop(0) 
    340         assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file' 
    341       elif next == "-p": 
    342         self.project = args.pop(0) 
    343       else: 
    344        print 'Unused argument: %s' % next 
    345        argu.append( next ) 
    346        nn+=1 
    347     if nn != 0: 
    348       print 'Unused arguments: ', argu 
    349       self.commandHints( argsIn ) 
    350  
    351     if self.project == 'CMIP5' and fltype != '-f': 
    352       fl0 = [] 
    353       for f in flist: 
    354         if string.find( f, '/latest/' ) != -1: 
    355           fl0.append(f) 
    356       flist = fl0 
    357  
    358     if forceLogOrg != None: 
    359       if forceLogOrg == 'single': 
    360         self.logByFile = False 
    361       else: 
    362         self.logByFile = True 
    363  
    364     if self.project[:2] == '__': 
    365        self.source = 'dummy' 
    366        flist = [] 
    367        ss = 'abcdefgijk' 
    368        ss = 'abcdefgijklmnopqrstuvwxyz' 
    369        ss = 'abc' 
    370        for i in range(10): 
    371          v = 'v%s' % i 
    372          for a in ss: 
    373            for b in ss: 
    374              flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) ) 
    375     flist.sort() 
    376     fnl = [] 
    377     for f in flist: 
    378       fn = string.split(f, '/')[-1] 
    379       fnl.append(fn) 
    380     nd = 0 
    381     dupl = [] 
    382     for k in range(len(fnl)-1): 
    383       if fnl[k] == fnl[k-1]: 
    384         nd += 1 
    385         dupl.append( fnl[k] ) 
    386     self.dupDict = {} 
    387     for f in dupl: 
    388       self.dupDict[f] = 0 
    389     if nd != 0: 
    390       self.errs.append( 'Duplicate file names encountered: %s' % nd ) 
    391       self.errs.append( dupl ) 
    392     self.flist = flist 
    393     self.fnl = fnl 
    394     if not os.path.isdir(   self.logDir ): 
    395        os.mkdir(   self.logDir ) 
    396  
    397     tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6] 
    398     self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1) 
    399 ## default appending to myapp.log; mode='w' forces a new file (deleting old contents). 
    400     self.logger = logging.getLogger('c4logger') 
    401     if self.policyBatchLogfileMode in ['n','np']: 
    402         assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile 
    403     m = self.policyBatchLogfileMode[0] 
    404     if m == 'n': 
    405       m = 'w' 
    406     if m == 'a': 
    407       self.hdlr = logging.FileHandler(self.batchLogfile) 
    408     else: 
    409       self.hdlr = logging.FileHandler(self.batchLogfile,mode=m) 
    410     formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s') 
    411     self.hdlr.setFormatter(formatter) 
    412     self.logger.setLevel(logging.INFO) 
    413     self.logger.addHandler(self.hdlr) 
    414  
    415     self.attributeMappings = [] 
    416     self.attributeMappingsLog = None 
    417     if self.attributeMappingFile != None: 
    418       for l in open( self.attributeMappingFile ).readlines(): 
    419         if l[0] != '#': 
    420           bb = string.split( string.strip(l), '|' )  
    421           assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l) 
    422           bits = string.split( bb[0], ';' ) 
    423           cl = [] 
    424           for b in bits: 
    425             cl.append( string.split(b, '=' ) ) 
    426           self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) ) 
    427       self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' ) 
    428  
    429   def commandHints(self, args): 
    430     if args[0] in ['-h','--sum']: 
    431       print 'Arguments look OK' 
    432     elif args[0] == '--copy-config': 
    433       print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>' 
    434     else: 
    435       if not( '-f' in args or '-d' in args or '-D' in args): 
    436         print 'No file or target directory specified' 
    437         print """USAGE: 
    438 ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options] 
    439  
    440 With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked. 
    441 """ 
    442     raise SystemExit(0) 
    443     
    444  
    445   def getFileLog( self, fn, flf=None ): 
    446     if flf == None: 
    447       tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3] 
    448       if fn in self.dupDict.keys(): 
    449         tag = '__%2.2i' % self.dupDict[fn] 
    450         self.dupDict[fn] += 1 
    451       else: 
    452         tag = '' 
    453       self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2) 
    454       if self.policyFileLogfileMode in ['n','np']: 
    455         assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile 
    456       m = self.policyFileLogfileMode[0] 
    457       if m == 'n': 
    458         m = 'w' 
    459     else: 
    460       m = 'a' 
    461       self.fileLogfile = flf 
    462  
    463     self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m)) 
    464     if m == 'a': 
    465       self.fHdlr = logging.FileHandler(self.fileLogfile) 
    466     else: 
    467       self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m) 
    468     fileFormatter = logging.Formatter('%(message)s') 
    469     self.fHdlr.setFormatter(fileFormatter) 
    470     self.fLogger.addHandler(self.fHdlr) 
    471     self.fLogger.setLevel(logging.INFO) 
    472     return self.fLogger 
    473  
    474   def closeFileLog(self): 
    475     self.fLogger.removeHandler(self.fHdlr) 
    476     self.fHdlr.close() 
    477     if self.policyFileLogfileMode in ['wo','np']: 
    478       os.popen( 'chmod %s %s;' % (444, self.fileLogfile) ) 
    479  
    480   def closeBatchLog(self): 
    481     self.logger.removeHandler(self.hdlr) 
    482     self.hdlr.close() 
    483     if self.policyBatchLogfileMode in ['wo','np']: 
    484       os.popen( 'chmod %s %s;' % (444, self.batchLogfile) ) 
    485  
    486243 
    487244class main(object): 

  • CCCC/trunk/ceda_cc/unitTestsS2.py

    r287 r304  
    5050  except: 
    5151    print 'Failed [%s]: dummy run triggered exception' % testId 
     52    raise 
    5253    raise baseException( 'Failed [%s]: dummy run triggered exception' % testId ) 
    5354 

  • CCCC/trunk/ceda_cc/utils_c4.py

    r302 r304  
     1"""A set of classes running checks and providing utilities to support checks""" 
    12import string, re, os, sys, traceback, ctypes 
    23 
    … …  
    6970 
    7071class checkBase(object): 
     72  """Base class for checks, containing a set of standard methods for managing operation of checks and logging of results""" 
    7173 
    7274  def  __init__(self,cls="CORDEX",reportPass=True,parent=None,monitor=None): 
     75    """Creat class instance: set defaults, link arguments to instance, create a range of compiled regular expressions""" 
    7376    self.cls = cls 
    7477    self.project = cls 
    7578    self.abortMessageCount = parent.abortMessageCount 
    7679    self.monitor = monitor 
    77     ## check done earlier 
    78     ## assert cls in ['CORDEX','SPECS'],'This version of the checker only supports CORDEX, SPECS' 
    7980    self.re_isInt = re.compile( '[0-9]+' ) 
    8081    self.errorCount = 0 
    … …  
    103104 
    104105  def isInt(self,x): 
     106    """Check that a string is a representation of an integer""" 
    105107    return self.re_isInt.match( x ) != None 
    106108 
    107109  def logMessage(self, msg, error=False ): 
     110    """Log messages and count messages""" 
    108111    self.messageCount += 1 
    109112    assert self.abortMessageCount < 0 or self.abortMessageCount > self.messageCount, 'Raising error [TESTX01], perhaps for testing' 
    … …  
    132135 
    133136  def log_exception( self, msg): 
     137    """Logging of exceptions -- putting trace information in log files""" 
    134138    if self.parent != None and self.parent.log != None: 
    135139        self.parent.log.error("Exception has occured" ,exc_info=1) 
    … …  
    138142 
    139143  def log_error( self, msg ): 
     144    """Create an error log message and call logMessage; count errors;""" 
    140145    self.lastError = msg 
    141146    self.errorCount += 1 
    … …  
    143148 
    144149  def log_pass( self ): 
     150    """Create a pass log message and call logMessage; count passes;""" 
    145151    self.passCount = True 
    146152    if self.reportPass: 
    … …  
    165171 
    166172  def test(self,res,msg,abort=False,part=False,appendLogfile=(None,None)): 
     173    """Handle test results.  
     174      :param res: [True/False] result of test; 
     175      :param msg: Message describing the test; 
     176      :param abort: {optional} Set True if checks should be aborted when test fails; 
     177      :param part: {optional} Set True if this is a component of a test (logging of pass suppressed); 
     178      :param appendLogfile: {optional} Allows results to be appended to pre-existing log file; 
     179    """ 
    167180    self.appendLogfile = appendLogfile 
    168181    if res: 
    … …  
    176189 
    177190  def runChecks(self): 
     191    """Run all the checks registered in this instance (in self.checks) and handle exceptions""" 
    178192 
    179193    try: 
    … …  
    189203     
    190204class checkFileName(checkBase): 
     205  """Check basic syntax of file names (i.e. checks properties of the text string, it does not attempt to access the file). 
     206Inherits :class:`checkBase` class. Checks are run by the :meth:`check` method.""" 
    191207 
    192208  def init(self): 
    … …  
    201217 
    202218  def check(self,fn): 
     219    """Initiate checks: manage arguments and then call *runChecks* (inherited from checkBase class). 
     220  Arguments: fn: file name: the file name to be checked.""" 
    203221    self.errorCount = 0 
    204222    assert type(fn) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(fn) 
    … …  
    210228### 
    211229  def do_check_fn(self): 
     230    """Basic file name checks: 
     231       (1) Check suffix; 
     232       (1b) [for ESA-CCI files] check presence of "ESACCI" and identify file naming convention; 
     233       (2) Split file name into components and check number of such components; 
     234       (3) Additional specialist checks for ESA-CCI, CORDEX, CMIP-type (for the time range). 
     235    """ 
    212236    fn = self.fn 
    213237    self.errorCount = 0 
    … …  
    343367 
    344368  def do_check_fnextra(self): 
     369    """Check whether file name components match constraints -- but only if those constraints are not implicitly verified through comparison with global attributes in later checks""" 
    345370    self.checkId = ('004','file_name_extra' ) 
    346371    vocabs = self.pcfg.vocabs 
    … …  
    353378        except: 
    354379          print 'failed trying to check file name component %s' % a 
    355           raise 
    356           ##raise baseException1( 'failed trying to check file name component %s' % a ) 
     380          raise baseException( 'failed trying to check file name component %s' % a ) 
    357381 
    358382    self.test( len(m)  == 0, 'File name components do not match constraints: %s' % str(m) ) 
    … …  
    360384 
    361385class checkGlobalAttributes(checkBase): 
     386  """Check global and variable attributes, using tables of valid values""" 
    362387 
    363388  def init(self): 
    … …  
    609634        
    610635class checkStandardDims(checkBase): 
     636  """Check the dimensions which are defined in the specifications""" 
    611637 
    612638  def init(self): 
    … …  
    10491075 
    10501076class checkByVar(checkBase): 
     1077  """Run some checks on groups of files with a common variable. Checks for continuity of time in group""" 
    10511078 
    10521079  def init(self,fileNameSeparator='_'): 
    … …  
    10611088 
    10621089  def impt(self,flist): 
     1090    """Scan a list of files and identify groups which a common variable and extract time ranges into a dictionary of lists, keyed on group identifiers. 
     1091     :param flist: List of file names. 
     1092 
     1093 This routine has rather obscure nested logical tests used to identify the group to which a file belongs. The complexity arises from the fact that the identification of the files that should form a continuous time series from the file names alone is not a standardised feature of the file names.""" 
    10631094    ee = {} 
    10641095    elist = [] 
    … …  
    10731104          freq = None 
    10741105 
    1075         ### isFixed = freq  in ['fx','fixed'] 
    10761106        group = fnParts[ self.pcfg.groupIndex ] 
    10771107 
    10781108        if self.parent.fileIsFixed: 
    1079        ## if isFixed: 
    10801109          trange = None 
    10811110        else: 
    … …  
    11361165 
    11371166  def checkTrange(self): 
     1167    """Manage time range checks: loop over groups of files identified by :meth:`impt`""" 
    11381168    keys = self.ee.keys() 
    11391169    keys.sort() 
    … …  
    11461176 
    11471177  def checkThisTrange( self, tt, group): 
     1178    """Check consistency across a list of time ranges""" 
    11481179 
    11491180    if group in ['3hr','6hr']: 

  • CCCC/trunk/ceda_cc/xceptions.py

    r262 r304  
    1  
     1"""Some exceptions used in the code""" 
    22 
    33class abortChecks(Exception): 
     4  """Raised when checks are aborted following failure of a critical test (e.g. file name cannot be parsed).""" 
    45  pass 
    56class loggedException(Exception): 
    6   pass 
    7 class baseException1(Exception): 
     7  """Raised after an exception has been caught and logged in a checking class, allowing execution to fall back to the loop over files.""" 
    88  pass 
    99 
    1010class baseException(Exception): 
     11  """Basic exception for general use in code.""" 
    1112 
    1213  def __init__(self,msg): 
Note: See TracChangeset for help on using the changeset viewer.