Changeset 75


Ignore:
Timestamp:
18/10/13 16:06:05 (7 years ago)
Author:
mjuckes
Message:

merged support for SPECS back into main trunk

Location:
CCCC/trunk
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/c4.py

    r68 r75  
    77reload( utils ) 
    88import cdms2 
    9  
    10 vocabs = { 'variable':utils.mipVocab(), \ 
     9project='SPECS' 
     10project='CORDEX' 
     11 
     12 
     13if project == 'SPECS': 
     14  vocabs = { 'variable':utils.mipVocab(project=project) } 
     15else: 
     16  vocabs = { 'variable':utils.mipVocab(), \ 
    1117           'driving_experiment_name':utils.listControl( 'driving_experiment_name', config.validExperiment ), \ 
    1218           'project_id':utils.listControl( 'project_id', ['CORDEX'] ), \ 
     
    8894 
    8995class checker: 
    90   def __init__(self): 
     96  def __init__(self,cls='CORDEX'): 
    9197    self.info = dummy() 
    9298    self.calendar = 'None' 
    93     self.cfn = utils.checkFileName(parent=self.info) 
    94     self.cga = utils.checkGlobalAttributes(parent=self.info) 
    95     self.cgd = utils.checkStandardDims(parent=self.info) 
    96     self.cgg = utils.checkGrids(parent=self.info) 
     99    self.cfn = utils.checkFileName(parent=self.info,cls=cls) 
     100    self.cga = utils.checkGlobalAttributes(parent=self.info,cls=cls) 
     101    self.cgd = utils.checkStandardDims(parent=self.info,cls=cls) 
     102    self.cgg = utils.checkGrids(parent=self.info,cls=cls) 
     103    self.cls = cls 
    97104 
    98105  def checkFile(self,fpath,log=None): 
     
    148155      return 
    149156 
    150     self.cgg.rotatedPoleGrids = config.rotatedPoleGrids 
    151     self.cgg.interpolatedGrids = config.interpolatedGrids 
    152     self.cgg.check( self.cfn.var, self.cfn.domain, self.da, self.va ) 
     157    if self.cls == 'CORDEX': 
     158      self.cgg.rotatedPoleGrids = config.rotatedPoleGrids 
     159      self.cgg.interpolatedGrids = config.interpolatedGrids 
     160      self.cgg.check( self.cfn.var, self.cfn.domain, self.da, self.va ) 
    153161     
    154     if not self.cgg.completed: 
    155       self.completed = False 
    156       return 
     162      if not self.cgg.completed: 
     163        self.completed = False 
     164        return 
    157165    self.completed = True 
    158166    self.drs = self.cga.getDrs() 
     
    235243    self.fHdlr.close() 
    236244 
    237 cc = checker() 
     245cc = checker(cls=project) 
    238246 
    239247cal = None 
     
    246254  c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) ) 
    247255 
    248   cbv = utils.checkByVar( parent=cc.info) 
     256  cbv = utils.checkByVar( parent=cc.info,cls=project) 
    249257  cbv.impt( c4i.flist ) 
    250258 
     
    285293 
    286294  cc.info.log = c4i.logger 
    287   cbv.check( recorder=rec, calendar=cc.calendar) 
     295   
     296  if project != 'SPECS': 
     297     cbv.check( recorder=rec, calendar=cc.calendar) 
    288298  rec.dumpAll() 
    289299  c4i.hdlr.close() 
  • CCCC/trunk/utils_c4.py

    r72 r75  
    8080cs = checkSeq() 
    8181 
     82class readVocab: 
     83 
     84  def __init__(self,dir): 
     85    self.dir = dir 
     86 
     87  def getSimpleList(self,file): 
     88    ii = open('%s/%s' % (self.dir,file) ) 
     89    return map( string.strip, ii.readlines() ) 
     90 
    8291class checkBase: 
    8392 
    8493  def  __init__(self,cls="CORDEX",reportPass=True,parent=None): 
    8594    self.cls = cls 
    86     assert cls == 'CORDEX','This version of the checker only supports CORDEX' 
     95    assert cls in ['CORDEX','SPECS'],'This version of the checker only supports CORDEX, SPECS' 
    8796    self.re_isInt = re.compile( '[0-9]+' ) 
    8897    self.errorCount = 0 
     
    91100    self.parent = parent 
    92101    self.reportPass=reportPass 
    93     self.requiredGlobalAttributes = [ 'institute_id', 'contact', 'rcm_version_id', 'product', 'CORDEX_domain', 'creation_date', \ 
     102    if self.cls == 'CORDEX': 
     103      self.requiredGlobalAttributes = [ 'institute_id', 'contact', 'rcm_version_id', 'product', 'CORDEX_domain', 'creation_date', \ 
    94104             'frequency', 'model_id', 'driving_model_id', 'driving_experiment', 'driving_model_ensemble_member', 'experiment_id'] 
    95     self.controlledGlobalAttributes = ['frequency', 'driving_experiment_name', 'project_id', 'CORDEX_domain', 'driving_model_id', 'model_id', 'institute_id','driving_model_ensemble_member','rcm_version_id'] 
    96     self.globalAttributesInFn = [None,'CORDEX_domain','driving_model_id','experiment_id','driving_model_ensemble_member','model_id','rcm_version_id'] 
    97     self.requiredVarAttributes = ['long_name', 'standard_name', 'units', 'missing_value', '_FillValue'] 
    98     self.checks = () 
    99     self.drsMappings = {'variable':'@var','institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \ 
     105      self.controlledGlobalAttributes = ['frequency', 'driving_experiment_name', 'project_id', 'CORDEX_domain', 'driving_model_id', 'model_id', 'institute_id','driving_model_ensemble_member','rcm_version_id'] 
     106      self.globalAttributesInFn = [None,'CORDEX_domain','driving_model_id','experiment_id','driving_model_ensemble_member','model_id','rcm_version_id'] 
     107      self.requiredVarAttributes = ['long_name', 'standard_name', 'units', 'missing_value', '_FillValue'] 
     108      self.drsMappings = {'variable':'@var','institute':'institute_id', 'product':'product', 'experiment':'experiment_id', \ 
    100109                        'ensemble':'driving_model_ensemble_member', 'model':'model_id', 'driving_model':'driving_model_id', \ 
    101110                        'frequency':'frequency', \ 
    102111                        'project':'project_id', 'domain':'CORDEX_domain', 'model_version':'rcm_version_id' } 
     112    elif self.cls == 'SPECS': 
     113      lrdr = readVocab( 'specs_vocabs/') 
     114      self.requiredGlobalAttributes = [ 'institute_id', 'contact', 'product', 'creation_date', 'tracking_id', \ 
     115              'experiment_id'] 
     116      self.requiredGlobalAttributes = lrdr.getSimpleList( 'globalAts.txt' ) 
     117      self.controlledGlobalAttributes = [ ] 
     118      self.globalAttributesInFn = [None,'CORDEX_domain','driving_model_id','experiment_id','driving_model_ensemble_member','model_id','rcm_version_id'] 
     119      self.requiredVarAttributes = ['long_name', 'standard_name', 'units', 'missing_value', '_FillValue'] 
     120      self.drsMappings = {'variable':'@var'} 
     121    self.checks = () 
    103122    self.init() 
    104123 
     
    189208    self.test( fn[-3:] == '.nc', 'File name ending ".nc" expected', abort=True, part=True ) 
    190209    bits = string.split( fn[:-3], '_' ) 
    191     self.test( len(bits) in [8,9], 'File name not parsed into 8 or 9 elements [%s]' % str(bits), abort=True ) 
    192  
    193210    self.fnParts = bits[:] 
    194     self.domain = self.fnParts[1] 
    195     self.freq = self.fnParts[7] 
     211 
     212    if self.cls == 'CORDEX': 
     213      self.fnPartsOkLen = [8,9] 
     214      self.fnPartsOkFixedLen = [8,] 
     215      self.fnPartsOkUnfixedLen = [9,] 
     216      checkTrangeLen = True 
     217      self.domain = self.fnParts[1] 
     218      self.freq = self.fnParts[7] 
     219    elif self.cls == 'SPECS': 
     220      self.domain = None 
     221      self.freq = self.fnParts[1] 
     222      self.fnPartsOkLen = [6,7] 
     223      self.fnPartsOkFixedLen = [6,] 
     224      self.fnPartsOkUnfixedLen = [7,] 
     225      checkTrangeLen = False 
     226    self.test( len(bits) in self.fnPartsOkLen, 'File name not parsed in %s elements [%s]' % (str(self.fnPartsOkLen),str(bits)), abort=True ) 
     227 
    196228    self.var = self.fnParts[0] 
    197229 
     230    self.isFixed = self.freq == 'fx' 
    198231 
    199232    self.checkId = '002' 
    200     if len(self.fnParts) == 9: 
     233    if not self.isFixed: 
    201234 
    202235## test time segment 
     
    212245 
    213246    self.checkId = '003' 
    214     if len(self.fnParts) == 8: 
    215       self.test( self.fnParts[7] == 'fx', 'Time range not specified and frequency not fx' ) 
     247    if self.isFixed: 
     248      self.test( len(self.fnParts) in self.fnPartsOkFixedLen, 'Number of file name elements not acceptable for fixed data' ) 
    216249 
    217250    self.checkId, ok = ('004',True) 
    218     if len(self.fnParts) == 9: 
     251    if len(self.fnParts) == 9 and checkTrangeLen: 
    219252      ltr = { 'mon':6, 'sem':6, 'day':8, '3hr':10, '6hr':10 } 
    220253      ok &=self.test( self.freq in ltr.keys(), 'Frequency [%s] not recognised' % self.freq, part=True ) 
     
    316349        mm.append( k ) 
    317350 
    318     ok &= self.test( len(mm)  == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, str(mm)), part=True ) 
     351    ok &= self.test( len(m)  == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, str(mm)), part=True ) 
    319352    if ok: 
    320353       self.log_pass() 
     
    563596class mipVocab: 
    564597 
    565   def __init__(self): 
     598  def __init__(self,project='CORDEX'): 
     599     assert project in ['CORDEX','SPECS'],'Project %s not recognised' % project 
     600     if project == 'CORDEX': 
     601       dir = 'cordex_vocabs/mip/' 
     602       tl = ['fx','sem','mon','day','6h','3h'] 
     603       vgmap = {'6h':'6hr','3h':'3hr'} 
     604       fnpat = 'CORDEX_%s' 
     605     elif project == 'SPECS': 
     606       dir = 'specs_vocabs/mip/' 
     607       tl = ['fx','Omon','Amon','Lmon','OImon','day','6hrLev'] 
     608       vgmap = {} 
     609       fnpat = 'SPECS_%s' 
    566610     ms = mipTableScan() 
    567      dir = 'cordex_vocabs/mip/' 
    568611     self.varInfo = {} 
    569612     self.varcons = {} 
    570      for f in ['fx','sem','mon','day','6h','3h']: 
    571         vg = f 
    572         if f in ['6h','3h']: 
    573           vg += 'r' 
     613     for f in tl: 
     614        vg = vgmap.get( f, f ) 
    574615        self.varcons[vg] = {} 
    575         fn = 'CORDEX_%s' % f 
     616        fn = fnpat % f 
    576617        ll = open( '%s%s' % (dir,fn) ).readlines() 
    577618        ee = ms.scan_table(ll,None,asDict=True) 
     
    640681      fn = string.split(f, '/' )[-1] 
    641682      fnParts = string.split( fn[:-3], '_' ) 
    642       if fnParts[7] == 'fx': 
     683      if self.cls == 'CORDEX': 
     684        isFixed = fnParts[7] == 'fx' 
     685        group = fnParts[7] 
     686      elif self.cls == 'SPECS': 
     687        isFixed = fnParts[1] == 'fx' 
     688        group = fnParts[1] 
     689 
     690      if isFixed: 
    643691        trange = None 
    644692      else: 
    645         trange = string.split( fnParts[8], '-' ) 
    646       group = fnParts[7] 
     693        trange = string.split( fnParts[-1], '-' ) 
    647694      var = fnParts[0] 
    648695      if group not in ee.keys(): 
     
    650697      if var not in ee[group].keys(): 
    651698        ee[group][var] = [] 
    652       ee[group][var].append( (f,fn,fnParts[7],trange) ) 
     699      ee[group][var].append( (f,fn,group,trange) ) 
    653700 
    654701    nn = len(flist) 
Note: See TracChangeset for help on using the changeset viewer.