Changeset 253 for CCCC


Ignore:
Timestamp:
12/11/14 13:03:58 (6 years ago)
Author:
mjuckes
Message:

Updated vocabulary management

Location:
CCCC/trunk/ceda_cc
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/ceda_cc/c4.py

    r252 r253  
    142142 
    143143class checker(object): 
    144   def __init__(self, pcfg, cls,reader,abortMessageCount=-1): 
     144  def __init__(self, pcfg, cls,reader,abortMessageCount=-1,experimental=False): 
    145145    self.info = dummy() 
     146    self.info.experimental=experimental 
    146147    self.info.pcfg = pcfg 
    147148    self.info.abortMessageCount = abortMessageCount 
     
    204205    ##self.cgd.heightValues = config.heightValues 
    205206    ##self.cgd.heightRange = config.heightRange 
    206     self.cgd.check( self.cfn.var, self.cfn.freq, self.da, self.va, self.cga.isInstantaneous ) 
     207    self.cgd.check( self.cfn.var, self.cfn.freq, self.da, self.va, self.cga.isInstantaneous, self.vocabs ) 
    207208    self.calendar = self.cgd.calendar 
    208209    if not self.cgd.completed: 
     
    262263      return 
    263264 
     265    self.experimental = False 
    264266    self.forceNetcdfLib = None 
    265267    fltype = None 
     
    279281        elif x in ['single','s']: 
    280282           forceLogOrg = 'single' 
     283      elif next == '--experimental': 
     284        self.experimental = True 
    281285      elif next == '--force-ncq': 
    282286        self.forceNetcdfLib = 'ncq3' 
     
    476480    assert pcfg.projectV.v == -1, 'Cannot handle anything other than latest version at present' 
    477481    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog,forceLib=c4i.forceNetcdfLib) 
    478     self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount) 
     482    self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount, experimental=c4i.experimental) 
    479483    rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy ) 
     484    self.ncLib = ncLib 
    480485 
    481486    # This list will record the drs dictionaries of all checked files for export to JSON 
  • CCCC/trunk/ceda_cc/config_c4.py

    r252 r253  
    278278    if self.projectV.id == 'CORDEX': 
    279279      self.fnParts = NT_fnParts( len=[8,9], fxLen=[8,],  unfLen=[9,], checkTLen=True, ixDomain=1, ixFreq=7 ) 
    280       self.fnPartsOkLen = [8,9] 
    281       self.fnPartsOkFixedLen = [8,] 
    282       self.fnPartsOkUnfixedLen = [9,] 
    283       self.checkTrangeLen = True 
    284       self.domainIndex = 1 
    285       self.freqIndex = 7 
     280    ##  self.fnPartsOkLen = [5,6] 
     281      ##self.fnPartsOkFixedLen = [5,] 
     282      ##self.fnPartsOkUnfixedLen = [6,] 
     283      ##self.checkTrangeLen = True 
     284      ##self.domainIndex = 1 
     285      ##self.freqIndex = 7 
    286286    elif self.projectV.id == 'CMIP5': 
    287 ## cRoot_Lmon_CESM1-WACCM_rcp85_r3i1p1_200601-205512.nc 
    288       self.fnPartsOkLen = [5,6] 
    289       self.fnPartsOkFixedLen = [5,] 
    290       self.fnPartsOkUnfixedLen = [6,] 
    291       self.checkTrangeLen = False 
    292       self.domainIndex = None 
    293       self.freqIndex = None 
     287      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None ) 
     288      ##self.fnPartsOkLen = [5,6] 
     289      ##self.fnPartsOkFixedLen = [5,] 
     290      ##self.fnPartsOkUnfixedLen = [6,] 
     291      ##self.checkTrangeLen = False 
     292      ##self.domainIndex = None 
     293      ##self.freqIndex = None 
    294294    elif self.projectV.id == 'SPECS': 
    295       self.fnPartsOkLen = [6,7] 
    296       self.fnPartsOkFixedLen = [6,] 
    297       self.fnPartsOkUnfixedLen = [7,] 
    298       self.checkTrangeLen = False 
    299       self.domainIndex = None 
    300       self.freqIndex = 1 
     295      self.fnParts = NT_fnParts( len=[6,7], fxLen=[6,],  unfLen=[7,], checkTLen=False, ixDomain=None, ixFreq=1 ) 
     296      ##self.fnPartsOkLen = [6,7] 
     297      ##self.fnPartsOkFixedLen = [6,] 
     298      ##self.fnPartsOkUnfixedLen = [7,] 
     299      ##self.checkTrangeLen = False 
     300      ##self.domainIndex = None 
     301      ##self.freqIndex = 1 
    301302    elif self.projectV.id == 'CCMI': 
    302       self.fnPartsOkLen = [5,6] 
    303       self.fnPartsOkFixedLen = [5,] 
    304       self.fnPartsOkUnfixedLen = [6,] 
    305       self.checkTrangeLen = False 
    306       self.domainIndex = None 
    307       self.freqIndex = None 
     303      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None ) 
     304      ##self.fnPartsOkLen = [5,6] 
     305      ##self.fnPartsOkFixedLen = [5,] 
     306      ##self.fnPartsOkUnfixedLen = [6,] 
     307      ##self.checkTrangeLen = False 
     308      ##self.domainIndex = None 
     309      ##self.freqIndex = None 
    308310    elif self.projectV.id == '__dummy': 
    309       self.fnPartsOkLen = [4,5] 
    310       self.fnPartsOkFixedLen = [4,] 
    311       self.fnPartsOkUnfixedLen = [5,] 
    312       self.checkTrangeLen = False 
    313       self.domainIndex = None 
    314       self.freqIndex = 1 
     311      self.fnParts = NT_fnParts( len=[4,5], fxLen=[4,],  unfLen=[5,], checkTLen=False, ixDomain=None, ixFreq=1 ) 
     312      ##self.fnPartsOkLen = [4,5] 
     313      ##self.fnPartsOkFixedLen = [4,] 
     314      ##self.fnPartsOkUnfixedLen = [5,] 
     315      ##self.checkTrangeLen = False 
     316      ##self.domainIndex = None 
     317      ##self.freqIndex = 1 
     318    self.fnPartsOkLen = self.fnParts.len 
     319    self.fnPartsOkFixedLen = self.fnParts.fxLen 
     320    self.fnPartsOkUnfixedLen = self.fnParts.unfLen 
     321    self.checkTrangeLen = self.fnParts.checkTLen 
     322    self.domainIndex = self.fnParts.ixDomain 
     323    self.freqIndex = self.fnParts.ixFreq 
    315324 
    316325 
     
    352361 
    353362    self.vocabs = getVocabs(self) 
     363    test = False 
     364    if test: 
     365      for k in self.vocabs['variable'].varcons.keys(): 
     366        for k2 in self.vocabs['variable'].varcons[k].keys(): 
     367          if "height2m" in self.vocabs['variable'].varcons[k][k2].get( '_dimension',[]): 
     368            print 'config_c4: %s , %s: %s' % (k,k2,str(self.vocabs['variable'].varcons[k][k2]['_dimension'] ) ) 
     369       
    354370 
    355371    ##assert self.project != 'CCMI', 'Not completely set up for CCMI yet' 
  • CCCC/trunk/ceda_cc/extractMipInfo.py

    r249 r253  
    1111      l0.append(l) 
    1212  return l0 
    13    
     13 
     14heightRequired = ['tas','tasmax','tasmin','huss','sfcWind','sfcWindmax','wsgsmax','uas','vas'] 
    1415 
    1516ms = mipTableScan() 
     
    255256      ix = 0 
    256257      xxx = 'dim: ' 
     258 
     259    self.ix = ix 
     260    self.vars = vars 
     261    self.vdict = vdict 
     262# dictionary td[tab][var][ix][attribute] 
     263    self.td = td 
    257264    vd2 = {} 
    258265    for v in vars: 
     
    347354     else: 
    348355      print "WARNING[X4]: ",xxx, 'Zero length element: %s' % v 
     356 
     357  def chkDims( self, reqh=None): 
     358    pass 
    349359    
    350360class typecheck1: 
     
    565575cmip5AxesAtts = ['axis', 'bounds_values', 'climatology', 'coords_attrib', 'formula', 'index_only', 'long_name', 'must_call_cmor_grid', 'must_have_bounds', 'out_name', 'positive', 'requested', 'requested_bounds', 'standard_name', 'stored_direction', 'tolerance', 'type', 'units', 'valid_max', 'valid_min', 'value', 'z_bounds_factors', 'z_factors'] 
    566576 
    567 ## check consistency of dimensions 
     577v = runcheck1( m, ald, isAxes=False ) 
     578## check consistency of dimension 
    568579r = runcheck1( m, ald, isAxes=True ) 
    569580for e in r.errors: 
  • CCCC/trunk/ceda_cc/file_utils.py

    r248 r253  
    7575    if self.forceLib == 'ncq3': 
    7676      import ncq3 
     77      print 'Using ncq3' 
    7778      self.ncq3 = ncq3 
    7879      self.loadNc__ncq(fpath) 
    7980    elif self.forceLib == 'cdms2': 
    8081      import cdms2 
     82      print 'Using using cdms2' 
    8183      self.cdms2 = cdms2 
    8284      self.loadNc__Cdms(fpath) 
    8385    elif self.forceLib == 'netCDF4': 
    8486      import netCDF4 
     87      print 'Using netCDF4' 
    8588      self.netCDF4 = netCDF4 
    8689      self.loadNc__Netcdf4(fpath) 
    8790    elif self.forceLib == 'Scientific': 
    8891      import Scientific 
     92      print 'Using scientific python' 
    8993      from Scientific.IO import NetCDF as ncdf 
    9094      self.ncdf = ncdf 
  • CCCC/trunk/ceda_cc/unitTestsS1.py

    r180 r253  
    2323p = dummy() 
    2424ps = dummy() 
    25 p.log = log 
    26 ps.log = log 
     25pcmip5 = dummy() 
     26pccmi = dummy() 
     27for x in (p,ps,pcmip5,pccmi): 
     28  x.log = log 
     29  x.abortMessageCount = -1 
    2730p.pcfg = config.projectConfig( "CORDEX" ) 
    2831ps.pcfg = config.projectConfig( "SPECS" ) 
    29 p.abortMessageCount = -1 
    30 ps.abortMessageCount = -1 
     32pcmip5.pcfg = config.projectConfig( "CMIP5" ) 
     33pccmi.pcfg = config.projectConfig( "CCMI" ) 
    3134 
    3235 
     
    3841c.check( fn ) 
    3942if c.errorCount == 0: 
    40   print 'OK: [%s] %s: valid file name' % (module,fn) 
    41 else: 
    42   print 'Failed [%s] %s: valid file name' % (module,fn) 
     43  print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
     44else: 
     45  print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    4346 
    4447testId = '#01.002' 
     
    4649c.check(fn) 
    4750if c.errorCount == 0 and c.isFixed: 
    48   print 'OK: [%s] %s: valid file name' % (module,fn) 
    49 else: 
    50   print 'Failed [%s] %s: valid file name' % (module,fn) 
     51  print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
     52else: 
     53  print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    5154 
    5255testId = '#01.003' 
     
    5457c.check(fn) 
    5558if c.errorCount == 0: 
    56   print 'OK: [%s] %s: valid file name' % (module,fn) 
    57 else: 
    58   print 'Failed [%s] %s: valid file name' % (module,fn) 
     59  print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
     60else: 
     61  print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    5962 
    6063testId = '#01.004' 
     
    6265c.check(fn) 
    6366if c.errorCount == 0: 
    64   print 'OK: [%s] %s: valid file name' % (module,fn) 
    65 else: 
    66   print 'Failed [%s] %s: valid file name' % (module,fn) 
     67  print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
     68else: 
     69  print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    6770 
    6871testId = '#01.005' 
     
    7073c.check(fn) 
    7174if c.errorCount == 0: 
    72   print 'Failed to detect bad file name: [%s] %s ' % (module,fn) 
    73 else: 
    74   print 'OK: -- detected bad file name: [%s] %s' % (module,fn) 
     75  print 'Failed to detect bad CORDEX file name: [%s] %s ' % (module,fn) 
     76else: 
     77  print 'OK: -- detected bad CORDEX file name: [%s] %s' % (module,fn) 
     78 
     79fn = "pr_3hr_HadGEM2-ES_historical_r2i1p1_196001010130-196412302230.nc" 
     80c = utils_c4.checkFileName(parent=pcmip5) 
     81c.check(fn) 
     82if c.errorCount == 0: 
     83  print 'OK: [%s] %s: valid CMIP5 file name' % (module,fn) 
     84else: 
     85  print 'Failed [%s] %s: valid CMIP5 file name' % (module,fn) 
     86 
     87fn = "clivi_monthly_CESM1-CAM4Chem_refC1sd_r1i1p1_197501-197912.nc" 
     88c = utils_c4.checkFileName(parent=pccmi) 
     89c.check(fn) 
     90if c.errorCount == 0: 
     91  print 'OK: [%s] %s: valid CCMI file name' % (module,fn) 
     92else: 
     93  print 'Failed [%s] %s: valid CCMI file name' % (module,fn) 
     94 
     95fn = "tas_Amon_EC-EARTH3_seaIceClimInit_series2_S19930501_r1i1p1_199306-199306.nc" 
     96c = utils_c4.checkFileName(parent=ps) 
     97c.check(fn) 
     98if c.errorCount == 0: 
     99  print 'Failed [%s] %s: passed invalid SPECS file name' % (module,fn) 
     100else: 
     101  print 'OK: [%s] %s: detected invalid SPECS file name' % (module,fn) 
     102 
     103fn = "tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199306.nc" 
     104c = utils_c4.checkFileName(parent=ps) 
     105c.check(fn) 
     106if c.errorCount == 0: 
     107  print 'OK: [%s] %s: valid SPECS file name' % (module,fn) 
     108else: 
     109  print 'Failed [%s] %s: valid SPECS file name' % (module,fn) 
     110 
    75111 
    76112c = utils_c4.checkStandardDims(parent=p) 
    77113module = 'checkStandardDims' 
    78 c.check( 'tas', 'day', {},{}, False ) 
     114## note last argument is "vocabs", but only used in "experimental" mode 
     115c.check( 'tas', 'day', {},{}, False, None ) 
    79116if c.errorCount == 0: 
    80117  print 'Failed [%s]: failed to detect empty dictionaries' % module 
  • CCCC/trunk/ceda_cc/unitTestsS2.py

    r210 r253  
    6666extras = [ 
    6767( '/data/work/cmip5/output1/pr_20110323/pr_3hr_HadGEM2-ES_historical_r2i1p1_196001010130-196412302230.nc', 'CMIP5', 0 ), 
    68 ('/badc/cmip5/data/cmip5/output1/MOHC/HadGEM2-ES/historical/mon/atmos/Amon/r1i1p1/files/pr_20110329/pr_Amon_HadGEM2-ES_historical_r1i1p1_185912-188411.nc', 'CMIP5', 0 ) ] 
     68('/data/work/cmip5/output1/pr_20110323/pr_3hr_HadGEM2-ES_historical_r2i1p1_200001010130-200412302230.nc', 'CMIP5', 0 ) ] 
    6969 
    7070 
  • CCCC/trunk/ceda_cc/utils_c4.py

    r252 r253  
    469469                 thisVal = globalAts[ "table_id" ] 
    470470                 self.test( False, 'Global attribute table_id does not conform to CMOR pattern ["Table ......"]: %s' % thisVal, part=True) 
     471           elif self.globalAttributesInFn[i][1:] == "ensemble": 
     472               thisVal = "r%si%sp%s" % (globalAts["realization"],globalAts["initialization_method"],globalAts["physics_version"]) 
     473## following mappings are depricated -- introduced for SPECS and withdrawn --- 
    471474           elif self.globalAttributesInFn[i][1:] == "experiment_family": 
    472475               thisVal = globalAts["experiment_id"][:-4] 
    473            elif self.globalAttributesInFn[i][1:] == "ensemble": 
    474                thisVal = "r%si%sp%s" % (globalAts["realization"],globalAts["initialization_method"],globalAts["physics_version"]) 
    475476           elif self.globalAttributesInFn[i][1:] == "forecast_reference_time": 
    476477               x = self.globalAts.get("forecast_reference_time",'yyyy-mm-dd Thh:mm:ssZ' ) 
     
    478479           elif self.globalAttributesInFn[i][1:] == "series": 
    479480               thisVal = 'series%s' % globalAts["series"] 
    480            elif self.globalAttributesInFn[i][1:] == "experiment_family": 
    481              thisVal = globalAts["experiment_id"][:-4] 
    482481           else: 
    483482               assert False, "Not coded to deal with this configuration: globalAttributesInFn[%s]=%s" % (i,self.globalAttributesInFn[i]) 
     
    507506    self.heightRange = self.pcfg.heightRange 
    508507 
    509   def check(self,varName,varGroup, da, va, isInsta): 
     508  def check(self,varName,varGroup, da, va, isInsta,vocabs): 
    510509    self.errorCount = 0 
    511510    assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName) 
     
    515514    self.va = va 
    516515    self.isInsta = isInsta 
     516    self.vocabs = vocabs 
    517517    self.runChecks() 
    518518 
     
    595595 
    596596    self.checkId = ('003','height_levels') 
    597     if varName in self.heightRequired: 
     597    hreq = varName in self.heightRequired 
     598    if self.parent.experimental: 
     599      print 'utils_c4: ', varName, self.vocabs['variable'].varcons[varGroup][varName].get( '_dimension',[]) 
     600      hreq = "height2m" in self.vocabs['variable'].varcons[varGroup][varName].get( '_dimension',[]) 
     601      if hreq: 
     602        print 'testing height, var=%s' % varName 
     603    if hreq: 
    598604      heightAtDict = {'long_name':"height", 'standard_name':"height", 'units':"m", 'positive':"up", 'axis':"Z" } 
    599605      ok = True 
     
    780786## set global default: type float 
    781787          eeee = { 'type':pcfg.defaults.get( 'variableDataType', 'float' ) } 
     788          eeee['_dimension'] = ee[v][0] 
    782789          ar = [] 
    783790          ac = [] 
Note: See TracChangeset for help on using the changeset viewer.