Changeset 263 for CCCC


Ignore:
Timestamp:
22/12/14 09:16:09 (5 years ago)
Author:
mjuckes
Message:

added esa-cci support

Location:
CCCC/trunk/ceda_cc
Files:
7 added
5 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/ceda_cc/c4.py

    r258 r263  
    4242                 'CMIP5':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \ 
    4343                 'CCMI':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \ 
     44                 'ESA-CCI':'%(level)s/%(platform)s/%(sensor)s/%(variable)s/', \ 
    4445                 '__def__':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \ 
    4546               } 
     
    502503    self.cc.info.amapListDraft = [] 
    503504    cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor) 
    504     cbv.impt( c4i.flist ) 
    505     if printInfo: 
    506       print cbv.info 
     505    if c4i.project not in ['ESA-CCI']: 
     506      cbv.impt( c4i.flist ) 
     507      if printInfo: 
     508        print cbv.info 
    507509 
    508510    fileLogOpen = False 
     
    587589    self.cc.info.log = c4i.logger 
    588590     
    589     if c4i.project not in ['SPECS','CCMI','CMIP5']: 
     591    if c4i.project not in ['SPECS','CCMI','CMIP5','ESA-CCI']: 
    590592       cbv.c4i = c4i 
    591593       cbv.setLogDict( logDict ) 
  • CCCC/trunk/ceda_cc/config_c4.py

    r262 r263  
    175175  elif pcfg.projectV.id == 'ESA-CCI': 
    176176    lrdr = readVocab( 'esacci_vocabs/') 
    177     vocabs = { 'variable':utils.mipVocab(pcfg,dummy=True), \ 
    178                'level':utils.listControl( 'level', lrdr.getSimpleList( 'procLevel01.txt', bit=0 ) ) \ 
     177    vocabs = { 'variable':utils.mipVocab(pcfg), \ 
     178               'version':utils.patternControl( 'version',  '^(fv[0-9]+(\.[0-9]+){0,1})$' ), \ 
     179               'level':utils.listControl( 'level', lrdr.getSimpleList( 'procLevel01.txt', bit=0 ) ), \ 
     180               'platform':utils.listControl( 'platforms', lrdr.getSimpleList( 'platforms.txt', bit=0 ) ), \ 
     181               'Conventions':utils.patternControl( 'Conventions', '^CF-1.[56789](,.*){0,1}$' ), \ 
     182               'sensor':utils.listControl( 'sensors', lrdr.getSimpleList( 'sensors.txt', bit=0 ) ), \ 
     183               'project':utils.listControl( 'project', ['Climate Change Initiative - European Space Agency'] ), \ 
     184               'cciProject':utils.listControl( 'project', lrdr.getSimpleList( 'cciProject.txt', bit=-1 ) ), \ 
     185               'var':utils.listControl( 'var', lrdr.getSimpleList( 'variables.txt', bit=-1 ) ) \ 
    179186             } 
    180187  elif pcfg.projectV.id == '__dummy': 
     
    202209    self.project = project 
    203210    self.fNameSep = '_' 
     211    self.varIndex = 0 
     212    self.fnvdict = None 
     213    self.varTables='CMIP' 
     214    self.checkVarType = True 
    204215    self.projectV = NT_project(project,version) 
    205216    self.gridSpecTol = 0.01 
     
    264275 
    265276    elif project == 'ESA-CCI': 
     277      lrdr = readVocab( 'esacci_vocabs/') 
     278      self.varTables='FLAT' 
    266279      self.fNameSep = '-' 
    267       self.requiredGlobalAttributes = map( lambda x: 'ga%s' % x, range(10) ) 
    268       self.controlledGlobalAttributes = [ ] 
    269       self.controlledFnParts = ['level'] 
    270       self.requiredVarAttributes = ['long_name', 'standard_name', 'units'] 
    271       self.drsMappings = {'variable':'@var'} 
     280      self.checkVarType = False 
     281      self.requiredGlobalAttributes = lrdr.getSimpleList( 'requiredGlobalAts.txt', bit=0 ) 
     282      self.controlledGlobalAttributes = ['platform','sensor','project','Conventions' ] 
     283      self.controlledFnParts = ['level','cciProject','var','version'] 
     284      self.requiredVarAttributes = ['long_name', 'standard_name', 'units'] 
     285      self.drsMappings = {'variable':'@var','platform':'platform','sensor':'sensor','level':'@level'} 
    272286      self.globalAttributesInFn = [None,] 
    273287    elif project == '__dummy': 
     
    362376       self.mipVocabVgmap = {'fixed':'fx','annual':'yr','monthly':'mon','daily':'day','hourly':'hr'} 
    363377       self.mipVocabFnpat = 'CCMI1_%s' 
     378    elif self.projectV.id == 'ESA-CCI': 
     379       self.mipVocabDir = op.join(CC_CONFIG_DIR, 'esacci_vocabs/') 
     380       self.mipVocabTl = [] 
     381       self.mipVocabVgmap = 'ESACCI' 
     382       self.mipVocabFnpat = 'variableInFile.txt' 
    364383    else: 
    365384       self.mipVocabDir = None 
     
    375394      self.groupIndex = 1 
    376395    elif self.project in ['ESA-CCI']: 
     396      self.fnvdict = { 'SSTskin':{'v':'sea_surface_temperature', 'sn':'sea_surface_skin_temperature'} } 
    377397      self.fnoptions = {'groupIndex':[3,1], 'trangeIndex':[0,-2] } 
    378398      self.fnoptions['inFn'] = [[None,'*activity','*level','*project','*var','*product','*additional','*gdsv','*version'], 
    379399                                ['*activity','*project','*level','*var','*additional',None,'*version']] 
     400      self.fnoptions['varIndex'] = [4,3] 
    380401##Indicative Date>[<Indicative Time>]-ESACCI-<Processing Level>_<CCI Project>-<Data Type>-<Product String>[- <Additional Segregator>][-v<GDS version>]-fv<File version>.nc 
    381402##ESACCI-<CCI Project>-<Processing Level>-<Data Type>-<Product String>[-<Additional Segregator>]-<IndicativeDate>[<Indicative Time>]-fv<File version>.nc 
     
    397418      self.trangeIndex = self.fnoptions['trangeIndex'][id] 
    398419      self.globalAttributesInFn = self.fnoptions['inFn'][id] 
     420      self.varIndex = self.fnoptions['varIndex'][id] 
    399421 
    400422 
  • CCCC/trunk/ceda_cc/file_utils.py

    r255 r263  
    144144      if len( self.ga[k] ) == 1: 
    145145        self.ga[k] = self.ga[k][0] 
     146## nasty fix to deal with fact that cdms2 does not read the 'id' global attribute 
     147    try: 
     148      thisid = self.nc.id 
     149      self.ga['id'] = thisid 
     150    except: 
     151      pass 
    146152    for v in self.nc.variables.keys(): 
    147153      self.va[v] = {} 
  • CCCC/trunk/ceda_cc/unitTestsS1.py

    r262 r263  
    111111  print 'Failed [%s] %s: valid SPECS file name' % (module,fn) 
    112112 
     113fn = "20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc" 
     114c = utils_c4.checkFileName(parent=pcci) 
     115c.check(fn) 
     116if c.errorCount == 0: 
     117  print 'OK: [%s] %s: valid ESA-CCI file name' % (module,fn) 
     118else: 
     119  print 'Failed [%s] %s: valid ESA-CCI file name' % (module,fn) 
     120 
    113121fn = "20120101015548-ESACCI-L3U-GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc" 
    114122c = utils_c4.checkFileName(parent=pcci) 
    115123c.check(fn) 
    116124if c.errorCount == 0: 
    117   print 'OK: [%s] %s: valid ESA-CCI file name' % (module,fn) 
    118 else: 
    119   print 'Failed [%s] %s: valid ESA-CCI file name' % (module,fn) 
    120  
    121 fn = "20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc" 
     125  print 'Failed: [%s] %s: Passed invalid ESA-CCI file name' % (module,fn) 
     126else: 
     127  print 'OK [%s] %s: Detected invalid ESA-CCI file name' % (module,fn) 
     128 
     129fn = "20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv.1.nc" 
    122130c = utils_c4.checkFileName(parent=pcci) 
    123131c.check(fn) 
  • CCCC/trunk/ceda_cc/utils_c4.py

    r262 r263  
    198198    self.checks = (self.do_check_fn,self.do_check_fnextra) 
    199199    self.re_c1 = re.compile( '^[0-9]*$' ) 
     200    self.fnDict = {} 
    200201#### 
    201202 
     
    207208 
    208209    self.runChecks() 
     210    self.parent.fnDict = self.fnDict 
    209211### 
    210212  def do_check_fn(self): 
     
    217219    self.test( fn[-3:] == '.nc', 'File name ending ".nc" expected', abort=True, part=True ) 
    218220    bits = string.split( fn[:-3], self.fnsep ) 
     221 
    219222    self.fnParts = bits[:] 
    220  
    221223    if self.pcfg.domainIndex != None: 
    222224      self.domain = self.fnParts[self.pcfg.domainIndex] 
     
    224226      self.domain = None 
    225227 
    226     self.test( len(bits) in self.pcfg.fnPartsOkLen, 'File name not parsed in %s elements [%s]' % (str(self.pcfg.fnPartsOkLen),str(bits)), abort=True ) 
    227228 
    228229    if self.pcfg.projectV.id in ['ESA-CCI']: 
     
    232233      else: 
    233234        self.esaFnId = 0 
     235        bb = string.split( bits[2], '_' ) 
     236        self.test( bits[2][0] == 'L' and len(bb) == 2, 'Cannot parse ESA-CCI file name: %s' % fn, abort=True ) 
     237        bits = bits[:2] + bb + bits[3:] 
     238        self.fnParts = bits[:] 
     239         
    234240      self.pcfg.setEsaCciFNType(self.esaFnId) 
     241    self.test( len(bits) in self.pcfg.fnPartsOkLen, 'File name not parsed in %s elements [%s]' % (str(self.pcfg.fnPartsOkLen),str(bits)), abort=True ) 
    235242 
    236243    self.fnDict = {} 
     
    272279      ##self.freq = self.fnParts[1] 
    273280 
    274     self.var = self.fnParts[0] 
     281    self.var = self.fnParts[self.pcfg.varIndex] 
     282 
     283    if self.pcfg.fnvdict != None: 
     284      if self.pcfg.fnvdict.has_key( self.var ): 
     285        self.var = self.pcfg.fnvdict.get( self.var )['v'] 
    275286 
    276287    self.isFixed = self.freq == 'fx' 
     
    379390      elif self.drsMappings[k] == '@mip_id': 
    380391        ee[k] = string.split( self.globalAts["table_id"] )[1] 
     392      elif self.drsMappings[k] == '@level': 
     393        ee[k] = self.parent.fnDict['level'] 
    381394      else: 
    382395        ee[k] = self.globalAts[ self.drsMappings[k] ] 
     
    415428    self.test( vocabs['variable'].isInTable( varName, varGroup ), msg, abort=True, part=True ) 
    416429 
    417     self.checkId = ('003','variable_type') 
    418  
    419     mipType = vocabs['variable'].getAttr( varName, varGroup, 'type' ) 
    420     thisType = {'real':'float32', 'integer':'int32', 'float':'float32', 'double':'float64' }.get( mipType, mipType ) 
    421     self.test( mipType == None or varAts[varName]['_type'] == thisType, 'Variable [%s/%s] not of type %s [%s]' % (varName,varGroup,str(thisType),varAts[varName]['_type']) ) 
     430    if self.pcfg.checkVarType: 
     431      self.checkId = ('003','variable_type') 
     432 
     433      mipType = vocabs['variable'].getAttr( varName, varGroup, 'type' ) 
     434      thisType = {'real':'float32', 'integer':'int32', 'float':'float32', 'double':'float64' }.get( mipType, mipType ) 
     435      self.test( mipType == None or varAts[varName]['_type'] == thisType, 'Variable [%s/%s] not of type %s [%s]' % (varName,varGroup,str(thisType),varAts[varName]['_type']) ) 
     436    else: 
     437      mipType = None 
    422438 
    423439    self.checkId = ('004','variable_ncattribute_present') 
     
    445461    hf = varAts[varName].has_key( '_FillValue' ) 
    446462    if hm or hf: 
    447       ok &= self.test( hm, 'missing_value must be present if _FillValue is [%s]' % varName ) 
    448       ok &= self.test( hf, '_FillValue must be present if missing_value is [%s]' % varName ) 
     463      if self.pcfg.varTables=='CMIP': 
     464        ok &= self.test( hm, 'missing_value must be present if _FillValue is [%s]' % varName ) 
     465        ok &= self.test( hf, '_FillValue must be present if missing_value is [%s]' % varName ) 
     466      else: 
     467        ok = True 
    449468      if mipType == 'real': 
    450469        if varAts[varName].has_key( 'missing_value' ): 
     
    460479    mm = [] 
    461480     
    462     contAts = ['long_name', 'standard_name', 'units'] 
    463     if varGroup != 'fx': 
    464       contAts.append( 'cell_methods' ) 
     481    if self.pcfg.varTables=='CMIP': 
     482      contAts = ['long_name', 'standard_name', 'units'] 
     483      if varGroup != 'fx': 
     484        contAts.append( 'cell_methods' ) 
     485    else: 
     486      contAts = ['standard_name'] 
    465487    hcm = varAts[varName].has_key( "cell_methods" ) 
    466488    for k in contAts + vocabs['variable'].lists(varName,'addControlledAttributes'): 
     
    547569           else: 
    548570               assert False, "Not coded to deal with this configuration: globalAttributesInFn[%s]=%s" % (i,self.globalAttributesInFn[i]) 
    549            ##print "Generated text val: %s:: %s" % (self.globalAttributesInFn[i], thisVal) 
    550571          
    551572         else: 
     
    593614    self.completed = False 
    594615    self.checkId = ('001','time_attributes') 
     616    self.calendar = 'None' 
    595617    if varGroup != 'fx': 
    596618      ok = True 
    597619      self.test( 'time' in da.keys(), 'Time dimension not found' , abort=True, part=True ) 
    598       if not isInsta: 
    599         ok &= self.test(  da['time'].get( 'bounds', 'xxx' ) == 'time_bnds', 'Required bounds attribute not present or not correct value', part=True ) 
     620      if self.pcfg.varTables=='CMIP': 
     621        if not isInsta: 
     622          ok &= self.test(  da['time'].get( 'bounds', 'xxx' ) == 'time_bnds', 'Required bounds attribute not present or not correct value', part=True ) 
    600623 
    601624## is time zone designator needed? 
    602       tunits = da['time'].get( 'units', 'xxx' ) 
    603       if self.project  == 'CORDEX': 
    604         ok &= self.test( tunits in ["days since 1949-12-01 00:00:00Z", "days since 1949-12-01 00:00:00", "days since 1949-12-01"], 
     625        tunits = da['time'].get( 'units', 'xxx' ) 
     626        if self.project  == 'CORDEX': 
     627          ok &= self.test( tunits in ["days since 1949-12-01 00:00:00Z", "days since 1949-12-01 00:00:00", "days since 1949-12-01"], 
    605628               'Time units [%s] attribute not set correctly to "days since 1949-12-01 00:00:00Z"' % tunits, part=True ) 
    606       else: 
    607         ok &= self.test( tunits[:10] == "days since", 'time units [%s] attribute not set correctly to "days since ....."' % tunits, part=True ) 
    608  
    609       ok &= self.test(  da['time'].has_key( 'calendar' ), 'Time: required attribute calendar missing', part=True ) 
    610  
    611       ok &= self.test( da['time']['_type'] in ["float64","double"], 'Time: data type not float64 [%s]' % da['time']['_type'], part=True ) 
     629        else: 
     630          ok &= self.test( tunits[:10] == "days since", 'time units [%s] attribute not set correctly to "days since ....."' % tunits, part=True ) 
     631 
     632        ok &= self.test(  da['time'].has_key( 'calendar' ), 'Time: required attribute calendar missing', part=True ) 
     633 
     634        ok &= self.test( da['time']['_type'] in ["float64","double"], 'Time: data type not float64 [%s]' % da['time']['_type'], part=True ) 
    612635        
    613       if ok: 
    614         self.log_pass() 
    615       self.calendar = da['time'].get( 'calendar', 'None' ) 
    616     else: 
    617       self.calendar = 'None' 
     636        if ok: 
     637          self.log_pass() 
     638        self.calendar = da['time'].get( 'calendar', 'None' ) 
    618639 
    619640    self.checkId = ('002','pressure_levels') 
     
    824845 
    825846  def __init__(self,pcfg,dummy=False): 
    826      project = pcfg.projectV.id 
     847     self.pcfg = pcfg 
    827848     if dummy: 
    828        self.pcfg = pcfg 
    829        return self.dummyMipTable() 
    830      ##assert project in ['CORDEX','SPECS'],'Project %s not recognised' % project 
    831      ##if project == 'CORDEX': 
    832        ##dir = 'cordex_vocabs/mip/' 
    833        ##tl = ['fx','sem','mon','day','6h','3h'] 
    834        ##vgmap = {'6h':'6hr','3h':'3hr'} 
    835        ##fnpat = 'CORDEX_%s' 
    836      ##elif project == 'SPECS': 
    837        ##dir = 'specs_vocabs/mip/' 
    838        ##tl = ['fx','Omon','Amon','Lmon','OImon','day','6hrLev'] 
    839        ##vgmap = {} 
    840        ##fnpat = 'SPECS_%s' 
    841      dir, tl, vgmap, fnpat = pcfg.mipVocabPars 
     849       self.dummyMipTable() 
     850     elif pcfg.varTables=='CMIP': 
     851       self.ingestMipTables() 
     852     elif pcfg.varTables=='FLAT': 
     853       self.flatTable() 
     854     
     855  def ingestMipTables(self): 
     856     dir, tl, vgmap, fnpat = self.pcfg.mipVocabPars 
    842857     ms = mipTableScan() 
    843858     self.varInfo = {} 
     
    851866        for v in ee.keys(): 
    852867## set global default: type float 
    853           eeee = { 'type':pcfg.defaults.get( 'variableDataType', 'float' ) } 
     868          eeee = { 'type':self.pcfg.defaults.get( 'variableDataType', 'float' ) } 
    854869          eeee['_dimension'] = ee[v][0] 
    855870          ar = [] 
     
    884899          self.varcons[vg][v] = eeee 
    885900 
     901  def flatTable(self): 
     902     self.varInfo = {} 
     903     self.varcons = {} 
     904     dir, tl, vg, fn = self.pcfg.mipVocabPars 
     905     ee = { 'standard_name':'sn%s', 'long_name':'n%s', 'units':'1' } 
     906     ll = open( '%s%s' % (dir,fn) ).readlines() 
     907     self.varcons[vg] = {} 
     908     for l in ll: 
     909       if l[0] != '#': 
     910          dt, v, sn = string.split( string.strip(l) ) 
     911          ar = [] 
     912          ac = [] 
     913          self.varInfo[v] = {'ar':ar, 'ac':ac } 
     914          self.varcons[vg][v] = {'standard_name':sn, 'type':'float' } 
     915 
    886916  def lists( self, k, k2 ): 
    887917     if k2 == 'addRequiredAttributes': 
     
    9831013        else: 
    9841014          trange = string.split( fnParts[-1], '-' ) 
    985         var = fnParts[0] 
     1015        var = fnParts[self.pcfg.varIndex] 
    9861016        thisKey = string.join( fnParts[:-1], '.' ) 
    9871017        if group not in ee.keys(): 
     
    10711101          lok &= self.test( x != None, 'Cannot match time range %s: %s [%s/%s]' % (i,fn,j,n), part=True, appendLogfile=(self.fLogDict.get(fn,None),fn) ) 
    10721102        if not lok: 
    1073           ### print 'Cannot match time range %s:' % t[1] 
    10741103          if self.recorder != None: 
    10751104            self.recorder.modify( t[1], 'ERROR: time range' ) 
Note: See TracChangeset for help on using the changeset viewer.