Changeset 302 for CCCC


Ignore:
Timestamp:
02/03/15 17:40:39 (5 years ago)
Author:
mjuckes
Message:

updated to deal with ocean colour CCI data -- some bigs fixed

Location:
CCCC/trunk/ceda_cc
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/ceda_cc/config/esacci_vocabs/extraAtts.txt

    r280 r302  
    11#Additional attributes for data publication 
    22org.ghrsst, AATSR-ESACCI-L3U-v1, algorithm=OE2, frequency=1sec 
     3uk.ac.pml, ESACCI-OC-L3S-OC_PRODUCTS-MERGED-1M_MONTHLY_4km_GEO_PML_OC4v6_QAA, algorithm=SeaDAS(v6.4-OCCCI), frequency=mon 
  • CCCC/trunk/ceda_cc/config/esacci_vocabs/sensors.txt

    r272 r302  
    1111MERIS-FRS 
    1212MERIS-RR 
     13MERIS 
    1314MIPAS 
    1415RA-2 
  • CCCC/trunk/ceda_cc/config/esacci_vocabs/variableInFile.txt

    r268 r302  
    44SSTdepth analysed_sst       sea_water_temperature 
    55BA    burned_area burned_area 
     6OC_PRODUCTS chlor_a mass_concentration_of_chlorophyll_a_in_sea_water 
  • CCCC/trunk/ceda_cc/config_c4.py

    r299 r302  
    298298      self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None ) 
    299299    elif self.projectV.id == 'ESA-CCI': 
    300       self.fnParts = NT_fnParts( len=[7,8,9], fxLen=[0,],  unfLen=[7,8,9,], checkTLen=False, ixDomain=None, ixFreq=1 ) 
     300      self.fnParts = NT_fnParts( len=[7,8,9], fxLen=[0,],  unfLen=[7,8,9,], checkTLen=False, ixDomain=None, ixFreq=None ) 
    301301      self.trangeType = 'ESA-CCI' 
    302302    elif self.projectV.id == '__dummy': 
     
    337337       self.mipVocabDir = op.join(CC_CONFIG_DIR, 'esacci_vocabs/') 
    338338       self.mipVocabTl = [] 
    339        self.mipVocabVgmap = 'ESACCI' 
     339       self.mipVocabVgmap = { 'ESACCI':'ESACCI' } 
    340340       self.mipVocabFnpat = 'variableInFile.txt' 
    341341    else: 
     
    353353    elif self.project in ['ESA-CCI']: 
    354354      self.fnvdict = { 'SSTskin':{'v':'sea_surface_temperature', 'sn':'sea_surface_skin_temperature'} } 
    355       self.fnoptions = {'groupIndex':[3,1], 'trangeIndex':[0,-2] } 
     355      self.fnoptions = {'groupIndex':[1,0], 'trangeIndex':[0,-2] } 
    356356      self.fnoptions['inFn'] = [[None,'*activity','*level','*project','*var','*product','*additional','*gdsv','*version'], 
    357357                                ['*activity','*project','*level','*var','*additional',None,'*version']] 
     
    439439               'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'institutions.txt', omt='last' ) ), \ 
    440440               'Conventions':utils.patternControl( 'Conventions', '^CF-1.[56789](,.*){0,1}$', examples=['CF-1.6'] ), \ 
    441                'sensor':utils.listControl( 'sensors', lrdr.getSimpleList( 'sensors.txt', bit=0 ) ), \ 
     441               'sensor':utils.listControl( 'sensors', lrdr.getSimpleList( 'sensors.txt', bit=0 ), enumeration=True, split=True, splitVal=',' ), \ 
    442442               'cdm_data_type':utils.listControl( 'cdm_data_type', lrdr.getSimpleList( 'threddsDataType.txt', bit=0 ) ), \ 
    443443               'time_coverage_duration':utils.patternControl( 'time_coverage_duration',  'ISO8601 duration', cls='ISO',examples=['P1Y'] ), \ 
  • CCCC/trunk/ceda_cc/extractMipInfo.py

    r277 r302  
    618618 
    619619mipl = ['cmip5'] 
     620mipl = mips.keys() 
    620621mips = map( lambda x: mips[x], mipl ) 
    621622r = run() 
  • CCCC/trunk/ceda_cc/unitTestsS1.py

    r279 r302  
    5252SPECS tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199306.nc True 
    5353ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc True CCIplus 
     54ESA-CCI ESACCI-OC-L3S-OC_PRODUCTS-MERGED-1M_MONTHLY_4km_GEO_PML_OC4v6_QAA-200005-fv1.0.nc True CCIplus 
    5455ESA-CCI 20120101015548-ESACCI-L3U-GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc False 
    5556ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv.1.nc False""" 
     
    8687      if len(l) > 2 and l[2]  == 'CCIplus': 
    8788        cga.globalAts =  {'platform':'platform','sensor':'sensor', \ 
    88                  'naming_authority':'org.ghrsst', 'id':'AATSR-ESACCI-L3U-v1', \ 
     89                'naming_authority':'org.ghrsst', 'id':'AATSR-ESACCI-L3U-v1', \ 
    8990                'spatial_resolution':'1 km'} 
    9091        cga.varAts = { 'var':{ 'standard_name':'dummy'} } 
     
    9293        cga.completed = True 
    9394        ee1 = cga.getDrs() 
    94         if ee1['ecv'] == 'Sea Surface Temperature': 
    95           print 'OK: [%sb] %s: DRS dictionary generated' % (testId,fn) 
     95        if ee1['ecv'] in ['Sea Surface Temperature','Ocean Colour']: 
     96          print 'OK: [%sb] %s: DRS dictionary generated (%s)' % (testId,fn,ee1['ecv']) 
    9697        else: 
    97           print 'Failed: [%sb] %s: DRS dictionary not generated correctly' % (testId,fn) 
     98          print 'Failed: [%sb] %s: DRS dictionary not generated correctly (%s)' % (testId,fn,ee1['ecv']) 
     99  
     100        if c.group == 'ESACCI': 
     101          print 'OK: group ESACCI detected'  
     102        else: 
     103          print 'Failed: group read as "%s" (not ESACCI) %s' % (c.group,fn) 
    98104         
    99105    else: 
  • CCCC/trunk/ceda_cc/utils_c4.py

    r298 r302  
    187187      self.log_exception( 'Exception caught by runChecks' ) 
    188188      raise loggedException 
    189       ##traceback.print_exc(file=open("errlog.txt","a")) 
    190       ##logger.error("Exception has occured" ,exc_info=1) 
    191189     
    192190class checkFileName(checkBase): 
     
    250248          self.fnDict[x[1:]] = bits[i] 
    251249      self.fnDict['version'] = bits[-1] 
     250      self.fnDict['gdsv'] = 'na' 
    252251      if self.esaFnId == 0: 
    253252        if len(bits) == 9: 
     
    315314              ll.append( tt[:2] ) 
    316315              tt = tt[2:] 
     316            elif j in [1,2]: 
     317              ll.append( '01' ) 
    317318            else: 
    318319              ll.append( '00' ) 
     
    381382    if self.fileId == None: 
    382383      self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id']) 
     384      if self.globalAts['naming_authority'] == 'uk.ac.pml': 
     385        i0 = string.find(self.globalAts['id'],'OC4v6_QAA') 
     386        if i0 != -1: 
     387          self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id'][:i0+9]) 
    383388 
    384389  def getDrs( self ): 
     
    404409      elif self.drsMappings[k][0] == '$': 
    405410        self.pcfg.getExtraAtts() 
    406         print self.pcfg.extraAtts 
    407411        self.getId() 
    408412        if string.find(self.drsMappings[k],':') != -1: 
     
    478482        self.parent.amapListDraft.append( '#@var=%s;%s=%s|%s=%s' % (varName,k,'__absent__',k,'<insert attribute value and uncomment>') ) 
    479483        ## print self.parent.amapListDraft[-1] 
    480     ##vaerr = not self.test( len(m)  == 0, 'Required variable attributes missing: %s' % str(m) ) 
    481  
    482     ##if vaerr or gaerr: 
    483       ##self.log_abort() 
    484484 
    485485## need to insert a check that variable is present 
     
    497497        if varAts[varName].has_key( 'missing_value' ): 
    498498           msg = 'Variable [%s] has incorrect attribute missing_value=%s [correct: %s]' % (varName,varAts[varName]['missing_value'],self.missingValue) 
    499            ## print varAts[varName]['missing_value'], type(varAts[varName]['missing_value']) 
    500            ## print self.missingValue, type(self.missingValue) 
    501499### need to use ctypes here when using ncq3 to read files -- appears OK for other libraries. 
    502500           ok &= self.test( ctypes.c_float(varAts[varName]['missing_value']).value == ctypes.c_float(self.missingValue).value, msg, part=True ) 
     
    933931     self.varInfo = {} 
    934932     self.varcons = {} 
    935      dir, tl, vg, fn = self.pcfg.mipVocabPars 
     933     dir, tl, vgm, fn = self.pcfg.mipVocabPars 
     934     vg = vgm.keys()[0] 
    936935     ee = { 'standard_name':'sn%s', 'long_name':'n%s', 'units':'1' } 
    937936     ll = open( '%s%s' % (dir,fn) ).readlines() 
Note: See TracChangeset for help on using the changeset viewer.