Changeset 302

Timestamp:

02/03/15 17:40:39 (6 years ago)

Author:

mjuckes

Message:

updated to deal with ocean colour CCI data -- some bigs fixed

Location:

CCCC/trunk/ceda_cc

Files:

• config/esacci_vocabs/extraAtts.txt (modified) (1 diff)
• config/esacci_vocabs/sensors.txt (modified) (1 diff)
• config/esacci_vocabs/variableInFile.txt (modified) (1 diff)
• config_c4.py (modified) (4 diffs)
• extractMipInfo.py (modified) (1 diff)
• unitTestsS1.py (modified) (3 diffs)
• utils_c4.py (modified) (8 diffs)

CCCC/trunk/ceda_cc/config/esacci_vocabs/extraAtts.txt

@@ -1 +1 @@
 #Additional attributes for data publication
 org.ghrsst, AATSR-ESACCI-L3U-v1, algorithm=OE2, frequency=1sec
+uk.ac.pml, ESACCI-OC-L3S-OC_PRODUCTS-MERGED-1M_MONTHLY_4km_GEO_PML_OC4v6_QAA, algorithm=SeaDAS(v6.4-OCCCI), frequency=mon

CCCC/trunk/ceda_cc/config/esacci_vocabs/sensors.txt

@@ -11 +11 @@
 MERIS-FRS
 MERIS-RR
+MERIS
 MIPAS
 RA-2

CCCC/trunk/ceda_cc/config/esacci_vocabs/variableInFile.txt

@@ -4 +4 @@
 SSTdepth analysed_sst       sea_water_temperature
 BA    burned_area burned_area
+OC_PRODUCTS chlor_a mass_concentration_of_chlorophyll_a_in_sea_water

CCCC/trunk/ceda_cc/config_c4.py

@@ -298 +298 @@
       self.fnParts = NT_fnParts( len=[5,6], fxLen=[5,],  unfLen=[6,], checkTLen=False, ixDomain=None, ixFreq=None )
     elif self.projectV.id == 'ESA-CCI':
-      self.fnParts = NT_fnParts( len=[7,8,9], fxLen=[0,],  unfLen=[7,8,9,], checkTLen=False, ixDomain=None, ixFreq=1 )
+      self.fnParts = NT_fnParts( len=[7,8,9], fxLen=[0,],  unfLen=[7,8,9,], checkTLen=False, ixDomain=None, ixFreq=None )
       self.trangeType = 'ESA-CCI'
     elif self.projectV.id == '__dummy':
@@ -337 +337 @@
        self.mipVocabDir = op.join(CC_CONFIG_DIR, 'esacci_vocabs/')
        self.mipVocabTl = []
-       self.mipVocabVgmap = 'ESACCI'
+       self.mipVocabVgmap = { 'ESACCI':'ESACCI' }
        self.mipVocabFnpat = 'variableInFile.txt'
     else:
@@ -353 +353 @@
     elif self.project in ['ESA-CCI']:
       self.fnvdict = { 'SSTskin':{'v':'sea_surface_temperature', 'sn':'sea_surface_skin_temperature'} }
-      self.fnoptions = {'groupIndex':[3,1], 'trangeIndex':[0,-2] }
+      self.fnoptions = {'groupIndex':[1,0], 'trangeIndex':[0,-2] }
       self.fnoptions['inFn'] = [[None,'*activity','*level','*project','*var','*product','*additional','*gdsv','*version'],
                                 ['*activity','*project','*level','*var','*additional',None,'*version']]
@@ -439 +439 @@
                'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'institutions.txt', omt='last' ) ), \
                'Conventions':utils.patternControl( 'Conventions', '^CF-1.[56789](,.*){0,1}$', examples=['CF-1.6'] ), \
-               'sensor':utils.listControl( 'sensors', lrdr.getSimpleList( 'sensors.txt', bit=0 ) ), \
+               'sensor':utils.listControl( 'sensors', lrdr.getSimpleList( 'sensors.txt', bit=0 ), enumeration=True, split=True, splitVal=',' ), \
                'cdm_data_type':utils.listControl( 'cdm_data_type', lrdr.getSimpleList( 'threddsDataType.txt', bit=0 ) ), \
                'time_coverage_duration':utils.patternControl( 'time_coverage_duration',  'ISO8601 duration', cls='ISO',examples=['P1Y'] ), \

CCCC/trunk/ceda_cc/extractMipInfo.py

@@ -618 +618 @@
 
 mipl = ['cmip5']
+mipl = mips.keys()
 mips = map( lambda x: mips[x], mipl )
 r = run()

CCCC/trunk/ceda_cc/unitTestsS1.py

@@ -52 +52 @@
 SPECS tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199306.nc True
 ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc True CCIplus
+ESA-CCI ESACCI-OC-L3S-OC_PRODUCTS-MERGED-1M_MONTHLY_4km_GEO_PML_OC4v6_QAA-200005-fv1.0.nc True CCIplus
 ESA-CCI 20120101015548-ESACCI-L3U-GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc False
 ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv.1.nc False"""
@@ -86 +87 @@
       if len(l) > 2 and l[2]  == 'CCIplus':
         cga.globalAts =  {'platform':'platform','sensor':'sensor', \
-                 'naming_authority':'org.ghrsst', 'id':'AATSR-ESACCI-L3U-v1', \
+                'naming_authority':'org.ghrsst', 'id':'AATSR-ESACCI-L3U-v1', \
                 'spatial_resolution':'1 km'}
         cga.varAts = { 'var':{ 'standard_name':'dummy'} }
@@ -92 +93 @@
         cga.completed = True
         ee1 = cga.getDrs()
-        if ee1['ecv'] == 'Sea Surface Temperature':
-          print 'OK: [%sb] %s: DRS dictionary generated' % (testId,fn)
+        if ee1['ecv'] in ['Sea Surface Temperature','Ocean Colour']:
+          print 'OK: [%sb] %s: DRS dictionary generated (%s)' % (testId,fn,ee1['ecv'])
         else:
-          print 'Failed: [%sb] %s: DRS dictionary not generated correctly' % (testId,fn)
+          print 'Failed: [%sb] %s: DRS dictionary not generated correctly (%s)' % (testId,fn,ee1['ecv'])
+ 
+        if c.group == 'ESACCI':
+          print 'OK: group ESACCI detected' 
+        else:
+          print 'Failed: group read as "%s" (not ESACCI) %s' % (c.group,fn)
         
     else:

CCCC/trunk/ceda_cc/utils_c4.py

@@ -187 +187 @@
       self.log_exception( 'Exception caught by runChecks' )
       raise loggedException
-      ##traceback.print_exc(file=open("errlog.txt","a"))
-      ##logger.error("Exception has occured" ,exc_info=1)
     
 class checkFileName(checkBase):
@@ -250 +248 @@
           self.fnDict[x[1:]] = bits[i]
       self.fnDict['version'] = bits[-1]
+      self.fnDict['gdsv'] = 'na'
       if self.esaFnId == 0:
         if len(bits) == 9:
@@ -315 +314 @@
               ll.append( tt[:2] )
               tt = tt[2:]
+            elif j in [1,2]:
+              ll.append( '01' )
             else:
               ll.append( '00' )
@@ -381 +382 @@
     if self.fileId == None:
       self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id'])
+      if self.globalAts['naming_authority'] == 'uk.ac.pml':
+        i0 = string.find(self.globalAts['id'],'OC4v6_QAA')
+        if i0 != -1:
+          self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id'][:i0+9])
 
   def getDrs( self ):
@@ -404 +409 @@
       elif self.drsMappings[k][0] == '$':
         self.pcfg.getExtraAtts()
-        print self.pcfg.extraAtts
         self.getId()
         if string.find(self.drsMappings[k],':') != -1:
@@ -478 +482 @@
         self.parent.amapListDraft.append( '#@var=%s;%s=%s|%s=%s' % (varName,k,'__absent__',k,'<insert attribute value and uncomment>') )
         ## print self.parent.amapListDraft[-1]
-    ##vaerr = not self.test( len(m)  == 0, 'Required variable attributes missing: %s' % str(m) )
-
-    ##if vaerr or gaerr:
-      ##self.log_abort()
 
 ## need to insert a check that variable is present
@@ -497 +497 @@
         if varAts[varName].has_key( 'missing_value' ):
            msg = 'Variable [%s] has incorrect attribute missing_value=%s [correct: %s]' % (varName,varAts[varName]['missing_value'],self.missingValue)
-           ## print varAts[varName]['missing_value'], type(varAts[varName]['missing_value'])
-           ## print self.missingValue, type(self.missingValue)
 ### need to use ctypes here when using ncq3 to read files -- appears OK for other libraries.
            ok &= self.test( ctypes.c_float(varAts[varName]['missing_value']).value == ctypes.c_float(self.missingValue).value, msg, part=True )
@@ -933 +931 @@
      self.varInfo = {}
      self.varcons = {}
-     dir, tl, vg, fn = self.pcfg.mipVocabPars
+     dir, tl, vgm, fn = self.pcfg.mipVocabPars
+     vg = vgm.keys()[0]
      ee = { 'standard_name':'sn%s', 'long_name':'n%s', 'units':'1' }
      ll = open( '%s%s' % (dir,fn) ).readlines()