Changeset 269 for CCCC


Ignore:
Timestamp:
07/01/15 13:27:00 (5 years ago)
Author:
mjuckes
Message:

added checks on ESA-CCI cdm_data_type and duration attributes

Location:
CCCC/trunk/ceda_cc
Files:
3 added
5 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/ceda_cc/config/specs_vocabs/globalAtsSample001.txt

    r163 r269  
    1 zos_Omon_EC-EARTH_decadal_series1_r1i1p1_1990-1990.nc 
     1zos_Omon_EC-EARTH_decadal_S199010101_r1i1p1_1990-1990.nc 
    22PASS 
    33batch  aa001 
     
    55Conventions   CF-1.6 
    66creation_date   19th JUly, 2014 
    7 experiment_id   decadal1980 
     7experiment_id   decadal 
    88forecast_reference_time   0001 
    99frequency  day 
    1010institution  ECMWF 
     11institute_id ECMWF 
    1112initialization_method  1 
    1213initialization_description  first 
     
    1920realization  1 
    2021source   misc 
    21 table_id  Table Omon 
     22table_id  Table Omon aghhajakakka 
    2223title   sample 
    2324tracking_id 1093890138391 
    2425series @int 1 
     26associated_experiment  N/A 
     27startdate  S199010101 
  • CCCC/trunk/ceda_cc/config/specs_vocabs/globalAtsSample002.txt

    r163 r269  
    1 zos_Omon_EC-EARTH_decadal_series1_r1i1p1_1990-1990.nc 
    2 FAIL 
     1zos_Omon_EC-EARTH_decadal_S199010101_r1i1p1_1990-1990.nc 
     2PASS 
    33batch  aa001 
    44contact   john@home.uk 
    55Conventions   CF-1.6 
    66creation_date   19th JUly, 2014 
    7 experiment_id   decada1980 
     7experiment_id   decadal 
    88forecast_reference_time   0001 
    99frequency  day 
    1010institution  ECMWF 
     11institute_id ECMWF 
    1112initialization_method  1 
    1213initialization_description  first 
    13 model_id  EC-EARTH 
     14model_id  Ec-EARTH 
    1415associated_model  None 
    15 modeling_realm   atmos 
     16modeling_realm   atm 
    1617physics_version   1 
    1718physics_description  first 
     
    1920realization  1 
    2021source   misc 
    21 table_id  Table Omon 
     22table_id  Table Omon aghhajakakka 
    2223title   sample 
    2324tracking_id 1093890138391 
    2425series @int 1 
     26associated_experiment  N/A 
     27startdate  S199010101 
  • CCCC/trunk/ceda_cc/config_c4.py

    r263 r269  
    116116    self.dir = dir 
    117117 
    118   def getSimpleList(self,file,bit=None): 
     118  def getSimpleList(self,file,bit=None,omt=None): 
    119119    ii = open('%s/%s/%s' % (CC_CONFIG_DIR, self.dir,file) ) 
    120120    oo = [] 
     
    122122      if l[0] != '#': 
    123123        ll = string.strip(l) 
    124         if bit == None: 
     124        if omt == 'last': 
     125          oo.append(string.join(string.split(ll)[:-1])) 
     126        elif bit == None: 
    125127          oo.append(ll) 
    126128        else: 
     
    179181               'level':utils.listControl( 'level', lrdr.getSimpleList( 'procLevel01.txt', bit=0 ) ), \ 
    180182               'platform':utils.listControl( 'platforms', lrdr.getSimpleList( 'platforms.txt', bit=0 ) ), \ 
     183               'institution':utils.listControl( 'institution', lrdr.getSimpleList( 'institutions.txt', omt='last' ) ), \ 
    181184               'Conventions':utils.patternControl( 'Conventions', '^CF-1.[56789](,.*){0,1}$' ), \ 
    182185               'sensor':utils.listControl( 'sensors', lrdr.getSimpleList( 'sensors.txt', bit=0 ) ), \ 
     186               'cdm_data_type':utils.listControl( 'cdm_data_type', lrdr.getSimpleList( 'threddsDataType.txt', bit=0 ) ), \ 
     187               'time_coverage_duration':utils.patternControl( 'time_coverage_duration',  'ISO8601 duration', cls='ISO' ), \ 
    183188               'project':utils.listControl( 'project', ['Climate Change Initiative - European Space Agency'] ), \ 
    184189               'cciProject':utils.listControl( 'project', lrdr.getSimpleList( 'cciProject.txt', bit=-1 ) ), \ 
     
    280285      self.checkVarType = False 
    281286      self.requiredGlobalAttributes = lrdr.getSimpleList( 'requiredGlobalAts.txt', bit=0 ) 
    282       self.controlledGlobalAttributes = ['platform','sensor','project','Conventions' ] 
     287      self.controlledGlobalAttributes = ['platform','sensor','project','Conventions','institution','cdm_data_type','time_coverage_duration' ] 
    283288      self.controlledFnParts = ['level','cciProject','var','version'] 
    284289      self.requiredVarAttributes = ['long_name', 'standard_name', 'units'] 
  • CCCC/trunk/ceda_cc/unitTestsS1.py

    r263 r269  
    208208  print 'OK: -- detected a bad grid' 
    209209 
    210 ii = open( op.join(config.CC_CONFIG_DIR, 'specs_vocabs/globalAtsSample001.txt') ) 
    211 fn = string.strip( ii.readline() ) 
    212 res = string.strip( ii.readline() ) 
     210##ii = open( op.join(config.CC_CONFIG_DIR, 'specs_vocabs/globalAtsSample001.txt') ) 
     211def readSampleGa( ifile ): 
     212  ii = open( op.join(config.CC_CONFIG_DIR, ifile) ) 
     213  fn = string.strip( ii.readline() ) 
     214  res = string.strip( ii.readline() ) 
     215  ga = {} 
     216  for l in ii.readlines(): 
     217    bits = string.split( string.strip(l) ) 
     218    if bits[1] == '@int': 
     219      ga[bits[0]] = int( bits[2] ) 
     220    else: 
     221      ga[bits[0]] = string.join( bits[1:] ) 
     222  return res, fn, ga 
     223 
     224gafile = 'specs_vocabs/globalAtsSample001.txt' 
     225res, fn, ga = readSampleGa( gafile ) 
     226 
    213227testId = '#04.001' 
    214228## switch to project = SPECS 
     
    220234  print 'Failed [%s] %s: valid file name' % (module,fn) 
    221235 
    222  
    223236## note test is done on string'ed values. 
    224 ga = {} 
    225 for l in ii.readlines(): 
    226   bits = string.split( string.strip(l) ) 
    227   if bits[1] == '@int': 
    228     ga[bits[0]] = int( bits[2] ) 
    229   else: 
    230     ga[bits[0]] = string.join( bits[1:] ) 
    231  
    232237va = { "tas":{ "_type":"float32", "standard_name":"air_temperature", 'long_name':'Air Temperature', 'units':'K', 'cell_methods':'time: mean'} } 
    233238 
     239testId = '#04.002' 
    234240cga = utils_c4.checkGlobalAttributes( parent=ps,cls='SPECS') 
    235241cga.check( ga, va, "tas", "day", ps.pcfg.vocabs, c.fnParts ) 
    236  
    237 ii = open( op.join(config.CC_CONFIG_DIR, 'specs_vocabs/globalAtsSample002.txt') ) 
    238 fn = string.strip( ii.readline() ) 
    239 res = string.strip( ii.readline() ) 
    240 testId = '#04.002' 
     242if cga.errorCount == 0: 
     243  print 'OK: [%s]: global attributes check (%s)' % (testId,gafile) 
     244else: 
     245  print 'Failed [%s]: global attributes check (%s)' % (testId,gafile) 
     246 
     247gafile = 'specs_vocabs/globalAtsSample002.txt' 
     248res, fn, ga = readSampleGa( gafile ) 
     249testId = '#04.003' 
    241250## switch to project = SPECS 
    242251c = utils_c4.checkFileName(parent=ps) 
    243252c.check( fn ) 
    244253if c.errorCount == 0: 
    245   print 'OK: [%s] %s: valid file name' % (module,fn) 
    246 else: 
    247   print 'Failed [%s] %s: valid file name' % (module,fn) 
    248  
     254  print 'OK: [%s] %s: valid file name' % (testId,fn) 
     255else: 
     256  print 'Failed [%s] %s: valid file name' % (testId,fn) 
    249257 
    250258## note test is done on string'ed values. 
    251 ga = {} 
    252 for l in ii.readlines(): 
    253   bits = string.split( string.strip(l) ) 
    254   if bits[1] == '@int': 
    255     ga[bits[0]] = int( bits[2] ) 
    256   else: 
    257     ga[bits[0]] = string.join( bits[1:] ) 
    258259 
    259260va = { "tas":{ "_type":"float32", "standard_name":"air_temperature", 'long_name':'Air Temperature', 'units':'K', 'cell_methods':'time: mean'} } 
    260261 
     262testId = '#04.004' 
    261263cga = utils_c4.checkGlobalAttributes( parent=ps,cls='SPECS') 
    262264cga.check( ga, va, "tas", "day", ps.pcfg.vocabs, c.fnParts ) 
     265if cga.errorCount == 0: 
     266  print 'Failed: [%s]: passed bad global attributes (%s)' % (testId,gafile) 
     267else: 
     268  print 'OK: [%s]: detected bad global attributes (%s)' % (testId,gafile) 
     269 
     270gafile = 'esacci_vocabs/sampleGlobalAts.txt' 
     271res, fn, ga = readSampleGa( gafile ) 
     272testId = '#04.005' 
     273## switch to project = ESA-CCI 
     274c = utils_c4.checkFileName(parent=pcci) 
     275c.check( fn ) 
     276if c.errorCount == 0: 
     277  print 'OK: [%s] %s: valid file name' % (testId,fn) 
     278else: 
     279  print 'Failed [%s] %s: valid file name' % (testId,fn) 
     280 
     281## note test is done on string'ed values. 
     282 
     283va = { "burned_area":{ "_type":"float32", "standard_name":"burned_area"} } 
     284 
     285testId = '#04.004' 
     286cga = utils_c4.checkGlobalAttributes( parent=pcci,cls='ESA-CCI') 
     287cga.check( ga, va, "burned_area", "ESACCI", pcci.pcfg.vocabs, c.fnParts ) 
     288if cga.errorCount == 0: 
     289  print 'Failed: [%s]: passed bad global attributes (%s)' % (testId,gafile) 
     290else: 
     291  print 'OK: [%s]: detected bad global attributes (%s)' % (testId,gafile) 
  • CCCC/trunk/ceda_cc/utils_c4.py

    r267 r269  
    935935class patternControl(object): 
    936936 
    937   def __init__(self,tag,pattern,list=None): 
    938     try: 
    939       self.re_pat = re.compile( pattern ) 
    940     except: 
    941       print "Failed to compile pattern >>%s<< (%s)" % (pattern, tag) 
    942     self.pattern = pattern 
     937  def __init__(self,tag,pattern,list=None,cls=None): 
     938    if cls != None: 
     939      assert cls in ['ISO'], 'value of cls [%s] not recognised' % cls 
     940      if cls == 'ISO': 
     941        assert pattern in ['ISO8601 duration'], 'value of pattern [%s] for ISO constraint not recognised' % pattern 
     942        if pattern == 'ISO8601 duration': 
     943          thispat = '^(P([0-9]+Y){0,1}([0-9]+M){0,1}([0-9]+D){0,1}(T([0-9]+H){0,1}([0-9]+M){0,1}([0-9]+(.[0-9]+){0,1}S){0,1}){0,1})$' 
     944        self.re_pat = re.compile( thispat ) 
     945        self.pattern = thispat 
     946        self.pattern_src = pattern 
     947    else: 
     948      try: 
     949        self.re_pat = re.compile( pattern ) 
     950      except: 
     951        print "Failed to compile pattern >>%s<< (%s)" % (pattern, tag) 
     952      self.pattern = pattern 
    943953    self.list = list 
     954    self.cls = cls 
    944955 
    945956  def check(self,val): 
Note: See TracChangeset for help on using the changeset viewer.