Changeset 276 for CCCC


Ignore:
Timestamp:
21/01/15 09:43:15 (5 years ago)
Author:
mjuckes
Message:

cleaned up unit tests and handling of fixed files

Location:
CCCC/trunk/ceda_cc
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/ceda_cc/c4.py

    r263 r276  
    3636 
    3737class dummy(object): 
    38    pass 
     38  def __init__(self): 
     39     self.experimental = None 
     40     self.parent = None 
    3941 
    4042pathTmplDict = { 'CORDEX':'%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/',   \ 
     
    145147  def __init__(self, pcfg, cls,reader,abortMessageCount=-1,experimental=False): 
    146148    self.info = dummy() 
    147     self.info.experimental=experimental 
    148149    self.info.pcfg = pcfg 
     150    self.info.fileIsFixed = None 
    149151    self.info.abortMessageCount = abortMessageCount 
     152    self.info.experimental = experimental 
    150153    self.calendar = 'None' 
    151154    self.ncReader = reader 
  • CCCC/trunk/ceda_cc/unitTestsS1.py

    r275 r276  
    11 
     2import collections 
    23import os.path as op 
    34import logging, time, string 
     
    2930  x.log = log 
    3031  x.abortMessageCount = -1 
     32  x.experimental = False 
    3133p.pcfg = config.projectConfig( "CORDEX" ) 
    3234ps.pcfg = config.projectConfig( "SPECS" ) 
     
    3537pcci.pcfg = config.projectConfig( "ESA-CCI" ) 
    3638 
     39cfgd = { 'CORDEX':p, 'SPECS':ps, 'CMIP5':pcmip5, 'CCMI':pccmi, 'ESA-CCI':pcci } 
     40 
     41fnlist = """CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_day_20060101-20101231.nc True 
     42CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_fx.nc True  fixed 
     43CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_3hr_2006010100-2010123100.nc True 
     44CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_3hr_200601010030-201012310030.nc True 
     45CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_day_200601010030-201012310030.nc False 
     46CMIP5 pr_3hr_HadGEM2-ES_historical_r2i1p1_196001010130-196412302230.nc True 
     47CMIP5 pr_3hr_HadGEM2-ES_historical_r2i1p1.nc False 
     48CCMI clivi_monthly_CESM1-CAM4Chem_refC1sd_r1i1p1_197501-197912.nc True 
     49CCMI areacella_fixed_SOCOL3_refC1_r0i0p0.nc True 
     50SPECS orog_fx_EC-EARTH3_seaIceClimInit_S19930501_r0i0p0.nc True fixed 
     51SPECS tas_Amon_EC-EARTH3_seaIceClimInit_series2_S19930501_r1i1p1_199306-199306.nc False 
     52SPECS tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199306.nc True 
     53ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc True 
     54ESA-CCI 20120101015548-ESACCI-L3U-GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc False 
     55ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv.1.nc False""" 
     56fntl = map( lambda x: tuple( string.split(x) ), string.split( fnlist, '\n' ) ) 
     57ee = collections.defaultdict( list ) 
     58for t in fntl: 
     59  ee[t[0]].append( t[1:] ) 
    3760 
    3861module = 'checkFileName' 
    39 c = utils_c4.checkFileName(parent=p) 
    40  
    41 fn = 'ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_day_20060101-20101231.nc' 
    42 testId = '#01.001' 
    43 c.check( fn ) 
    44 if c.errorCount == 0: 
    45   print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
    46 else: 
    47   print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    48  
    49 testId = '#01.002' 
    50 fn = 'ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_fx.nc' 
    51 c.check(fn) 
    52 if c.errorCount == 0 and c.isFixed: 
    53   print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
    54 else: 
    55   print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    56  
    57 testId = '#01.003' 
    58 fn = 'ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_3hr_2006010100-2010123100.nc' 
    59 c.check(fn) 
    60 if c.errorCount == 0: 
    61   print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
    62 else: 
    63   print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    64  
    65 testId = '#01.004' 
    66 fn = 'ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_3hr_200601010030-201012310030.nc' 
    67 c.check(fn) 
    68 if c.errorCount == 0: 
    69   print 'OK: [%s] %s: valid CORDEX file name' % (module,fn) 
    70 else: 
    71   print 'Failed [%s] %s: valid CORDEX file name' % (module,fn) 
    72  
    73 testId = '#01.005' 
    74 fn = 'ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_day_200601010030-201012310030.nc' 
    75 c.check(fn) 
    76 if c.errorCount == 0: 
    77   print 'Failed to detect bad CORDEX file name: [%s] %s ' % (module,fn) 
    78 else: 
    79   print 'OK: -- detected bad CORDEX file name: [%s] %s' % (module,fn) 
    80  
    81 fn = "pr_3hr_HadGEM2-ES_historical_r2i1p1_196001010130-196412302230.nc" 
    82 c = utils_c4.checkFileName(parent=pcmip5) 
    83 c.check(fn) 
    84 if c.errorCount == 0: 
    85   print 'OK: [%s] %s: valid CMIP5 file name' % (module,fn) 
    86 else: 
    87   print 'Failed [%s] %s: valid CMIP5 file name' % (module,fn) 
    88  
    89 fn = "clivi_monthly_CESM1-CAM4Chem_refC1sd_r1i1p1_197501-197912.nc" 
    90 c = utils_c4.checkFileName(parent=pccmi) 
    91 c.check(fn) 
    92 if c.errorCount == 0: 
    93   print 'OK: [%s] %s: valid CCMI file name' % (module,fn) 
    94 else: 
    95   print 'Failed [%s] %s: valid CCMI file name' % (module,fn) 
    96  
    97 fn = "areacella_fixed_SOCOL3_refC1_r0i0p0.nc" 
    98 c = utils_c4.checkFileName(parent=pccmi) 
    99 c.check(fn) 
    100 if c.errorCount == 0: 
    101   print 'OK: [%s] %s: valid CCMI file name' % (module,fn) 
    102 else: 
    103   print 'Failed [%s] %s: valid CCMI file name' % (module,fn) 
    104  
    105 fn = "tas_Amon_EC-EARTH3_seaIceClimInit_series2_S19930501_r1i1p1_199306-199306.nc" 
    106 c = utils_c4.checkFileName(parent=ps) 
    107 c.check(fn) 
    108 if c.errorCount == 0: 
    109   print 'Failed [%s] %s: passed invalid SPECS file name' % (module,fn) 
    110 else: 
    111   print 'OK: [%s] %s: detected invalid SPECS file name' % (module,fn) 
    112  
    113 fn = "tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199306.nc" 
    114 c = utils_c4.checkFileName(parent=ps) 
    115 c.check(fn) 
    116 if c.errorCount == 0: 
    117   print 'OK: [%s] %s: valid SPECS file name' % (module,fn) 
    118 else: 
    119   print 'Failed [%s] %s: valid SPECS file name' % (module,fn) 
    120  
    121 fn = "20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc" 
    122 c = utils_c4.checkFileName(parent=pcci) 
    123 c.check(fn) 
    124 if c.errorCount == 0: 
    125   print 'OK: [%s] %s: valid ESA-CCI file name' % (module,fn) 
    126 else: 
    127   print 'Failed [%s] %s: valid ESA-CCI file name' % (module,fn) 
    128  
    129 fn = "20120101015548-ESACCI-L3U-GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc" 
    130 c = utils_c4.checkFileName(parent=pcci) 
    131 c.check(fn) 
    132 if c.errorCount == 0: 
    133   print 'Failed: [%s] %s: Passed invalid ESA-CCI file name' % (module,fn) 
    134 else: 
    135   print 'OK [%s] %s: Detected invalid ESA-CCI file name' % (module,fn) 
    136  
    137 fn = "20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv.1.nc" 
    138 c = utils_c4.checkFileName(parent=pcci) 
    139 c.check(fn) 
    140 if c.errorCount == 0: 
    141   print 'Failed: [%s] %s: Passed invalid ESA-CCI file name' % (module,fn) 
    142 else: 
    143   print 'OK [%s] %s: Detected invalid ESA-CCI file name' % (module,fn) 
    144  
     62keys = ee.keys() 
     63keys.sort() 
     64kk = 0 
     65for k in keys: 
     66  c = utils_c4.checkFileName(parent=cfgd[k]) 
     67  for l in ee[k]: 
     68    fn = l[0] 
     69    isValid = l[1] == 'True' 
     70    if len(l) > 2: 
     71      chckFx = l[2] == 'fixed' 
     72    else: 
     73      chckFx = False 
     74    kk += 1 
     75    testId = '#01.%3.3i' % kk 
     76    c.check( fn ) 
     77    passed = c.errorCount == 0 
     78    if chckFx: 
     79      passed &= c.isFixed 
     80    ok = (isValid == passed) 
     81    vstr = { True:'Valid', False:'Invalid' }[isValid] 
     82    pstr = { True:'passed', False:'rejected' }[passed] 
     83    if ok: 
     84      print 'OK: [%s] %s: %s %s file name %s' % (testId,fn,vstr,k,pstr) 
     85    else: 
     86      print 'Failed: [%s] %s: %s %s file name %s' % (testId,fn,vstr,k,pstr) 
    14587 
    14688c = utils_c4.checkStandardDims(parent=p) 
     
    299241  print 'OK: [%s]: detected bad global attributes (%s)' % (testId,gafile) 
    300242 
    301  
    302243ls = utils_c4.listControl('test',['a','b','c1','c2'],split=True,splitVal=',',enumeration=True) 
    303244testId = '#05.001' 
  • CCCC/trunk/ceda_cc/unitTestsS2.py

    r253 r276  
    3838c.check( fn ) 
    3939if c.errorCount == 0: 
    40   print 'OK [%s] %s: valid file name with project=__dummy' % (module,fn) 
     40  print 'OK: [%s] %s: valid file name with project=__dummy' % (module,fn) 
    4141else: 
    4242  print 'Failed [%s] %s: valid file name' % (module,fn) 
     
    4545try: 
    4646  m = main( args=['-p', '__dummy'], monitorFileHandles=True ) 
    47   print 'OK [%s]: dummy run completed without exception' % testId 
     47  print 'OK: [%s]: dummy run completed without exception' % testId 
    4848except: 
    4949  print 'Failed [%s]: dummy run triggered exception' % testId 
     
    5252testId = '#11.002' 
    5353if m.monitor.fhCountMax < 10: 
    54   print 'OK [%s]: fhCountMax = %s' % ( testId, m.monitor.fhCountMax ) 
     54  print 'OK: [%s]: fhCountMax = %s' % ( testId, m.monitor.fhCountMax ) 
    5555else: 
    5656  print 'Failed [%s]: fhCountMax = %s' % ( testId, m.monitor.fhCountMax ) 
     
    6161  print 'Failed [%s]: did not trigger exception' % testId 
    6262except: 
    63   print 'OK [%s]: attempt to trigger exception successful' % testId 
     63  print 'OK: [%s]: attempt to trigger exception successful' % testId 
    6464 
    6565 
     
    7777      m = main( args=['-p', e[1], '-f', e[0], '--force-cdms2','--ld', 'ld_test1' ], abortMessageCount=10 ) 
    7878      if m.ok: 
    79          print 'OK [%s]: successfully checked test file with cdms2' % testId 
     79         print 'OK: [%s]: successfully checked test file with cdms2' % testId 
    8080      else: 
    8181         print 'Failed [%s]: incorrect test results' % testId 
     
    8484    m = main( args=['-p', e[1], '-f', e[0], '--force-ncq','--ld', 'ld_test2' ], abortMessageCount=10 ) 
    8585    if m.ok: 
    86        print 'OK [%s]: successfully checked test file with ncq3' % testId 
     86       print 'OK: [%s]: successfully checked test file with ncq3' % testId 
    8787    else: 
    8888       print 'Failed [%s]: incorrect test results' % testId 
     
    9292      m = main( args=['-p', e[1], '-f', e[0], '--force-pync4','--ld', 'ld_test3' ], abortMessageCount=10 ) 
    9393      if m.ok: 
    94          print 'OK [%s]: successfully checked test file with python NetCDF4' % testId 
     94         print 'OK: [%s]: successfully checked test file with python NetCDF4' % testId 
    9595      else: 
    9696         print 'Failed [%s]: incorrect test results' % testId 
     
    100100      m = main( args=['-p', e[1], '-f', e[0], '--force-scientific','--ld', 'ld_test4' ], abortMessageCount=10 ) 
    101101      if m.ok: 
    102          print 'OK [%s]: successfully checked test file with python Scientific' % testId 
     102         print 'OK: [%s]: successfully checked test file with python Scientific' % testId 
    103103      else: 
    104104         print 'Failed [%s]: incorrect test results' % testId 
  • CCCC/trunk/ceda_cc/utils_c4.py

    r274 r276  
    195195    self.id = 'C4.001' 
    196196    self.checkId = 'unset' 
     197    self.isFixed = False 
    197198    self.step = 'Initialised' 
    198199    self.checks = (self.do_check_fn,self.do_check_fnextra) 
     
    286287 
    287288    self.isFixed = self.freq in ['fx','fixed'] 
     289    self.parent.fileIsFixed = True 
    288290    if self.isFixed: 
    289291      self.test( len(self.fnParts) in self.pcfg.fnPartsOkFixedLen, 'Number of file name elements not acceptable for fixed data' ) 
     
    440442    m = [] 
    441443    reqAts = self.requiredVarAttributes[:] 
    442     if varGroup != 'fx' and self.pcfg.projectV.id in ['CORDEX']: 
     444    if (not self.parent.fileIsFixed) and self.pcfg.projectV.id in ['CORDEX']: 
    443445      reqAts.append( 'cell_methods' ) 
    444446    for k in reqAts + vocabs['variable'].lists(varName, 'addRequiredAttributes'): 
     
    449451      for k in m: 
    450452        self.parent.amapListDraft.append( '#@var=%s;%s=%s|%s=%s' % (varName,k,'__absent__',k,'<insert attribute value and uncomment>') ) 
    451         print self.parent.amapListDraft[-1] 
     453        ## print self.parent.amapListDraft[-1] 
    452454    ##vaerr = not self.test( len(m)  == 0, 'Required variable attributes missing: %s' % str(m) ) 
    453455 
     
    481483    if self.pcfg.varTables=='CMIP': 
    482484      contAts = ['long_name', 'standard_name', 'units'] 
    483       if varGroup not in ['fx','fixed']: 
     485      if not self.parent.fileIsFixed: 
     486      ##if varGroup not in ['fx','fixed']: 
    484487        contAts.append( 'cell_methods' ) 
    485488    else: 
     
    524527       self.log_pass() 
    525528 
    526     if (varGroup not in ['fx','fixed']) and hcm: 
     529    if (not self.parent.fileIsFixed) and hcm: 
     530    ## if (varGroup not in ['fx','fixed']) and hcm: 
    527531      self.isInstantaneous = string.find( varAts[varName]['cell_methods'], 'time: point' ) != -1 
    528532    else: 
     
    615619    self.checkId = ('001','time_attributes') 
    616620    self.calendar = 'None' 
    617     if varGroup not in ['fx','fixed']: 
     621    if not self.parent.fileIsFixed: 
     622    ## if varGroup not in ['fx','fixed']: 
    618623      ok = True 
    619624      self.test( 'time' in da.keys(), 'Time dimension not found' , abort=True, part=True ) 
     
    10351040          freq = None 
    10361041 
    1037         isFixed = freq  in ['fx','fixed'] 
     1042        ### isFixed = freq  in ['fx','fixed'] 
    10381043        group = fnParts[ self.pcfg.groupIndex ] 
    10391044 
    1040         if isFixed: 
     1045        if self.parent.fileIsFixed: 
     1046       ## if isFixed: 
    10411047          trange = None 
    10421048        else: 
Note: See TracChangeset for help on using the changeset viewer.