source: CCCC/trunk/ceda_cc/unitTestsS1.py @ 340

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/unitTestsS1.py
Revision 340, 10.6 KB checked in by mjuckes, 5 years ago (diff)

updates to iso 8601 check and esa-cci config

Line 
1
2import collections
3import os.path as op
4import logging, time, string
5import utils_c4
6import config_c4 as config
7
8
9#### set up log file ####
10tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
11testLogFile = '%s__qclog_%s.txt' % ('unitTests',tstring2)
12log = logging.getLogger(testLogFile)
13fHdlr = logging.FileHandler(testLogFile,mode='w')
14fileFormatter = logging.Formatter('%(message)s')
15fHdlr.setFormatter(fileFormatter)
16log.addHandler(fHdlr)
17log.setLevel(logging.INFO)
18
19class dummy(object):
20
21  def __init__(self):
22     pass
23
24p = dummy()
25ps = dummy()
26pcmip5 = dummy()
27pccmi = dummy()
28pcci = dummy()
29for x in (p,ps,pcmip5,pccmi,pcci):
30  x.log = log
31  x.abortMessageCount = -1
32  x.experimental = False
33p.pcfg = config.projectConfig( "CORDEX" )
34ps.pcfg = config.projectConfig( "SPECS" )
35pcmip5.pcfg = config.projectConfig( "CMIP5" )
36pccmi.pcfg = config.projectConfig( "CCMI" )
37pcci.pcfg = config.projectConfig( "ESA-CCI" )
38
39cfgd = { 'CORDEX':p, 'SPECS':ps, 'CMIP5':pcmip5, 'CCMI':pccmi, 'ESA-CCI':pcci }
40
41fnlist = """CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_day_20060101-20101231.nc True
42CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_fx.nc True  fixed
43CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_3hr_2006010100-2010123100.nc True
44CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_3hr_200601010030-201012310030.nc True
45CORDEX ps_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_x1_day_200601010030-201012310030.nc False
46CMIP5 pr_3hr_HadGEM2-ES_historical_r2i1p1_196001010130-196412302230.nc True
47CMIP5 pr_3hr_HadGEM2-ES_historical_r2i1p1.nc False
48CCMI clivi_monthly_CESM1-CAM4Chem_refC1sd_r1i1p1_197501-197912.nc True
49CCMI areacella_fixed_SOCOL3_refC1_r0i0p0.nc True
50SPECS orog_fx_EC-EARTH3_seaIceClimInit_S19930501_r0i0p0.nc True fixed
51SPECS tas_Amon_EC-EARTH3_seaIceClimInit_series2_S19930501_r1i1p1_199306-199306.nc False
52SPECS tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199306.nc True
53SPECS tas_Amon_EC-EARTH3_seaIceClimInit_S19930501_r1i1p1_199306-199305.nc False
54ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc True CCIplus
55ESA-CCI ESACCI-OC-L3S-OC_PRODUCTS-MERGED-1M_MONTHLY_4km_GEO_PML_OC4v6_QAA-200005-fv1.0.nc True CCIplus
56ESA-CCI 20120101015548-ESACCI-L3U-GHRSST-SSTskin-AATSR-LT-v02.0-fv01.1.nc False
57ESA-CCI 20120101015548-ESACCI-L3U_GHRSST-SSTskin-AATSR-LT-v02.0-fv.1.nc False"""
58fntl = map( lambda x: tuple( string.split(x) ), string.split( fnlist, '\n' ) )
59ee = collections.defaultdict( list )
60for t in fntl:
61  ee[t[0]].append( t[1:] )
62
63cga = utils_c4.checkGlobalAttributes(parent=cfgd['ESA-CCI'])
64module = 'checkFileName'
65keys = ee.keys()
66keys.sort()
67kk = 0
68for k in keys:
69  c = utils_c4.checkFileName(parent=cfgd[k])
70  for l in ee[k]:
71    fn = l[0]
72    isValid = l[1] == 'True'
73    if len(l) > 2:
74      chckFx = l[2] == 'fixed'
75    else:
76      chckFx = False
77    kk += 1
78    testId = '#01.%3.3i' % kk
79    c.check( fn )
80    passed = c.errorCount == 0
81    if chckFx:
82      passed &= c.isFixed
83    ok = (isValid == passed)
84    vstr = { True:'Valid', False:'Invalid' }[isValid]
85    pstr = { True:'passed', False:'rejected' }[passed]
86    if ok:
87      print 'OK: [%s] %s: %s %s file name %s' % (testId,fn,vstr,k,pstr)
88      if len(l) > 2 and l[2]  == 'CCIplus':
89        cga.globalAts =  {'platform':'platform','sensor':'sensor', \
90                'naming_authority':'org.ghrsst', 'id':'AATSR-ESACCI-L3U-v1', \
91                'spatial_resolution':'1 km'}
92        cga.varAts = { 'var':{ 'standard_name':'dummy'} }
93        cga.var = 'var'
94        cga.completed = True
95        ee1 = cga.getDrs()
96        if ee1['ecv'] in ['Sea Surface Temperature','Ocean Colour']:
97          print 'OK: [%sb] %s: DRS dictionary generated (%s)' % (testId,fn,ee1['ecv'])
98        else:
99          print 'Failed: [%sb] %s: DRS dictionary not generated correctly (%s)' % (testId,fn,ee1['ecv'])
100 
101        if c.group == 'ESACCI':
102          print 'OK: group ESACCI detected' 
103        else:
104          print 'Failed: group read as "%s" (not ESACCI) %s' % (c.group,fn)
105       
106    else:
107      print 'Failed: [%s] %s: %s %s file name %s' % (testId,fn,vstr,k,pstr)
108
109c = utils_c4.checkStandardDims(parent=p)
110module = 'checkStandardDims'
111## note last argument is "vocabs", but only used in "experimental" mode
112c.check( 'tas', 'day', {},{}, False, None )
113if c.errorCount == 0:
114  print 'Failed [%s]: failed to detect empty dictionaries' % module
115else:
116  print 'OK: -- detected error in standard dims'
117
118c = utils_c4.checkByVar(parent=p)
119module = 'checkByVar (regex)'
120c.check( norun=True )
121import re
122r1 = re.compile( c.pats['subd'][0] )
123for x in ['200401010000','2004010100']:
124  m = r1.match( x )
125  if m:
126     print 'OK: -- passed [%s] %s for sub-daily data' % (module,x)
127  else:
128     print 'Failed to match correct sub-daily time range element [%s] %s' % (module,x)
129
130for x in ['200401010040','2004010200']:
131  m = r1.match( x )
132  if not m:
133     print 'OK: -- correctly failed [%s] %s for sub-daily data' % (module,x)
134  else:
135     print 'Failed to detect bad sub-daily time range element [%s] %s' % (module,x)
136
137r1 = re.compile( c.pats['sem'][0] )
138for x in ['199012','199101']:
139  m = r1.match( x )
140  if m:
141     print 'OK: -- passed [%s] %s for seasonal data' % (module,x)
142  else:
143     print 'Failed to match correct seasonal time range element [%s] %s' % (module,x)
144
145for x in ['199011','199112']:
146  m = r1.match( x )
147  if not m:
148     print 'OK: -- correctly failed [%s] %s for seasonal data' % (module,x)
149  else:
150     print 'Failed to detect bad seasonal time range element [%s] %s' % (module,x)
151
152r1 = re.compile( c.pats['mon'][0] )
153for x in ['199101']:
154  m = r1.match( x )
155  if m:
156     print 'OK: -- passed [%s] %s for daily data' % (module,x)
157  else:
158     print 'Failed to match correct daily time range element [%s] %s -- %s' % (module,x,c.pats['day'][0])
159
160
161c = utils_c4.checkGrids(parent=p)
162c.interpolatedGrids = config.interpolatedGrids
163
164lat = map( lambda x: -46.25 + x*0.5, range(185) )
165lon = map( lambda x: -25.25 + x*0.5, range(173) )
166da = {'lat':{'_data':lat,'units':'degrees_north','long_name':'latitude','standard_name':'latitude','_type':'float64'}, 'lon':{'_data':lon,'units':'degrees_east','long_name':'longitude','standard_name':'longitude','_type':'float64'} }
167c.check( 'tas','AFR-44i', da, {'tas':{} } )
168if c.errorCount == 0:
169  print 'OK: -- passed a correct grid'
170else:
171  print 'Failed -- reported errors on correct grid'
172lat = map( lambda x: -46.25 + x*0.5, range(180) )
173lon = map( lambda x: -25.25 + x*0.5, range(172) )
174da = {'lat':{'_data':lat,'units':'degrees_north','long_name':'latitude','standard_name':'latitude','_type':'float64'}, 'lon':{'_data':lon,'units':'degrees_east','long_name':'longitude','standard_name':'longitude','_type':'float64'} }
175c.check( 'tas','AFR-44i', da, {'tas':{} } )
176if c.errorCount == 0:
177  print 'Failed -- passed a bad grid'
178else:
179  print 'OK: -- detected a bad grid'
180
181##ii = open( op.join(config.CC_CONFIG_DIR, 'specs_vocabs/globalAtsSample001.txt') )
182def readSampleGa( ifile ):
183  ii = open( op.join(config.CC_CONFIG_DIR, ifile) )
184  fn = string.strip( ii.readline() )
185  res = string.strip( ii.readline() )
186  ga = {}
187  for l in ii.readlines():
188    bits = string.split( string.strip(l) )
189    if bits[1] == '@int':
190      ga[bits[0]] = int( bits[2] )
191    else:
192      ga[bits[0]] = string.join( bits[1:] )
193  return res, fn, ga
194
195gafile = 'specs_vocabs/globalAtsSample001.txt'
196res, fn, ga = readSampleGa( gafile )
197
198testId = '#04.001'
199## switch to project = SPECS
200c = utils_c4.checkFileName(parent=ps)
201c.check( fn )
202if c.errorCount == 0:
203  print 'OK: [%s] %s: valid file name' % (module,fn)
204else:
205  print 'Failed [%s] %s: valid file name' % (module,fn)
206
207## note test is done on string'ed values.
208va = { "tas":{ "_type":"float32", "standard_name":"air_temperature", 'long_name':'Air Temperature', 'units':'K', 'cell_methods':'time: mean'} }
209
210testId = '#04.002'
211cga = utils_c4.checkGlobalAttributes( parent=ps,cls='SPECS')
212cga.check( ga, va, "tas", "day", ps.pcfg.vocabs, c.fnParts )
213if cga.errorCount == 0:
214  print 'OK: [%s]: global attributes check (%s)' % (testId,gafile)
215else:
216  print 'Failed [%s]: global attributes check (%s)' % (testId,gafile)
217
218gafile = 'specs_vocabs/globalAtsSample002.txt'
219res, fn, ga = readSampleGa( gafile )
220testId = '#04.003'
221## switch to project = SPECS
222c = utils_c4.checkFileName(parent=ps)
223c.check( fn )
224if c.errorCount == 0:
225  print 'OK: [%s] %s: valid file name' % (testId,fn)
226else:
227  print 'Failed [%s] %s: valid file name' % (testId,fn)
228
229## note test is done on string'ed values.
230
231va = { "tas":{ "_type":"float32", "standard_name":"air_temperature", 'long_name':'Air Temperature', 'units':'K', 'cell_methods':'time: mean'} }
232
233testId = '#04.004'
234cga = utils_c4.checkGlobalAttributes( parent=ps,cls='SPECS')
235cga.check( ga, va, "tas", "day", ps.pcfg.vocabs, c.fnParts )
236if cga.errorCount == 0:
237  print 'Failed: [%s]: passed bad global attributes (%s)' % (testId,gafile)
238else:
239  print 'OK: [%s]: detected bad global attributes (%s)' % (testId,gafile)
240
241gafile = 'esacci_vocabs/sampleGlobalAts.txt'
242res, fn, ga = readSampleGa( gafile )
243testId = '#04.005'
244## switch to project = ESA-CCI
245c = utils_c4.checkFileName(parent=pcci)
246c.check( fn )
247if c.errorCount == 0:
248  print 'OK: [%s] %s: valid file name' % (testId,fn)
249else:
250  print 'Failed [%s] %s: valid file name' % (testId,fn)
251
252## note test is done on string'ed values.
253
254va = { "burned_area":{ "_type":"float32", "standard_name":"burned_area"} }
255
256testId = '#04.004'
257cga = utils_c4.checkGlobalAttributes( parent=pcci,cls='ESA-CCI')
258cga.check( ga, va, "burned_area", "ESACCI", pcci.pcfg.vocabs, c.fnParts )
259if cga.errorCount == 0:
260  print 'Failed: [%s]: passed bad global attributes (%s)' % (testId,gafile)
261else:
262  print 'OK: [%s]: detected bad global attributes (%s)' % (testId,gafile)
263
264ls = utils_c4.listControl('test',['a','b','c1','c2'],split=True,splitVal=',',enumeration=True)
265testId = '#05.001'
266res = ls.essplit.findall( 'a, b, c<1,2>')
267if res == ['a', 'b', 'c<1,2>']:
268  print 'OK: [%s] Split of list with enumeration passed' % (testId)
269else:
270  print 'Failed: [%s] Split of list with enumeration failed %s' % (testId,str(res))
271
272testId = '#05.002'
273res = ls.check( 'a, c<1,2>' )
274if res:
275  print 'OK: [%s] Valid value passed' % (testId)
276else:
277  print 'Failed: [%s] valid value rejected %s' % (testId,str(res))
278
279testId = '#05.003'
280res = ls.check( 'a, c<1,3>' )
281if res:
282  print 'Failed: [%s] Invalid value passed' % (testId)
283else:
284  print 'OK: [%s] Invalid value rejected %s' % (testId,str(res))
285
286
287testId = '#06.001'
288pc = utils_c4.patternControl( 'pattern test', 'ISO8601 duration', cls='ISO' )
289res = pc.check( 'P1Z' )
290if res:
291  print 'Failed: [%s] Invalid value passed' % (testId)
292else:
293  print 'OK: [%s] Invalid value rejected %s' % (testId,str(res))
294res = pc.check( 'P1W' )
295if res:
296  print 'OK: [%s] Valid value passed' % (testId)
297else:
298  print 'Failed: [%s] Valid value rejected %s' % (testId,str(res))
299
Note: See TracBrowser for help on using the repository browser.