source: CCCC/trunk/c4.py @ 68

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/c4.py@68
Revision 68, 10.0 KB checked in by mjuckes, 7 years ago (diff)

fixed bug in exception handling in checkStandardDims

Line 
1
2import utils_c4 as utils
3import config_c4 as config
4import os, string, time
5import logging
6
7reload( utils )
8import cdms2
9
10vocabs = { 'variable':utils.mipVocab(), \
11           'driving_experiment_name':utils.listControl( 'driving_experiment_name', config.validExperiment ), \
12           'project_id':utils.listControl( 'project_id', ['CORDEX'] ), \
13           'CORDEX_domain':utils.listControl( 'CORDEX_domain',  config.validCordexDomains ), \
14           'driving_model_id':utils.listControl( 'driving_model_id',  config.validGcmNames ), \
15           'driving_model_ensemble_member':utils.patternControl( 'driving_model_ensemble_member',  'r[0-9]+i[0-9]+p[0-9]+' ), \
16           'rcm_version_id':utils.patternControl( 'rcm_version_id',  '[a-zA-Z0-9-]+' ), \
17           'model_id':utils.listControl( 'model_id',  config.validRcmNames ), \
18           'institute_id':utils.listControl( 'institute_id',  config.validInstNames ), \
19           'frequency':utils.listControl( 'frequency', config.validCordexFrequecies ) }
20
21#driving_model_ensemble_member = <CMIP5Ensemble_member>
22#rcm_version_id = <RCMVersionID>                     
23
24class dummy:
25   pass
26
27class recorder:
28
29  def __init__(self,fileName,type='map'):
30    self.file = fileName
31    self.type = type
32    self.pathTmpl = '%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/'
33    self.records = {}
34
35  def open(self):
36    if self.type == 'map':
37      self.fh = open( self.file, 'a' )
38    else:
39      self.sh = shelve.open( self.file )
40
41  def close(self):
42    if self.type == 'map':
43      self.fh.close()
44    else:
45      self.sh.close()
46
47  def add(self,fpath,drs,safe=True):
48    assert self.type == 'map','Can only do map files at present'
49    assert os.path.isfile( fpath ), 'File %s not found' % fpath
50    assert type(drs) == type( {} ), '2nd user argument to method add should be a dictionary [%s]' % type(drs)
51    tpath = self.pathTmpl % drs
52    fdate = time.ctime(os.path.getmtime(fpath))
53    sz = os.stat(fpath).st_size
54    record = '%s | OK | %s | modTime = %s | target = %s ' % (fpath,sz,fdate,tpath)
55    for k in ['creation_date','tracking_id']:
56      if k in drs.keys():
57        record += ' | %s = %s' % (k,drs[k])
58
59    fn = string.split( fpath, '/' )[-1]
60    self.records[fn] = record
61 
62  def modify(self,fn,msg):
63    assert fn in self.records.keys(),'Attempt to modify non-existent record %s, %s' % [fn,str(self.records.keys()[0:10])]
64    if string.find( self.records[fn], '| OK |') == -1:
65      print 'File %s already flagged with errors' % fn
66      return
67    s = string.replace( self.records[fn], '| OK |', '| %s |' % msg )
68    print '--> ',s
69    self.records[fn] = s
70
71  def dumpAll(self,safe=True):
72    keys = self.records.keys()
73    keys.sort()
74    for k in keys:
75      self.dump( self.records[k], safe=safe )
76
77  def dump( self, record, safe=True ):
78    if safe:
79      self.open()
80    self.fh.write( record + '\n' )
81    if safe:
82      self.close()
83
84  def addErr(self,fpath,reason,safe=True):
85    record = '%s | %s' % (fpath, reason)
86    fn = string.split( fpath, '/' )[-1]
87    self.records[fn] = record
88
89class checker:
90  def __init__(self):
91    self.info = dummy()
92    self.calendar = 'None'
93    self.cfn = utils.checkFileName(parent=self.info)
94    self.cga = utils.checkGlobalAttributes(parent=self.info)
95    self.cgd = utils.checkStandardDims(parent=self.info)
96    self.cgg = utils.checkGrids(parent=self.info)
97
98  def checkFile(self,fpath,log=None):
99    self.calendar = 'None'
100    self.info.log = log
101
102    fn = string.split( fpath, '/' )[-1]
103    self.cfn.check( fn )
104    if not self.cfn.completed:
105      self.completed = False
106      return
107    if not os.path.isfile( fpath ):
108      print 'File %s not found' % fpath
109      self.completed = False
110      return
111
112    self.nc = cdms2.open( fpath )
113    self.ga = {}
114    self.va = {}
115    self.da = {}
116    for k in self.nc.attributes.keys():
117      self.ga[k] = self.nc.attributes[k]
118    for v in self.nc.variables.keys():
119      self.va[v] = {}
120      for k in self.nc.variables[v].attributes.keys():
121        self.va[v][k] = self.nc.variables[v].attributes[k]
122      self.va[v]['_type'] = str( self.nc.variables[v].dtype )
123      if v in ['plev','plev_bnds','height']:
124        self.va[v]['_data'] = self.nc.variables[v].getValue().tolist()
125
126    for v in self.nc.axes.keys():
127      self.da[v] = {}
128      for k in self.nc.axes[v].attributes.keys():
129        self.da[v][k] = self.nc.axes[v].attributes[k]
130      self.da[v]['_type'] = str( self.nc.axes[v].getValue().dtype )
131      self.da[v]['_data'] = self.nc.axes[v].getValue().tolist()
132     
133    self.nc.close()
134
135    self.cga.check( self.ga, self.va, self.cfn.var, self.cfn.freq, vocabs, self.cfn.fnParts )
136    if not self.cga.completed:
137      self.completed = False
138      return
139
140    self.cgd.plevRequired = config.plevRequired
141    self.cgd.plevValues = config.plevValues
142    self.cgd.heightRequired = config.heightRequired
143    self.cgd.heightValues = config.heightValues
144    self.cgd.check( self.cfn.var, self.cfn.freq, self.da, self.va, self.cga.isInstantaneous )
145    self.calendar = self.cgd.calendar
146    if not self.cgd.completed:
147      self.completed = False
148      return
149
150    self.cgg.rotatedPoleGrids = config.rotatedPoleGrids
151    self.cgg.interpolatedGrids = config.interpolatedGrids
152    self.cgg.check( self.cfn.var, self.cfn.domain, self.da, self.va )
153   
154    if not self.cgg.completed:
155      self.completed = False
156      return
157    self.completed = True
158    self.drs = self.cga.getDrs()
159    self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount
160
161class c4_init:
162
163  def __init__(self):
164    self.logByFile = True
165    args = sys.argv[1:]
166    nn = 0
167
168    self.recordFile = 'Rec.txt'
169    self.logDir = 'logs_02'
170    while len(args) > 0:
171      next = args.pop(0)
172      if next == '-f':
173        flist = [args.pop(0),]
174        self.logByFile = False
175      elif next == '-d':
176        import glob
177        fdir = args.pop(0)
178        flist = glob.glob( '%s/*.nc' % fdir  )
179      elif next == '-D':
180        flist  = []
181        fdir = args.pop(0)
182        for root, dirs, files in os.walk( fdir ):
183          for f in files:
184            fpath = '%s/%s' % (root,f)
185            if os.path.isfile( fpath ) and f[-3:] == '.nc':
186              flist.append( fpath )
187      elif next == '-R':
188        self.recordFile = args.pop(0)
189      elif next == '--ld':
190        self.logDir = args.pop(0)
191      else:
192       print 'Unused argument: %s' % next
193       nn+=1
194    assert nn==0, 'Aborting because of unused arguments'
195
196    flist.sort()
197    fnl = []
198    nd = 0
199    for f in flist:
200      fn = string.split(f, '/')[-1]
201      if fn in fnl:
202        print 'ERROR: file name duplicated %s' % fn
203        nd += 0
204      else:
205        fnl.append(fn)
206    assert nd == 0, 'Duplicate file names encountered'
207    self.flist = flist
208    self.fnl = fnl
209    if not os.path.isdir(   self.logDir ):
210       os.mkdir(   self.logDir )
211
212    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
213    batchLogFile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
214## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
215    self.logger = logging.getLogger('c4logger')
216    self.hdlr = logging.FileHandler(batchLogFile,mode='w')
217    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
218    self.hdlr.setFormatter(formatter)
219    self.logger.setLevel(logging.INFO)
220    self.logger.addHandler(self.hdlr)
221
222
223  def getFileLog( self, fn ):
224    tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
225    self.fileLogFile = '%s/%s__qclog_%s.txt' % (self.logDir,fn[:-3],tstring2)
226    fLogger = logging.getLogger('fileLog_%s' % fn)
227    self.fHdlr = logging.FileHandler(self.fileLogFile,mode='w')
228    fileFormatter = logging.Formatter('%(message)s')
229    self.fHdlr.setFormatter(fileFormatter)
230    fLogger.addHandler(self.fHdlr)
231    fLogger.setLevel(logging.INFO)
232    return fLogger
233
234  def closeFileLog(self):
235    self.fHdlr.close()
236
237cc = checker()
238
239cal = None
240
241if __name__ == '__main__':
242  import sys
243  c4i = c4_init()
244  rec = recorder( c4i.recordFile )
245
246  c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) )
247
248  cbv = utils.checkByVar( parent=cc.info)
249  cbv.impt( c4i.flist )
250
251  for f in c4i.flist:
252    fn = string.split(f,'/')[-1]
253    c4i.logger.info( 'Starting: %s' % fn )
254    try:
255### need to have a unique name, otherwise get mixing of logs despite close statement below.
256      if c4i.logByFile:
257        fLogger = c4i.getFileLog( fn )
258        c4i.logger.info( 'Log file: %s' % c4i.fileLogFile )
259      else:
260        fLogger = c4i.logger
261
262      fLogger.info( 'Starting file %s' % fn )
263## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
264      cc.checkFile( f, log=fLogger )
265      if c4i.logByFile:
266        c4i.closeFileLog( )
267
268      if cc.completed:
269        if cal != None:
270          if cal != cc.calendar:
271            c4i.logger.info( 'Error: change in calendar attribute %s --> %s' % (cal, cc.calendar) )
272            cc.errorCount += 1
273        cal = cc.calendar
274        c4i.logger.info( 'Done -- error count %s' % cc.errorCount ) 
275        if cc.errorCount == 0:
276          rec.add( f, cc.drs )
277        else:
278          rec.addErr( f, 'ERRORS FOUND | errorCount = %s' % cc.errorCount )
279      else:
280        c4i.logger.info( 'Done -- testing aborted because of severity of errors' )
281        rec.addErr( f, 'ERRORS FOUND AND CHECKS ABORTED' )
282    except:
283      c4i.logger.error("Exception has occured" ,exc_info=1)
284      rec.addErr( f, 'ERROR: Exception' )
285
286  cc.info.log = c4i.logger
287  cbv.check( recorder=rec, calendar=cc.calendar)
288  rec.dumpAll()
289  c4i.hdlr.close()
290else:
291  f1 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/day/clh/clh_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_day_19810101-19851231.nc'
292  f2 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/sem/tas/tas_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_sem_200012-201011.nc'
293  f3 = '/data/u10/cordex/AFR-44i/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/mon/tas/tas_AFR-44i_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_mon_199101-200012.nc'
294  cc.checkFile( f3 )
Note: See TracBrowser for help on using the repository browser.