source: CCCC/trunk/ceda_cc/c4.py @ 291

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/c4.py@291
Revision 291, 23.6 KB checked in by mjuckes, 5 years ago (diff)

bringing fix to setup back from 1.3.1

Line 
1
2import sys
3
4testmain=False
5## callout to summary.py: if this option is selected, imports of libraries are not needed.
6if not testmain:
7  if __name__ == '__main__' and sys.argv[1] == '--sum':
8      import summary
9      summary.main()
10      raise SystemExit(0)
11  elif __name__ == '__main__' and sys.argv[1] == '-v':
12      from versionConfig import version, versionComment
13      print 'ceda-cc version %s [%s]' % (version,versionComment)
14      raise SystemExit(0)
15  elif __name__ == '__main__' and sys.argv[1] == '--unitTest':
16      print "Starting test suite 1"
17      import unitTestsS1
18      print "Starting test suite 2"
19      import unitTestsS2
20      print "Tests completed"
21      raise SystemExit(0)
22
23# Standard library imports
24import os, string, time, logging, sys, glob, pkgutil
25import shutil
26## pkgutil is used in file_utils
27# Third party imports
28
29## Local imports with 3rd party dependencies
30#### netcdf --- currently only support for cmds2 -- re-arranged to facilitate support for alternative modules
31
32import file_utils
33
34from file_utils import fileMetadata, ncLib
35
36# Local imports
37import utils_c4 as utils
38import config_c4 as config
39
40reload( utils )
41
42from xceptions import baseException
43
44from fcc_utils2 import tupsort
45
46
47#driving_model_ensemble_member = <CMIP5Ensemble_member>
48#rcm_version_id = <RCMVersionID>                     
49
50class dummy(object):
51  def __init__(self):
52     self.experimental = None
53     self.parent = None
54
55pathTmplDict = { 'CORDEX':'%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/',   \
56                 'SPECS':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(start_date)s/%(frequency)s/%(realm)s/%(table)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \
57                 'CMIP5':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \
58                 'CCMI':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \
59                 'ESA-CCI':'%(level)s/%(platform)s/%(sensor)s/%(variable)s/', \
60                 '__def__':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \
61               }
62
63## Core DRS: list of vocab names
64## Path template: -- current version puts upper case in "project"
65## Dataset template: 
66
67class recorder(object):
68
69  def __init__(self,project,fileName,type='map',dummy=False):
70    self.dummy = dummy
71    self.file = fileName
72    self.type = type
73    self.pathTmpl = '%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/'
74    self.pathTmpl = pathTmplDict.get(project,pathTmplDict['__def__'])
75    self.records = {}
76    self.tidtupl = []
77
78  def open(self):
79    if self.type == 'map':
80      self.fh = open( self.file, 'a' )
81    else:
82      self.sh = shelve.open( self.file )
83
84  def close(self):
85    if self.type == 'map':
86      self.fh.close()
87    else:
88      self.sh.close()
89
90  def add(self,fpath,drs,safe=True):
91    assert self.type == 'map','Can only do map files at present'
92    assert type(drs) == type( {} ), '2nd user argument to method add should be a dictionary [%s]' % type(drs)
93    tpath = self.pathTmpl % drs
94    if not self.dummy:
95      assert os.path.isfile( fpath ), 'File %s not found' % fpath
96      fdate = time.ctime(os.path.getmtime(fpath))
97      sz = os.stat(fpath).st_size
98    else:
99      fdate = "na"
100      sz = 0
101    record = '%s | OK | %s | modTime = %s | target = %s ' % (fpath,sz,fdate,tpath)
102    fn = string.split( fpath, '/' )[-1]
103    for k in ['creation_date','tracking_id']:
104      if k in drs.keys():
105        record += ' | %s = %s' % (k,drs[k])
106        if k == 'tracking_id':
107          self.tidtupl.append( (fn,drs[k]) )
108
109    self.records[fn] = record
110 
111  def modify(self,fn,msg):
112    assert fn in self.records.keys(),'Attempt to modify non-existent record %s, %s' % [fn,str(self.records.keys()[0:10])]
113    if string.find( self.records[fn], '| OK |') == -1:
114      ##print 'File %s already flagged with errors' % fn
115      return
116    s = string.replace( self.records[fn], '| OK |', '| %s |' % msg )
117    ##print '--> ',s
118    self.records[fn] = s
119
120  def checktids(self):
121## sort by tracking id
122    if len( self.tidtupl ) == 1:
123      return
124    self.tidtupl.sort( cmp=tupsort(k=1).cmp )
125    nd = 0
126    fnl = []
127    for k in range(len(self.tidtupl)-1):
128      if self.tidtupl[k][1] == self.tidtupl[k+1][1]:
129        print 'Duplicate tracking_id: %s, %s:: %s' % (self.tidtupl[k][0],self.tidtupl[k+1][0],self.tidtupl[k][1])
130        nd += 1
131        if len(fnl) == 0 or fnl[-1] != self.tidtupl[k][0]:
132          fnl.append( self.tidtupl[k][0])
133        fnl.append( self.tidtupl[k+1][0])
134    if nd == 0:
135      print 'No duplicate tracking ids found in %s files' % len(self.tidtupl)
136    else:
137      print '%s duplicate tracking ids' % nd
138      for f in fnl:
139        self.modify( f, 'ERROR: duplicate tid' )
140
141  def dumpAll(self,safe=True):
142    keys = self.records.keys()
143    keys.sort()
144    for k in keys:
145      self.dump( self.records[k], safe=safe )
146
147  def dump( self, record, safe=True ):
148    if safe:
149      self.open()
150    self.fh.write( record + '\n' )
151    if safe:
152      self.close()
153
154  def addErr(self,fpath,reason,safe=True):
155    record = '%s | %s' % (fpath, reason)
156    fn = string.split( fpath, '/' )[-1]
157    self.records[fn] = record
158
159class checker(object):
160  def __init__(self, pcfg, cls,reader,abortMessageCount=-1,experimental=False):
161    self.info = dummy()
162    self.info.pcfg = pcfg
163    self.info.fileIsFixed = None
164    self.info.abortMessageCount = abortMessageCount
165    self.info.experimental = experimental
166    self.calendar = 'None'
167    self.ncReader = reader
168    self.cfn = utils.checkFileName( parent=self.info,cls=cls)
169    self.cga = utils.checkGlobalAttributes( parent=self.info,cls=cls)
170    self.cgd = utils.checkStandardDims( parent=self.info,cls=cls)
171    self.cgg = utils.checkGrids( parent=self.info,cls=cls)
172    self.cls = cls
173
174    # Define vocabs based on project
175    ##self.vocabs = getVocabs(pcgf)
176    self.vocabs = pcfg.vocabs
177
178  def checkFile(self,fpath,log=None,attributeMappings=[]):
179    self.calendar = 'None'
180    self.info.log = log
181
182    fn = string.split( fpath, '/' )[-1]
183
184    if attributeMappings != []:
185      self.ncReader.loadNc( fpath )
186      self.ncReader.applyMap( attributeMappings, self.cfn.globalAttributesInFn, log=log )
187      ncRed = True
188      thisFn = self.ncReader.fn
189    else:
190      ncRed = False
191      thisFn = fn
192
193    self.cfn.check( thisFn )
194    if not self.cfn.completed:
195      self.completed = False
196      return
197    if not self.info.pcfg.projectV.id[:2] == '__':
198      if not os.path.isfile( fpath ):
199        print 'File %s not found [2]' % fpath
200        self.completed = False
201        return
202
203    if not ncRed:
204      ##print fpath
205      self.ncReader.loadNc( fpath )
206    self.ga = self.ncReader.ga
207    self.va = self.ncReader.va
208    self.da = self.ncReader.da
209
210    if self.cfn.freq != None:
211      vGroup = self.cfn.freq
212    else:
213      vGroup = self.info.pcfg.mipVocabVgmap.get(self.cfn.group,self.cfn.group)
214    self.cga.check( self.ga, self.va, self.cfn.var, vGroup, self.vocabs, self.cfn.fnParts )
215    if not self.cga.completed:
216      self.completed = False
217      return
218
219    ##self.cgd.plevRequired = config.plevRequired
220    ##self.cgd.plevValues = config.plevValues
221    ##self.cgd.heightRequired = config.heightRequired
222    ##self.cgd.heightValues = config.heightValues
223    ##self.cgd.heightRange = config.heightRange
224    self.cgd.check( self.cfn.var, self.cfn.freq, self.da, self.va, self.cga.isInstantaneous, self.vocabs )
225    self.calendar = self.cgd.calendar
226    if not self.cgd.completed:
227      self.completed = False
228      return
229
230    if self.info.pcfg.doCheckGrids:
231      ##self.cgg.rotatedPoleGrids = config.rotatedPoleGrids
232      ##self.cgg.interpolatedGrids = config.interpolatedGrids
233      self.cgg.check( self.cfn.var, self.cfn.domain, self.da, self.va )
234   
235      if not self.cgg.completed:
236        self.completed = False
237        return
238    self.completed = True
239    self.drs = self.cga.getDrs()
240    self.drs['project'] = self.info.pcfg.projectV.id
241    self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount
242
243class c4_init(object):
244
245  def __init__(self,args=None):
246    self.logByFile = True
247    self.policyFileLogfileMode = 'w'
248    self.policyBatchLogfileMode = 'np'
249    if args==None:
250       args = sys.argv[1:]
251    nn = 0
252
253    self.attributeMappingFile = None
254    self.recordFile = 'Rec.txt'
255    self.logDir = 'logs_02'
256    self.errs = []
257   
258    # Set default project to "CORDEX"
259    self.project = "CORDEX"
260    self.holdExceptions = False
261    forceLogOrg = None
262    argsIn = args[:]
263
264    # The --copy-config option must be the first argument if it is present.
265    if args[0] == '--copy-config':
266       if len(args) < 2:
267         self.commandHints( argsIn )
268       args.pop(0)
269       dest_dir = args.pop(0)
270       config.copy_config(dest_dir)
271       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
272       print
273       raise SystemExit(0)
274    elif args[0] == '-h':
275       print 'Help command not implemented yet'
276       raise SystemExit(0)
277
278    self.summarymode = args[0] == '--sum'
279    if self.summarymode:
280      return
281
282    self.experimental = False
283    self.forceNetcdfLib = None
284    fltype = None
285    argu = []
286    while len(args) > 0:
287      next = args.pop(0)
288      if next == '-f':
289        flist = [args.pop(0),]
290        self.logByFile = False
291        fltype = '-f'
292        self.source = flist[0]
293      elif next == '--log':
294        x = args.pop(0)
295        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
296        if x in ['multi','m']:
297           forceLogOrg = 'multi'
298        elif x in ['single','s']:
299           forceLogOrg = 'single'
300      elif next == '--experimental':
301        self.experimental = True
302      elif next == '--force-ncq':
303        self.forceNetcdfLib = 'ncq3'
304      elif next == '--force-cdms2':
305        self.forceNetcdfLib = 'cdms2'
306      elif next == '--force-pync4':
307        self.forceNetcdfLib = 'netCDF4'
308      elif next == '--force-scientific':
309        self.forceNetcdfLib = 'Scientific'
310      elif next == '--flfmode':
311        lfmk = args.pop(0)
312        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
313        self.policyFileLogfileMode = lfmk
314      elif next == '--blfmode':
315        lfmk = args.pop(0)
316        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
317        self.policyBatchLogfileMode = lfmk
318      elif next == '-d':
319        fdir = args.pop(0)
320        flist = glob.glob( '%s/*.nc' % fdir  )
321        self.source = '%s/*.nc' % fdir
322      elif next == '-D':
323        flist  = []
324        fdir = args.pop(0)
325        for root, dirs, files in os.walk( fdir, followlinks=True ):
326          for f in files:
327            fpath = '%s/%s' % (root,f)
328            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
329              flist.append( fpath )
330        self.source = '%s/.....' % fdir
331      elif next == '-R':
332        self.recordFile = args.pop(0)
333      elif next == '--ld':
334        self.logDir = args.pop(0)
335      elif next in ['--catchAllExceptions','--cae']:
336        self.holdExceptions = True
337      elif next == '--aMap':
338        self.attributeMappingFile = args.pop(0)
339        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
340      elif next == "-p":
341        self.project = args.pop(0)
342      else:
343       print 'Unused argument: %s' % next
344       argu.append( next )
345       nn+=1
346    if nn != 0:
347      print 'Unused arguments: ', argu
348      self.commandHints( argsIn )
349
350    if self.project == 'CMIP5' and fltype != '-f':
351      fl0 = []
352      for f in flist:
353        if string.find( f, '/latest/' ) != -1:
354          fl0.append(f)
355      flist = fl0
356
357    if forceLogOrg != None:
358      if forceLogOrg == 'single':
359        self.logByFile = False
360      else:
361        self.logByFile = True
362
363    if self.project[:2] == '__':
364       self.source = 'dummy'
365       flist = []
366       ss = 'abcdefgijk'
367       ss = 'abcdefgijklmnopqrstuvwxyz'
368       ss = 'abc'
369       for i in range(10):
370         v = 'v%s' % i
371         for a in ss:
372           for b in ss:
373             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
374    flist.sort()
375    fnl = []
376    for f in flist:
377      fn = string.split(f, '/')[-1]
378      fnl.append(fn)
379    nd = 0
380    dupl = []
381    for k in range(len(fnl)-1):
382      if fnl[k] == fnl[k-1]:
383        nd += 1
384        dupl.append( fnl[k] )
385    self.dupDict = {}
386    for f in dupl:
387      self.dupDict[f] = 0
388    if nd != 0:
389      self.errs.append( 'Duplicate file names encountered: %s' % nd )
390      self.errs.append( dupl )
391    self.flist = flist
392    self.fnl = fnl
393    if not os.path.isdir(   self.logDir ):
394       os.mkdir(   self.logDir )
395
396    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
397    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
398## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
399    self.logger = logging.getLogger('c4logger')
400    if self.policyBatchLogfileMode in ['n','np']:
401        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
402    m = self.policyBatchLogfileMode[0]
403    if m == 'n':
404      m = 'w'
405    if m == 'a':
406      self.hdlr = logging.FileHandler(self.batchLogfile)
407    else:
408      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
409    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
410    self.hdlr.setFormatter(formatter)
411    self.logger.setLevel(logging.INFO)
412    self.logger.addHandler(self.hdlr)
413
414    self.attributeMappings = []
415    self.attributeMappingsLog = None
416    if self.attributeMappingFile != None:
417      for l in open( self.attributeMappingFile ).readlines():
418        if l[0] != '#':
419          bb = string.split( string.strip(l), '|' ) 
420          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
421          bits = string.split( bb[0], ';' )
422          cl = []
423          for b in bits:
424            cl.append( string.split(b, '=' ) )
425          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
426      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
427
428  def commandHints(self, args):
429    if args[0] in ['-h','--sum']:
430      print 'Arguments look OK'
431    elif args[0] == '--copy-config':
432      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
433    else:
434      if not( '-f' in args or '-d' in args or '-D' in args):
435        print 'No file or target directory specified'
436        print """USAGE:
437ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]
438
439With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked.
440"""
441    raise SystemExit(0)
442   
443
444  def getFileLog( self, fn, flf=None ):
445    if flf == None:
446      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
447      if fn in self.dupDict.keys():
448        tag = '__%2.2i' % self.dupDict[fn]
449        self.dupDict[fn] += 1
450      else:
451        tag = ''
452      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
453      if self.policyFileLogfileMode in ['n','np']:
454        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
455      m = self.policyFileLogfileMode[0]
456      if m == 'n':
457        m = 'w'
458    else:
459      m = 'a'
460      self.fileLogfile = flf
461
462    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
463    if m == 'a':
464      self.fHdlr = logging.FileHandler(self.fileLogfile)
465    else:
466      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
467    fileFormatter = logging.Formatter('%(message)s')
468    self.fHdlr.setFormatter(fileFormatter)
469    self.fLogger.addHandler(self.fHdlr)
470    self.fLogger.setLevel(logging.INFO)
471    return self.fLogger
472
473  def closeFileLog(self):
474    self.fLogger.removeHandler(self.fHdlr)
475    self.fHdlr.close()
476    if self.policyFileLogfileMode in ['wo','np']:
477      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
478
479  def closeBatchLog(self):
480    self.logger.removeHandler(self.hdlr)
481    self.hdlr.close()
482    if self.policyBatchLogfileMode in ['wo','np']:
483      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
484
485
486class main(object):
487
488  def __init__(self,args=None,abortMessageCount=-1,printInfo=False,monitorFileHandles = False):
489    logDict = {}
490    ecount = 0
491    c4i = c4_init(args=args)
492    c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) )
493    c4i.logger.info( 'Source: %s' % c4i.source )
494     
495    isDummy  = c4i.project[:2] == '__'
496    if (ncLib == None) and (not isDummy):
497       raise baseException( 'Cannot proceed with non-dummy [%s] project without a netcdf API' % (c4i.project) )
498    pcfg = config.projectConfig( c4i.project )
499    assert pcfg.projectV.v == -1, 'Cannot handle anything other than latest version at present'
500    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog,forceLib=c4i.forceNetcdfLib)
501    c4i.logger.info( 'Python netcdf: %s' % ncReader.ncLib )
502    self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount, experimental=c4i.experimental)
503    rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy )
504    self.ncLib = ncLib
505
506    # This list will record the drs dictionaries of all checked files for export to JSON
507    drs_list = []
508
509    if monitorFileHandles:
510      self.monitor = utils.sysMonitor()
511    else:
512      self.monitor = None
513
514    cal = None
515    if len( c4i.errs ) > 0:
516      for i in range(0,len( c4i.errs ), 2 ):
517        c4i.logger.info( c4i.errs[i] )
518 
519    self.cc.info.amapListDraft = []
520    cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor)
521    if c4i.project not in ['ESA-CCI']:
522      cbv.impt( c4i.flist )
523      if printInfo:
524        print cbv.info
525
526    fileLogOpen = False
527    self.resList =  []
528    stdoutsum = 2000
529    npass = 0
530    kf = 0
531    for f in c4i.flist:
532      kf += 1
533      rv = False
534      ec = None
535      if monitorFileHandles:
536        nofhStart = self.monitor.get_open_fds()
537      fn = string.split(f,'/')[-1]
538      c4i.logger.info( 'Starting: %s' % fn )
539      try:
540  ### need to have a unique name, otherwise get mixing of logs despite close statement below.
541  ### if duplicate file names are present, this will be recorded in the main log, tag appended to file level log name (not yet tested).
542        if c4i.logByFile:
543          fLogger = c4i.getFileLog( fn )
544          logDict[fn] = c4i.fileLogfile
545          c4i.logger.info( 'Log file: %s' % c4i.fileLogfile )
546          fileLogOpen = True
547        else:
548          fLogger = c4i.logger
549 
550        fLogger.info( 'Starting file %s' % fn )
551## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
552        self.cc.checkFile( f, log=fLogger,attributeMappings=c4i.attributeMappings )
553
554        if self.cc.completed:
555          if cal not in (None, 'None') and self.cc.cgd.varGroup != "fx":
556            if cal != self.cc.calendar:
557              cal_change_err_msg = 'Error: change in calendar attribute %s --> %s' % (cal, self.cc.calendar)
558              c4i.logger.info(cal_change_err_msg)
559              fLogger.info(cal_change_err_msg)
560              self.cc.errorCount += 1
561
562          cal = self.cc.calendar
563          ec = self.cc.errorCount
564        rv =  ec == 0
565        if rv:
566          npass += 1
567        self.resList.append( (rv,ec) )
568
569        if c4i.logByFile:
570          if self.cc.completed:
571            fLogger.info( 'Done -- error count %s' % self.cc.errorCount )
572          else:
573            fLogger.info( 'Done -- checks not completed' )
574          c4i.closeFileLog( )
575          fileLogOpen = False
576
577        if self.cc.completed:
578          c4i.logger.info( 'Done -- error count %s' % self.cc.errorCount ) 
579          ecount += self.cc.errorCount
580          if self.cc.errorCount == 0:
581            rec.add( f, self.cc.drs )
582            drs_list.append({'path': f, 'drs': self.cc.drs})
583          else:
584            rec.addErr( f, 'ERRORS FOUND | errorCount = %s' % self.cc.errorCount )
585        else:
586          ecount += 20
587          c4i.logger.info( 'Done -- testing aborted because of severity of errors' )
588          rec.addErr( f, 'ERRORS FOUND AND CHECKS ABORTED' )
589      except:
590        c4i.logger.error("Exception has occured" ,exc_info=1)
591        if fileLogOpen:
592          fLogger.error("C4.100.001: [exception]: FAILED:: Exception has occured" ,exc_info=1)
593          c4i.closeFileLog( )
594          fileLogOpen = False
595        rec.addErr( f, 'ERROR: Exception' )
596        if not c4i.holdExceptions:
597          raise
598      if stdoutsum > 0 and kf%stdoutsum == 0:
599         print '%s files checked; %s passed this round' % (kf,npass)
600      if monitorFileHandles:
601        nofhEnd = self.monitor.get_open_fds()
602        if nofhEnd > nofhStart:
603           print 'Open file handles: %s --- %s' % (nofhStart, nofhEnd)
604 
605    self.cc.info.log = c4i.logger
606   
607    if c4i.project not in ['SPECS','CCMI','CMIP5','ESA-CCI']:
608       cbv.c4i = c4i
609       cbv.setLogDict( logDict )
610       cbv.check( recorder=rec, calendar=self.cc.calendar)
611       try:
612         ecount += cbv.errorCount
613       except:
614         ecount = None
615    ncReader.close()
616    if type( self.cc.info.amapListDraft ) == type( [] ) and len(  self.cc.info.amapListDraft ) > 0:
617      ll =  self.cc.info.amapListDraft
618      ll.sort()
619      oo = open( 'amapDraft.txt', 'w' )
620      oo.write( ll[0] + '\n' )
621      for i in range( 1,len(ll) ):
622        if ll[i] != ll[i-1]:
623          oo.write( ll[i] + '\n' )
624      oo.close()
625    if c4i.project in ['SPECS','CCMI','CMIP5']:
626      rec.checktids()
627    rec.dumpAll()
628
629    #!TODO: the recorder class could export JSON if it recorded the full drs dictionaries.
630    #       This lightweight solution re-uses the filename from the rec class and dumps
631    #       JSON in a separate function.
632    json_file = os.path.splitext(rec.file)[0] + '.json'
633    dump_drs_list(drs_list, json_file)
634
635    if printInfo:
636      print 'Error count %s' % ecount
637    ##c4i.hdlr.close()
638    c4i.closeBatchLog()
639    self.ok = all( map( lambda x: x[0], self.resList ) )
640
641
642def dump_drs_list(drs_list, filename):
643    import json
644    fh = open(filename, 'a+')
645    for drs in drs_list:
646                fh.write(json.dumps(drs))
647                fh.write('\n')
648    fh.close()
649
650
651def main_entry():
652  """
653   Wrapper around main() for use with setuptools.
654
655  """
656  if sys.argv[1] == '--sum':
657      import summary
658      summary.main()
659  elif sys.argv[1] == '-v':
660      from versionConfig import version, versionComment
661      print 'ceda-cc version %s [%s]' % (version,versionComment)
662  elif sys.argv[1] == '--unitTest':
663      print "Starting test suite 1"
664      import unitTestsS1
665      print "Starting test suite 2"
666      import unitTestsS2
667      print "Tests completed"
668  else:
669     main(printInfo=True)
670
671if __name__ == '__main__':
672  main_entry()
673
674
675##else:
676  ##f1 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/day/clh/clh_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_day_19810101-19851231.nc'
677  ##f2 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/sem/tas/tas_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_sem_200012-201011.nc'
678  ##f3 = '/data/u10/cordex/AFR-44i/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/mon/tas/tas_AFR-44i_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_mon_199101-200012.nc'
679  ##cc.checkFile( f3 )
Note: See TracBrowser for help on using the repository browser.