source: CCCC/trunk/ceda_cc/c4.py @ 297

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/c4.py@297
Revision 297, 23.7 KB checked in by astephen, 5 years ago (diff)

Modified to test if no command-line args provided or if "-h", "-help" or "--help" are first arg. This will redirect all cases to showing help.

Line 
1
2import sys
3
4testmain=False
5## callout to summary.py: if this option is selected, imports of libraries are not needed.
6if not testmain:
7  if __name__ == '__main__' and sys.argv[1] == '--sum':
8      import summary
9      summary.main()
10      raise SystemExit(0)
11  elif __name__ == '__main__' and sys.argv[1] == '-v':
12      from versionConfig import version, versionComment
13      print 'ceda-cc version %s [%s]' % (version,versionComment)
14      raise SystemExit(0)
15  elif __name__ == '__main__' and sys.argv[1] == '--unitTest':
16      print "Starting test suite 1"
17      import unitTestsS1
18      print "Starting test suite 2"
19      import unitTestsS2
20      print "Tests completed"
21      raise SystemExit(0)
22
23# Standard library imports
24import os, string, time, logging, sys, glob, pkgutil
25import shutil
26## pkgutil is used in file_utils
27# Third party imports
28
29## Local imports with 3rd party dependencies
30#### netcdf --- currently only support for cmds2 -- re-arranged to facilitate support for alternative modules
31
32import file_utils
33
34from file_utils import fileMetadata, ncLib
35
36# Local imports
37import utils_c4 as utils
38import config_c4 as config
39
40reload( utils )
41
42from xceptions import baseException
43
44from fcc_utils2 import tupsort
45
46
47#driving_model_ensemble_member = <CMIP5Ensemble_member>
48#rcm_version_id = <RCMVersionID>                     
49
50class dummy(object):
51  def __init__(self):
52     self.experimental = None
53     self.parent = None
54
55pathTmplDict = { 'CORDEX':'%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/',   \
56                 'SPECS':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(start_date)s/%(frequency)s/%(realm)s/%(table)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \
57                 'CMIP5':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \
58                 'CCMI':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \
59                 'ESA-CCI':'%(level)s/%(platform)s/%(sensor)s/%(variable)s/', \
60                 '__def__':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \
61               }
62
63## Core DRS: list of vocab names
64## Path template: -- current version puts upper case in "project"
65## Dataset template: 
66
67class recorder(object):
68
69  def __init__(self,project,fileName,type='map',dummy=False):
70    self.dummy = dummy
71    self.file = fileName
72    self.type = type
73    self.pathTmpl = '%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/'
74    self.pathTmpl = pathTmplDict.get(project,pathTmplDict['__def__'])
75    self.records = {}
76    self.tidtupl = []
77
78  def open(self):
79    if self.type == 'map':
80      self.fh = open( self.file, 'a' )
81    else:
82      self.sh = shelve.open( self.file )
83
84  def close(self):
85    if self.type == 'map':
86      self.fh.close()
87    else:
88      self.sh.close()
89
90  def add(self,fpath,drs,safe=True):
91    assert self.type == 'map','Can only do map files at present'
92    assert type(drs) == type( {} ), '2nd user argument to method add should be a dictionary [%s]' % type(drs)
93    tpath = self.pathTmpl % drs
94    if not self.dummy:
95      assert os.path.isfile( fpath ), 'File %s not found' % fpath
96      fdate = time.ctime(os.path.getmtime(fpath))
97      sz = os.stat(fpath).st_size
98    else:
99      fdate = "na"
100      sz = 0
101    record = '%s | OK | %s | modTime = %s | target = %s ' % (fpath,sz,fdate,tpath)
102    fn = string.split( fpath, '/' )[-1]
103    for k in ['creation_date','tracking_id']:
104      if k in drs.keys():
105        record += ' | %s = %s' % (k,drs[k])
106        if k == 'tracking_id':
107          self.tidtupl.append( (fn,drs[k]) )
108
109    self.records[fn] = record
110 
111  def modify(self,fn,msg):
112    assert fn in self.records.keys(),'Attempt to modify non-existent record %s, %s' % [fn,str(self.records.keys()[0:10])]
113    if string.find( self.records[fn], '| OK |') == -1:
114      ##print 'File %s already flagged with errors' % fn
115      return
116    s = string.replace( self.records[fn], '| OK |', '| %s |' % msg )
117    ##print '--> ',s
118    self.records[fn] = s
119
120  def checktids(self):
121## sort by tracking id
122    if len( self.tidtupl ) == 1:
123      return
124    self.tidtupl.sort( cmp=tupsort(k=1).cmp )
125    nd = 0
126    fnl = []
127    for k in range(len(self.tidtupl)-1):
128      if self.tidtupl[k][1] == self.tidtupl[k+1][1]:
129        print 'Duplicate tracking_id: %s, %s:: %s' % (self.tidtupl[k][0],self.tidtupl[k+1][0],self.tidtupl[k][1])
130        nd += 1
131        if len(fnl) == 0 or fnl[-1] != self.tidtupl[k][0]:
132          fnl.append( self.tidtupl[k][0])
133        fnl.append( self.tidtupl[k+1][0])
134    if nd == 0:
135      print 'No duplicate tracking ids found in %s files' % len(self.tidtupl)
136    else:
137      print '%s duplicate tracking ids' % nd
138      for f in fnl:
139        self.modify( f, 'ERROR: duplicate tid' )
140
141  def dumpAll(self,safe=True):
142    keys = self.records.keys()
143    keys.sort()
144    for k in keys:
145      self.dump( self.records[k], safe=safe )
146
147  def dump( self, record, safe=True ):
148    if safe:
149      self.open()
150    self.fh.write( record + '\n' )
151    if safe:
152      self.close()
153
154  def addErr(self,fpath,reason,safe=True):
155    record = '%s | %s' % (fpath, reason)
156    fn = string.split( fpath, '/' )[-1]
157    self.records[fn] = record
158
159class checker(object):
160  def __init__(self, pcfg, cls,reader,abortMessageCount=-1,experimental=False):
161    self.info = dummy()
162    self.info.pcfg = pcfg
163    self.info.fileIsFixed = None
164    self.info.abortMessageCount = abortMessageCount
165    self.info.experimental = experimental
166    self.calendar = 'None'
167    self.ncReader = reader
168    self.cfn = utils.checkFileName( parent=self.info,cls=cls)
169    self.cga = utils.checkGlobalAttributes( parent=self.info,cls=cls)
170    self.cgd = utils.checkStandardDims( parent=self.info,cls=cls)
171    self.cgg = utils.checkGrids( parent=self.info,cls=cls)
172    self.cls = cls
173
174    # Define vocabs based on project
175    ##self.vocabs = getVocabs(pcgf)
176    self.vocabs = pcfg.vocabs
177
178  def checkFile(self,fpath,log=None,attributeMappings=[]):
179    self.calendar = 'None'
180    self.info.log = log
181
182    fn = string.split( fpath, '/' )[-1]
183
184    if attributeMappings != []:
185      self.ncReader.loadNc( fpath )
186      self.ncReader.applyMap( attributeMappings, self.cfn.globalAttributesInFn, log=log )
187      ncRed = True
188      thisFn = self.ncReader.fn
189    else:
190      ncRed = False
191      thisFn = fn
192
193    self.cfn.check( thisFn )
194    if not self.cfn.completed:
195      self.completed = False
196      return
197    if not self.info.pcfg.projectV.id[:2] == '__':
198      if not os.path.isfile( fpath ):
199        print 'File %s not found [2]' % fpath
200        self.completed = False
201        return
202
203    if not ncRed:
204      ##print fpath
205      self.ncReader.loadNc( fpath )
206    self.ga = self.ncReader.ga
207    self.va = self.ncReader.va
208    self.da = self.ncReader.da
209
210    if self.cfn.freq != None:
211      vGroup = self.cfn.freq
212    else:
213      vGroup = self.info.pcfg.mipVocabVgmap.get(self.cfn.group,self.cfn.group)
214    self.cga.check( self.ga, self.va, self.cfn.var, vGroup, self.vocabs, self.cfn.fnParts )
215    if not self.cga.completed:
216      self.completed = False
217      return
218
219    ##self.cgd.plevRequired = config.plevRequired
220    ##self.cgd.plevValues = config.plevValues
221    ##self.cgd.heightRequired = config.heightRequired
222    ##self.cgd.heightValues = config.heightValues
223    ##self.cgd.heightRange = config.heightRange
224    self.cgd.check( self.cfn.var, self.cfn.freq, self.da, self.va, self.cga.isInstantaneous, self.vocabs )
225    self.calendar = self.cgd.calendar
226    if not self.cgd.completed:
227      self.completed = False
228      return
229
230    if self.info.pcfg.doCheckGrids:
231      ##self.cgg.rotatedPoleGrids = config.rotatedPoleGrids
232      ##self.cgg.interpolatedGrids = config.interpolatedGrids
233      self.cgg.check( self.cfn.var, self.cfn.domain, self.da, self.va )
234   
235      if not self.cgg.completed:
236        self.completed = False
237        return
238    self.completed = True
239    self.drs = self.cga.getDrs()
240    self.drs['project'] = self.info.pcfg.projectV.id
241    self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount
242
243class c4_init(object):
244
245  def __init__(self,args=None):
246    self.logByFile = True
247    self.policyFileLogfileMode = 'w'
248    self.policyBatchLogfileMode = 'np'
249    if args==None:
250       args = sys.argv[1:]
251    nn = 0
252
253    self.attributeMappingFile = None
254    self.recordFile = 'Rec.txt'
255    self.logDir = 'logs_02'
256    self.errs = []
257   
258    # Set default project to "CORDEX"
259    self.project = "CORDEX"
260    self.holdExceptions = False
261    forceLogOrg = None
262    argsIn = args[:]
263   
264    # Show help if no args or help requested
265    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
266       print 'Help command not implemented yet'
267       raise SystemExit(0)     
268    # The --copy-config option must be the first argument if it is present.
269    elif args[0] == '--copy-config':
270       if len(args) < 2:
271         self.commandHints( argsIn )
272       args.pop(0)
273       dest_dir = args.pop(0)
274       config.copy_config(dest_dir)
275       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
276       print
277       raise SystemExit(0)
278
279    self.summarymode = args[0] == '--sum'
280    if self.summarymode:
281      return
282
283    self.experimental = False
284    self.forceNetcdfLib = None
285    fltype = None
286    argu = []
287    while len(args) > 0:
288      next = args.pop(0)
289      if next == '-f':
290        flist = [args.pop(0),]
291        self.logByFile = False
292        fltype = '-f'
293        self.source = flist[0]
294      elif next == '--log':
295        x = args.pop(0)
296        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
297        if x in ['multi','m']:
298           forceLogOrg = 'multi'
299        elif x in ['single','s']:
300           forceLogOrg = 'single'
301      elif next == '--experimental':
302        self.experimental = True
303      elif next == '--force-ncq':
304        self.forceNetcdfLib = 'ncq3'
305      elif next == '--force-cdms2':
306        self.forceNetcdfLib = 'cdms2'
307      elif next == '--force-pync4':
308        self.forceNetcdfLib = 'netCDF4'
309      elif next == '--force-scientific':
310        self.forceNetcdfLib = 'Scientific'
311      elif next == '--flfmode':
312        lfmk = args.pop(0)
313        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
314        self.policyFileLogfileMode = lfmk
315      elif next == '--blfmode':
316        lfmk = args.pop(0)
317        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
318        self.policyBatchLogfileMode = lfmk
319      elif next == '-d':
320        fdir = args.pop(0)
321        flist = glob.glob( '%s/*.nc' % fdir  )
322        self.source = '%s/*.nc' % fdir
323      elif next == '-D':
324        flist  = []
325        fdir = args.pop(0)
326        for root, dirs, files in os.walk( fdir, followlinks=True ):
327          for f in files:
328            fpath = '%s/%s' % (root,f)
329            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
330              flist.append( fpath )
331        self.source = '%s/.....' % fdir
332      elif next == '-R':
333        self.recordFile = args.pop(0)
334      elif next == '--ld':
335        self.logDir = args.pop(0)
336      elif next in ['--catchAllExceptions','--cae']:
337        self.holdExceptions = True
338      elif next == '--aMap':
339        self.attributeMappingFile = args.pop(0)
340        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
341      elif next == "-p":
342        self.project = args.pop(0)
343      else:
344       print 'Unused argument: %s' % next
345       argu.append( next )
346       nn+=1
347    if nn != 0:
348      print 'Unused arguments: ', argu
349      self.commandHints( argsIn )
350
351    if self.project == 'CMIP5' and fltype != '-f':
352      fl0 = []
353      for f in flist:
354        if string.find( f, '/latest/' ) != -1:
355          fl0.append(f)
356      flist = fl0
357
358    if forceLogOrg != None:
359      if forceLogOrg == 'single':
360        self.logByFile = False
361      else:
362        self.logByFile = True
363
364    if self.project[:2] == '__':
365       self.source = 'dummy'
366       flist = []
367       ss = 'abcdefgijk'
368       ss = 'abcdefgijklmnopqrstuvwxyz'
369       ss = 'abc'
370       for i in range(10):
371         v = 'v%s' % i
372         for a in ss:
373           for b in ss:
374             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
375    flist.sort()
376    fnl = []
377    for f in flist:
378      fn = string.split(f, '/')[-1]
379      fnl.append(fn)
380    nd = 0
381    dupl = []
382    for k in range(len(fnl)-1):
383      if fnl[k] == fnl[k-1]:
384        nd += 1
385        dupl.append( fnl[k] )
386    self.dupDict = {}
387    for f in dupl:
388      self.dupDict[f] = 0
389    if nd != 0:
390      self.errs.append( 'Duplicate file names encountered: %s' % nd )
391      self.errs.append( dupl )
392    self.flist = flist
393    self.fnl = fnl
394    if not os.path.isdir(   self.logDir ):
395       os.mkdir(   self.logDir )
396
397    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
398    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
399## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
400    self.logger = logging.getLogger('c4logger')
401    if self.policyBatchLogfileMode in ['n','np']:
402        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
403    m = self.policyBatchLogfileMode[0]
404    if m == 'n':
405      m = 'w'
406    if m == 'a':
407      self.hdlr = logging.FileHandler(self.batchLogfile)
408    else:
409      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
410    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
411    self.hdlr.setFormatter(formatter)
412    self.logger.setLevel(logging.INFO)
413    self.logger.addHandler(self.hdlr)
414
415    self.attributeMappings = []
416    self.attributeMappingsLog = None
417    if self.attributeMappingFile != None:
418      for l in open( self.attributeMappingFile ).readlines():
419        if l[0] != '#':
420          bb = string.split( string.strip(l), '|' ) 
421          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
422          bits = string.split( bb[0], ';' )
423          cl = []
424          for b in bits:
425            cl.append( string.split(b, '=' ) )
426          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
427      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
428
429  def commandHints(self, args):
430    if args[0] in ['-h','--sum']:
431      print 'Arguments look OK'
432    elif args[0] == '--copy-config':
433      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
434    else:
435      if not( '-f' in args or '-d' in args or '-D' in args):
436        print 'No file or target directory specified'
437        print """USAGE:
438ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]
439
440With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked.
441"""
442    raise SystemExit(0)
443   
444
445  def getFileLog( self, fn, flf=None ):
446    if flf == None:
447      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
448      if fn in self.dupDict.keys():
449        tag = '__%2.2i' % self.dupDict[fn]
450        self.dupDict[fn] += 1
451      else:
452        tag = ''
453      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
454      if self.policyFileLogfileMode in ['n','np']:
455        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
456      m = self.policyFileLogfileMode[0]
457      if m == 'n':
458        m = 'w'
459    else:
460      m = 'a'
461      self.fileLogfile = flf
462
463    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
464    if m == 'a':
465      self.fHdlr = logging.FileHandler(self.fileLogfile)
466    else:
467      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
468    fileFormatter = logging.Formatter('%(message)s')
469    self.fHdlr.setFormatter(fileFormatter)
470    self.fLogger.addHandler(self.fHdlr)
471    self.fLogger.setLevel(logging.INFO)
472    return self.fLogger
473
474  def closeFileLog(self):
475    self.fLogger.removeHandler(self.fHdlr)
476    self.fHdlr.close()
477    if self.policyFileLogfileMode in ['wo','np']:
478      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
479
480  def closeBatchLog(self):
481    self.logger.removeHandler(self.hdlr)
482    self.hdlr.close()
483    if self.policyBatchLogfileMode in ['wo','np']:
484      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
485
486
487class main(object):
488
489  def __init__(self,args=None,abortMessageCount=-1,printInfo=False,monitorFileHandles = False):
490    logDict = {}
491    ecount = 0
492    c4i = c4_init(args=args)
493    c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) )
494    c4i.logger.info( 'Source: %s' % c4i.source )
495     
496    isDummy  = c4i.project[:2] == '__'
497    if (ncLib == None) and (not isDummy):
498       raise baseException( 'Cannot proceed with non-dummy [%s] project without a netcdf API' % (c4i.project) )
499    pcfg = config.projectConfig( c4i.project )
500    assert pcfg.projectV.v == -1, 'Cannot handle anything other than latest version at present'
501    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog,forceLib=c4i.forceNetcdfLib)
502    c4i.logger.info( 'Python netcdf: %s' % ncReader.ncLib )
503    self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount, experimental=c4i.experimental)
504    rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy )
505    self.ncLib = ncLib
506
507    # This list will record the drs dictionaries of all checked files for export to JSON
508    drs_list = []
509
510    if monitorFileHandles:
511      self.monitor = utils.sysMonitor()
512    else:
513      self.monitor = None
514
515    cal = None
516    if len( c4i.errs ) > 0:
517      for i in range(0,len( c4i.errs ), 2 ):
518        c4i.logger.info( c4i.errs[i] )
519 
520    self.cc.info.amapListDraft = []
521    cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor)
522    if c4i.project not in ['ESA-CCI']:
523      cbv.impt( c4i.flist )
524      if printInfo:
525        print cbv.info
526
527    fileLogOpen = False
528    self.resList =  []
529    stdoutsum = 2000
530    npass = 0
531    kf = 0
532    for f in c4i.flist:
533      kf += 1
534      rv = False
535      ec = None
536      if monitorFileHandles:
537        nofhStart = self.monitor.get_open_fds()
538      fn = string.split(f,'/')[-1]
539      c4i.logger.info( 'Starting: %s' % fn )
540      try:
541  ### need to have a unique name, otherwise get mixing of logs despite close statement below.
542  ### if duplicate file names are present, this will be recorded in the main log, tag appended to file level log name (not yet tested).
543        if c4i.logByFile:
544          fLogger = c4i.getFileLog( fn )
545          logDict[fn] = c4i.fileLogfile
546          c4i.logger.info( 'Log file: %s' % c4i.fileLogfile )
547          fileLogOpen = True
548        else:
549          fLogger = c4i.logger
550 
551        fLogger.info( 'Starting file %s' % fn )
552## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
553        self.cc.checkFile( f, log=fLogger,attributeMappings=c4i.attributeMappings )
554
555        if self.cc.completed:
556          if cal not in (None, 'None') and self.cc.cgd.varGroup != "fx":
557            if cal != self.cc.calendar:
558              cal_change_err_msg = 'Error: change in calendar attribute %s --> %s' % (cal, self.cc.calendar)
559              c4i.logger.info(cal_change_err_msg)
560              fLogger.info(cal_change_err_msg)
561              self.cc.errorCount += 1
562
563          cal = self.cc.calendar
564          ec = self.cc.errorCount
565        rv =  ec == 0
566        if rv:
567          npass += 1
568        self.resList.append( (rv,ec) )
569
570        if c4i.logByFile:
571          if self.cc.completed:
572            fLogger.info( 'Done -- error count %s' % self.cc.errorCount )
573          else:
574            fLogger.info( 'Done -- checks not completed' )
575          c4i.closeFileLog( )
576          fileLogOpen = False
577
578        if self.cc.completed:
579          c4i.logger.info( 'Done -- error count %s' % self.cc.errorCount ) 
580          ecount += self.cc.errorCount
581          if self.cc.errorCount == 0:
582            rec.add( f, self.cc.drs )
583            drs_list.append({'path': f, 'drs': self.cc.drs})
584          else:
585            rec.addErr( f, 'ERRORS FOUND | errorCount = %s' % self.cc.errorCount )
586        else:
587          ecount += 20
588          c4i.logger.info( 'Done -- testing aborted because of severity of errors' )
589          rec.addErr( f, 'ERRORS FOUND AND CHECKS ABORTED' )
590      except:
591        c4i.logger.error("Exception has occured" ,exc_info=1)
592        if fileLogOpen:
593          fLogger.error("C4.100.001: [exception]: FAILED:: Exception has occured" ,exc_info=1)
594          c4i.closeFileLog( )
595          fileLogOpen = False
596        rec.addErr( f, 'ERROR: Exception' )
597        if not c4i.holdExceptions:
598          raise
599      if stdoutsum > 0 and kf%stdoutsum == 0:
600         print '%s files checked; %s passed this round' % (kf,npass)
601      if monitorFileHandles:
602        nofhEnd = self.monitor.get_open_fds()
603        if nofhEnd > nofhStart:
604           print 'Open file handles: %s --- %s' % (nofhStart, nofhEnd)
605 
606    self.cc.info.log = c4i.logger
607   
608    if c4i.project not in ['SPECS','CCMI','CMIP5','ESA-CCI']:
609       cbv.c4i = c4i
610       cbv.setLogDict( logDict )
611       cbv.check( recorder=rec, calendar=self.cc.calendar)
612       try:
613         ecount += cbv.errorCount
614       except:
615         ecount = None
616    ncReader.close()
617    if type( self.cc.info.amapListDraft ) == type( [] ) and len(  self.cc.info.amapListDraft ) > 0:
618      ll =  self.cc.info.amapListDraft
619      ll.sort()
620      oo = open( 'amapDraft.txt', 'w' )
621      oo.write( ll[0] + '\n' )
622      for i in range( 1,len(ll) ):
623        if ll[i] != ll[i-1]:
624          oo.write( ll[i] + '\n' )
625      oo.close()
626    if c4i.project in ['SPECS','CCMI','CMIP5']:
627      rec.checktids()
628    rec.dumpAll()
629
630    #!TODO: the recorder class could export JSON if it recorded the full drs dictionaries.
631    #       This lightweight solution re-uses the filename from the rec class and dumps
632    #       JSON in a separate function.
633    json_file = os.path.splitext(rec.file)[0] + '.json'
634    dump_drs_list(drs_list, json_file)
635
636    if printInfo:
637      print 'Error count %s' % ecount
638    ##c4i.hdlr.close()
639    c4i.closeBatchLog()
640    self.ok = all( map( lambda x: x[0], self.resList ) )
641
642
643def dump_drs_list(drs_list, filename):
644    import json
645    fh = open(filename, 'a+')
646    for drs in drs_list:
647                fh.write(json.dumps(drs))
648                fh.write('\n')
649    fh.close()
650
651
652def main_entry():
653  """
654   Wrapper around main() for use with setuptools.
655
656  """
657  if sys.argv[1] == '--sum':
658      import summary
659      summary.main()
660  elif sys.argv[1] == '-v':
661      from versionConfig import version, versionComment
662      print 'ceda-cc version %s [%s]' % (version,versionComment)
663  elif sys.argv[1] == '--unitTest':
664      print "Starting test suite 1"
665      import unitTestsS1
666      print "Starting test suite 2"
667      import unitTestsS2
668      print "Tests completed"
669  else:
670     main(printInfo=True)
671
672if __name__ == '__main__':
673  main_entry()
674
675
676##else:
677  ##f1 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/day/clh/clh_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_day_19810101-19851231.nc'
678  ##f2 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/sem/tas/tas_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_sem_200012-201011.nc'
679  ##f3 = '/data/u10/cordex/AFR-44i/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/mon/tas/tas_AFR-44i_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_mon_199101-200012.nc'
680  ##cc.checkFile( f3 )
Note: See TracBrowser for help on using the repository browser.