source: CCCC/tags/1.2.5/ceda_cc/c4.py @ 243

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/tags/1.2.5/ceda_cc/c4.py@1241
Revision 243, 22.2 KB checked in by astephen, 5 years ago (diff)

bugs in CCMI configuration fixed

Line 
1
2import sys
3
4## callout to summary.py: if this option is selected, imports of libraries are not needed.
5if __name__ == '__main__' and sys.argv[1] == '--sum':
6      import summary
7      summary.main()
8      raise SystemExit(0)
9
10# Standard library imports
11import os, string, time, logging, sys, glob, pkgutil
12import shutil
13## pkgutil is used in file_utils
14# Third party imports
15
16## Local imports with 3rd party dependencies
17#### netcdf --- currently only support for cmds2 -- re-arranged to facilitate support for alternative modules
18
19import file_utils
20
21from file_utils import fileMetadata, ncLib
22
23# Local imports
24import utils_c4 as utils
25import config_c4 as config
26
27reload( utils )
28
29from xceptions import baseException
30
31from fcc_utils2 import tupsort
32
33
34#driving_model_ensemble_member = <CMIP5Ensemble_member>
35#rcm_version_id = <RCMVersionID>                     
36
37class dummy(object):
38   pass
39
40pathTmplDict = { 'CORDEX':'%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/',   \
41                 'SPECS':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(start_date)s/%(frequency)s/%(realm)s/%(table)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \
42                 'CMIP5':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \
43                 'CCMI':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(table)s/%(ensemble)s/files/%%(version)s/%(variable)s/', \
44                 '__def__':'%(project)s/%(product)s/%(institute)s/%(model)s/%(experiment)s/%(frequency)s/%(realm)s/%(variable)s/%(ensemble)s/files/%%(version)s/', \
45               }
46
47## Core DRS: list of vocab names
48## Path template: -- current version puts upper case in "project"
49## Dataset template: 
50
51class recorder(object):
52
53  def __init__(self,project,fileName,type='map',dummy=False):
54    self.dummy = dummy
55    self.file = fileName
56    self.type = type
57    self.pathTmpl = '%(project)s/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(model)s/%(model_version)s/%(frequency)s/%(variable)s/files/%%(version)s/'
58    self.pathTmpl = pathTmplDict.get(project,pathTmplDict['__def__'])
59    self.records = {}
60    self.tidtupl = []
61
62  def open(self):
63    if self.type == 'map':
64      self.fh = open( self.file, 'a' )
65    else:
66      self.sh = shelve.open( self.file )
67
68  def close(self):
69    if self.type == 'map':
70      self.fh.close()
71    else:
72      self.sh.close()
73
74  def add(self,fpath,drs,safe=True):
75    assert self.type == 'map','Can only do map files at present'
76    assert type(drs) == type( {} ), '2nd user argument to method add should be a dictionary [%s]' % type(drs)
77    tpath = self.pathTmpl % drs
78    if not self.dummy:
79      assert os.path.isfile( fpath ), 'File %s not found' % fpath
80      fdate = time.ctime(os.path.getmtime(fpath))
81      sz = os.stat(fpath).st_size
82    else:
83      fdate = "na"
84      sz = 0
85    record = '%s | OK | %s | modTime = %s | target = %s ' % (fpath,sz,fdate,tpath)
86    fn = string.split( fpath, '/' )[-1]
87    for k in ['creation_date','tracking_id']:
88      if k in drs.keys():
89        record += ' | %s = %s' % (k,drs[k])
90        if k == 'tracking_id':
91          self.tidtupl.append( (fn,drs[k]) )
92
93    self.records[fn] = record
94 
95  def modify(self,fn,msg):
96    assert fn in self.records.keys(),'Attempt to modify non-existent record %s, %s' % [fn,str(self.records.keys()[0:10])]
97    if string.find( self.records[fn], '| OK |') == -1:
98      ##print 'File %s already flagged with errors' % fn
99      return
100    s = string.replace( self.records[fn], '| OK |', '| %s |' % msg )
101    ##print '--> ',s
102    self.records[fn] = s
103
104  def checktids(self):
105## sort by tracking id
106    if len( self.tidtupl ) == 1:
107      return
108    self.tidtupl.sort( cmp=tupsort(k=1).cmp )
109    nd = 0
110    fnl = []
111    for k in range(len(self.tidtupl)-1):
112      if self.tidtupl[k][1] == self.tidtupl[k+1][1]:
113        print 'Duplicate tracking_id: %s, %s:: %s' % (self.tidtupl[k][0],self.tidtupl[k+1][0],self.tidtupl[k][1])
114        nd += 1
115        if len(fnl) == 0 or fnl[-1] != self.tidtupl[k][0]:
116          fnl.append( self.tidtupl[k][0])
117        fnl.append( self.tidtupl[k+1][0])
118    if nd == 0:
119      print 'No duplicate tracking ids found in %s files' % len(self.tidtupl)
120    else:
121      print '%s duplicate tracking ids' % nd
122      for f in fnl:
123        self.modify( f, 'ERROR: duplicate tid' )
124
125  def dumpAll(self,safe=True):
126    keys = self.records.keys()
127    keys.sort()
128    for k in keys:
129      self.dump( self.records[k], safe=safe )
130
131  def dump( self, record, safe=True ):
132    if safe:
133      self.open()
134    self.fh.write( record + '\n' )
135    if safe:
136      self.close()
137
138  def addErr(self,fpath,reason,safe=True):
139    record = '%s | %s' % (fpath, reason)
140    fn = string.split( fpath, '/' )[-1]
141    self.records[fn] = record
142
143class checker(object):
144  def __init__(self, pcfg, cls,reader,abortMessageCount=-1):
145    self.info = dummy()
146    self.info.pcfg = pcfg
147    self.info.abortMessageCount = abortMessageCount
148    self.calendar = 'None'
149    self.ncReader = reader
150    self.cfn = utils.checkFileName( parent=self.info,cls=cls)
151    self.cga = utils.checkGlobalAttributes( parent=self.info,cls=cls)
152    self.cgd = utils.checkStandardDims( parent=self.info,cls=cls)
153    self.cgg = utils.checkGrids( parent=self.info,cls=cls)
154    self.cls = cls
155
156    # Define vocabs based on project
157    ##self.vocabs = getVocabs(pcgf)
158    self.vocabs = pcfg.vocabs
159
160  def checkFile(self,fpath,log=None,attributeMappings=[]):
161    self.calendar = 'None'
162    self.info.log = log
163
164    fn = string.split( fpath, '/' )[-1]
165
166    if attributeMappings != []:
167      self.ncReader.loadNc( fpath )
168      self.ncReader.applyMap( attributeMappings, self.cfn.globalAttributesInFn, log=log )
169      ncRed = True
170      thisFn = self.ncReader.fn
171    else:
172      ncRed = False
173      thisFn = fn
174
175    self.cfn.check( thisFn )
176    if not self.cfn.completed:
177      self.completed = False
178      return
179    if not self.info.pcfg.project[:2] == '__':
180      if not os.path.isfile( fpath ):
181        print 'File %s not found [2]' % fpath
182        self.completed = False
183        return
184
185    if not ncRed:
186      ##print fpath
187      self.ncReader.loadNc( fpath )
188    self.ga = self.ncReader.ga
189    self.va = self.ncReader.va
190    self.da = self.ncReader.da
191
192    if self.cfn.freq != None:
193      vGroup = self.cfn.freq
194    else:
195      vGroup = self.info.pcfg.mipVocabVgmap.get(self.cfn.group,self.cfn.group)
196    self.cga.check( self.ga, self.va, self.cfn.var, vGroup, self.vocabs, self.cfn.fnParts )
197    if not self.cga.completed:
198      self.completed = False
199      return
200
201    ##self.cgd.plevRequired = config.plevRequired
202    ##self.cgd.plevValues = config.plevValues
203    ##self.cgd.heightRequired = config.heightRequired
204    ##self.cgd.heightValues = config.heightValues
205    ##self.cgd.heightRange = config.heightRange
206    self.cgd.check( self.cfn.var, self.cfn.freq, self.da, self.va, self.cga.isInstantaneous )
207    self.calendar = self.cgd.calendar
208    if not self.cgd.completed:
209      self.completed = False
210      return
211
212    if self.info.pcfg.doCheckGrids:
213      ##self.cgg.rotatedPoleGrids = config.rotatedPoleGrids
214      ##self.cgg.interpolatedGrids = config.interpolatedGrids
215      self.cgg.check( self.cfn.var, self.cfn.domain, self.da, self.va )
216   
217      if not self.cgg.completed:
218        self.completed = False
219        return
220    self.completed = True
221    self.drs = self.cga.getDrs()
222    self.drs['project'] = self.info.pcfg.project
223    self.errorCount = self.cfn.errorCount + self.cga.errorCount + self.cgd.errorCount + self.cgg.errorCount
224
225class c4_init(object):
226
227  def __init__(self,args=None):
228    self.logByFile = True
229    self.policyFileLogfileMode = 'w'
230    self.policyBatchLogfileMode = 'np'
231    if args==None:
232       args = sys.argv[1:]
233    nn = 0
234
235    self.attributeMappingFile = None
236    self.recordFile = 'Rec.txt'
237    self.logDir = 'logs_02'
238    self.errs = []
239   
240    # Set default project to "CORDEX"
241    self.project = "CORDEX"
242    self.holdExceptions = False
243    forceLogOrg = None
244    argsIn = args[:]
245
246    # The --copy-config option must be the first argument if it is present.
247    if args[0] == '--copy-config':
248       if len(args) < 2:
249         self.commandHints( argsIn )
250       args.pop(0)
251       dest_dir = args.pop(0)
252       config.copy_config(dest_dir)
253       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
254       print
255       raise SystemExit(0)
256    elif args[0] == '-h':
257       print 'Help command not implemented yet'
258       raise SystemExit(0)
259
260    self.summarymode = args[0] == '--sum'
261    if self.summarymode:
262      return
263
264    self.forceNetcdfLib = None
265    fltype = None
266    argu = []
267    while len(args) > 0:
268      next = args.pop(0)
269      if next == '-f':
270        flist = [args.pop(0),]
271        self.logByFile = False
272        fltype = '-f'
273        self.source = flist[0]
274      elif next == '--log':
275        x = args.pop(0)
276        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
277        if x in ['multi','m']:
278           forceLogOrg = 'multi'
279        elif x in ['single','s']:
280           forceLogOrg = 'single'
281      elif next == '--force-ncq':
282        self.forceNetcdfLib = 'ncq3'
283      elif next == '--force-cdms2':
284        self.forceNetcdfLib = 'cdms2'
285      elif next == '--force-pync4':
286        self.forceNetcdfLib = 'netCDF4'
287      elif next == '--force-scientific':
288        self.forceNetcdfLib = 'Scientific'
289      elif next == '--flfmode':
290        lfmk = args.pop(0)
291        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
292        self.policyFileLogfileMode = lfmk
293      elif next == '--blfmode':
294        lfmk = args.pop(0)
295        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
296        self.policyBatchLogfileMode = lfmk
297      elif next == '-d':
298        fdir = args.pop(0)
299        flist = glob.glob( '%s/*.nc' % fdir  )
300        self.source = '%s/*.nc' % fdir
301      elif next == '-D':
302        flist  = []
303        fdir = args.pop(0)
304        for root, dirs, files in os.walk( fdir, followlinks=True ):
305          for f in files:
306            fpath = '%s/%s' % (root,f)
307            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
308              flist.append( fpath )
309        self.source = '%s/.....' % fdir
310      elif next == '-R':
311        self.recordFile = args.pop(0)
312      elif next == '--ld':
313        self.logDir = args.pop(0)
314      elif next in ['--catchAllExceptions','--cae']:
315        self.holdExceptions = True
316      elif next == '--aMap':
317        self.attributeMappingFile = args.pop(0)
318        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
319      elif next == "-p":
320        self.project = args.pop(0)
321      else:
322       print 'Unused argument: %s' % next
323       argu.append( next )
324       nn+=1
325    if nn != 0:
326      print 'Unused arguments: ', argu
327      self.commandHints( argsIn )
328
329    if self.project == 'CMIP5' and fltype != '-f':
330      fl0 = []
331      for f in flist:
332        if string.find( f, '/latest/' ) != -1:
333          fl0.append(f)
334      flist = fl0
335
336    if forceLogOrg != None:
337      if forceLogOrg == 'single':
338        self.logByFile = False
339      else:
340        self.logByFile = True
341
342    if self.project[:2] == '__':
343       self.source = 'dummy'
344       flist = []
345       ss = 'abcdefgijk'
346       ss = 'abcdefgijklmnopqrstuvwxyz'
347       ss = 'abc'
348       for i in range(10):
349         v = 'v%s' % i
350         for a in ss:
351           for b in ss:
352             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
353    flist.sort()
354    fnl = []
355    for f in flist:
356      fn = string.split(f, '/')[-1]
357      fnl.append(fn)
358    nd = 0
359    dupl = []
360    for k in range(len(fnl)-1):
361      if fnl[k] == fnl[k-1]:
362        nd += 1
363        dupl.append( fnl[k] )
364    self.dupDict = {}
365    for f in dupl:
366      self.dupDict[f] = 0
367    if nd != 0:
368      self.errs.append( 'Duplicate file names encountered: %s' % nd )
369      self.errs.append( dupl )
370    self.flist = flist
371    self.fnl = fnl
372    if not os.path.isdir(   self.logDir ):
373       os.mkdir(   self.logDir )
374
375    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
376    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
377## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
378    self.logger = logging.getLogger('c4logger')
379    if self.policyBatchLogfileMode in ['n','np']:
380        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
381    m = self.policyBatchLogfileMode[0]
382    if m == 'n':
383      m = 'w'
384    if m == 'a':
385      self.hdlr = logging.FileHandler(self.batchLogfile)
386    else:
387      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
388    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
389    self.hdlr.setFormatter(formatter)
390    self.logger.setLevel(logging.INFO)
391    self.logger.addHandler(self.hdlr)
392
393    self.attributeMappings = []
394    self.attributeMappingsLog = None
395    if self.attributeMappingFile != None:
396      for l in open( self.attributeMappingFile ).readlines():
397        if l[0] != '#':
398          bb = string.split( string.strip(l), '|' ) 
399          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
400          bits = string.split( bb[0], ';' )
401          cl = []
402          for b in bits:
403            cl.append( string.split(b, '=' ) )
404          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
405      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
406
407  def commandHints(self, args):
408    if args[0] in ['-h','--sum']:
409      print 'Arguments look OK'
410    elif args[0] == '--copy-config':
411      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
412    else:
413      if not( '-f' in args or '-d' in args or '-D' in args):
414        print 'No file or target directory specified'
415        print """USAGE:
416ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]
417
418With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked.
419"""
420    raise SystemExit(0)
421   
422
423  def getFileLog( self, fn, flf=None ):
424    if flf == None:
425      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
426      if fn in self.dupDict.keys():
427        tag = '__%2.2i' % self.dupDict[fn]
428        self.dupDict[fn] += 1
429      else:
430        tag = ''
431      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
432      if self.policyFileLogfileMode in ['n','np']:
433        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
434      m = self.policyFileLogfileMode[0]
435      if m == 'n':
436        m = 'w'
437    else:
438      m = 'a'
439      self.fileLogfile = flf
440
441    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
442    if m == 'a':
443      self.fHdlr = logging.FileHandler(self.fileLogfile)
444    else:
445      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
446    fileFormatter = logging.Formatter('%(message)s')
447    self.fHdlr.setFormatter(fileFormatter)
448    self.fLogger.addHandler(self.fHdlr)
449    self.fLogger.setLevel(logging.INFO)
450    return self.fLogger
451
452  def closeFileLog(self):
453    self.fLogger.removeHandler(self.fHdlr)
454    self.fHdlr.close()
455    if self.policyFileLogfileMode in ['wo','np']:
456      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
457
458  def closeBatchLog(self):
459    self.logger.removeHandler(self.hdlr)
460    self.hdlr.close()
461    if self.policyBatchLogfileMode in ['wo','np']:
462      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
463
464
465class main(object):
466
467  def __init__(self,args=None,abortMessageCount=-1,printInfo=False,monitorFileHandles = False):
468    logDict = {}
469    ecount = 0
470    c4i = c4_init(args=args)
471     
472    isDummy  = c4i.project[:2] == '__'
473    if (ncLib == None) and (not isDummy):
474       raise baseException( 'Cannot proceed with non-dummy [%s] project without a netcdf API' % (c4i.project) )
475    pcfg = config.projectConfig( c4i.project )
476    ncReader = fileMetadata(dummy=isDummy, attributeMappingsLog=c4i.attributeMappingsLog,forceLib=c4i.forceNetcdfLib)
477    self.cc = checker(pcfg, c4i.project, ncReader,abortMessageCount=abortMessageCount)
478    rec = recorder( c4i.project, c4i.recordFile, dummy=isDummy )
479
480    # This list will record the drs dictionaries of all checked files for export to JSON
481    drs_list = []
482
483    if monitorFileHandles:
484      self.monitor = utils.sysMonitor()
485    else:
486      self.monitor = None
487
488    cal = None
489    c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) )
490    c4i.logger.info( 'Source: %s' % c4i.source )
491    if len( c4i.errs ) > 0:
492      for i in range(0,len( c4i.errs ), 2 ):
493        c4i.logger.info( c4i.errs[i] )
494 
495    self.cc.info.amapListDraft = []
496    cbv = utils.checkByVar( parent=self.cc.info,cls=c4i.project,monitor=self.monitor)
497    cbv.impt( c4i.flist )
498    if printInfo:
499      print cbv.info
500
501    fileLogOpen = False
502    self.resList =  []
503    stdoutsum = 2000
504    npass = 0
505    kf = 0
506    for f in c4i.flist:
507      kf += 1
508      rv = False
509      ec = None
510      if monitorFileHandles:
511        nofhStart = self.monitor.get_open_fds()
512      fn = string.split(f,'/')[-1]
513      c4i.logger.info( 'Starting: %s' % fn )
514      try:
515  ### need to have a unique name, otherwise get mixing of logs despite close statement below.
516  ### if duplicate file names are present, this will be recorded in the main log, tag appended to file level log name (not yet tested).
517        if c4i.logByFile:
518          fLogger = c4i.getFileLog( fn )
519          logDict[fn] = c4i.fileLogfile
520          c4i.logger.info( 'Log file: %s' % c4i.fileLogfile )
521          fileLogOpen = True
522        else:
523          fLogger = c4i.logger
524 
525        fLogger.info( 'Starting file %s' % fn )
526## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
527        self.cc.checkFile( f, log=fLogger,attributeMappings=c4i.attributeMappings )
528
529        if self.cc.completed:
530          if cal not in (None, 'None') and self.cc.cgd.varGroup != "fx":
531            if cal != self.cc.calendar:
532              cal_change_err_msg = 'Error: change in calendar attribute %s --> %s' % (cal, self.cc.calendar)
533              c4i.logger.info(cal_change_err_msg)
534              fLogger.info(cal_change_err_msg)
535              self.cc.errorCount += 1
536
537          cal = self.cc.calendar
538          ec = self.cc.errorCount
539        rv =  ec == 0
540        if rv:
541          npass += 1
542        self.resList.append( (rv,ec) )
543
544        if c4i.logByFile:
545          if self.cc.completed:
546            fLogger.info( 'Done -- error count %s' % self.cc.errorCount )
547          else:
548            fLogger.info( 'Done -- checks not completed' )
549          c4i.closeFileLog( )
550          fileLogOpen = False
551
552        if self.cc.completed:
553          c4i.logger.info( 'Done -- error count %s' % self.cc.errorCount ) 
554          ecount += self.cc.errorCount
555          if self.cc.errorCount == 0:
556            rec.add( f, self.cc.drs )
557            drs_list.append({'path': f, 'drs': self.cc.drs})
558          else:
559            rec.addErr( f, 'ERRORS FOUND | errorCount = %s' % self.cc.errorCount )
560        else:
561          ecount += 20
562          c4i.logger.info( 'Done -- testing aborted because of severity of errors' )
563          rec.addErr( f, 'ERRORS FOUND AND CHECKS ABORTED' )
564      except:
565        c4i.logger.error("Exception has occured" ,exc_info=1)
566        if fileLogOpen:
567          fLogger.error("C4.100.001: [exception]: FAILED:: Exception has occured" ,exc_info=1)
568          c4i.closeFileLog( )
569          fileLogOpen = False
570        rec.addErr( f, 'ERROR: Exception' )
571        if not c4i.holdExceptions:
572          raise
573      if stdoutsum > 0 and kf%stdoutsum == 0:
574         print '%s files checked; %s passed this round' % (kf,npass)
575      if monitorFileHandles:
576        nofhEnd = self.monitor.get_open_fds()
577        if nofhEnd > nofhStart:
578           print 'Open file handles: %s --- %s' % (nofhStart, nofhEnd)
579 
580    self.cc.info.log = c4i.logger
581   
582    if c4i.project not in ['SPECS','CCMI','CMIP5']:
583       cbv.c4i = c4i
584       cbv.setLogDict( logDict )
585       cbv.check( recorder=rec, calendar=self.cc.calendar)
586       try:
587         ecount += cbv.errorCount
588       except:
589         ecount = None
590    ncReader.close()
591    if type( self.cc.info.amapListDraft ) == type( [] ) and len(  self.cc.info.amapListDraft ) > 0:
592      ll =  self.cc.info.amapListDraft
593      ll.sort()
594      oo = open( 'amapDraft.txt', 'w' )
595      oo.write( ll[0] + '\n' )
596      for i in range( 1,len(ll) ):
597        if ll[i] != ll[i-1]:
598          oo.write( ll[i] + '\n' )
599      oo.close()
600    if c4i.project in ['SPECS','CCMI','CMIP5']:
601      rec.checktids()
602    rec.dumpAll()
603
604    #!TODO: the recorder class could export JSON if it recorded the full drs dictionaries.
605    #       This lightweight solution re-uses the filename from the rec class and dumps
606    #       JSON in a separate function.
607    json_file = os.path.splitext(rec.file)[0] + '.json'
608    dump_drs_list(drs_list, json_file)
609
610    if printInfo:
611      print 'Error count %s' % ecount
612    ##c4i.hdlr.close()
613    c4i.closeBatchLog()
614    self.ok = all( map( lambda x: x[0], self.resList ) )
615
616
617def dump_drs_list(drs_list, filename):
618    import json
619    with open(filename, 'a+') as fh:
620          for drs in drs_list:
621                fh.write(json.dumps(drs))
622                fh.write('\n')
623
624
625def main_entry():
626   """
627   Wrapper around main() for use with setuptools.
628
629   """
630   main(printInfo=True)
631
632if __name__ == '__main__':
633  if sys.argv[1] == '--sum':
634      import summary
635      summary.main()
636      raise SystemExit(0)
637  main_entry()
638
639
640##else:
641  ##f1 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/day/clh/clh_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_day_19810101-19851231.nc'
642  ##f2 = '/data/u10/cordex/AFR-44/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/sem/tas/tas_AFR-44_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_sem_200012-201011.nc'
643  ##f3 = '/data/u10/cordex/AFR-44i/SMHI/ECMWF-ERAINT/evaluation/SMHI-RCA4/v1/mon/tas/tas_AFR-44i_ECMWF-ERAINT_evaluation_r1i1p1_SMHI-RCA4_v1_mon_199101-200012.nc'
644  ##cc.checkFile( f3 )
Note: See TracBrowser for help on using the repository browser.