source: CCCC/trunk/ceda_cc/ccinit.py @ 456

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/ccinit.py@456
Revision 456, 10.6 KB checked in by mjuckes, 4 years ago (diff)

additions for CCI

Line 
1"""
2Arguments:
3  :-p: project
4  :--copy-config: <target directory path>: copy configuration files to target directory path;
5  :--sum: <log file directory> create a summary of the results  logged in the specified directory.
6  :-f: <file>: check a single file;
7  :-d: <directory>: check all the NetCDF files in a directory;
8  :-D: <directory>: check all the NetCDF files in a directory tree (or latest versions if a recognised version managment system is defined for the project);
9  :--ld: <log file directory>:  ## directory to take log files;
10  :-R: <record file name>: file name for file to take one record per file checked;
11  :-l: <file list file>: a file containing one data file to check per line;
12  :--cae: "catch all errors": will trap exceptions and record in  log files, and then continue. Default is to stop after unrecognised exceptions.
13  :--log: <single|multi>:  Set log file management option -- see "Single log" and "Multi-log" below.
14  :--blfmode: <mode>:    Set mode for batch log file -- see log file modes
15  :--blfms:            Set milli-second mode for naming batch log files (instead of second mode);
16  :--flfmode: <mode>:  Set mode for file-level log file -- see log file modes
17  :--aMap:             Read in some attribute mappings and run tests with virtual substitutions;
18
19Log file modes
20--------------
21
22Valid modes are: 'a': append
23                 'n', 'np': new file, 'np': protect after closing (mode = 444)
24                 'w', 'wo': write (overwrite if present), 'wo': protect after closing (mode = 444)
25
26Note that the log files generated in multi-log mode will re-use file names. If running with --flfmode set to 'n','np' or 'wo' it will be necessary to change or clear the target directory. The names of batch log files include the time, to the nearest second, when the process is started, so will not generally suffer from re-use.
27"""
28
29"""
30ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>|-l <file list file>] [other options]
31
32With the "-D" option, all files in the directory tree beneath the given directory will be checked. With the "-d" option, only files in the given directory will be checked.
33"""
34import sys, os, string, time, logging, glob
35import config_c4
36
37class c4_init(object):
38
39  def __init__(self,args=None):
40    self.logByFile = True
41    self.policyFileLogfileMode = 'w'
42    self.policyBatchLogfileMode = 'np'
43    self.policyBatchLogfileMs = False
44    if args==None:
45       args = sys.argv[1:]
46    nn = 0
47
48    self.attributeMappingFile = None
49    self.recordFile = 'Rec.txt'
50    self.logDir = 'logs_02'
51    self.errs = []
52   
53    # Set default project to "CORDEX"
54    self.project = "CORDEX"
55    self.holdExceptions = False
56    forceLogOrg = None
57    argsIn = args[:]
58   
59    # Show help if no args or help requested
60    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
61       print __doc__
62       raise SystemExit(0)     
63    # The --copy-config option must be the first argument if it is present.
64    elif args[0] == '--copy-config':
65       if len(args) < 2:
66         self.commandHints( argsIn )
67       args.pop(0)
68       dest_dir = args.pop(0)
69       config_c4.copy_config(dest_dir)
70       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
71       print
72       raise SystemExit(0)
73
74    self.summarymode = args[0] == '--sum'
75    if self.summarymode:
76      return
77
78    self.experimental = False
79    self.forceNetcdfLib = None
80    self.getdrs = True
81    fltype = None
82    argu = []
83    while len(args) > 0:
84      next = args.pop(0)
85      if next == '-f':
86        flist = [args.pop(0),]
87        self.logByFile = False
88        fltype = '-f'
89        self.source = flist[0]
90      elif next == '--log':
91        x = args.pop(0)
92        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
93        if x in ['multi','m']:
94           forceLogOrg = 'multi'
95        elif x in ['single','s']:
96           forceLogOrg = 'single'
97      elif next == '--experimental':
98        self.experimental = True
99      elif next == '--force-ncq':
100        self.forceNetcdfLib = 'ncq3'
101      elif next == '--force-cdms2':
102        self.forceNetcdfLib = 'cdms2'
103      elif next == '--force-pync4':
104        self.forceNetcdfLib = 'netCDF4'
105      elif next == '--nodrs':
106        self.getdrs = False
107      elif next == '--force-scientific':
108        self.forceNetcdfLib = 'Scientific'
109      elif next == '--flfmode':
110        lfmk = args.pop(0)
111        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
112        self.policyFileLogfileMode = lfmk
113      elif next == '--blfmode':
114        lfmk = args.pop(0)
115        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
116        self.policyBatchLogfileMode = lfmk
117      elif next == '--blfms':
118        self.policyBatchLogfileMs = False
119      elif next == '-d':
120        fdir = args.pop(0)
121        flist = glob.glob( '%s/*.nc' % fdir  )
122        self.source = '%s/*.nc' % fdir
123      elif next == '-D':
124        flist  = []
125        fdir = args.pop(0)
126        for root, dirs, files in os.walk( fdir, followlinks=True ):
127          for f in files:
128            fpath = '%s/%s' % (root,f)
129            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
130              flist.append( fpath )
131        self.source = '%s/.....' % fdir
132      elif next == '-l':
133        flist = open(args.pop(0)).read().split()
134        self.source = flist[0]
135      elif next == '-R':
136        self.recordFile = args.pop(0)
137      elif next == '--ld':
138        self.logDir = args.pop(0)
139      elif next in ['--catchAllExceptions','--cae']:
140        self.holdExceptions = True
141      elif next == '--aMap':
142        self.attributeMappingFile = args.pop(0)
143        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
144      elif next == "-p":
145        self.project = args.pop(0)
146      else:
147       print 'Unused argument: %s' % next
148       argu.append( next )
149       nn+=1
150    if nn != 0:
151      print 'Unused arguments: ', argu
152      self.commandHints( argsIn )
153
154    if self.project == 'CMIP5' and fltype != '-f':
155      fl0 = []
156      for f in flist:
157        if string.find( f, '/latest/' ) != -1:
158          fl0.append(f)
159      flist = fl0
160
161    if forceLogOrg != None:
162      if forceLogOrg == 'single':
163        self.logByFile = False
164      else:
165        self.logByFile = True
166
167    if self.project[:2] == '__':
168       self.source = 'dummy'
169       flist = []
170       ss = 'abcdefgijk'
171       ss = 'abcdefgijklmnopqrstuvwxyz'
172       ss = 'abc'
173       for i in range(10):
174         v = 'v%s' % i
175         for a in ss:
176           for b in ss:
177             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
178    flist.sort()
179    fnl = []
180    for f in flist:
181      fn = string.split(f, '/')[-1]
182      fnl.append(fn)
183    nd = 0
184    dupl = []
185    for k in range(len(fnl)-1):
186      if fnl[k] == fnl[k-1]:
187        nd += 1
188        dupl.append( fnl[k] )
189    self.dupDict = {}
190    for f in dupl:
191      self.dupDict[f] = 0
192    if nd != 0:
193      self.errs.append( 'Duplicate file names encountered: %s' % nd )
194      self.errs.append( dupl )
195    self.flist = flist
196    self.fnl = fnl
197    if not os.path.isdir(   self.logDir ):
198       os.mkdir(   self.logDir )
199
200    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
201    if self.policyBatchLogfileMs:
202      t = time.time()
203      ms = int( ( t-int(t) )*1000. )
204      tsring1 += '.%3.3i' % ms
205    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
206## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
207    self.logger = logging.getLogger('c4logger')
208    if self.policyBatchLogfileMode in ['n','np']:
209        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
210    m = self.policyBatchLogfileMode[0]
211    if m == 'n':
212      m = 'w'
213    if m == 'a':
214      self.hdlr = logging.FileHandler(self.batchLogfile)
215    else:
216      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
217    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
218    self.hdlr.setFormatter(formatter)
219    self.logger.setLevel(logging.INFO)
220    self.logger.addHandler(self.hdlr)
221
222    self.attributeMappings = []
223    self.attributeMappingsLog = None
224    if self.attributeMappingFile != None:
225      for l in open( self.attributeMappingFile ).readlines():
226        if l[0] != '#':
227          bb = string.split( string.strip(l), '|' ) 
228          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
229          bits = string.split( bb[0], ';' )
230          cl = []
231          for b in bits:
232            cl.append( string.split(b, '=' ) )
233          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
234      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
235
236  def commandHints(self, args):
237    if args[0] in ['-h','--sum']:
238      print 'Arguments look OK'
239    elif args[0] == '--copy-config':
240      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
241    else:
242      if not( '-f' in args or '-d' in args or '-D' in args):
243        print 'No file or target directory specified'
244        print __doc__
245    raise SystemExit(0)
246   
247
248  def getFileLog( self, fn, flf=None ):
249    if flf == None:
250      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
251      if fn in self.dupDict.keys():
252        tag = '__%2.2i' % self.dupDict[fn]
253        self.dupDict[fn] += 1
254      else:
255        tag = ''
256      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
257      if self.policyFileLogfileMode in ['n','np']:
258        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
259      m = self.policyFileLogfileMode[0]
260      if m == 'n':
261        m = 'w'
262    else:
263      m = 'a'
264      self.fileLogfile = flf
265
266    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
267    if m == 'a':
268      self.fHdlr = logging.FileHandler(self.fileLogfile)
269    else:
270      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
271    fileFormatter = logging.Formatter('%(message)s')
272    self.fHdlr.setFormatter(fileFormatter)
273    self.fLogger.addHandler(self.fHdlr)
274    self.fLogger.setLevel(logging.INFO)
275    return self.fLogger
276
277  def closeFileLog(self):
278    self.fLogger.removeHandler(self.fHdlr)
279    self.fHdlr.close()
280    if self.policyFileLogfileMode in ['wo','np']:
281      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
282
283  def closeBatchLog(self):
284    self.logger.removeHandler(self.hdlr)
285    self.hdlr.close()
286    if self.policyBatchLogfileMode in ['wo','np']:
287      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
Note: See TracBrowser for help on using the repository browser.