source: CCCC/trunk/ceda_cc/ccinit.py @ 331

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/ccinit.py@331
Revision 331, 10.3 KB checked in by mjuckes, 6 years ago (diff)

cleared up misleading message for variable with no standard name in table; milli-second naming mode for log files
Line 
1"""
2Arguments:
3  :-p: project
4  :--copy-config: <target directory path>: copy configuration files to target directory path;
5  :--sum: <log file directory> create a summary of the results  logged in the specified directory.
6  :-f: <file>: check a single file;
7  :-d: <directory>: check all the NetCDF files in a directory;
8  :-D: <directory>: check all the NetCDF files in a directory tree (or latest versions if a recognised version managment system is defined for the project);
9  :--ld: <log file directory>:  ## directory to take log files;
10  :-R: <record file name>: file name for file to take one record per file checked;
11  :--cae: "catch all errors": will trap exceptions and record in  log files, and then continue. Default is to stop after unrecognised exceptions.
12  :--log: <single|multi>:  Set log file management option -- see "Single log" and "Multi-log" below.
13  :--blfmode: <mode>:    Set mode for batch log file -- see log file modes
14  :--blfms:            Set milli-second mode for naming batch log files (instead of second mode);
15  :--flfmode: <mode>:  Set mode for file-level log file -- see log file modes
16  :--aMap:             Read in some attribute mappings and run tests with virtual substitutions;
17
18Log file modes
19--------------
20
21Valid modes are: 'a': append
22                 'n', 'np': new file, 'np': protect after closing (mode = 444)
23                 'w', 'wo': write (overwrite if present), 'wo': protect after closing (mode = 444)
24
25Note that the log files generated in multi-log mode will re-use file names. If running with --flfmode set to 'n','np' or 'wo' it will be necessary to change or clear the target directory. The names of batch log files include the time, to the nearest second, when the process is started, so will not generally suffer from re-use.
26"""
27
28"""
29ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]
30
31With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked.
32"""
33import sys, os, string, time, logging
34
35class c4_init(object):
36
37  def __init__(self,args=None):
38    self.logByFile = True
39    self.policyFileLogfileMode = 'w'
40    self.policyBatchLogfileMode = 'np'
41    self.policyBatchLogfileMs = False
42    if args==None:
43       args = sys.argv[1:]
44    nn = 0
45
46    self.attributeMappingFile = None
47    self.recordFile = 'Rec.txt'
48    self.logDir = 'logs_02'
49    self.errs = []
50   
51    # Set default project to "CORDEX"
52    self.project = "CORDEX"
53    self.holdExceptions = False
54    forceLogOrg = None
55    argsIn = args[:]
56   
57    # Show help if no args or help requested
58    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
59       print __doc__
60       raise SystemExit(0)     
61    # The --copy-config option must be the first argument if it is present.
62    elif args[0] == '--copy-config':
63       if len(args) < 2:
64         self.commandHints( argsIn )
65       args.pop(0)
66       dest_dir = args.pop(0)
67       config.copy_config(dest_dir)
68       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
69       print
70       raise SystemExit(0)
71
72    self.summarymode = args[0] == '--sum'
73    if self.summarymode:
74      return
75
76    self.experimental = False
77    self.forceNetcdfLib = None
78    fltype = None
79    argu = []
80    while len(args) > 0:
81      next = args.pop(0)
82      if next == '-f':
83        flist = [args.pop(0),]
84        self.logByFile = False
85        fltype = '-f'
86        self.source = flist[0]
87      elif next == '--log':
88        x = args.pop(0)
89        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
90        if x in ['multi','m']:
91           forceLogOrg = 'multi'
92        elif x in ['single','s']:
93           forceLogOrg = 'single'
94      elif next == '--experimental':
95        self.experimental = True
96      elif next == '--force-ncq':
97        self.forceNetcdfLib = 'ncq3'
98      elif next == '--force-cdms2':
99        self.forceNetcdfLib = 'cdms2'
100      elif next == '--force-pync4':
101        self.forceNetcdfLib = 'netCDF4'
102      elif next == '--force-scientific':
103        self.forceNetcdfLib = 'Scientific'
104      elif next == '--flfmode':
105        lfmk = args.pop(0)
106        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
107        self.policyFileLogfileMode = lfmk
108      elif next == '--blfmode':
109        lfmk = args.pop(0)
110        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
111        self.policyBatchLogfileMode = lfmk
112      elif next == '--blfms':
113        self.policyBatchLogfileMs = False
114      elif next == '-d':
115        fdir = args.pop(0)
116        flist = glob.glob( '%s/*.nc' % fdir  )
117        self.source = '%s/*.nc' % fdir
118      elif next == '-D':
119        flist  = []
120        fdir = args.pop(0)
121        for root, dirs, files in os.walk( fdir, followlinks=True ):
122          for f in files:
123            fpath = '%s/%s' % (root,f)
124            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
125              flist.append( fpath )
126        self.source = '%s/.....' % fdir
127      elif next == '-R':
128        self.recordFile = args.pop(0)
129      elif next == '--ld':
130        self.logDir = args.pop(0)
131      elif next in ['--catchAllExceptions','--cae']:
132        self.holdExceptions = True
133      elif next == '--aMap':
134        self.attributeMappingFile = args.pop(0)
135        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
136      elif next == "-p":
137        self.project = args.pop(0)
138      else:
139       print 'Unused argument: %s' % next
140       argu.append( next )
141       nn+=1
142    if nn != 0:
143      print 'Unused arguments: ', argu
144      self.commandHints( argsIn )
145
146    if self.project == 'CMIP5' and fltype != '-f':
147      fl0 = []
148      for f in flist:
149        if string.find( f, '/latest/' ) != -1:
150          fl0.append(f)
151      flist = fl0
152
153    if forceLogOrg != None:
154      if forceLogOrg == 'single':
155        self.logByFile = False
156      else:
157        self.logByFile = True
158
159    if self.project[:2] == '__':
160       self.source = 'dummy'
161       flist = []
162       ss = 'abcdefgijk'
163       ss = 'abcdefgijklmnopqrstuvwxyz'
164       ss = 'abc'
165       for i in range(10):
166         v = 'v%s' % i
167         for a in ss:
168           for b in ss:
169             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
170    flist.sort()
171    fnl = []
172    for f in flist:
173      fn = string.split(f, '/')[-1]
174      fnl.append(fn)
175    nd = 0
176    dupl = []
177    for k in range(len(fnl)-1):
178      if fnl[k] == fnl[k-1]:
179        nd += 1
180        dupl.append( fnl[k] )
181    self.dupDict = {}
182    for f in dupl:
183      self.dupDict[f] = 0
184    if nd != 0:
185      self.errs.append( 'Duplicate file names encountered: %s' % nd )
186      self.errs.append( dupl )
187    self.flist = flist
188    self.fnl = fnl
189    if not os.path.isdir(   self.logDir ):
190       os.mkdir(   self.logDir )
191
192    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
193    if self.policyBatchLogfileMs:
194      t = time.time()
195      ms = int( ( t-int(t) )*1000. )
196      tsring1 += '.%3.3i' % ms
197    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
198## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
199    self.logger = logging.getLogger('c4logger')
200    if self.policyBatchLogfileMode in ['n','np']:
201        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
202    m = self.policyBatchLogfileMode[0]
203    if m == 'n':
204      m = 'w'
205    if m == 'a':
206      self.hdlr = logging.FileHandler(self.batchLogfile)
207    else:
208      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
209    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
210    self.hdlr.setFormatter(formatter)
211    self.logger.setLevel(logging.INFO)
212    self.logger.addHandler(self.hdlr)
213
214    self.attributeMappings = []
215    self.attributeMappingsLog = None
216    if self.attributeMappingFile != None:
217      for l in open( self.attributeMappingFile ).readlines():
218        if l[0] != '#':
219          bb = string.split( string.strip(l), '|' ) 
220          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
221          bits = string.split( bb[0], ';' )
222          cl = []
223          for b in bits:
224            cl.append( string.split(b, '=' ) )
225          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
226      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
227
228  def commandHints(self, args):
229    if args[0] in ['-h','--sum']:
230      print 'Arguments look OK'
231    elif args[0] == '--copy-config':
232      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
233    else:
234      if not( '-f' in args or '-d' in args or '-D' in args):
235        print 'No file or target directory specified'
236        print __doc__
237    raise SystemExit(0)
238   
239
240  def getFileLog( self, fn, flf=None ):
241    if flf == None:
242      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
243      if fn in self.dupDict.keys():
244        tag = '__%2.2i' % self.dupDict[fn]
245        self.dupDict[fn] += 1
246      else:
247        tag = ''
248      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
249      if self.policyFileLogfileMode in ['n','np']:
250        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
251      m = self.policyFileLogfileMode[0]
252      if m == 'n':
253        m = 'w'
254    else:
255      m = 'a'
256      self.fileLogfile = flf
257
258    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
259    if m == 'a':
260      self.fHdlr = logging.FileHandler(self.fileLogfile)
261    else:
262      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
263    fileFormatter = logging.Formatter('%(message)s')
264    self.fHdlr.setFormatter(fileFormatter)
265    self.fLogger.addHandler(self.fHdlr)
266    self.fLogger.setLevel(logging.INFO)
267    return self.fLogger
268
269  def closeFileLog(self):
270    self.fLogger.removeHandler(self.fHdlr)
271    self.fHdlr.close()
272    if self.policyFileLogfileMode in ['wo','np']:
273      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
274
275  def closeBatchLog(self):
276    self.logger.removeHandler(self.hdlr)
277    self.hdlr.close()
278    if self.policyBatchLogfileMode in ['wo','np']:
279      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
Note: See TracBrowser for help on using the repository browser.