source: CCCC/trunk/ceda_cc/ccinit.py @ 333

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/ccinit.py@333
Revision 333, 10.3 KB checked in by astephen, 6 years ago (diff)

Fixed import of config_c4 so that "copy_config" works correctly.

Line 
1"""
2Arguments:
3  :-p: project
4  :--copy-config: <target directory path>: copy configuration files to target directory path;
5  :--sum: <log file directory> create a summary of the results  logged in the specified directory.
6  :-f: <file>: check a single file;
7  :-d: <directory>: check all the NetCDF files in a directory;
8  :-D: <directory>: check all the NetCDF files in a directory tree (or latest versions if a recognised version managment system is defined for the project);
9  :--ld: <log file directory>:  ## directory to take log files;
10  :-R: <record file name>: file name for file to take one record per file checked;
11  :--cae: "catch all errors": will trap exceptions and record in  log files, and then continue. Default is to stop after unrecognised exceptions.
12  :--log: <single|multi>:  Set log file management option -- see "Single log" and "Multi-log" below.
13  :--blfmode: <mode>:    Set mode for batch log file -- see log file modes
14  :--blfms:            Set milli-second mode for naming batch log files (instead of second mode);
15  :--flfmode: <mode>:  Set mode for file-level log file -- see log file modes
16  :--aMap:             Read in some attribute mappings and run tests with virtual substitutions;
17
18Log file modes
19--------------
20
21Valid modes are: 'a': append
22                 'n', 'np': new file, 'np': protect after closing (mode = 444)
23                 'w', 'wo': write (overwrite if present), 'wo': protect after closing (mode = 444)
24
25Note that the log files generated in multi-log mode will re-use file names. If running with --flfmode set to 'n','np' or 'wo' it will be necessary to change or clear the target directory. The names of batch log files include the time, to the nearest second, when the process is started, so will not generally suffer from re-use.
26"""
27
28"""
29ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]
30
31With the "-D" option, all files in the directory tree beneath the given directory will be checked. With the "-d" option, only files in the given directory will be checked.
32"""
33import sys, os, string, time, logging
34import config_c4
35
36class c4_init(object):
37
38  def __init__(self,args=None):
39    self.logByFile = True
40    self.policyFileLogfileMode = 'w'
41    self.policyBatchLogfileMode = 'np'
42    self.policyBatchLogfileMs = False
43    if args==None:
44       args = sys.argv[1:]
45    nn = 0
46
47    self.attributeMappingFile = None
48    self.recordFile = 'Rec.txt'
49    self.logDir = 'logs_02'
50    self.errs = []
51   
52    # Set default project to "CORDEX"
53    self.project = "CORDEX"
54    self.holdExceptions = False
55    forceLogOrg = None
56    argsIn = args[:]
57   
58    # Show help if no args or help requested
59    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
60       print __doc__
61       raise SystemExit(0)     
62    # The --copy-config option must be the first argument if it is present.
63    elif args[0] == '--copy-config':
64       if len(args) < 2:
65         self.commandHints( argsIn )
66       args.pop(0)
67       dest_dir = args.pop(0)
68       config_c4.copy_config(dest_dir)
69       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
70       print
71       raise SystemExit(0)
72
73    self.summarymode = args[0] == '--sum'
74    if self.summarymode:
75      return
76
77    self.experimental = False
78    self.forceNetcdfLib = None
79    fltype = None
80    argu = []
81    while len(args) > 0:
82      next = args.pop(0)
83      if next == '-f':
84        flist = [args.pop(0),]
85        self.logByFile = False
86        fltype = '-f'
87        self.source = flist[0]
88      elif next == '--log':
89        x = args.pop(0)
90        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
91        if x in ['multi','m']:
92           forceLogOrg = 'multi'
93        elif x in ['single','s']:
94           forceLogOrg = 'single'
95      elif next == '--experimental':
96        self.experimental = True
97      elif next == '--force-ncq':
98        self.forceNetcdfLib = 'ncq3'
99      elif next == '--force-cdms2':
100        self.forceNetcdfLib = 'cdms2'
101      elif next == '--force-pync4':
102        self.forceNetcdfLib = 'netCDF4'
103      elif next == '--force-scientific':
104        self.forceNetcdfLib = 'Scientific'
105      elif next == '--flfmode':
106        lfmk = args.pop(0)
107        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
108        self.policyFileLogfileMode = lfmk
109      elif next == '--blfmode':
110        lfmk = args.pop(0)
111        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
112        self.policyBatchLogfileMode = lfmk
113      elif next == '--blfms':
114        self.policyBatchLogfileMs = False
115      elif next == '-d':
116        fdir = args.pop(0)
117        flist = glob.glob( '%s/*.nc' % fdir  )
118        self.source = '%s/*.nc' % fdir
119      elif next == '-D':
120        flist  = []
121        fdir = args.pop(0)
122        for root, dirs, files in os.walk( fdir, followlinks=True ):
123          for f in files:
124            fpath = '%s/%s' % (root,f)
125            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
126              flist.append( fpath )
127        self.source = '%s/.....' % fdir
128      elif next == '-R':
129        self.recordFile = args.pop(0)
130      elif next == '--ld':
131        self.logDir = args.pop(0)
132      elif next in ['--catchAllExceptions','--cae']:
133        self.holdExceptions = True
134      elif next == '--aMap':
135        self.attributeMappingFile = args.pop(0)
136        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
137      elif next == "-p":
138        self.project = args.pop(0)
139      else:
140       print 'Unused argument: %s' % next
141       argu.append( next )
142       nn+=1
143    if nn != 0:
144      print 'Unused arguments: ', argu
145      self.commandHints( argsIn )
146
147    if self.project == 'CMIP5' and fltype != '-f':
148      fl0 = []
149      for f in flist:
150        if string.find( f, '/latest/' ) != -1:
151          fl0.append(f)
152      flist = fl0
153
154    if forceLogOrg != None:
155      if forceLogOrg == 'single':
156        self.logByFile = False
157      else:
158        self.logByFile = True
159
160    if self.project[:2] == '__':
161       self.source = 'dummy'
162       flist = []
163       ss = 'abcdefgijk'
164       ss = 'abcdefgijklmnopqrstuvwxyz'
165       ss = 'abc'
166       for i in range(10):
167         v = 'v%s' % i
168         for a in ss:
169           for b in ss:
170             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
171    flist.sort()
172    fnl = []
173    for f in flist:
174      fn = string.split(f, '/')[-1]
175      fnl.append(fn)
176    nd = 0
177    dupl = []
178    for k in range(len(fnl)-1):
179      if fnl[k] == fnl[k-1]:
180        nd += 1
181        dupl.append( fnl[k] )
182    self.dupDict = {}
183    for f in dupl:
184      self.dupDict[f] = 0
185    if nd != 0:
186      self.errs.append( 'Duplicate file names encountered: %s' % nd )
187      self.errs.append( dupl )
188    self.flist = flist
189    self.fnl = fnl
190    if not os.path.isdir(   self.logDir ):
191       os.mkdir(   self.logDir )
192
193    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
194    if self.policyBatchLogfileMs:
195      t = time.time()
196      ms = int( ( t-int(t) )*1000. )
197      tsring1 += '.%3.3i' % ms
198    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
199## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
200    self.logger = logging.getLogger('c4logger')
201    if self.policyBatchLogfileMode in ['n','np']:
202        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
203    m = self.policyBatchLogfileMode[0]
204    if m == 'n':
205      m = 'w'
206    if m == 'a':
207      self.hdlr = logging.FileHandler(self.batchLogfile)
208    else:
209      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
210    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
211    self.hdlr.setFormatter(formatter)
212    self.logger.setLevel(logging.INFO)
213    self.logger.addHandler(self.hdlr)
214
215    self.attributeMappings = []
216    self.attributeMappingsLog = None
217    if self.attributeMappingFile != None:
218      for l in open( self.attributeMappingFile ).readlines():
219        if l[0] != '#':
220          bb = string.split( string.strip(l), '|' ) 
221          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
222          bits = string.split( bb[0], ';' )
223          cl = []
224          for b in bits:
225            cl.append( string.split(b, '=' ) )
226          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
227      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
228
229  def commandHints(self, args):
230    if args[0] in ['-h','--sum']:
231      print 'Arguments look OK'
232    elif args[0] == '--copy-config':
233      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
234    else:
235      if not( '-f' in args or '-d' in args or '-D' in args):
236        print 'No file or target directory specified'
237        print __doc__
238    raise SystemExit(0)
239   
240
241  def getFileLog( self, fn, flf=None ):
242    if flf == None:
243      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
244      if fn in self.dupDict.keys():
245        tag = '__%2.2i' % self.dupDict[fn]
246        self.dupDict[fn] += 1
247      else:
248        tag = ''
249      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
250      if self.policyFileLogfileMode in ['n','np']:
251        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
252      m = self.policyFileLogfileMode[0]
253      if m == 'n':
254        m = 'w'
255    else:
256      m = 'a'
257      self.fileLogfile = flf
258
259    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
260    if m == 'a':
261      self.fHdlr = logging.FileHandler(self.fileLogfile)
262    else:
263      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
264    fileFormatter = logging.Formatter('%(message)s')
265    self.fHdlr.setFormatter(fileFormatter)
266    self.fLogger.addHandler(self.fHdlr)
267    self.fLogger.setLevel(logging.INFO)
268    return self.fLogger
269
270  def closeFileLog(self):
271    self.fLogger.removeHandler(self.fHdlr)
272    self.fHdlr.close()
273    if self.policyFileLogfileMode in ['wo','np']:
274      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
275
276  def closeBatchLog(self):
277    self.logger.removeHandler(self.hdlr)
278    self.hdlr.close()
279    if self.policyBatchLogfileMode in ['wo','np']:
280      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
Note: See TracBrowser for help on using the repository browser.