source: CCCC/trunk/ceda_cc/ccinit.py @ 869

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/ccinit.py@869
Revision 869, 10.8 KB checked in by mjuckes, 3 years ago (diff)

added option to specify files in a text file, one file name per line

Line 
1"""
2Arguments:
3  :-p: project
4  :--copy-config: <target directory path>: copy configuration files to target directory path;
5  :--sum: <log file directory> create a summary of the results  logged in the specified directory.
6  :-f: <file>: check a single file;
7  :-d: <directory>: check all the NetCDF files in a directory;
8  :-D: <directory>: check all the NetCDF files in a directory tree (or latest versions if a recognised version managment system is defined for the project);
9  :--ld: <log file directory>:  ## directory to take log files;
10  :-R: <record file name>: file name for file to take one record per file checked;
11  :-l: <file list file>: a file containing one data file to check per line;
12  :--cae: "catch all errors": will trap exceptions and record in  log files, and then continue. Default is to stop after unrecognised exceptions.
13  :--log: <single|multi>:  Set log file management option -- see "Single log" and "Multi-log" below.
14  :--blfmode: <mode>:    Set mode for batch log file -- see log file modes
15  :--blfms:            Set milli-second mode for naming batch log files (instead of second mode);
16  :--flfmode: <mode>:  Set mode for file-level log file -- see log file modes
17  :--aMap:             Read in some attribute mappings and run tests with virtual substitutions;
18
19Log file modes
20--------------
21
22Valid modes are: 'a': append
23                 'n', 'np': new file, 'np': protect after closing (mode = 444)
24                 'w', 'wo': write (overwrite if present), 'wo': protect after closing (mode = 444)
25
26Note that the log files generated in multi-log mode will re-use file names. If running with --flfmode set to 'n','np' or 'wo' it will be necessary to change or clear the target directory. The names of batch log files include the time, to the nearest second, when the process is started, so will not generally suffer from re-use.
27"""
28
29"""
30ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>|-l <file list file>] [other options]
31
32With the "-D" option, all files in the directory tree beneath the given directory will be checked. With the "-d" option, only files in the given directory will be checked.
33"""
34import sys, os, string, time, logging, glob
35import config_c4
36
37class c4_init(object):
38
39  def __init__(self,args=None):
40    self.logByFile = True
41    self.policyFileLogfileMode = 'w'
42    self.policyBatchLogfileMode = 'np'
43    self.policyBatchLogfileMs = False
44    if args==None:
45       args = sys.argv[1:]
46    nn = 0
47
48    self.attributeMappingFile = None
49    self.recordFile = 'Rec.txt'
50    self.logDir = 'logs_02'
51    self.errs = []
52   
53    # Set default project to "CORDEX"
54    self.project = "CORDEX"
55    self.holdExceptions = False
56    forceLogOrg = None
57    argsIn = args[:]
58   
59    # Show help if no args or help requested
60    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
61       print __doc__
62       raise SystemExit(0)     
63    # The --copy-config option must be the first argument if it is present.
64    elif args[0] == '--copy-config':
65       if len(args) < 2:
66         self.commandHints( argsIn )
67       args.pop(0)
68       dest_dir = args.pop(0)
69       config_c4.copy_config(dest_dir)
70       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
71       print
72       raise SystemExit(0)
73
74    self.summarymode = args[0] == '--sum'
75    if self.summarymode:
76      return
77
78    self.experimental = False
79    self.forceNetcdfLib = None
80    self.getdrs = True
81    fltype = None
82    argu = []
83    while len(args) > 0:
84      next = args.pop(0)
85      if next == '-f':
86        flist = [args.pop(0),]
87        self.logByFile = False
88        fltype = '-f'
89        self.source = flist[0]
90      elif next == '--log':
91        x = args.pop(0)
92        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
93        if x in ['multi','m']:
94           forceLogOrg = 'multi'
95        elif x in ['single','s']:
96           forceLogOrg = 'single'
97      elif next == '--experimental':
98        self.experimental = True
99      elif next == '--force-ncq':
100        self.forceNetcdfLib = 'ncq3'
101      elif next == '--force-cdms2':
102        self.forceNetcdfLib = 'cdms2'
103      elif next == '--force-pync4':
104        self.forceNetcdfLib = 'netCDF4'
105      elif next == '--nodrs':
106        self.getdrs = False
107      elif next == '--force-scientific':
108        self.forceNetcdfLib = 'Scientific'
109      elif next == '--flfmode':
110        lfmk = args.pop(0)
111        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
112        self.policyFileLogfileMode = lfmk
113      elif next == '--blfmode':
114        lfmk = args.pop(0)
115        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
116        self.policyBatchLogfileMode = lfmk
117      elif next == '--blfms':
118        self.policyBatchLogfileMs = False
119      elif next == '-fl':
120        flf = args.pop(0)
121        flist = []
122        for l in open( flf ).readlines():
123          flist.append( l.strip() )
124        self.source = flf 
125      elif next == '-d':
126        fdir = args.pop(0)
127        flist = glob.glob( '%s/*.nc' % fdir  )
128        self.source = '%s/*.nc' % fdir
129      elif next == '-D':
130        flist  = []
131        fdir = args.pop(0)
132        for root, dirs, files in os.walk( fdir, followlinks=True ):
133          for f in files:
134            fpath = '%s/%s' % (root,f)
135            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
136              flist.append( fpath )
137        self.source = '%s/.....' % fdir
138      elif next == '-l':
139        flist = open(args.pop(0)).read().split()
140        self.source = flist[0]
141      elif next == '-R':
142        self.recordFile = args.pop(0)
143      elif next == '--ld':
144        self.logDir = args.pop(0)
145      elif next in ['--catchAllExceptions','--cae']:
146        self.holdExceptions = True
147      elif next == '--aMap':
148        self.attributeMappingFile = args.pop(0)
149        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
150      elif next == "-p":
151        self.project = args.pop(0)
152      else:
153       print 'Unused argument: %s' % next
154       argu.append( next )
155       nn+=1
156    if nn != 0:
157      print 'Unused arguments: ', argu
158      self.commandHints( argsIn )
159
160    if self.project == 'CMIP5' and fltype != '-f':
161      fl0 = []
162      for f in flist:
163        if string.find( f, '/latest/' ) != -1:
164          fl0.append(f)
165      flist = fl0
166
167    if forceLogOrg != None:
168      if forceLogOrg == 'single':
169        self.logByFile = False
170      else:
171        self.logByFile = True
172
173    if self.project[:2] == '__':
174       self.source = 'dummy'
175       flist = []
176       ss = 'abcdefgijk'
177       ss = 'abcdefgijklmnopqrstuvwxyz'
178       ss = 'abc'
179       for i in range(10):
180         v = 'v%s' % i
181         for a in ss:
182           for b in ss:
183             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
184    flist.sort()
185    fnl = []
186    for f in flist:
187      fn = string.split(f, '/')[-1]
188      fnl.append(fn)
189    nd = 0
190    dupl = []
191    for k in range(len(fnl)-1):
192      if fnl[k] == fnl[k-1]:
193        nd += 1
194        dupl.append( fnl[k] )
195    self.dupDict = {}
196    for f in dupl:
197      self.dupDict[f] = 0
198    if nd != 0:
199      self.errs.append( 'Duplicate file names encountered: %s' % nd )
200      self.errs.append( dupl )
201    self.flist = flist
202    self.fnl = fnl
203    if not os.path.isdir(   self.logDir ):
204       os.mkdir(   self.logDir )
205
206    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
207    if self.policyBatchLogfileMs:
208      t = time.time()
209      ms = int( ( t-int(t) )*1000. )
210      tsring1 += '.%3.3i' % ms
211    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
212## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
213    self.logger = logging.getLogger('c4logger')
214    if self.policyBatchLogfileMode in ['n','np']:
215        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
216    m = self.policyBatchLogfileMode[0]
217    if m == 'n':
218      m = 'w'
219    if m == 'a':
220      self.hdlr = logging.FileHandler(self.batchLogfile)
221    else:
222      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
223    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
224    self.hdlr.setFormatter(formatter)
225    self.logger.setLevel(logging.INFO)
226    self.logger.addHandler(self.hdlr)
227
228    self.attributeMappings = []
229    self.attributeMappingsLog = None
230    if self.attributeMappingFile != None:
231      for l in open( self.attributeMappingFile ).readlines():
232        if l[0] != '#':
233          bb = string.split( string.strip(l), '|' ) 
234          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
235          bits = string.split( bb[0], ';' )
236          cl = []
237          for b in bits:
238            cl.append( string.split(b, '=' ) )
239          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
240      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
241
242  def commandHints(self, args):
243    if args[0] in ['-h','--sum']:
244      print 'Arguments look OK'
245    elif args[0] == '--copy-config':
246      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
247    else:
248      if not( '-f' in args or '-d' in args or '-D' in args):
249        print 'No file or target directory specified'
250        print __doc__
251    raise SystemExit(0)
252   
253
254  def getFileLog( self, fn, flf=None ):
255    if flf == None:
256      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
257      if fn in self.dupDict.keys():
258        tag = '__%2.2i' % self.dupDict[fn]
259        self.dupDict[fn] += 1
260      else:
261        tag = ''
262      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
263      if self.policyFileLogfileMode in ['n','np']:
264        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
265      m = self.policyFileLogfileMode[0]
266      if m == 'n':
267        m = 'w'
268    else:
269      m = 'a'
270      self.fileLogfile = flf
271
272    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
273    if m == 'a':
274      self.fHdlr = logging.FileHandler(self.fileLogfile)
275    else:
276      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
277    fileFormatter = logging.Formatter('%(message)s')
278    self.fHdlr.setFormatter(fileFormatter)
279    self.fLogger.addHandler(self.fHdlr)
280    self.fLogger.setLevel(logging.INFO)
281    return self.fLogger
282
283  def closeFileLog(self):
284    self.fLogger.removeHandler(self.fHdlr)
285    self.fHdlr.close()
286    if self.policyFileLogfileMode in ['wo','np']:
287      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
288
289  def closeBatchLog(self):
290    self.logger.removeHandler(self.hdlr)
291    self.hdlr.close()
292    if self.policyBatchLogfileMode in ['wo','np']:
293      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
Note: See TracBrowser for help on using the repository browser.