source: CCCC/trunk/ceda_cc/ccinit.py @ 327

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/ccinit.py@327
Revision 327, 10.0 KB checked in by mjuckes, 6 years ago (diff)

Additional documentation; missing import added in c4.py

Line 
1"""
2Arguments:
3  :-p: project
4  :--copy-config: <target directory path>: copy configuration files to target directory path;
5  :--sum: <log file directory> create a summary of the results  logged in the specified directory.
6  :-f: <file>: check a single file;
7  :-d: <directory>: check all the NetCDF files in a directory;
8  :-D: <directory>: check all the NetCDF files in a directory tree (or latest versions if a recognised version managment system is defined for the project);
9  :--ld: <log file directory>:  ## directory to take log files;
10  :-R: <record file name>: file name for file to take one record per file checked;
11  :--cae: "catch all errors": will trap exceptions and record in  log files, and then continue. Default is to stop after unrecognised exceptions.
12  :--log: <single|multi>:  Set log file management option -- see "Single log" and "Multi-log" below.
13  :--blfmode: <mode>:    Set mode for batch log file -- see log file modes
14  :--flfmode: <mode>:  Set mode for file-level log file -- see log file modes
15  :--aMap:             Read in some attribute mappings and run tests with virtual substitutions;
16
17Log file modes
18--------------
19
20Valid modes are: 'a': append
21                 'n', 'np': new file, 'np': protect after closing (mode = 444)
22                 'w', 'wo': write (overwrite if present), 'wo': protect after closing (mode = 444)
23
24Note that the log files generated in multi-log mode will re-use file names. If running with --flfmode set to 'n','np' or 'wo' it will be necessary to change or clear the target directory. The names of batch log files include the time, to the nearest second, when the process is started, so will not generally suffer from re-use.
25"""
26
27"""
28ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>] [other options]
29
30With the "-D" option, all files in the directory tree beneath the given diretory will be checked. With the "-d" option, only files in the given directory will be checked.
31"""
32import sys, os, string, time, logging
33
34class c4_init(object):
35
36  def __init__(self,args=None):
37    self.logByFile = True
38    self.policyFileLogfileMode = 'w'
39    self.policyBatchLogfileMode = 'np'
40    if args==None:
41       args = sys.argv[1:]
42    nn = 0
43
44    self.attributeMappingFile = None
45    self.recordFile = 'Rec.txt'
46    self.logDir = 'logs_02'
47    self.errs = []
48   
49    # Set default project to "CORDEX"
50    self.project = "CORDEX"
51    self.holdExceptions = False
52    forceLogOrg = None
53    argsIn = args[:]
54   
55    # Show help if no args or help requested
56    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
57       print __doc__
58       raise SystemExit(0)     
59    # The --copy-config option must be the first argument if it is present.
60    elif args[0] == '--copy-config':
61       if len(args) < 2:
62         self.commandHints( argsIn )
63       args.pop(0)
64       dest_dir = args.pop(0)
65       config.copy_config(dest_dir)
66       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
67       print
68       raise SystemExit(0)
69
70    self.summarymode = args[0] == '--sum'
71    if self.summarymode:
72      return
73
74    self.experimental = False
75    self.forceNetcdfLib = None
76    fltype = None
77    argu = []
78    while len(args) > 0:
79      next = args.pop(0)
80      if next == '-f':
81        flist = [args.pop(0),]
82        self.logByFile = False
83        fltype = '-f'
84        self.source = flist[0]
85      elif next == '--log':
86        x = args.pop(0)
87        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
88        if x in ['multi','m']:
89           forceLogOrg = 'multi'
90        elif x in ['single','s']:
91           forceLogOrg = 'single'
92      elif next == '--experimental':
93        self.experimental = True
94      elif next == '--force-ncq':
95        self.forceNetcdfLib = 'ncq3'
96      elif next == '--force-cdms2':
97        self.forceNetcdfLib = 'cdms2'
98      elif next == '--force-pync4':
99        self.forceNetcdfLib = 'netCDF4'
100      elif next == '--force-scientific':
101        self.forceNetcdfLib = 'Scientific'
102      elif next == '--flfmode':
103        lfmk = args.pop(0)
104        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
105        self.policyFileLogfileMode = lfmk
106      elif next == '--blfmode':
107        lfmk = args.pop(0)
108        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
109        self.policyBatchLogfileMode = lfmk
110      elif next == '-d':
111        fdir = args.pop(0)
112        flist = glob.glob( '%s/*.nc' % fdir  )
113        self.source = '%s/*.nc' % fdir
114      elif next == '-D':
115        flist  = []
116        fdir = args.pop(0)
117        for root, dirs, files in os.walk( fdir, followlinks=True ):
118          for f in files:
119            fpath = '%s/%s' % (root,f)
120            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
121              flist.append( fpath )
122        self.source = '%s/.....' % fdir
123      elif next == '-R':
124        self.recordFile = args.pop(0)
125      elif next == '--ld':
126        self.logDir = args.pop(0)
127      elif next in ['--catchAllExceptions','--cae']:
128        self.holdExceptions = True
129      elif next == '--aMap':
130        self.attributeMappingFile = args.pop(0)
131        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
132      elif next == "-p":
133        self.project = args.pop(0)
134      else:
135       print 'Unused argument: %s' % next
136       argu.append( next )
137       nn+=1
138    if nn != 0:
139      print 'Unused arguments: ', argu
140      self.commandHints( argsIn )
141
142    if self.project == 'CMIP5' and fltype != '-f':
143      fl0 = []
144      for f in flist:
145        if string.find( f, '/latest/' ) != -1:
146          fl0.append(f)
147      flist = fl0
148
149    if forceLogOrg != None:
150      if forceLogOrg == 'single':
151        self.logByFile = False
152      else:
153        self.logByFile = True
154
155    if self.project[:2] == '__':
156       self.source = 'dummy'
157       flist = []
158       ss = 'abcdefgijk'
159       ss = 'abcdefgijklmnopqrstuvwxyz'
160       ss = 'abc'
161       for i in range(10):
162         v = 'v%s' % i
163         for a in ss:
164           for b in ss:
165             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
166    flist.sort()
167    fnl = []
168    for f in flist:
169      fn = string.split(f, '/')[-1]
170      fnl.append(fn)
171    nd = 0
172    dupl = []
173    for k in range(len(fnl)-1):
174      if fnl[k] == fnl[k-1]:
175        nd += 1
176        dupl.append( fnl[k] )
177    self.dupDict = {}
178    for f in dupl:
179      self.dupDict[f] = 0
180    if nd != 0:
181      self.errs.append( 'Duplicate file names encountered: %s' % nd )
182      self.errs.append( dupl )
183    self.flist = flist
184    self.fnl = fnl
185    if not os.path.isdir(   self.logDir ):
186       os.mkdir(   self.logDir )
187
188    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
189    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
190## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
191    self.logger = logging.getLogger('c4logger')
192    if self.policyBatchLogfileMode in ['n','np']:
193        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
194    m = self.policyBatchLogfileMode[0]
195    if m == 'n':
196      m = 'w'
197    if m == 'a':
198      self.hdlr = logging.FileHandler(self.batchLogfile)
199    else:
200      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
201    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
202    self.hdlr.setFormatter(formatter)
203    self.logger.setLevel(logging.INFO)
204    self.logger.addHandler(self.hdlr)
205
206    self.attributeMappings = []
207    self.attributeMappingsLog = None
208    if self.attributeMappingFile != None:
209      for l in open( self.attributeMappingFile ).readlines():
210        if l[0] != '#':
211          bb = string.split( string.strip(l), '|' ) 
212          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
213          bits = string.split( bb[0], ';' )
214          cl = []
215          for b in bits:
216            cl.append( string.split(b, '=' ) )
217          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
218      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
219
220  def commandHints(self, args):
221    if args[0] in ['-h','--sum']:
222      print 'Arguments look OK'
223    elif args[0] == '--copy-config':
224      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
225    else:
226      if not( '-f' in args or '-d' in args or '-D' in args):
227        print 'No file or target directory specified'
228        print __doc__
229    raise SystemExit(0)
230   
231
232  def getFileLog( self, fn, flf=None ):
233    if flf == None:
234      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
235      if fn in self.dupDict.keys():
236        tag = '__%2.2i' % self.dupDict[fn]
237        self.dupDict[fn] += 1
238      else:
239        tag = ''
240      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
241      if self.policyFileLogfileMode in ['n','np']:
242        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
243      m = self.policyFileLogfileMode[0]
244      if m == 'n':
245        m = 'w'
246    else:
247      m = 'a'
248      self.fileLogfile = flf
249
250    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
251    if m == 'a':
252      self.fHdlr = logging.FileHandler(self.fileLogfile)
253    else:
254      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
255    fileFormatter = logging.Formatter('%(message)s')
256    self.fHdlr.setFormatter(fileFormatter)
257    self.fLogger.addHandler(self.fHdlr)
258    self.fLogger.setLevel(logging.INFO)
259    return self.fLogger
260
261  def closeFileLog(self):
262    self.fLogger.removeHandler(self.fHdlr)
263    self.fHdlr.close()
264    if self.policyFileLogfileMode in ['wo','np']:
265      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
266
267  def closeBatchLog(self):
268    self.logger.removeHandler(self.hdlr)
269    self.hdlr.close()
270    if self.policyBatchLogfileMode in ['wo','np']:
271      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
Note: See TracBrowser for help on using the repository browser.