source: CCCC/trunk/ceda_cc/ccinit.py @ 342

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CCCC/trunk/ceda_cc/ccinit.py@342
Revision 342, 10.5 KB checked in by astephen, 5 years ago (diff)

Added import of 'glob' module to overcome ImportError?.

Line 
1"""
2Arguments:
3  :-p: project
4  :--copy-config: <target directory path>: copy configuration files to target directory path;
5  :--sum: <log file directory> create a summary of the results  logged in the specified directory.
6  :-f: <file>: check a single file;
7  :-d: <directory>: check all the NetCDF files in a directory;
8  :-D: <directory>: check all the NetCDF files in a directory tree (or latest versions if a recognised version managment system is defined for the project);
9  :--ld: <log file directory>:  ## directory to take log files;
10  :-R: <record file name>: file name for file to take one record per file checked;
11  :-l: <file list file>: a file containing one data file to check per line;
12  :--cae: "catch all errors": will trap exceptions and record in  log files, and then continue. Default is to stop after unrecognised exceptions.
13  :--log: <single|multi>:  Set log file management option -- see "Single log" and "Multi-log" below.
14  :--blfmode: <mode>:    Set mode for batch log file -- see log file modes
15  :--blfms:            Set milli-second mode for naming batch log files (instead of second mode);
16  :--flfmode: <mode>:  Set mode for file-level log file -- see log file modes
17  :--aMap:             Read in some attribute mappings and run tests with virtual substitutions;
18
19Log file modes
20--------------
21
22Valid modes are: 'a': append
23                 'n', 'np': new file, 'np': protect after closing (mode = 444)
24                 'w', 'wo': write (overwrite if present), 'wo': protect after closing (mode = 444)
25
26Note that the log files generated in multi-log mode will re-use file names. If running with --flfmode set to 'n','np' or 'wo' it will be necessary to change or clear the target directory. The names of batch log files include the time, to the nearest second, when the process is started, so will not generally suffer from re-use.
27"""
28
29"""
30ceda_cc -p <project> [-f <NetCDF file>|-d <directory containing files>|-D <root of directory tree>|-l <file list file>] [other options]
31
32With the "-D" option, all files in the directory tree beneath the given directory will be checked. With the "-d" option, only files in the given directory will be checked.
33"""
34import sys, os, string, time, logging, glob
35import config_c4
36
37class c4_init(object):
38
39  def __init__(self,args=None):
40    self.logByFile = True
41    self.policyFileLogfileMode = 'w'
42    self.policyBatchLogfileMode = 'np'
43    self.policyBatchLogfileMs = False
44    if args==None:
45       args = sys.argv[1:]
46    nn = 0
47
48    self.attributeMappingFile = None
49    self.recordFile = 'Rec.txt'
50    self.logDir = 'logs_02'
51    self.errs = []
52   
53    # Set default project to "CORDEX"
54    self.project = "CORDEX"
55    self.holdExceptions = False
56    forceLogOrg = None
57    argsIn = args[:]
58   
59    # Show help if no args or help requested
60    if len(args) == 0 or args[0] in ('-h', '-help', '--help'):
61       print __doc__
62       raise SystemExit(0)     
63    # The --copy-config option must be the first argument if it is present.
64    elif args[0] == '--copy-config':
65       if len(args) < 2:
66         self.commandHints( argsIn )
67       args.pop(0)
68       dest_dir = args.pop(0)
69       config_c4.copy_config(dest_dir)
70       print 'Configuration directory copied to %s.  Set CC_CONFIG_DIR to use this configuration.' % dest_dir
71       print
72       raise SystemExit(0)
73
74    self.summarymode = args[0] == '--sum'
75    if self.summarymode:
76      return
77
78    self.experimental = False
79    self.forceNetcdfLib = None
80    fltype = None
81    argu = []
82    while len(args) > 0:
83      next = args.pop(0)
84      if next == '-f':
85        flist = [args.pop(0),]
86        self.logByFile = False
87        fltype = '-f'
88        self.source = flist[0]
89      elif next == '--log':
90        x = args.pop(0)
91        assert x in ['single','multi','s','m'], 'unrecognised logging option (--log): %s' % (x)
92        if x in ['multi','m']:
93           forceLogOrg = 'multi'
94        elif x in ['single','s']:
95           forceLogOrg = 'single'
96      elif next == '--experimental':
97        self.experimental = True
98      elif next == '--force-ncq':
99        self.forceNetcdfLib = 'ncq3'
100      elif next == '--force-cdms2':
101        self.forceNetcdfLib = 'cdms2'
102      elif next == '--force-pync4':
103        self.forceNetcdfLib = 'netCDF4'
104      elif next == '--force-scientific':
105        self.forceNetcdfLib = 'Scientific'
106      elif next == '--flfmode':
107        lfmk = args.pop(0)
108        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised file logfile mode (--flfmode): %s' % lfmk
109        self.policyFileLogfileMode = lfmk
110      elif next == '--blfmode':
111        lfmk = args.pop(0)
112        assert lfmk in ['a','n','np','w','wo'], 'Unrecognised batch logfile mode (--blfmode): %s' % lfmk
113        self.policyBatchLogfileMode = lfmk
114      elif next == '--blfms':
115        self.policyBatchLogfileMs = False
116      elif next == '-d':
117        fdir = args.pop(0)
118        flist = glob.glob( '%s/*.nc' % fdir  )
119        self.source = '%s/*.nc' % fdir
120      elif next == '-D':
121        flist  = []
122        fdir = args.pop(0)
123        for root, dirs, files in os.walk( fdir, followlinks=True ):
124          for f in files:
125            fpath = '%s/%s' % (root,f)
126            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc':
127              flist.append( fpath )
128        self.source = '%s/.....' % fdir
129      elif next == '-l':
130        flist = open(args.pop(0)).read().split()
131        self.source = flist[0]
132      elif next == '-R':
133        self.recordFile = args.pop(0)
134      elif next == '--ld':
135        self.logDir = args.pop(0)
136      elif next in ['--catchAllExceptions','--cae']:
137        self.holdExceptions = True
138      elif next == '--aMap':
139        self.attributeMappingFile = args.pop(0)
140        assert os.path.isfile( self.attributeMappingFile ), 'The token "--aMap" should be followed by the path or name of a file'
141      elif next == "-p":
142        self.project = args.pop(0)
143      else:
144       print 'Unused argument: %s' % next
145       argu.append( next )
146       nn+=1
147    if nn != 0:
148      print 'Unused arguments: ', argu
149      self.commandHints( argsIn )
150
151    if self.project == 'CMIP5' and fltype != '-f':
152      fl0 = []
153      for f in flist:
154        if string.find( f, '/latest/' ) != -1:
155          fl0.append(f)
156      flist = fl0
157
158    if forceLogOrg != None:
159      if forceLogOrg == 'single':
160        self.logByFile = False
161      else:
162        self.logByFile = True
163
164    if self.project[:2] == '__':
165       self.source = 'dummy'
166       flist = []
167       ss = 'abcdefgijk'
168       ss = 'abcdefgijklmnopqrstuvwxyz'
169       ss = 'abc'
170       for i in range(10):
171         v = 'v%s' % i
172         for a in ss:
173           for b in ss:
174             flist.append( '%s_day_%s_%s_1900-1909.nc' % (v,a,b) )
175    flist.sort()
176    fnl = []
177    for f in flist:
178      fn = string.split(f, '/')[-1]
179      fnl.append(fn)
180    nd = 0
181    dupl = []
182    for k in range(len(fnl)-1):
183      if fnl[k] == fnl[k-1]:
184        nd += 1
185        dupl.append( fnl[k] )
186    self.dupDict = {}
187    for f in dupl:
188      self.dupDict[f] = 0
189    if nd != 0:
190      self.errs.append( 'Duplicate file names encountered: %s' % nd )
191      self.errs.append( dupl )
192    self.flist = flist
193    self.fnl = fnl
194    if not os.path.isdir(   self.logDir ):
195       os.mkdir(   self.logDir )
196
197    tstring1 = '%4.4i%2.2i%2.2i_%2.2i%2.2i%2.2i' % time.gmtime()[0:6]
198    if self.policyBatchLogfileMs:
199      t = time.time()
200      ms = int( ( t-int(t) )*1000. )
201      tsring1 += '.%3.3i' % ms
202    self.batchLogfile = '%s/qcBatchLog_%s.txt' % (  self.logDir,tstring1)
203## default appending to myapp.log; mode='w' forces a new file (deleting old contents).
204    self.logger = logging.getLogger('c4logger')
205    if self.policyBatchLogfileMode in ['n','np']:
206        assert not os.path.isfile( self.batchLogfile ), '%s exists and policy set to new file' % self.batchLogfile
207    m = self.policyBatchLogfileMode[0]
208    if m == 'n':
209      m = 'w'
210    if m == 'a':
211      self.hdlr = logging.FileHandler(self.batchLogfile)
212    else:
213      self.hdlr = logging.FileHandler(self.batchLogfile,mode=m)
214    formatter = logging.Formatter('%(asctime)s %(levelname)s %(message)s')
215    self.hdlr.setFormatter(formatter)
216    self.logger.setLevel(logging.INFO)
217    self.logger.addHandler(self.hdlr)
218
219    self.attributeMappings = []
220    self.attributeMappingsLog = None
221    if self.attributeMappingFile != None:
222      for l in open( self.attributeMappingFile ).readlines():
223        if l[0] != '#':
224          bb = string.split( string.strip(l), '|' ) 
225          assert len(bb) ==2, "Error in experimental module attributeMapping -- configuration line not scanned [%s]" % str(l)
226          bits = string.split( bb[0], ';' )
227          cl = []
228          for b in bits:
229            cl.append( string.split(b, '=' ) )
230          self.attributeMappings.append( ('am001',cl, string.split(bb[1],'=') ) )
231      self.attributeMappingsLog = open( 'attributeMappingsLog.txt', 'w' )
232
233  def commandHints(self, args):
234    if args[0] in ['-h','--sum']:
235      print 'Arguments look OK'
236    elif args[0] == '--copy-config':
237      print 'Usage [configuration copy]: ceda_cc --copy-config <target directory path>'
238    else:
239      if not( '-f' in args or '-d' in args or '-D' in args):
240        print 'No file or target directory specified'
241        print __doc__
242    raise SystemExit(0)
243   
244
245  def getFileLog( self, fn, flf=None ):
246    if flf == None:
247      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3]
248      if fn in self.dupDict.keys():
249        tag = '__%2.2i' % self.dupDict[fn]
250        self.dupDict[fn] += 1
251      else:
252        tag = ''
253      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2)
254      if self.policyFileLogfileMode in ['n','np']:
255        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile
256      m = self.policyFileLogfileMode[0]
257      if m == 'n':
258        m = 'w'
259    else:
260      m = 'a'
261      self.fileLogfile = flf
262
263    self.fLogger = logging.getLogger('fileLog_%s_%s' % (fn,m))
264    if m == 'a':
265      self.fHdlr = logging.FileHandler(self.fileLogfile)
266    else:
267      self.fHdlr = logging.FileHandler(self.fileLogfile,mode=m)
268    fileFormatter = logging.Formatter('%(message)s')
269    self.fHdlr.setFormatter(fileFormatter)
270    self.fLogger.addHandler(self.fHdlr)
271    self.fLogger.setLevel(logging.INFO)
272    return self.fLogger
273
274  def closeFileLog(self):
275    self.fLogger.removeHandler(self.fHdlr)
276    self.fHdlr.close()
277    if self.policyFileLogfileMode in ['wo','np']:
278      os.popen( 'chmod %s %s;' % (444, self.fileLogfile) )
279
280  def closeBatchLog(self):
281    self.logger.removeHandler(self.hdlr)
282    self.hdlr.close()
283    if self.policyBatchLogfileMode in ['wo','np']:
284      os.popen( 'chmod %s %s;' % (444, self.batchLogfile) )
Note: See TracBrowser for help on using the repository browser.