source: FCC/run_qc.py @ 29

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/FCC/run_qc.py@29
Revision 29, 4.9 KB checked in by mjuckes, 7 years ago (diff)

updated README

  • Property svn:executable set to *
Line 
1#!/usr/bin/python
2
3import os, string, glob, shelve, time, sys
4import stat, re, logging, getopt
5import qc_utils
6
7args = getopt.getopt( sys.argv[1:], 'c:b:d:l:q', ['R','repeatable',
8           'noLinksOnly','linksOnly','omit-infile-checks','oic','uc=','user-config=',
9           'no-log-passes','nlp','exception-on-fail', 'ef', 'append-log'] )
10##args = getopt.getopt( sys.argv[1:], 'c:b:d:l:q', ['no-log-passes','nlp','exception-on-fail', 'ef', 'omit-infile-checks','oic','with-infile-checks','wic','append-log','omitVars='] )
11
12deferredMessages = []
13base = '/badc/cmip5/data/cmip5/'
14thisDir = 'output1/MOHC/HadGEM2-ES/rcp45/day/atmos/'
15appendLog = False
16logFilename = None
17quietMode = False
18linksOnly=True
19linksOnly=False
20inFileChecks=True
21configFile = 'config_cmip5.txt'
22exceptionOnFail = False
23logPasses = True
24repeatable = False
25forceInFileChecks=False
26userConfigFile=None
27
28for a in args[0]:
29    neat=0
30    if a[0] == '-b':
31      base = a[1]
32    elif a[0] == '-c':
33      configFile = a[1]
34    elif a[0] == '-d':
35      thisDir = a[1]
36    elif a[0] == '-q':
37      quietMode = True
38    elif a[0] == '-l':
39      logFilename = a[1]
40    elif a[0] in ['--user-config','--uc']:
41      userConfigFile=a[1]
42      userConfigFileArgUsed =a[0]
43    elif a[0] in ['--omit-infile-checks','--oic']:
44      inFileChecks=False
45    elif a[0] in ['--exception-on-fail','--ef']:
46      exceptionOnFail = True
47    elif a[0] in ['--no-log-passes','--nlp']:
48      logPasses = False
49    elif a[0] in ['--repeatable','--R']:
50      repeatable = True
51    elif a[0] == '--append-log':
52      appendLog = True
53    elif a[0] == '--noLinksOnly':
54      linksOnly=False
55    elif a[0] == '--linksOnly':
56      linksOnly=True
57
58if logFilename == None:
59  logFilename = string.replace( string.strip(thisDir,'/'), '/', '_' ) + '.log'
60
61if userConfigFile != None:
62  assert os.path.isfile( userConfigFile ),'User configuration file %s (given by %s=) not found' % (userConfigFile,userConfigFileArgUsed)
63
64
65appendingLog = os.path.isfile( logFilename ) and appendLog
66if os.path.isfile( logFilename ) and not appendLog:
67  os.rename( logFilename, logFilename + '_bck' )
68  if not repeatable:
69    deferredMessages.append( 'Previous log file moved to %s_bck' % logFilename )
70
71assert os.path.isdir( base ), '%s is not a directory [use "-b" argument] ' % base
72baseDir = base + thisDir
73scriptVersion = 0.5
74scriptName = 'run_qc'
75
76log = logging.getLogger( __name__ )
77fh = logging.FileHandler( logFilename )
78log.level = logging.INFO
79log.addHandler( fh )
80creation_date = time.ctime()
81if appendingLog:
82  log.info( '#############################################################' )
83log.info( 'Starting %s script, version %5.2f' % (scriptName, scriptVersion) )
84if not repeatable:
85  log.info( 'Date: %s' % creation_date )
86log.info( 'Scanning: %s' % baseDir )
87for m in deferredMessages:
88  log.info( m )
89
90
91assert os.path.isdir( baseDir ), '%s is not a directory ' % baseDir
92             
93qccp = qc_utils.qcConfigParse( configFile )
94qccp.parse_l0()
95qccp.parse_l1()
96if userConfigFile != None:
97  qccp.open( userConfigFile )
98  qccp.parse_l0()
99  qccp.parse_l1()
100
101pi =  qc_utils.dirParser( qccp, linksOnly=linksOnly )
102
103handler = qc_utils.qcHandler( qccp, log, baseDir, logPasses=logPasses )
104
105os.path.walk( baseDir, pi.parse, handler )
106
107k0 = handler.datasets.keys()[0]
108
109handler._log( 'CQC.200.003.001', 'ALL', str( pi.nclevs ), ok=len( pi.nclevs )<= 1 )
110
111##
112## roles of groups and datasets currently overlap -- this needs to be cleared up.
113dsp = qc_utils.dataSetParser(qccp,log,handler)
114dskeys = handler.datasets.keys()
115dskeys.sort()
116gkeys = handler.groups.keys()
117gkeys.sort()
118
119nofail = True
120noerror = True
121
122ifcg = qccp._parse_l1.gc.get( 'inFileCheckGroup', None )
123##print dskeys
124for dsclass in dskeys:
125  ifc = inFileChecks and ifcg == dsclass
126  fifc = forceInFileChecks and ifc
127  ##print dsclass, ifcg == dsclass
128  fks = handler.datasets[dsclass].keys()
129  ##print '%s:: %s' % (dsclass, len(fks) )
130  fks.sort()
131  log.info( 'Parsing %s "%s" datasets' % (len(fks),dsclass) )
132  for k in fks:
133    dsp.parse( dsclass, k, handler.datasets[dsclass][k], inFileChecks=ifc, forceInFileChecks=fifc )
134    noerror &= handler.noerror
135    if handler.nofail:
136      if logPasses:
137        log.info( '%s::%s: All tests passed' % (dsclass,k) )
138    else:
139      nofail = False
140      log.info( '%s::%s: test(s) failed' % (dsclass,k) )
141
142log.info( 'Parsed %s NetCDF files' % pi.count_nc )
143keys =  handler.msgk.keys()
144keys.sort()
145for k in keys:
146  if handler.msgk[k] >0:
147    log.info( '%s: %s: %s' % (k,handler.msg[k], handler.msgk[k]))
148
149cj = { True:'appended to', False:'in' }[appendingLog]
150ca = { True:'NO TESTS FAILED', False:'TEST(S) FAILED' }[nofail]
151if not repeatable:
152  log.info( 'Date: %s' % time.ctime() )
153  endNote = 'SCRIPT COMPLETED:: %s:: %s:: log %s %s' % (time.ctime(),ca,cj,logFilename)
154else:
155  endNote = 'SCRIPT COMPLETED:: %s' % ca
156
157if not quietMode:
158  print endNote
159log.info( endNote )
160fh.close()
161
162if not nofail:
163   assert not exceptionOnFail, 'Tests failed and exceptionOnFail True'
Note: See TracBrowser for help on using the repository browser.