source: FCC/run_qc.py @ 38

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/FCC/run_qc.py@38
Revision 38, 4.9 KB checked in by mjuckes, 7 years ago (diff)

add constant constraint

  • Property svn:executable set to *
Line 
1#!/usr/bin/python
2
3import os, string, glob, shelve, time, sys
4import stat, re, logging, getopt
5import qc_utils
6
7args = getopt.getopt( sys.argv[1:], 'c:b:d:l:q', ['R','repeatable',
8           'noLinksOnly','linksOnly','omit-infile-checks','oic','uc=','user-config=',
9           'no-log-passes','nlp','exception-on-fail', 'ef', 'append-log'] )
10##args = getopt.getopt( sys.argv[1:], 'c:b:d:l:q', ['no-log-passes','nlp','exception-on-fail', 'ef', 'omit-infile-checks','oic','with-infile-checks','wic','append-log','omitVars='] )
11
12deferredMessages = []
13base = '/badc/cmip5/data/cmip5/'
14thisDir = 'output1/MOHC/HadGEM2-ES/rcp45/day/atmos/'
15appendLog = False
16logFilename = None
17quietMode = False
18linksOnly=True
19linksOnly=False
20inFileChecks=True
21configFile = 'config_cmip5.txt'
22exceptionOnFail = False
23logPasses = True
24repeatable = False
25forceInFileChecks=False
26userConfigFile=None
27
28for a in args[0]:
29    neat=0
30    if a[0] == '-b':
31      base = a[1]
32    elif a[0] == '-c':
33      configFile = a[1]
34    elif a[0] == '-d':
35      thisDir = a[1]
36    elif a[0] == '-q':
37      quietMode = True
38    elif a[0] == '-l':
39      logFilename = a[1]
40    elif a[0] in ['--user-config','--uc']:
41      userConfigFile=a[1]
42      userConfigFileArgUsed =a[0]
43    elif a[0] in ['--omit-infile-checks','--oic']:
44      inFileChecks=False
45    elif a[0] in ['--exception-on-fail','--ef']:
46      exceptionOnFail = True
47    elif a[0] in ['--no-log-passes','--nlp']:
48      logPasses = False
49    elif a[0] in ['--repeatable','--R']:
50      repeatable = True
51    elif a[0] == '--append-log':
52      appendLog = True
53    elif a[0] == '--noLinksOnly':
54      linksOnly=False
55    elif a[0] == '--linksOnly':
56      linksOnly=True
57
58if logFilename == None:
59  logFilename = string.replace( string.strip(thisDir,'/'), '/', '_' ) + '.log'
60
61if userConfigFile != None:
62  assert os.path.isfile( userConfigFile ),'User configuration file %s (given by %s=) not found' % (userConfigFile,userConfigFileArgUsed)
63
64
65appendingLog = os.path.isfile( logFilename ) and appendLog
66if os.path.isfile( logFilename ) and not appendLog:
67  os.rename( logFilename, logFilename + '_bck' )
68  if not repeatable:
69    deferredMessages.append( 'Previous log file moved to %s_bck' % logFilename )
70
71assert os.path.isdir( base ), '%s is not a directory [use "-b" argument] ' % base
72baseDir = base + thisDir
73scriptVersion = 0.5
74scriptName = 'run_qc'
75
76log = logging.getLogger( __name__ )
77fh = logging.FileHandler( logFilename )
78log.level = logging.INFO
79log.addHandler( fh )
80creation_date = time.ctime()
81if appendingLog:
82  log.info( '#############################################################' )
83log.info( 'Starting %s script, version %5.2f' % (scriptName, scriptVersion) )
84if not repeatable:
85  log.info( 'Date: %s' % creation_date )
86log.info( 'Scanning: %s' % baseDir )
87for m in deferredMessages:
88  log.info( m )
89
90
91assert os.path.isdir( baseDir ), '%s is not a directory ' % baseDir
92             
93qccp = qc_utils.qcConfigParse( configFile )
94qccp.parse_l0()
95qccp.parse_l1()
96if userConfigFile != None:
97  qccp.open( userConfigFile )
98  qccp.parse_l0()
99  qccp.parse_l1()
100
101pi =  qc_utils.dirParser( qccp, linksOnly=linksOnly )
102
103handler = qc_utils.qcHandler( qccp, log, baseDir, logPasses=logPasses )
104
105os.path.walk( baseDir, pi.parse, handler )
106
107k0 = handler.datasets.keys()[0]
108
109handler._log( 'CQC.200.003.001', 'ALL', str( pi.nclevs ), ok=len( pi.nclevs )<= 1 )
110
111##
112## roles of groups and datasets currently overlap -- this needs to be cleared up.
113dsp = qc_utils.dataSetParser(qccp,log,handler)
114dskeys = handler.datasets.keys()
115dskeys.sort()
116##gkeys = handler.groups.keys()
117##gkeys.sort()
118
119nofail = True
120noerror = True
121
122ifcg = qccp._parse_l1.gc.get( 'inFileCheckGroup', None )
123##print dskeys
124log.info( 'Analysing by dataset classes: %s' % str( dskeys ) )
125for dsclass in dskeys:
126  ifc = inFileChecks and ifcg == dsclass
127  fifc = forceInFileChecks and ifc
128  ##print dsclass, ifcg == dsclass
129  fks = handler.datasets[dsclass].keys()
130  ##print '%s:: %s' % (dsclass, len(fks) )
131  fks.sort()
132  log.info( 'Parsing %s "%s" datasets' % (len(fks),dsclass) )
133  for k in fks:
134    dsp.parse( dsclass, k, handler.datasets[dsclass][k], inFileChecks=ifc, forceInFileChecks=fifc )
135    noerror &= handler.noerror
136    if handler.nofail:
137      if logPasses:
138        log.info( '%s::%s: All tests passed' % (dsclass,k) )
139    else:
140      nofail = False
141      log.info( '%s::%s: test(s) failed' % (dsclass,k) )
142
143log.info( 'Parsed %s NetCDF files' % pi.count_nc )
144keys =  handler.msgk.keys()
145keys.sort()
146for k in keys:
147  if handler.msgk[k] >0:
148    log.info( '%s: %s: %s' % (k,handler.msg[k], handler.msgk[k]))
149
150cj = { True:'appended to', False:'in' }[appendingLog]
151ca = { True:'NO TESTS FAILED', False:'TEST(S) FAILED' }[nofail]
152if not repeatable:
153  log.info( 'Date: %s' % time.ctime() )
154  endNote = 'SCRIPT COMPLETED:: %s:: %s:: log %s %s' % (time.ctime(),ca,cj,logFilename)
155else:
156  endNote = 'SCRIPT COMPLETED:: %s' % ca
157
158if not quietMode:
159  print endNote
160log.info( endNote )
161fh.close()
162
163if not nofail:
164   assert not exceptionOnFail, 'Tests failed and exceptionOnFail True'
Note: See TracBrowser for help on using the repository browser.