1 | """A set of classes running checks and providing utilities to support checks""" |
---|
2 | import string, re, os, sys, traceback, ctypes |
---|
3 | |
---|
4 | def strmm3( mm ): |
---|
5 | return string.join( map( lambda x: '%s="%s" [correct: "%s"]' % x, mm ), '; ' ) |
---|
6 | |
---|
7 | from fcc_utils import mipTableScan |
---|
8 | from xceptions import * |
---|
9 | |
---|
10 | class timeInt(object): |
---|
11 | |
---|
12 | vc = {'gregorian':0, 'standard':0, 'proleptic_gregorian':0, 'noleap':1, '365_day':1, 'all_leap':2, '366_day':2, '360_day':3, 'julian':0, 'none':None} |
---|
13 | mnmx = [ (365,366),(365,365),(366,366),(360,360) ] |
---|
14 | mmnmmx = [ (28,31),(28,31),(29,31),(30,30) ] |
---|
15 | def __init__(self,cal='proleptic_gregorian',dpymn=None, dpymx=None,dpmmn=None,dpmmx=None,tol=1.e-6): |
---|
16 | self.tol = tol |
---|
17 | if not self.vc.has_key(cal) or cal == None: |
---|
18 | assert dpymx != None and dpymn != None, 'If standard calendar is not use, dpymn and dpymx must be set' |
---|
19 | assert dpmmx != None and dpmmn != None, 'If standard calendar is not use, dpmmn and dpmmx must be set' |
---|
20 | self.dpymn = dpymn - tol |
---|
21 | self.dpymx = dpymx + tol |
---|
22 | self.dpmmn = dpmmn - tol |
---|
23 | self.dpmmx = dpmmx + tol |
---|
24 | else: |
---|
25 | self.dpymn = self.mnmx[self.vc[cal]][0] - tol |
---|
26 | self.dpymx = self.mnmx[self.vc[cal]][1] + tol |
---|
27 | self.dpmmn = self.mmnmmx[self.vc[cal]][0] - tol |
---|
28 | self.dpmmx = self.mmnmmx[self.vc[cal]][1] + tol |
---|
29 | self.map = { 'yr':'P1Y','monClim':'P1M','mon':'P1M','day':'P1D','6hr':'P6H','3hr':'P3H'} |
---|
30 | self.nd = { 'x':'y' } |
---|
31 | |
---|
32 | def setUnit(self,u): |
---|
33 | if u not in ['days','months','years']: |
---|
34 | print 'Time unit %s not supported' % u |
---|
35 | self.u = None |
---|
36 | else: |
---|
37 | self.u = u |
---|
38 | |
---|
39 | def chk(self,v,u,f): |
---|
40 | if not self.map.has_key(f): |
---|
41 | return (0,'No frequency check available for f = %s' % f ) |
---|
42 | if u not in ['days']: |
---|
43 | return (0,'No frequency check available for units = %s' % u ) |
---|
44 | x = self.map(f) |
---|
45 | if x == 'P1Y': |
---|
46 | return (v > self.dpymn) and (v < self.dpymx) |
---|
47 | elif x == 'P1M': |
---|
48 | return (v > self.dpmmn) and (v < self.dpmmx) |
---|
49 | elif x == 'P1D': |
---|
50 | return (v > 1.-self.tol) and (v < 1.+self.tol) |
---|
51 | elif x == 'P6H': |
---|
52 | return (v > 0.25-self.tol) and (v < 0.25+self.tol) |
---|
53 | elif x == 'P3H': |
---|
54 | return (v > 0.125-self.tol) and (v < 0.125+self.tol) |
---|
55 | |
---|
56 | |
---|
57 | class reportSection(object): |
---|
58 | |
---|
59 | def __init__(self,id,cls,parent=None, description=None): |
---|
60 | self.id = id |
---|
61 | self.cls = cls |
---|
62 | self.parent = parent |
---|
63 | self.description = description |
---|
64 | self.records = [] |
---|
65 | self.subsections = [] |
---|
66 | self.closed = False |
---|
67 | self.npass = 0 |
---|
68 | self.fail = 0 |
---|
69 | self.auditDone = True |
---|
70 | |
---|
71 | def addSubSection( self, id, cls, description=None): |
---|
72 | assert not self.closed, 'Attempt to add sub-section to closed report section' |
---|
73 | self.subsections.append( reportSection(id, cls, parent=self, description=description ) ) |
---|
74 | self.auditDone = False |
---|
75 | return self.subsections[-1] |
---|
76 | |
---|
77 | def addRecord( self, id, cls, res, msg ): |
---|
78 | assert not self.closed, 'Attempt to add record to closed report section' |
---|
79 | self.records.append( (id, cls, res, msg) ) |
---|
80 | self.auditDone = False |
---|
81 | |
---|
82 | def close(self): |
---|
83 | self.closed = True |
---|
84 | |
---|
85 | def reopen(self): |
---|
86 | self.closed = False |
---|
87 | |
---|
88 | def audit(self): |
---|
89 | if self.auditDone: |
---|
90 | return |
---|
91 | self.closed = True |
---|
92 | self.fail = 0 |
---|
93 | self.npass = 0 |
---|
94 | for ss in self.subsections: |
---|
95 | ss.audit() |
---|
96 | self.fail += ss.fail |
---|
97 | self.npass += ss.npass |
---|
98 | |
---|
99 | for r in self.records: |
---|
100 | if r[2]: |
---|
101 | self.npass += 1 |
---|
102 | else: |
---|
103 | self.fail += 1 |
---|
104 | |
---|
105 | class checkSeq(object): |
---|
106 | def __init__(self): |
---|
107 | pass |
---|
108 | |
---|
109 | def check(self,x): |
---|
110 | d = map( lambda i: x[i+1] - x[i], range(len(x)-1) ) |
---|
111 | self.delt = sum(d)/len(d) |
---|
112 | self.dmx = max(d) |
---|
113 | self.dmn = min(d) |
---|
114 | return self.dmx - self.dmn < abs(self.delt)*1.e-4 |
---|
115 | |
---|
116 | cs = checkSeq() |
---|
117 | |
---|
118 | class checkBase(object): |
---|
119 | """Base class for checks, containing a set of standard methods for managing operation of checks and logging of results""" |
---|
120 | |
---|
121 | def __init__(self,cls="CORDEX",reportPass=True,parent=None,monitor=None): |
---|
122 | """Creat class instance: set defaults, link arguments to instance, create a range of compiled regular expressions""" |
---|
123 | self.cls = cls |
---|
124 | self.project = cls |
---|
125 | self.abortMessageCount = parent.abortMessageCount |
---|
126 | self.monitor = monitor |
---|
127 | self.re_isInt = re.compile( '[0-9]+' ) |
---|
128 | self.errorCount = 0 |
---|
129 | self.passCount = 0 |
---|
130 | self.missingValue = 1.e20 |
---|
131 | self.missingValue = ctypes.c_float(1.00000002004e+20).value |
---|
132 | from file_utils import ncLib |
---|
133 | if ncLib == 'netCDF4': |
---|
134 | import numpy |
---|
135 | self.missingValue = numpy.float32(self.missingValue) |
---|
136 | self.parent = parent |
---|
137 | self.reportPass=reportPass |
---|
138 | self.pcfg = parent.pcfg |
---|
139 | ################################ |
---|
140 | self.requiredGlobalAttributes = self.pcfg.requiredGlobalAttributes |
---|
141 | self.controlledGlobalAttributes = self.pcfg.controlledGlobalAttributes |
---|
142 | self.globalAttributesInFn = self.pcfg.globalAttributesInFn |
---|
143 | self.requiredVarAttributes = self.pcfg.requiredVarAttributes |
---|
144 | self.drsMappings = self.pcfg.drsMappings |
---|
145 | ####################################### |
---|
146 | self.checks = () |
---|
147 | self.messageCount = 0 |
---|
148 | self.init() |
---|
149 | if not hasattr( self.parent, 'amapListDraft' ): |
---|
150 | self.parent.amapListDraft = [] |
---|
151 | |
---|
152 | def isInt(self,x): |
---|
153 | """Check that a string is a representation of an integer""" |
---|
154 | return self.re_isInt.match( x ) != None |
---|
155 | |
---|
156 | def logMessage(self, msg, error=False ): |
---|
157 | """Log messages and count messages""" |
---|
158 | self.messageCount += 1 |
---|
159 | assert self.abortMessageCount < 0 or self.abortMessageCount > self.messageCount, 'Raising error [TESTX01], perhaps for testing' |
---|
160 | if self.parent != None and self.parent.log != None: |
---|
161 | if error: |
---|
162 | self.parent.log.error( msg ) |
---|
163 | else: |
---|
164 | self.parent.log.info( msg ) |
---|
165 | else: |
---|
166 | print msg |
---|
167 | |
---|
168 | doThis = True |
---|
169 | if self.appendLogfile[0] != None and doThis: |
---|
170 | if self.monitor != None: |
---|
171 | nofh0 = self.monitor.get_open_fds() |
---|
172 | xlog = self.c4i.getFileLog( self.appendLogfile[1], flf=self.appendLogfile[0] ) |
---|
173 | if error: |
---|
174 | xlog.error( msg ) |
---|
175 | else: |
---|
176 | xlog.info( msg ) |
---|
177 | self.c4i.closeFileLog() |
---|
178 | if self.monitor != None: |
---|
179 | nofh9 = self.monitor.get_open_fds() |
---|
180 | if nofh9 > nofh0: |
---|
181 | print 'Leaking file handles [1]: %s --- %s' % (nofh0, nofh9) |
---|
182 | |
---|
183 | def log_exception( self, msg): |
---|
184 | """Logging of exceptions -- putting trace information in log files""" |
---|
185 | if self.parent != None and self.parent.log != None: |
---|
186 | self.parent.log.error("Exception has occured" ,exc_info=1) |
---|
187 | else: |
---|
188 | traceback.print_exc(file=sys.stdout) |
---|
189 | |
---|
190 | def log_error( self, msg ): |
---|
191 | """Create an error log message and call logMessage; count errors;""" |
---|
192 | self.lastError = msg |
---|
193 | self.errorCount += 1 |
---|
194 | self.logMessage( '%s.%s: FAILED:: %s' % (self.id,self.getCheckId(),msg), error=True ) |
---|
195 | |
---|
196 | def log_pass( self ): |
---|
197 | """Create a pass log message and call logMessage; count passes;""" |
---|
198 | self.passCount = True |
---|
199 | if self.reportPass: |
---|
200 | self.logMessage( '%s.%s: OK' % (self.id,self.getCheckId()) ) |
---|
201 | |
---|
202 | def log_abort( self ): |
---|
203 | self.completed = False |
---|
204 | self.logMessage( '%s.%s: ABORTED:: Errors too severe to complete further checks in this module' % (self.id,'xxx') ) |
---|
205 | raise abortChecks |
---|
206 | |
---|
207 | def status(self): |
---|
208 | return '%s.%s' % (self.id,self.getCheckId()) |
---|
209 | |
---|
210 | def getCheckId(self,full=True): |
---|
211 | if type( self.checkId ) == type( 'x' ): |
---|
212 | return self.checkId |
---|
213 | else: |
---|
214 | if full: |
---|
215 | return '%s: [%s]' % self.checkId |
---|
216 | else: |
---|
217 | return self.checkId[0] |
---|
218 | |
---|
219 | def test(self,res,msg,abort=False,part=False,appendLogfile=(None,None)): |
---|
220 | """Handle test results. |
---|
221 | :param res: [True/False] result of test; |
---|
222 | :param msg: Message describing the test; |
---|
223 | :param abort: {optional} Set True if checks should be aborted when test fails; |
---|
224 | :param part: {optional} Set True if this is a component of a test (logging of pass suppressed); |
---|
225 | :param appendLogfile: {optional} Allows results to be appended to pre-existing log file; |
---|
226 | """ |
---|
227 | self.appendLogfile = appendLogfile |
---|
228 | if res: |
---|
229 | if not part: |
---|
230 | self.log_pass() |
---|
231 | else: |
---|
232 | self.log_error(msg) |
---|
233 | if abort: |
---|
234 | self.log_abort() |
---|
235 | return res |
---|
236 | |
---|
237 | def runChecks(self): |
---|
238 | """Run all the checks registered in this instance (in self.checks) and handle exceptions""" |
---|
239 | |
---|
240 | try: |
---|
241 | for c in self.checks: |
---|
242 | c() # run check |
---|
243 | self.completed = True |
---|
244 | except abortChecks: |
---|
245 | ## error logging done before raising this exception |
---|
246 | return |
---|
247 | except: |
---|
248 | self.log_exception( 'Exception caught by runChecks' ) |
---|
249 | raise loggedException |
---|
250 | |
---|
251 | class checkFileName(checkBase): |
---|
252 | """Check basic syntax of file names (i.e. checks properties of the text string, it does not attempt to access the file). |
---|
253 | Inherits :class:`checkBase` class. Checks are run by the :meth:`check` method.""" |
---|
254 | |
---|
255 | def init(self): |
---|
256 | self.id = 'C4.001' |
---|
257 | self.checkId = 'unset' |
---|
258 | self.isFixed = False |
---|
259 | self.step = 'Initialised' |
---|
260 | self.checks = (self.do_check_fn,self.do_check_fnextra) |
---|
261 | self.re_c1 = re.compile( '^[0-9]*$' ) |
---|
262 | self.fnDict = {} |
---|
263 | #### |
---|
264 | |
---|
265 | def check(self,fn): |
---|
266 | """Initiate checks: manage arguments and then call *runChecks* (inherited from checkBase class). |
---|
267 | Arguments: fn: file name: the file name to be checked.""" |
---|
268 | self.errorCount = 0 |
---|
269 | assert type(fn) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(fn) |
---|
270 | self.fn = fn |
---|
271 | self.fnsep = self.pcfg.fNameSep |
---|
272 | |
---|
273 | self.runChecks() |
---|
274 | self.parent.fnDict = self.fnDict |
---|
275 | ### |
---|
276 | def do_check_fn(self): |
---|
277 | """Basic file name checks: |
---|
278 | (1) Check suffix; |
---|
279 | (1b) [for ESA-CCI files] check presence of "ESACCI" and identify file naming convention; |
---|
280 | (2) Split file name into components and check number of such components; |
---|
281 | (3) Additional specialist checks for ESA-CCI, CORDEX, CMIP-type (for the time range). |
---|
282 | """ |
---|
283 | fn = self.fn |
---|
284 | self.errorCount = 0 |
---|
285 | self.completed = False |
---|
286 | |
---|
287 | ## check basic parsing of file name |
---|
288 | self.checkId = ('001','parse_filename') |
---|
289 | self.test( fn[-3:] == '.nc', 'File name ending ".nc" expected', abort=True, part=True ) |
---|
290 | bits = string.split( fn[:-3], self.fnsep ) |
---|
291 | |
---|
292 | self.fnParts = bits[:] |
---|
293 | if self.pcfg.domainIndex != None: |
---|
294 | self.domain = self.fnParts[self.pcfg.domainIndex] |
---|
295 | else: |
---|
296 | self.domain = None |
---|
297 | |
---|
298 | |
---|
299 | if self.pcfg.projectV.id in ['ESA-CCI']: |
---|
300 | self.test( 'ESACCI' in bits[:2] or 'ESA' == bits[0], 'File name not a valid ESA-CCI file name: %s' % fn, abort=True ) |
---|
301 | if bits[0] == 'ESA': |
---|
302 | self.esaFnId = 2 |
---|
303 | elif bits[0] == 'ESACCI': |
---|
304 | self.esaFnId = 1 |
---|
305 | else: |
---|
306 | self.esaFnId = 0 |
---|
307 | bb = string.split( bits[2], '_' ) |
---|
308 | self.test( bits[2][0] == 'L' and len(bb) == 2, 'Cannot parse ESA-CCI file name: %s' % fn, abort=True ) |
---|
309 | bits = bits[:2] + bb + bits[3:] |
---|
310 | self.fnParts = bits[:] |
---|
311 | |
---|
312 | self.pcfg.setEsaCciFNType(self.esaFnId) |
---|
313 | self.test( len(bits) in self.pcfg.fnPartsOkLen, 'File name not parsed in %s elements [%s]' % (str(self.pcfg.fnPartsOkLen),str(bits)), abort=True ) |
---|
314 | |
---|
315 | self.fnDict = {} |
---|
316 | if self.pcfg.projectV.id in ['ESA-CCI']: |
---|
317 | l0 = {0:6, 1:5, 2:5}[self.esaFnId] |
---|
318 | for i in range(l0): |
---|
319 | x = self.pcfg.globalAttributesInFn[i] |
---|
320 | if x != None and x[0] == '*': |
---|
321 | self.fnDict[x[1:]] = bits[i] |
---|
322 | self.fnDict['version'] = bits[-1] |
---|
323 | self.fnDict['gdsv'] = 'na' |
---|
324 | if self.esaFnId == 0: |
---|
325 | if len(bits) == 9: |
---|
326 | self.fnDict['additional'] = bits[-3] |
---|
327 | self.fnDict['gdsv'] = bits[-2] |
---|
328 | elif len(bits) == 8: |
---|
329 | if bits[-2][0] == 'v': |
---|
330 | self.fnDict['gdsv'] = bits[-2] |
---|
331 | else: |
---|
332 | self.fnDict['additional'] = bits[-2] |
---|
333 | elif self.esaFnId in [1,2]: |
---|
334 | if len(bits) == 8: |
---|
335 | self.fnDict['additional'] = bits[-3] |
---|
336 | |
---|
337 | if self.pcfg.groupIndex != None: |
---|
338 | self.group = self.fnParts[self.pcfg.groupIndex] |
---|
339 | else: |
---|
340 | self.group = None |
---|
341 | |
---|
342 | if self.pcfg.freqIndex != None: |
---|
343 | self.freq = self.fnParts[self.pcfg.freqIndex] |
---|
344 | elif self.group in ['fx','fixed']: |
---|
345 | self.freq = 'fx' |
---|
346 | else: |
---|
347 | self.freq = None |
---|
348 | |
---|
349 | ##if self.cls == 'CORDEX': |
---|
350 | ##self.freq = self.fnParts[7] |
---|
351 | ##elif self.cls == 'SPECS': |
---|
352 | ##self.freq = self.fnParts[1] |
---|
353 | |
---|
354 | self.var = self.fnParts[self.pcfg.varIndex] |
---|
355 | |
---|
356 | if self.pcfg.fnvdict != None: |
---|
357 | if self.pcfg.fnvdict.has_key( self.var ): |
---|
358 | self.var = self.pcfg.fnvdict.get( self.var )['v'] |
---|
359 | |
---|
360 | self.isFixed = self.freq in ['fx','fixed'] |
---|
361 | self.parent.fileIsFixed = True |
---|
362 | if self.isFixed: |
---|
363 | self.test( len(self.fnParts) in self.pcfg.fnPartsOkFixedLen, 'Number of file name elements not acceptable for fixed data' ) |
---|
364 | |
---|
365 | self.checkId = ('002','parse_filename_timerange') |
---|
366 | if not self.isFixed: |
---|
367 | |
---|
368 | ## test time segment |
---|
369 | if self.pcfg.trangeType == 'CMIP': |
---|
370 | bits = string.split( self.fnParts[-1], '-' ) |
---|
371 | self.test( len(bits) == 2, 'File time segment [%s] will not parse into 2 elements' % (self.fnParts[-1] ), abort=True, part=True ) |
---|
372 | |
---|
373 | self.test( len(bits[0]) == len(bits[1]), 'Start and end time specified in file name [%s] of unequal length' % (self.fnParts[-1] ), abort=True, part=True ) |
---|
374 | self.test( int(bits[0]) <= int(bits[1]), 'Start and end time specified in file name [%s] in wrong order' % (self.fnParts[-1] ), abort=True, part=True ) |
---|
375 | |
---|
376 | for b in bits: |
---|
377 | self.test( self.isInt(b), 'Time segment in filename [%s] contains non integer characters' % (self.fnParts[-1] ), abort=True, part=True ) |
---|
378 | self.log_pass() |
---|
379 | self.fnTimeParts = bits[:] |
---|
380 | elif self.pcfg.trangeType == 'ESA-CCI': |
---|
381 | self.pcfg.checkTrangeLen = False |
---|
382 | tt = self.fnParts[self.pcfg.trangeIndex] |
---|
383 | if self.test( len(tt) in [4,6,8,10,12,14] and self.re_c1.match(tt) != None, 'Length of indicative date/time not consistent with YYYY[MM[DD[HH[MM[SS]]]]] specification: %s' % self.fnParts[-1], part=True ): |
---|
384 | ll = [tt[:4],] |
---|
385 | tt = tt[4:] |
---|
386 | for j in range(5): |
---|
387 | if len(tt) > 0: |
---|
388 | ll.append( tt[:2] ) |
---|
389 | tt = tt[2:] |
---|
390 | elif j in [1,2]: |
---|
391 | ll.append( '01' ) |
---|
392 | else: |
---|
393 | ll.append( '00' ) |
---|
394 | indDateTime = map( int, ll ) |
---|
395 | self.test( indDateTime[1] in range(1,13), 'Invalid Month in indicative date time %s' % str(ll), part=True ) |
---|
396 | self.test( indDateTime[2] in range(1,32), 'Invalid Day in indicative date time %s' % str(ll), part=True ) |
---|
397 | self.test( indDateTime[3] in range(25), 'Invalid hour in indicative date time %s' % str(ll), part=True ) |
---|
398 | self.test( indDateTime[4] in range(60), 'Invalid minute in indicative date time %s' % str(ll), part=True ) |
---|
399 | self.test( indDateTime[5] in range(60), 'Invalid second in indicative date time %s' % str(ll), part=True ) |
---|
400 | |
---|
401 | self.checkId = '003' |
---|
402 | |
---|
403 | self.checkId, ok = (('004','filename_timerange_length'),True) |
---|
404 | if (not self.isFixed) and self.pcfg.checkTrangeLen: |
---|
405 | ltr = { 'mon':6, 'sem':6, 'day':8, '3hr':[10,12], '6hr':10 } |
---|
406 | ok &=self.test( self.freq in ltr.keys(), 'Frequency [%s] not recognised' % self.freq, part=True ) |
---|
407 | if ok: |
---|
408 | if type( ltr[self.freq] ) == type(0): |
---|
409 | msg = 'Length of time range parts [%s,%s] not equal to required length [%s] for frequency %s' % (self.fnTimeParts[0],self.fnTimeParts[1],ltr[self.freq],self.freq) |
---|
410 | ok &= self.test( len(self.fnTimeParts[0]) == ltr[self.freq], msg, part=True ) |
---|
411 | elif type( ltr[self.freq] ) in [type([]),type( () )]: |
---|
412 | msg = 'Length of time range parts [%s,%s] not in acceptable list [%s] for frequency %s' % (self.fnTimeParts[0],self.fnTimeParts[1],str(ltr[self.freq]),self.freq) |
---|
413 | ok &= self.test( len(self.fnTimeParts[0]) in ltr[self.freq], msg, part=True ) |
---|
414 | |
---|
415 | if ok: |
---|
416 | self.log_pass() |
---|
417 | |
---|
418 | def do_check_fnextra(self): |
---|
419 | """Check whether file name components match constraints -- but only if those constraints are not implicitly verified through comparison with global attributes in later checks""" |
---|
420 | self.checkId = ('004','file_name_extra' ) |
---|
421 | vocabs = self.pcfg.vocabs |
---|
422 | m = [] |
---|
423 | for a in self.pcfg.controlledFnParts: |
---|
424 | if self.fnDict.has_key(a): |
---|
425 | try: |
---|
426 | if not vocabs[a].check( str(self.fnDict[a]) ): |
---|
427 | m.append( (a,self.fnDict[a],vocabs[a].note) ) |
---|
428 | except: |
---|
429 | print 'failed trying to check file name component %s' % a |
---|
430 | raise baseException( 'failed trying to check file name component %s' % a ) |
---|
431 | |
---|
432 | self.test( len(m) == 0, 'File name components do not match constraints: %s' % str(m) ) |
---|
433 | |
---|
434 | |
---|
435 | class checkGlobalAttributes(checkBase): |
---|
436 | """Check global and variable attributes, using tables of valid values""" |
---|
437 | |
---|
438 | def init(self): |
---|
439 | self.id = 'C4.002' |
---|
440 | self.checkId = 'unset' |
---|
441 | self.step = 'Initialised' |
---|
442 | self.checks = (self.do_check_ga,) |
---|
443 | self.fileId = None |
---|
444 | |
---|
445 | def check(self,globalAts, varAts,varName,varGroup, vocabs, fnParts): |
---|
446 | self.errorCount = 0 |
---|
447 | assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName) |
---|
448 | self.var = varName |
---|
449 | self.globalAts = globalAts |
---|
450 | self.varAts = varAts |
---|
451 | self.varGroup = varGroup |
---|
452 | self.vocabs = vocabs |
---|
453 | self.fnParts = fnParts |
---|
454 | self.runChecks() |
---|
455 | |
---|
456 | def getId(self): |
---|
457 | if self.fileId == None: |
---|
458 | self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id']) |
---|
459 | if self.globalAts['naming_authority'] == 'uk.ac.pml': |
---|
460 | i0 = string.find(self.globalAts['id'],'OC4v6_QAA') |
---|
461 | if i0 != -1: |
---|
462 | self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id'][:i0+9]) |
---|
463 | |
---|
464 | def getDrs( self ): |
---|
465 | assert self.completed, 'method getDrs should not be called if checks have not been completed successfully' |
---|
466 | ee = {} |
---|
467 | drsDefaults = { 'convention_version':'n/a'} |
---|
468 | if not self.globalAts.has_key('product'): |
---|
469 | self.globalAts['product'] = 'output' |
---|
470 | for k in self.drsMappings: |
---|
471 | if self.drsMappings[k] == '@var': |
---|
472 | ee[k] = self.var |
---|
473 | elif self.drsMappings[k][0] == '=': |
---|
474 | ee[k] = self.drsMappings[k][1:] |
---|
475 | elif self.drsMappings[k] == '@ensemble': |
---|
476 | ee[k] = "r%si%sp%s" % (self.globalAts["realization"],self.globalAts["initialization_method"],self.globalAts["physics_version"]) |
---|
477 | elif self.drsMappings[k] == '@forecast_reference_time': |
---|
478 | x = self.globalAts.get("forecast_reference_time",'yyyy-mm-dd Thh:mm:ssZ' ) |
---|
479 | ee[k] = "%s%s%s" % (x[:4],x[5:7],x[8:10]) |
---|
480 | elif self.drsMappings[k] == '@mip_id': |
---|
481 | ee[k] = string.split( self.globalAts["table_id"] )[1] |
---|
482 | elif self.drsMappings[k] == '@ecv': |
---|
483 | ee[k] = self.pcfg.ecvMappings[ self.parent.fnDict['project'] ] |
---|
484 | elif self.drsMappings[k][0] == '$': |
---|
485 | self.pcfg.getExtraAtts() |
---|
486 | self.getId() |
---|
487 | if string.find(self.drsMappings[k],':') != -1: |
---|
488 | k2,dflt = string.split( self.drsMappings[k][1:],':') |
---|
489 | ee[k] = self.pcfg.extraAtts[self.fileId].get( k2, dflt ) |
---|
490 | else: |
---|
491 | ee[k] = self.pcfg.extraAtts[self.fileId][self.drsMappings[k][1:]] |
---|
492 | elif self.drsMappings[k][0] == '*': |
---|
493 | thisk = self.drsMappings[k][1:] |
---|
494 | ee[k] = self.varAts[self.var][thisk] |
---|
495 | elif self.drsMappings[k][0] == '#': |
---|
496 | thisk = self.drsMappings[k][1:] |
---|
497 | if drsDefaults.has_key( thisk ): |
---|
498 | ee[k] = self.parent.fnDict.get(thisk, drsDefaults[thisk] ) |
---|
499 | else: |
---|
500 | ee[k] = self.parent.fnDict[thisk] |
---|
501 | else: |
---|
502 | ee[k] = self.globalAts[ self.drsMappings[k] ] |
---|
503 | |
---|
504 | for k in ['creation_date','tracking_id']: |
---|
505 | if k in self.globalAts.keys(): |
---|
506 | ee[k] = self.globalAts[k] |
---|
507 | |
---|
508 | return ee |
---|
509 | |
---|
510 | def do_check_ga(self): |
---|
511 | varName = self.var |
---|
512 | globalAts = self.globalAts |
---|
513 | varAts = self.varAts |
---|
514 | varGroup = self.varGroup |
---|
515 | vocabs = self.vocabs |
---|
516 | fnParts = self.fnParts |
---|
517 | |
---|
518 | self.completed = False |
---|
519 | self.checkId = ('001','global_ncattribute_present') |
---|
520 | m = [] |
---|
521 | for k in self.requiredGlobalAttributes: |
---|
522 | if not globalAts.has_key(k): |
---|
523 | m.append(k) |
---|
524 | self.globalAts[k] = '__errorReported__' |
---|
525 | |
---|
526 | if not self.test( len(m) == 0, 'Required global attributes missing: %s' % str(m) ): |
---|
527 | gaerr = True |
---|
528 | for k in m: |
---|
529 | self.parent.amapListDraft.append( '#@;%s=%s|%s=%s' % (k,'__absent__',k,'<insert attribute value and uncomment>') ) |
---|
530 | |
---|
531 | self.checkId = ('002','variable_in_group') |
---|
532 | |
---|
533 | self.test( varAts.has_key( varName ), 'Expected variable [%s] not present' % varName, abort=True, part=True ) |
---|
534 | msg = 'Variable %s not in table %s' % (varName,varGroup) |
---|
535 | self.test( vocabs['variable'].isInTable( varName, varGroup ), msg, abort=True, part=True ) |
---|
536 | |
---|
537 | if self.pcfg.checkVarType: |
---|
538 | self.checkId = ('003','variable_type') |
---|
539 | |
---|
540 | mipType = vocabs['variable'].getAttr( varName, varGroup, 'type' ) |
---|
541 | thisType = {'real':'float32', 'integer':'int32', 'float':'float32', 'double':'float64' }.get( mipType, mipType ) |
---|
542 | self.test( mipType == None or varAts[varName]['_type'] == thisType, 'Variable [%s/%s] not of type %s [%s]' % (varName,varGroup,str(thisType),varAts[varName]['_type']) ) |
---|
543 | else: |
---|
544 | mipType = None |
---|
545 | |
---|
546 | self.checkId = ('004','variable_ncattribute_present') |
---|
547 | m = [] |
---|
548 | reqAts = self.requiredVarAttributes[:] |
---|
549 | if (not self.parent.fileIsFixed) and self.pcfg.projectV.id in ['CORDEX']: |
---|
550 | reqAts.append( 'cell_methods' ) |
---|
551 | for k in reqAts + vocabs['variable'].lists(varName, 'addRequiredAttributes'): |
---|
552 | if not varAts[varName].has_key(k): |
---|
553 | m.append(k) |
---|
554 | if not self.test( len(m) == 0, 'Required variable attributes missing: %s' % str(m) ): |
---|
555 | vaerr = True |
---|
556 | for k in m: |
---|
557 | self.parent.amapListDraft.append( '#@var=%s;%s=%s|%s=%s' % (varName,k,'__absent__',k,'<insert attribute value and uncomment>') ) |
---|
558 | ## print self.parent.amapListDraft[-1] |
---|
559 | |
---|
560 | ## need to insert a check that variable is present |
---|
561 | self.checkId = ('005','variable_ncattribute_mipvalues') |
---|
562 | ok = True |
---|
563 | hm = varAts[varName].get( 'missing_value', None ) != None |
---|
564 | hf = varAts[varName].has_key( '_FillValue' ) |
---|
565 | if hm or hf: |
---|
566 | if self.pcfg.varTables=='CMIP': |
---|
567 | ok &= self.test( hm, 'missing_value must be present if _FillValue is [%s]' % varName ) |
---|
568 | ok &= self.test( hf, '_FillValue must be present if missing_value is [%s]' % varName ) |
---|
569 | else: |
---|
570 | ok = True |
---|
571 | if mipType == 'real': |
---|
572 | if varAts[varName].has_key( 'missing_value' ): |
---|
573 | msg = 'Variable [%s] has incorrect attribute missing_value=%s [correct: %s]' % (varName,varAts[varName]['missing_value'],self.missingValue) |
---|
574 | ### need to use ctypes here when using ncq3 to read files -- appears OK for other libraries. |
---|
575 | ok &= self.test( ctypes.c_float(varAts[varName]['missing_value']).value == ctypes.c_float(self.missingValue).value, msg, part=True ) |
---|
576 | if varAts[varName].has_key( '_FillValue' ): |
---|
577 | msg = 'Variable [%s] has incorrect attribute _FillValue=%s [correct: %s]' % (varName,varAts[varName]['_FillValue'],self.missingValue) |
---|
578 | ok &= self.test( varAts[varName]['_FillValue'] == self.missingValue, msg, part=True ) |
---|
579 | |
---|
580 | mm = [] |
---|
581 | |
---|
582 | if self.pcfg.varTables=='CMIP': |
---|
583 | contAts = ['long_name', 'standard_name', 'units'] |
---|
584 | if not self.parent.fileIsFixed: |
---|
585 | ##if varGroup not in ['fx','fixed']: |
---|
586 | contAts.append( 'cell_methods' ) |
---|
587 | else: |
---|
588 | contAts = ['standard_name'] |
---|
589 | hcm = varAts[varName].has_key( "cell_methods" ) |
---|
590 | for k in contAts + vocabs['variable'].lists(varName,'addControlledAttributes'): |
---|
591 | targ = varAts[varName].get( k, 'Attribute not present' ) |
---|
592 | val = vocabs['variable'].getAttr( varName, varGroup, k ) |
---|
593 | |
---|
594 | if k == "cell_methods": |
---|
595 | if val != None: |
---|
596 | parenthesies1 = [] |
---|
597 | targ0 = targ[:] |
---|
598 | while string.find( targ, '(' ) != -1: |
---|
599 | i0 = targ.index( '(' ) |
---|
600 | i1 = targ.index( ')' ) |
---|
601 | parenthesies1.append( targ[i0:i1+1] ) |
---|
602 | targ = targ[:i0-1] + targ[i1+1:] |
---|
603 | parenthesies2 = [] |
---|
604 | val0 = val[:] |
---|
605 | while string.find( val, '(' ) != -1: |
---|
606 | i0 = val.index( '(' ) |
---|
607 | i1 = val.index( ')' ) |
---|
608 | parenthesies2.append( val[i0:i1+1] ) |
---|
609 | val = val[:i0-1] + val[i1+1:] |
---|
610 | for p in parenthesies2: |
---|
611 | if p not in parenthesies1: |
---|
612 | mm.append( (k,parenthesies1,p) ) |
---|
613 | if string.find( targ, val): |
---|
614 | mm.append( (k,targ,val) ) |
---|
615 | elif targ != 'Attribute not present' and targ != val: |
---|
616 | mm.append( (k,targ,val) ) |
---|
617 | |
---|
618 | ok &= self.test( len(mm) == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, strmm3(mm)), part=True ) |
---|
619 | if len( mm ) != 0: |
---|
620 | if self.parent.amapListDraft == None: |
---|
621 | self.parent.amapListDraft = [] |
---|
622 | for m in mm: |
---|
623 | self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % (varName,m[0],m[1],m[0],m[2]) ) |
---|
624 | |
---|
625 | if ok: |
---|
626 | self.log_pass() |
---|
627 | |
---|
628 | if (not self.parent.fileIsFixed) and hcm: |
---|
629 | ## if (varGroup not in ['fx','fixed']) and hcm: |
---|
630 | self.isInstantaneous = string.find( varAts[varName]['cell_methods'], 'time: point' ) != -1 |
---|
631 | else: |
---|
632 | self.isInstantaneous = True |
---|
633 | |
---|
634 | self.checkId = ('006','global_ncattribute_cv' ) |
---|
635 | m = [] |
---|
636 | for a in self.controlledGlobalAttributes: |
---|
637 | if globalAts.has_key(a): |
---|
638 | try: |
---|
639 | if not vocabs[a].check( str(globalAts[a]) ): |
---|
640 | m.append( (a,globalAts[a],vocabs[a].note) ) |
---|
641 | except: |
---|
642 | print 'failed trying to check global attribute %s' % a |
---|
643 | raise baseException( 'failed trying to check global attribute %s' % a ) |
---|
644 | |
---|
645 | if not self.test( len(m) == 0, 'Global attributes do not match constraints: %s' % str(m) ): |
---|
646 | for t in m: |
---|
647 | self.parent.amapListDraft.append( '#@;%s=%s|%s=%s' % (t[0],str(t[1]),t[0],'<insert attribute value and uncomment>' + str(t[2]) ) ) |
---|
648 | |
---|
649 | self.checkId = ('007','filename_filemetadata_consistency') |
---|
650 | m = [] |
---|
651 | for i in range(len(self.globalAttributesInFn)): |
---|
652 | if self.globalAttributesInFn[i] != None and self.globalAttributesInFn[i][0] != '*': |
---|
653 | targVal = fnParts[i] |
---|
654 | if self.globalAttributesInFn[i][0] == "@": |
---|
655 | if self.globalAttributesInFn[i][1:] == "mip_id": |
---|
656 | bits = string.split( globalAts[ "table_id" ] ) |
---|
657 | if len( bits ) > 2 and bits[0] == "Table": |
---|
658 | thisVal = bits[1] |
---|
659 | else: |
---|
660 | thisVal = globalAts[ "table_id" ] |
---|
661 | self.test( False, 'Global attribute table_id does not conform to CMOR pattern ["Table ......"]: %s' % thisVal, part=True) |
---|
662 | elif self.globalAttributesInFn[i][1:] == "ensemble": |
---|
663 | thisVal = "r%si%sp%s" % (globalAts["realization"],globalAts["initialization_method"],globalAts["physics_version"]) |
---|
664 | ## following mappings are depricated -- introduced for SPECS and withdrawn --- |
---|
665 | elif self.globalAttributesInFn[i][1:] == "experiment_family": |
---|
666 | thisVal = globalAts["experiment_id"][:-4] |
---|
667 | elif self.globalAttributesInFn[i][1:] == "forecast_reference_time": |
---|
668 | x = self.globalAts.get("forecast_reference_time",'yyyy-mm-dd Thh:mm:ssZ' ) |
---|
669 | thisVal = "S%s%s%s" % (x[:4],x[5:7],x[8:10]) |
---|
670 | elif self.globalAttributesInFn[i][1:] == "series": |
---|
671 | thisVal = 'series%s' % globalAts["series"] |
---|
672 | else: |
---|
673 | assert False, "Not coded to deal with this configuration: globalAttributesInFn[%s]=%s" % (i,self.globalAttributesInFn[i]) |
---|
674 | |
---|
675 | else: |
---|
676 | thisVal = globalAts[self.globalAttributesInFn[i]] |
---|
677 | |
---|
678 | if thisVal not in [targVal,'__errorReported__']: |
---|
679 | m.append( (i,self.globalAttributesInFn[i]) ) |
---|
680 | |
---|
681 | self.test( len(m) == 0,'File name segments do not match corresponding global attributes: %s' % str(m) ) |
---|
682 | |
---|
683 | self.completed = True |
---|
684 | |
---|
685 | class checkStandardDims(checkBase): |
---|
686 | """Check the dimensions which are defined in the specifications""" |
---|
687 | |
---|
688 | def init(self): |
---|
689 | self.id = 'C4.003' |
---|
690 | self.checkId = 'unset' |
---|
691 | self.step = 'Initialised' |
---|
692 | self.checks = (self.do_check,) |
---|
693 | self.plevRequired = self.pcfg.plevRequired |
---|
694 | self.plevValues = self.pcfg.plevValues |
---|
695 | self.heightRequired = self.pcfg.heightRequired |
---|
696 | self.heightValues = self.pcfg.heightValues |
---|
697 | self.heightRange = self.pcfg.heightRange |
---|
698 | |
---|
699 | def check(self,varName,varGroup, da, va, isInsta,vocabs): |
---|
700 | self.errorCount = 0 |
---|
701 | assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName) |
---|
702 | self.var = varName |
---|
703 | self.varGroup = varGroup |
---|
704 | self.da = da |
---|
705 | self.va = va |
---|
706 | self.isInsta = isInsta |
---|
707 | self.vocabs = vocabs |
---|
708 | self.runChecks() |
---|
709 | |
---|
710 | def do_check(self): |
---|
711 | varName = self.var |
---|
712 | varGroup = self.varGroup |
---|
713 | da = self.da |
---|
714 | va = self.va |
---|
715 | isInsta = self.isInsta |
---|
716 | |
---|
717 | self.errorCount = 0 |
---|
718 | self.completed = False |
---|
719 | self.checkId = ('001','time_attributes') |
---|
720 | self.calendar = 'None' |
---|
721 | if not self.parent.fileIsFixed: |
---|
722 | ## if varGroup not in ['fx','fixed']: |
---|
723 | ok = True |
---|
724 | self.test( 'time' in da.keys(), 'Time dimension not found' , abort=True, part=True ) |
---|
725 | if self.pcfg.varTables=='CMIP': |
---|
726 | if not isInsta: |
---|
727 | ok &= self.test( da['time'].get( 'bounds', 'xxx' ) == 'time_bnds', 'Required bounds attribute not present or not correct value', part=True ) |
---|
728 | |
---|
729 | ## is time zone designator needed? |
---|
730 | tunits = da['time'].get( 'units', 'xxx' ) |
---|
731 | if self.project == 'CORDEX': |
---|
732 | ok &= self.test( tunits in ["days since 1949-12-01 00:00:00Z", "days since 1949-12-01 00:00:00", "days since 1949-12-01"], |
---|
733 | 'Time units [%s] attribute not set correctly to "days since 1949-12-01 00:00:00Z"' % tunits, part=True ) |
---|
734 | else: |
---|
735 | ok &= self.test( tunits[:10] == "days since", 'time units [%s] attribute not set correctly to "days since ....."' % tunits, part=True ) |
---|
736 | |
---|
737 | ok &= self.test( da['time'].has_key( 'calendar' ), 'Time: required attribute calendar missing', part=True ) |
---|
738 | |
---|
739 | ok &= self.test( da['time']['_type'] in ["float64","double"], 'Time: data type not float64 [%s]' % da['time']['_type'], part=True ) |
---|
740 | |
---|
741 | if ok: |
---|
742 | self.log_pass() |
---|
743 | self.calendar = da['time'].get( 'calendar', 'None' ) |
---|
744 | |
---|
745 | self.checkId = ('002','pressure_levels') |
---|
746 | if varName in self.plevRequired: |
---|
747 | ok = True |
---|
748 | self.test( 'plev' in va.keys(), 'plev coordinate not found %s' % str(va.keys()), abort=True, part=True ) |
---|
749 | |
---|
750 | ok &= self.test( int( va['plev']['_data'][0] ) == self.plevValues[varName], \ |
---|
751 | 'plev value [%s] does not match required [%s]' % (va['plev']['_data'],self.plevValues[varName] ), part=True ) |
---|
752 | |
---|
753 | plevAtDict = {'standard_name':"air_pressure", \ |
---|
754 | 'long_name':"pressure", \ |
---|
755 | 'units':"Pa", \ |
---|
756 | 'positive':"down", \ |
---|
757 | 'axis':"Z" } |
---|
758 | |
---|
759 | if varName in ['clh','clm','cll']: |
---|
760 | plevAtDict['bounds']= "plev_bnds" |
---|
761 | |
---|
762 | for k in plevAtDict.keys(): |
---|
763 | ok &= self.test( va['plev'].get( k, None ) == plevAtDict[k], |
---|
764 | 'plev attribute %s absent or wrong value (should be %s)' % (k,plevAtDict[k]), part=True ) |
---|
765 | |
---|
766 | if varName in ['clh','clm','cll']: |
---|
767 | self.test( "plev_bnds" in va.keys(), 'plev_bnds variable not found %s' % str(va.keys()), abort=True, part=True ) |
---|
768 | mm = [] |
---|
769 | for k in plevAtDict.keys(): |
---|
770 | if k != 'bounds' and k in va['plev_bnds'].keys(): |
---|
771 | if va['plev_bnds'][k] != va['plev'][k]: |
---|
772 | mm.append(k) |
---|
773 | ok &= self.test( len(mm) == 0, 'Attributes of plev_bnds do not match those of plev: %s' % str(mm), part=True ) |
---|
774 | |
---|
775 | bndsVals = {'clh':[44000, 0], 'clm':[68000, 44000], 'cll':[100000, 68000] } |
---|
776 | res = self.test( len( va['plev_bnds']['_data'] ) == 2, 'plev_bnds array is of wrong length', part=True ) |
---|
777 | ok &= res |
---|
778 | if res: |
---|
779 | kk = 0 |
---|
780 | for i in [0,1]: |
---|
781 | if int(va['plev_bnds']['_data'][i]) != bndsVals[varName][i]: |
---|
782 | kk+=1 |
---|
783 | ok &= self.test( kk == 0, 'plev_bnds values not correct: should be %s' % str(bndsVals[varName]), part=True ) |
---|
784 | |
---|
785 | if ok: |
---|
786 | self.log_pass() |
---|
787 | |
---|
788 | self.checkId = ('003','height_levels') |
---|
789 | hreq = varName in self.heightRequired |
---|
790 | if self.parent.experimental: |
---|
791 | print 'utils_c4: ', varName, self.vocabs['variable'].varcons[varGroup][varName].get( '_dimension',[]) |
---|
792 | hreq = "height2m" in self.vocabs['variable'].varcons[varGroup][varName].get( '_dimension',[]) |
---|
793 | if hreq: |
---|
794 | print 'testing height, var=%s' % varName |
---|
795 | if hreq: |
---|
796 | heightAtDict = {'long_name':"height", 'standard_name':"height", 'units':"m", 'positive':"up", 'axis':"Z" } |
---|
797 | ok = True |
---|
798 | ok &= self.test( 'height' in va.keys(), 'height coordinate not found %s' % str(va.keys()), abort=True, part=True ) |
---|
799 | ##ok &= self.test( abs( va['height']['_data'] - self.heightValues[varName]) < 0.001, \ |
---|
800 | ##'height value [%s] does not match required [%s]' % (va['height']['_data'],self.heightValues[varName] ), part=True ) |
---|
801 | |
---|
802 | ok1 = self.test( len( va['height']['_data'] ) == 1, 'More height values (%s) than expected (1)' % (len( va['height']['_data'])), part=True ) |
---|
803 | if ok1: |
---|
804 | r = self.heightRange[varName] |
---|
805 | ok1 &= self.test( r[0] <= va['height']['_data'][0] <= r[1], \ |
---|
806 | 'height value [%s] not in specified range [%s]' % (va['height']['_data'], (self.heightRange[varName] ) ), part=True ) |
---|
807 | |
---|
808 | ok &= ok1 |
---|
809 | |
---|
810 | for k in heightAtDict.keys(): |
---|
811 | val = va['height'].get( k, "none" ) |
---|
812 | if not self.test( val == heightAtDict[k], \ |
---|
813 | 'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ): |
---|
814 | self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % ('height',k,val,k,heightAtDict[k]) ) |
---|
815 | ok = False |
---|
816 | |
---|
817 | if ok: |
---|
818 | self.log_pass() |
---|
819 | |
---|
820 | self.completed = True |
---|
821 | |
---|
822 | class checkGrids(checkBase): |
---|
823 | |
---|
824 | def init(self): |
---|
825 | self.id = 'C4.004' |
---|
826 | self.checkId = 'unset' |
---|
827 | self.step = 'Initialised' |
---|
828 | self.checks = (self.do_check_rp,self.do_check_intd) |
---|
829 | |
---|
830 | def check(self,varName, domain, da, va): |
---|
831 | self.errorCount = 0 |
---|
832 | assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName) |
---|
833 | self.var = varName |
---|
834 | self.domain = domain |
---|
835 | self.da = da |
---|
836 | self.va = va |
---|
837 | |
---|
838 | self.runChecks() |
---|
839 | ##for c in self.checks: |
---|
840 | ##c() |
---|
841 | ##self.do_check_rp() |
---|
842 | ##self.do_check_intd() |
---|
843 | |
---|
844 | def do_check_rp(self): |
---|
845 | varName = self.var |
---|
846 | domain = self.domain |
---|
847 | da = self.da |
---|
848 | va = self.va |
---|
849 | if va[varName].get( 'grid_mapping', None ) == "rotated_pole": |
---|
850 | self.checkId = ('001','grid_mapping') |
---|
851 | atDict = { 'grid_mapping_name':'rotated_latitude_longitude' } |
---|
852 | atDict['grid_north_pole_latitude'] = self.pcfg.rotatedPoleGrids[domain]['grid_np_lat'] |
---|
853 | if self.pcfg.rotatedPoleGrids[domain]['grid_np_lon'] != 'N/A': |
---|
854 | atDict['grid_north_pole_longitude'] = self.pcfg.rotatedPoleGrids[domain]['grid_np_lon'] |
---|
855 | |
---|
856 | self.checkId = ('002','rotated_latlon_attributes') |
---|
857 | self.test( 'rlat' in da.keys() and 'rlon' in da.keys(), 'rlat and rlon not found (required for grid_mapping = rotated_pole )', abort=True, part=True ) |
---|
858 | |
---|
859 | atDict = {'rlat':{'long_name':"rotated latitude", 'standard_name':"grid_latitude", 'units':"degrees", 'axis':"Y", '_type':'float64'}, |
---|
860 | 'rlon':{'long_name':"rotated longitude", 'standard_name':"grid_longitude", 'units':"degrees", 'axis':"X", '_type':'float64'} } |
---|
861 | mm = [] |
---|
862 | for k in ['rlat','rlon']: |
---|
863 | for k2 in atDict[k].keys(): |
---|
864 | if atDict[k][k2] != da[k].get(k2, None ): |
---|
865 | mm.append( (k,k2) ) |
---|
866 | record = '#@ax=%s;%s=%s|%s=%s <uncomment if correct>' % (k,k2,da[k].get(k2, '__missing__'),k2,atDict[k][k2] ) |
---|
867 | self.parent.amapListDraft.append( record ) |
---|
868 | self.test( len(mm) == 0, 'Required attributes of grid coordinate arrays not correct: %s' % str(mm) ) |
---|
869 | |
---|
870 | self.checkId = ('003','rotated_latlon_domain') |
---|
871 | ok = True |
---|
872 | for k in ['rlat','rlon']: |
---|
873 | res = len(da[k]['_data']) == self.pcfg.rotatedPoleGrids[domain][ {'rlat':'nlat','rlon':'nlon' }[k] ] |
---|
874 | if not res: |
---|
875 | self.test( res, 'Size of %s dimension does not match specification (%s,%s)' % (k,a,b), part=True ) |
---|
876 | ok = False |
---|
877 | |
---|
878 | a = ( da['rlat']['_data'][0], da['rlat']['_data'][-1], da['rlon']['_data'][0], da['rlon']['_data'][-1] ) |
---|
879 | b = map( lambda x: self.pcfg.rotatedPoleGrids[domain][x], ['s','n','w','e'] ) |
---|
880 | mm = [] |
---|
881 | for i in range(4): |
---|
882 | if abs(a[i] - b[i]) > self.pcfg.gridSpecTol: |
---|
883 | mm.append( (a[i],b[i]) ) |
---|
884 | |
---|
885 | ok &= self.test( len(mm) == 0, 'Domain boundaries for rotated pole grid do not match %s within tolerance (%s)' % (str(mm),self.pcfg.gridSpecTol), part=True ) |
---|
886 | |
---|
887 | for k in ['rlat','rlon']: |
---|
888 | ok &= self.test( cs.check( da[k]['_data'] ), '%s values not evenly spaced -- min/max delta = %s, %s' % (k,cs.dmn,cs.dmx), part=True ) |
---|
889 | |
---|
890 | if ok: |
---|
891 | self.log_pass() |
---|
892 | |
---|
893 | def do_check_intd(self): |
---|
894 | varName = self.var |
---|
895 | domain = self.domain |
---|
896 | da = self.da |
---|
897 | va = self.va |
---|
898 | if domain[-1] == 'i': |
---|
899 | self.checkId = ('004','regular_grid_attributes') |
---|
900 | self.test( 'lat' in da.keys() and 'lon' in da.keys(), 'lat and lon not found (required for interpolated data)', abort=True, part=True ) |
---|
901 | |
---|
902 | atDict = {'lat':{'long_name':"latitude", 'standard_name':"latitude", 'units':"degrees_north", '_type':'float64'}, |
---|
903 | 'lon':{'long_name':"longitude", 'standard_name':"longitude", 'units':"degrees_east", '_type':'float64'} } |
---|
904 | mm = [] |
---|
905 | for k in ['lat','lon']: |
---|
906 | for k2 in atDict[k].keys(): |
---|
907 | if atDict[k][k2] != da[k].get(k2, None ): |
---|
908 | mm.append( (k,k2) ) |
---|
909 | record = '#@ax=%s;%s=%s|%s=%s <uncomment if correct>' % (k,k2,da[k].get(k2, '__missing__'),k2,atDict[k][k2] ) |
---|
910 | self.parent.amapListDraft.append( record ) |
---|
911 | |
---|
912 | self.test( len(mm) == 0, 'Required attributes of grid coordinate arrays not correct: %s' % str(mm), part=True ) |
---|
913 | |
---|
914 | ok = True |
---|
915 | self.checkId = ('005','regular_grid_domain') |
---|
916 | for k in ['lat','lon']: |
---|
917 | res = len(da[k]['_data']) >= self.pcfg.interpolatedGrids[domain][ {'lat':'nlat','lon':'nlon' }[k] ] |
---|
918 | if not res: |
---|
919 | a,b = len(da[k]['_data']), self.pcfg.interpolatedGrids[domain][ {'lat':'nlat','lon':'nlon' }[k] ] |
---|
920 | self.test( res, 'Size of %s dimension does not match specification (%s,%s)' % (k,a,b), part=True ) |
---|
921 | ok = False |
---|
922 | |
---|
923 | a = ( da['lat']['_data'][0], da['lat']['_data'][-1], da['lon']['_data'][0], da['lon']['_data'][-1] ) |
---|
924 | b = map( lambda x: self.pcfg.interpolatedGrids[domain][x], ['s','n','w','e'] ) |
---|
925 | rs = self.pcfg.interpolatedGrids[domain]['res'] |
---|
926 | c = [-rs,rs,-rs,rs] |
---|
927 | mm = [] |
---|
928 | for i in range(4): |
---|
929 | if a[i] != b[i]: |
---|
930 | x = (a[i]-b[i])/c[i] |
---|
931 | if x < 0 or abs( x - int(x) ) > 0.001: |
---|
932 | skipThis = False |
---|
933 | if self.project == 'CORDEX': |
---|
934 | if domain[:3] == 'ANT': |
---|
935 | if i == 2 and abs( a[i] - 0.25 ) < 0.001: |
---|
936 | skipThis = True |
---|
937 | elif i == 3 and abs( a[i] - 359.75 ) < 0.001: |
---|
938 | skipThis = True |
---|
939 | if not skipThis: |
---|
940 | mm.append( (a[i],b[i]) ) |
---|
941 | |
---|
942 | ok &= self.test( len(mm) == 0, 'Interpolated grid boundary error: File %s; Req. %s' % (str(a),str(b)), part=True ) |
---|
943 | |
---|
944 | for k in ['lat','lon']: |
---|
945 | ok &= self.test( cs.check( da[k]['_data'] ), '%s values not evenly spaced -- min/max delta = %s, %s' % (k,cs.dmn,cs.dmx), part=True ) |
---|
946 | if ok: |
---|
947 | self.log_pass() |
---|
948 | |
---|
949 | class mipVocab(object): |
---|
950 | |
---|
951 | def __init__(self,pcfg,dummy=False): |
---|
952 | self.pcfg = pcfg |
---|
953 | if dummy: |
---|
954 | self.dummyMipTable() |
---|
955 | elif pcfg.varTables=='CMIP': |
---|
956 | self.ingestMipTables() |
---|
957 | elif pcfg.varTables=='FLAT': |
---|
958 | self.flatTable() |
---|
959 | |
---|
960 | def ingestMipTables(self): |
---|
961 | dir, tl, vgmap, fnpat = self.pcfg.mipVocabPars |
---|
962 | ms = mipTableScan() |
---|
963 | self.varInfo = {} |
---|
964 | self.varcons = {} |
---|
965 | for f in tl: |
---|
966 | vg = vgmap.get( f, f ) |
---|
967 | self.varcons[vg] = {} |
---|
968 | fn = fnpat % f |
---|
969 | ll = open( '%s%s' % (dir,fn) ).readlines() |
---|
970 | ee = ms.scan_table(ll,None,asDict=True) |
---|
971 | for v in ee.keys(): |
---|
972 | ## set global default: type float |
---|
973 | eeee = { 'type':self.pcfg.defaults.get( 'variableDataType', 'float' ) } |
---|
974 | eeee['_dimension'] = ee[v][0] |
---|
975 | ar = [] |
---|
976 | ac = [] |
---|
977 | for a in ee[v][1].keys(): |
---|
978 | eeee[a] = ee[v][1][a] |
---|
979 | ##if 'positive' in eeee.keys(): |
---|
980 | ##ar.append( 'positive' ) |
---|
981 | ##ac.append( 'positive' ) |
---|
982 | self.varInfo[v] = {'ar':ar, 'ac':ac } |
---|
983 | self.varcons[vg][v] = eeee |
---|
984 | |
---|
985 | def dummyMipTable(self): |
---|
986 | self.varInfo = {} |
---|
987 | self.varcons = {} |
---|
988 | ee = { 'standard_name':'sn%s', 'long_name':'n%s', 'units':'1' } |
---|
989 | dir, tl, vgmap, fnpat = self.pcfg.mipVocabPars |
---|
990 | for f in tl: |
---|
991 | vg = vgmap.get( f, f ) |
---|
992 | self.varcons[vg] = {} |
---|
993 | for i in range(12): |
---|
994 | v = 'v%s' % i |
---|
995 | eeee = {} |
---|
996 | eeee['standard_name'] = ee['standard_name'] % i |
---|
997 | eeee['long_name'] = ee['long_name'] % i |
---|
998 | eeee['cell_methods'] = 'time: point' |
---|
999 | eeee['units'] = ee['units'] |
---|
1000 | eeee['type'] = 'float' |
---|
1001 | ar = [] |
---|
1002 | ac = [] |
---|
1003 | self.varInfo[v] = {'ar':ar, 'ac':ac } |
---|
1004 | self.varcons[vg][v] = eeee |
---|
1005 | |
---|
1006 | def flatTable(self): |
---|
1007 | self.varInfo = {} |
---|
1008 | self.varcons = {} |
---|
1009 | dir, tl, vgm, fn = self.pcfg.mipVocabPars |
---|
1010 | vg = vgm.keys()[0] |
---|
1011 | ee = { 'standard_name':'sn%s', 'long_name':'n%s', 'units':'1' } |
---|
1012 | ll = open( '%s%s' % (dir,fn) ).readlines() |
---|
1013 | self.varcons[vg] = {} |
---|
1014 | for l in ll: |
---|
1015 | if l[0] != '#': |
---|
1016 | dt, v, sn = string.split( string.strip(l) ) |
---|
1017 | self.pcfg.fnvdict[dt] = { 'v':v, 'sn':sn } |
---|
1018 | ar = [] |
---|
1019 | ac = [] |
---|
1020 | self.varInfo[v] = {'ar':ar, 'ac':ac } |
---|
1021 | self.varcons[vg][v] = {'standard_name':sn, 'type':'float' } |
---|
1022 | |
---|
1023 | def lists( self, k, k2 ): |
---|
1024 | if k2 == 'addRequiredAttributes': |
---|
1025 | return self.varInfo[k]['ar'] |
---|
1026 | elif k2 == 'addControlledAttributes': |
---|
1027 | return self.varInfo[k]['ac'] |
---|
1028 | else: |
---|
1029 | raise baseException( 'mipVocab.lists called with bad list specifier %s' % k2 ) |
---|
1030 | |
---|
1031 | def isInTable( self, v, vg1 ): |
---|
1032 | vg = vg1 |
---|
1033 | if vg == 'ESA': |
---|
1034 | vg = 'ESACCI' |
---|
1035 | assert vg in self.varcons.keys(), '%s not found in self.varcons.keys() [%s]' % (vg,str(self.varcons.keys()) ) |
---|
1036 | return (v in self.varcons[vg].keys()) |
---|
1037 | |
---|
1038 | def getAttr( self, v, vg1, a ): |
---|
1039 | vg = vg1 |
---|
1040 | if vg == 'ESA': |
---|
1041 | vg = 'ESACCI' |
---|
1042 | assert vg in self.varcons.keys(), '%s not found in self.varcons.keys()' |
---|
1043 | assert v in self.varcons[vg].keys(), '%s not found in self.varcons[%s].keys()' % (v,vg) |
---|
1044 | |
---|
1045 | return self.varcons[vg][v][a] |
---|
1046 | |
---|
1047 | class patternControl(object): |
---|
1048 | |
---|
1049 | def __init__(self,tag,pattern,list=None,cls=None,examples=None,badExamples=None,runTest=True): |
---|
1050 | if cls != None: |
---|
1051 | assert cls in ['ISO'], 'value of cls [%s] not recognised' % cls |
---|
1052 | if cls == 'ISO': |
---|
1053 | assert pattern in ['ISO8601 duration'], 'value of pattern [%s] for ISO constraint not recognised' % pattern |
---|
1054 | if pattern == 'ISO8601 duration': |
---|
1055 | thispat = '^(P([0-9]+Y){0,1}([0-9]+M){0,1}([0-9]+D){0,1}(T([0-9]+H){0,1}([0-9]+M){0,1}([0-9]+(.[0-9]+){0,1}S){0,1}){0,1})$' |
---|
1056 | self.re_pat = re.compile( thispat ) |
---|
1057 | self.pattern = thispat |
---|
1058 | self.pattern_src = pattern |
---|
1059 | else: |
---|
1060 | try: |
---|
1061 | self.re_pat = re.compile( pattern ) |
---|
1062 | except: |
---|
1063 | print "Failed to compile pattern >>%s<< (%s)" % (pattern, tag) |
---|
1064 | self.pattern = pattern |
---|
1065 | |
---|
1066 | self.examples = examples |
---|
1067 | self.badExamples = badExamples |
---|
1068 | self.list = list |
---|
1069 | self.cls = cls |
---|
1070 | |
---|
1071 | if runTest: |
---|
1072 | if examples != None: |
---|
1073 | for e in examples: |
---|
1074 | assert self.check(e), 'Internal check failed: example %s does not fit pattern %s' % (e,self.pattern) |
---|
1075 | |
---|
1076 | def check(self,val): |
---|
1077 | self.note = '-' |
---|
1078 | m = self.re_pat.match( val ) |
---|
1079 | if self.list == None: |
---|
1080 | self.note = "simple test" |
---|
1081 | return m != None |
---|
1082 | else: |
---|
1083 | if m == None: |
---|
1084 | self.note = "no match %s::%s" % (val,self.pattern) |
---|
1085 | return False |
---|
1086 | if not m.groupdict().has_key("val"): |
---|
1087 | self.note = "no 'val' in match" |
---|
1088 | return False |
---|
1089 | self.note = "val=%s" % m.groupdict()["val"] |
---|
1090 | return m.groupdict()["val"] in self.list |
---|
1091 | |
---|
1092 | class listControl(object): |
---|
1093 | def __init__(self,tag,list,split=False,splitVal=None,enumeration=False): |
---|
1094 | self.list = list |
---|
1095 | self.tag = tag |
---|
1096 | self.split = split |
---|
1097 | self.splitVal = splitVal |
---|
1098 | self.enumeration = enumeration |
---|
1099 | self.etest = re.compile( '(.*)<([0-9]+(,[0-9]+)*)>' ) |
---|
1100 | self.essplit = re.compile(r'(?:[^\s,<]|<(?:\\.|[^>])*>)+') |
---|
1101 | |
---|
1102 | def check(self,val): |
---|
1103 | self.note = '-' |
---|
1104 | if len(self.list) < 4: |
---|
1105 | self.note = str( self.list ) |
---|
1106 | else: |
---|
1107 | self.note = str( self.list[:4] ) |
---|
1108 | if self.split: |
---|
1109 | if self.splitVal == None: |
---|
1110 | vs = string.split( val ) |
---|
1111 | elif self.enumeration: |
---|
1112 | vs = map( string.strip, self.essplit.findall( val ) ) |
---|
1113 | else: |
---|
1114 | vs = map( string.strip, string.split( val, self.splitVal ) ) |
---|
1115 | else: |
---|
1116 | vs = [val,] |
---|
1117 | if self.enumeration: |
---|
1118 | vs2 = [] |
---|
1119 | for v in vs: |
---|
1120 | m = self.etest.findall( v ) |
---|
1121 | if m in [None,[]]: |
---|
1122 | vs2.append( v ) |
---|
1123 | else: |
---|
1124 | opts = string.split( m[0][1], ',' ) |
---|
1125 | for o in opts: |
---|
1126 | vs2.append( '%s%s' % (m[0][0],o) ) |
---|
1127 | vs = vs2[:] |
---|
1128 | |
---|
1129 | return all( map( lambda x: x in self.list, vs ) ) |
---|
1130 | |
---|
1131 | |
---|
1132 | class checkByVar(checkBase): |
---|
1133 | """Run some checks on groups of files with a common variable. Checks for continuity of time in group""" |
---|
1134 | |
---|
1135 | def init(self,fileNameSeparator='_'): |
---|
1136 | self.id = 'C5.001' |
---|
1137 | self.checkId = 'unset' |
---|
1138 | self.step = 'Initialised' |
---|
1139 | self.checks = (self.checkTrange,) |
---|
1140 | self.fnsep = fileNameSeparator |
---|
1141 | |
---|
1142 | def setLogDict( self,fLogDict ): |
---|
1143 | self.fLogDict = fLogDict |
---|
1144 | |
---|
1145 | def impt(self,flist): |
---|
1146 | """Scan a list of files and identify groups which a common variable and extract time ranges into a dictionary of lists, keyed on group identifiers. |
---|
1147 | :param flist: List of file names. |
---|
1148 | |
---|
1149 | This routine has rather obscure nested logical tests used to identify the group to which a file belongs. The complexity arises from the fact that the identification of the files that should form a continuous time series from the file names alone is not a standardised feature of the file names.""" |
---|
1150 | ee = {} |
---|
1151 | elist = [] |
---|
1152 | for f in flist: |
---|
1153 | fn = string.split(f, '/' )[-1] |
---|
1154 | fnParts = string.split( fn[:-3], self.fnsep ) |
---|
1155 | |
---|
1156 | try: |
---|
1157 | if self.pcfg.freqIndex != None: |
---|
1158 | freq = fnParts[self.pcfg.freqIndex] |
---|
1159 | else: |
---|
1160 | freq = None |
---|
1161 | |
---|
1162 | group = fnParts[ self.pcfg.groupIndex ] |
---|
1163 | |
---|
1164 | if self.parent.fileIsFixed: |
---|
1165 | trange = None |
---|
1166 | else: |
---|
1167 | trange = string.split( fnParts[-1], '-' ) |
---|
1168 | var = fnParts[self.pcfg.varIndex] |
---|
1169 | thisKey = string.join( fnParts[:-1], '.' ) |
---|
1170 | if group not in ee.keys(): |
---|
1171 | ee[group] = {} |
---|
1172 | if thisKey not in ee[group].keys(): |
---|
1173 | ee[group][thisKey] = [] |
---|
1174 | ee[group][thisKey].append( (f,fn,group,trange) ) |
---|
1175 | except: |
---|
1176 | print 'Cannot parse file name: %s' % (f) |
---|
1177 | elist.append(f) |
---|
1178 | ## this ee entry is not used, except in bookkeeping check below. |
---|
1179 | ## parsing of file name is repeated later, and a error log entry is created at that stage -- this could be improved. |
---|
1180 | ## in order to improve, need to clarify flow of program: the list here is used to provide preliminary info before log files etc are set up. |
---|
1181 | group = '__error__' |
---|
1182 | thisKey = fn |
---|
1183 | if group not in ee.keys(): |
---|
1184 | ee[group] = {} |
---|
1185 | if thisKey not in ee[group].keys(): |
---|
1186 | ee[group][thisKey] = [] |
---|
1187 | ee[group][thisKey].append( (f,fn,group) ) |
---|
1188 | |
---|
1189 | nn = len(flist) |
---|
1190 | n2 = 0 |
---|
1191 | for k in ee.keys(): |
---|
1192 | for k2 in ee[k].keys(): |
---|
1193 | n2 += len( ee[k][k2] ) |
---|
1194 | |
---|
1195 | assert nn==n2, 'some file lost!!!!!!' |
---|
1196 | if len(elist) == 0: |
---|
1197 | self.info = '%s files, %s' % (nn,str(ee.keys()) ) |
---|
1198 | else: |
---|
1199 | self.info = '%s files, %s frequencies, severe errors in file names: %s' % (nn,len(ee.keys()),len(elist) ) |
---|
1200 | for e in elist: |
---|
1201 | self.info += '\n%s' % e |
---|
1202 | self.ee = ee |
---|
1203 | |
---|
1204 | def check(self, recorder=None,calendar='None',norun=False): |
---|
1205 | self.errorCount = 0 |
---|
1206 | self.recorder=recorder |
---|
1207 | self.calendar=calendar |
---|
1208 | if calendar == '360-day': |
---|
1209 | self.enddec = 30 |
---|
1210 | else: |
---|
1211 | self.enddec = 31 |
---|
1212 | mm = { 'enddec':self.enddec } |
---|
1213 | self.pats = {'mon':('(?P<d>[0-9]{3})101','(?P<e>[0-9]{3})012'), \ |
---|
1214 | 'sem':('(?P<d>[0-9]{3})(012|101)','(?P<e>[0-9]{3})(011|010)'), \ |
---|
1215 | 'day':('(?P<d>[0-9]{3}[16])0101','(?P<e>[0-9]{3}[50])12%(enddec)s' % mm), \ |
---|
1216 | 'subd':('(?P<d>[0-9]{4})0101(?P<h1>[0-9]{2})(?P<mm>[30]0){0,1}$', '(?P<e>[0-9]{4})12%(enddec)s(?P<h2>[0-9]{2})([30]0){0,1}$' % mm ), \ |
---|
1217 | 'subd2':('(?P<d>[0-9]{4})0101(?P<h1>[0-9]{2})', '(?P<e>[0-9]{4})010100' ) } |
---|
1218 | |
---|
1219 | if not norun: |
---|
1220 | self.runChecks() |
---|
1221 | |
---|
1222 | def checkTrange(self): |
---|
1223 | """Manage time range checks: loop over groups of files identified by :meth:`impt`""" |
---|
1224 | keys = self.ee.keys() |
---|
1225 | keys.sort() |
---|
1226 | for k in keys: |
---|
1227 | if k not in ['fx','fixed']: |
---|
1228 | keys2 = self.ee[k].keys() |
---|
1229 | keys2.sort() |
---|
1230 | for k2 in keys2: |
---|
1231 | self.checkThisTrange( self.ee[k][k2], k ) |
---|
1232 | |
---|
1233 | def checkThisTrange( self, tt, group): |
---|
1234 | """Check consistency across a list of time ranges""" |
---|
1235 | |
---|
1236 | if group in ['3hr','6hr']: |
---|
1237 | kg = 'subd' |
---|
1238 | else: |
---|
1239 | kg = group |
---|
1240 | ps = self.pats[kg] |
---|
1241 | rere = (re.compile( ps[0] ), re.compile( ps[1] ) ) |
---|
1242 | |
---|
1243 | n = len(tt) |
---|
1244 | self.checkId = ('001','filename_timerange_value') |
---|
1245 | for j in range(n): |
---|
1246 | if self.monitor != None: |
---|
1247 | nofh0 = self.monitor.get_open_fds() |
---|
1248 | t = tt[j] |
---|
1249 | fn = t[1] |
---|
1250 | isFirst = j == 0 |
---|
1251 | isLast = j == n-1 |
---|
1252 | lok = True |
---|
1253 | for i in [0,1]: |
---|
1254 | if not (i==0 and isFirst or i==1 and isLast): |
---|
1255 | x = rere[i].match( t[3][i] ) |
---|
1256 | lok &= self.test( x != None, 'Cannot match time range %s: %s [%s/%s]' % (i,fn,j,n), part=True, appendLogfile=(self.fLogDict.get(fn,None),fn) ) |
---|
1257 | if not lok: |
---|
1258 | if self.recorder != None: |
---|
1259 | self.recorder.modify( t[1], 'ERROR: time range' ) |
---|
1260 | if self.monitor != None: |
---|
1261 | nofh9 = self.monitor.get_open_fds() |
---|
1262 | if nofh9 > nofh0: |
---|
1263 | print 'Open file handles: %s --- %s [%s]' % (nofh0, nofh9, j ) |
---|
1264 | |
---|
1265 | ### http://stackoverflow.com/questions/2023608/check-what-files-are-open-in-python |
---|
1266 | class sysMonitor(object): |
---|
1267 | |
---|
1268 | def __init__(self): |
---|
1269 | self.fhCountMax = 0 |
---|
1270 | |
---|
1271 | def get_open_fds(self): |
---|
1272 | ''' |
---|
1273 | return the number of open file descriptors for current process |
---|
1274 | .. warning: will only work on UNIX-like os-es. |
---|
1275 | ''' |
---|
1276 | import subprocess |
---|
1277 | import os |
---|
1278 | |
---|
1279 | pid = os.getpid() |
---|
1280 | self.procs = subprocess.check_output( |
---|
1281 | [ "lsof", '-w', '-Ff', "-p", str( pid ) ] ) |
---|
1282 | |
---|
1283 | self.ps = filter( |
---|
1284 | lambda s: s and s[ 0 ] == 'f' and s[1: ].isdigit(), |
---|
1285 | self.procs.split( '\n' ) ) |
---|
1286 | self.fhCountMax = max( self.fhCountMax, len(self.ps) ) |
---|
1287 | return len( self.ps ) |
---|