1 | |
---|
2 | from dreqPy import dreq |
---|
3 | from dreqPy import misc_utils |
---|
4 | from ingest import mipTab |
---|
5 | import cmchecl |
---|
6 | import collections, sys, os |
---|
7 | |
---|
8 | python2 = True |
---|
9 | if sys.version_info[0] == 3: |
---|
10 | python2 = False |
---|
11 | |
---|
12 | if len(sys.argv) > 1: |
---|
13 | lname = sys.argv[1] |
---|
14 | else: |
---|
15 | lname = 'audit' |
---|
16 | logFarm = misc_utils.dreqLog(dir='logs') |
---|
17 | log = logFarm.getLog(lname) |
---|
18 | |
---|
19 | class c1(object): |
---|
20 | def __init__(self): |
---|
21 | self.a = collections.defaultdict( list ) |
---|
22 | |
---|
23 | class checkUnits(object): |
---|
24 | |
---|
25 | def __init__(self, cc, extra=None): |
---|
26 | assert python2, 'The checkUnits module requires cf, which is not available in python 3' |
---|
27 | import cf |
---|
28 | if extra != None: |
---|
29 | for k in sorted( extra.keys() ): |
---|
30 | try: |
---|
31 | a = cf.units.Units( k ) |
---|
32 | except: |
---|
33 | log.warn( 'WARN.001.0004: bad units %s (%s)' % (k,str(extra[k])) ) |
---|
34 | |
---|
35 | for k in sorted( cc.keys() ): |
---|
36 | try: |
---|
37 | a = cf.units.Units( k ) |
---|
38 | except: |
---|
39 | log.warn( 'Bad CF units: %s' % k ) |
---|
40 | a = None |
---|
41 | |
---|
42 | if a != None: |
---|
43 | for k2 in cc[k].a: |
---|
44 | try: |
---|
45 | b = cf.units.Units( k2 ) |
---|
46 | except: |
---|
47 | b = None |
---|
48 | log.warn( 'WARN.001.0003: unit invalid : %s .... %s: %s' % (k,k2,str( cc[k].a[k2]) ) ) |
---|
49 | if b != None and not a.equivalent(b): |
---|
50 | log.warn( 'WARN.001.0002: unit mismatch: %s .... %s: %s' % (k,k2,str( cc[k].a[k2]) ) ) |
---|
51 | |
---|
52 | def splitStr( s ): |
---|
53 | if s == '': |
---|
54 | return [''] |
---|
55 | if s.find( ' ') == -1: |
---|
56 | return [s,] |
---|
57 | return s.split() |
---|
58 | |
---|
59 | class checkDims(object): |
---|
60 | |
---|
61 | def __init__(self, dq): |
---|
62 | """Generate list of used dimensions in ss; |
---|
63 | Check against defined dimenions [todo]""" |
---|
64 | ss = set() |
---|
65 | self.cellm = set() |
---|
66 | cc = collections.defaultdict(list) |
---|
67 | duds = ('', '****', '?',) |
---|
68 | for i in dq.coll['spatialShape'].items: |
---|
69 | if i.dimensions != '': |
---|
70 | for x in i.dimensions.split( '|' ): |
---|
71 | if x in duds: |
---|
72 | cc[x].append( i.uid ) |
---|
73 | else: |
---|
74 | ss.add(x) |
---|
75 | for i in dq.coll['temporalShape'].items: |
---|
76 | if i.dimensions != '': |
---|
77 | for x in i.dimensions.split( '|' ): |
---|
78 | if x in duds: |
---|
79 | cc[x].append( i.uid ) |
---|
80 | else: |
---|
81 | ss.add(x) |
---|
82 | for i in dq.coll['structure'].items: |
---|
83 | self.cellm.add( i.cell_methods ) |
---|
84 | if i.odims != '': |
---|
85 | for x in i.odims.split( '|' ): |
---|
86 | if x in duds: |
---|
87 | cc[x].append( i.uid ) |
---|
88 | else: |
---|
89 | ss.add(x) |
---|
90 | if i.coords != '': |
---|
91 | for x in i.coords.split( '|' ): |
---|
92 | if x in duds: |
---|
93 | cc[x].append( i.uid ) |
---|
94 | else: |
---|
95 | ss.add(x) |
---|
96 | |
---|
97 | cellm = sorted( list( self.cellm ) ) |
---|
98 | for c in cellm: |
---|
99 | log.info( 'cell_methods\t %s' % c ) |
---|
100 | log.info( str( sorted( list(ss) ) ) ) |
---|
101 | ll = set( [i.label for i in dq.coll['grids'].items] ) |
---|
102 | for x in ss: |
---|
103 | if x not in ll: |
---|
104 | log.error( 'ERROR: dimension not found: %s' % x ) |
---|
105 | |
---|
106 | class auditor(object): |
---|
107 | def __init__(self,dq): |
---|
108 | self.dq = dq |
---|
109 | self.run() |
---|
110 | |
---|
111 | def run(self): |
---|
112 | self.loadDraftNewVar() |
---|
113 | self.audit_var() |
---|
114 | self.audit_sn() |
---|
115 | self.audit_cmv() |
---|
116 | self.audit_rqv() |
---|
117 | self.audit_rql() |
---|
118 | self.audit_rqg() |
---|
119 | self.audit_str() |
---|
120 | self.audit_cm() |
---|
121 | |
---|
122 | def audit_cm(self): |
---|
123 | log.info( 'Running Cell Methods Audit' ) |
---|
124 | cc = cmchecl.check_cm() |
---|
125 | nf = 0 |
---|
126 | for i in self.dq.coll['cellMethods'].items: |
---|
127 | res = cc.test( i.cell_methods ) |
---|
128 | if not res: |
---|
129 | log.error( 'ERROR.cm.00001: cannot parse cell methods: "%s" [%s]' % (i.cell_methods,i.label) ) |
---|
130 | nf += 1 |
---|
131 | if nf == 0: |
---|
132 | log.info( 'INFO.cm.00001: all cell methods strings parsed by cmchecl' ) |
---|
133 | |
---|
134 | l1 = [i for i in self.dq.coll['structure'].items if i.cell_methods == ""] |
---|
135 | ss = set() |
---|
136 | if len( l1 ) > 0: |
---|
137 | log.error( 'ERROR.cm.00010: %s blank cell methods in structure records: ' % len(l1) ) |
---|
138 | for i in l1: |
---|
139 | log.error( 'ERROR.cm.00011: %s: %s' % ( i.label, i.title ) ) |
---|
140 | for i in l1: |
---|
141 | if 'CMORvar' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
142 | for u in dq.inx.iref_by_sect[i.uid].a['CMORvar']: |
---|
143 | ss.add(u) |
---|
144 | if len(ss) > 0: |
---|
145 | log.error( 'ERROR.cm.00020: %s CMOR vars affected ' % len(ss) ) |
---|
146 | |
---|
147 | def audit_var(self): |
---|
148 | cc = collections.defaultdict( list ) |
---|
149 | ss = collections.defaultdict( c1 ) |
---|
150 | ssx = collections.defaultdict( set ) |
---|
151 | umap = { 'Wm-2':'W m-2', 'string':''} |
---|
152 | for i in self.dq.coll['var'].items: |
---|
153 | cc[i.label].append(i) |
---|
154 | isn = self.dq.inx.uid[i.sn] |
---|
155 | if isn._h.label != 'remarks': |
---|
156 | u = umap.get( isn.units, isn.units ) |
---|
157 | ss[u].a[i.units].append( i.label ) |
---|
158 | else: |
---|
159 | ssx[i.units].add( i.label ) |
---|
160 | |
---|
161 | ii = [k for k in cc if len(cc[k]) > 1] |
---|
162 | log.warn( 'var.0001: %48s [%s]: %s' % ('Duplicate variable names',len( self.dq.coll['var'].items ),len(ii)) ) |
---|
163 | showDupVar=False |
---|
164 | showDupVar=True |
---|
165 | showDupVarBrief=False |
---|
166 | if showDupVar: |
---|
167 | for i in ii: |
---|
168 | log.info( '----- %s -----' % i ) |
---|
169 | for x in cc[i]: |
---|
170 | log.info( '%s, %s, %s, %s' % ( x.label,x.title,x.sn,x.prov )) |
---|
171 | elif showDupVarBrief: |
---|
172 | lg.info( str( ii )) |
---|
173 | self.cc = cc |
---|
174 | |
---|
175 | if not python2: |
---|
176 | log.warn( 'Skipping the units check .. not available in python3' ) |
---|
177 | else: |
---|
178 | checkUnits( ss, extra=ssx ) |
---|
179 | |
---|
180 | def audit_str(self): |
---|
181 | nms = 0 |
---|
182 | ss = set() |
---|
183 | cc = collections.defaultdict( list ) |
---|
184 | for i in self.dq.coll['structure'].items: |
---|
185 | cc[ (i.spid, i.tmid,i.odims,i.coords,i.cmid) ].append( i ) |
---|
186 | if i.cmid != '__unset__' and self.dq.inx.uid[i.cmid]._h.label == 'remarks' and i.cell_methods != '': |
---|
187 | ss.add(i.cmid) |
---|
188 | nms += 1 |
---|
189 | if nms > 0: |
---|
190 | log.error( 'str.00010: missing cell methods records for %s records (%s)' % (nms, len(ss) ) ) |
---|
191 | log.error( 'str.00011: %s' % str( sorted(list(ss)) ) ) |
---|
192 | else: |
---|
193 | log.info( 'str.00010: all structure methods have valid cmid' ) |
---|
194 | |
---|
195 | ks = [k for k in cc if len(cc[k]) > 1] |
---|
196 | log.info( 'str.00020: count of repeated space/time structure: %s' % len(ks) ) |
---|
197 | if len(ks) > 0: |
---|
198 | oo = open( 'structureRepeats.csv', 'w' ) |
---|
199 | for k in ks: |
---|
200 | oo.write( '\t'.join( ['NEXT', k[0], k[1], k[2], k[3], self.dq.inx.uid[k[0]].label, self.dq.inx.uid[k[1]].label] ) + '\n' ) |
---|
201 | for i in cc[k]: |
---|
202 | oo.write( '\t'.join( ['+',i.uid, i.label, i.title, i.odims, i.coords, i.cell_methods, i.cell_measures, i.flag_meanings, i.spid ] ) + '\n' ) |
---|
203 | oo.close() |
---|
204 | |
---|
205 | def saveRevisedNewVar(self,ll): |
---|
206 | oo = open( 'revised_newvars.csv', 'w' ) |
---|
207 | for v in sorted( list( ll ) ): |
---|
208 | oo.write( self.draftNewVar[v] ) |
---|
209 | oo.close() |
---|
210 | |
---|
211 | def loadDraftNewVar(self): |
---|
212 | self.draftNewVar = {} |
---|
213 | for f in ['draft_newvars.csv','draft_newvars02.csv']: |
---|
214 | ii = open( f, 'r' ).readlines() |
---|
215 | ss = set() |
---|
216 | for l in ii: |
---|
217 | r = l.split( '\t' ) |
---|
218 | vn = r[0].strip() |
---|
219 | assert vn not in ss, 'Duplicate variable name in %s: %s' % (f,vn) |
---|
220 | ss.add(vn) |
---|
221 | if vn in self.draftNewVar: |
---|
222 | log.info( 'INFO.newvar.0001: overwriting new var: %s' % vn ) |
---|
223 | self.draftNewVar[vn] = l |
---|
224 | |
---|
225 | def audit_cmv(self): |
---|
226 | zmtabs = ['AERmonZ', 'E6hrZ', 'EdayZ', 'EmonZ'] |
---|
227 | mipt = mipTab.mipt() |
---|
228 | zmtabsi = [mipt.invmap[x] for x in zmtabs] |
---|
229 | ccmv = collections.defaultdict( list ) |
---|
230 | ccmv2 = collections.defaultdict( list ) |
---|
231 | ccmv4 = collections.defaultdict( list ) |
---|
232 | ccmv6 = collections.defaultdict( list ) |
---|
233 | ccp = collections.defaultdict( set ) |
---|
234 | |
---|
235 | cmlk = {} |
---|
236 | zmstr = set() |
---|
237 | for i in self.dq.coll['structure'].items: |
---|
238 | cmlk[i.uid] = i.cell_methods |
---|
239 | if i.cell_methods.find( 'longitude: mean' ) != -1 or i.cell_methods.find( 'global: mean' ): |
---|
240 | zmstr.add( i.uid ) |
---|
241 | print 'ZMSTR', zmstr |
---|
242 | |
---|
243 | ## |
---|
244 | ## check that vars in AERmonZ, E6hrZ, EdayZ, EmonZ have cell_methods with longitude mean. |
---|
245 | ## |
---|
246 | nms = 0 |
---|
247 | ss = set() |
---|
248 | badRealm = set() |
---|
249 | badzm = set() |
---|
250 | badzmt = set() |
---|
251 | spidmap = {'XY-AH':'XY-A/AH','XY-A':'XY-A/AH'} |
---|
252 | for i in self.dq.coll['CMORvar'].items: |
---|
253 | if i.mipTable in zmtabsi and (i.stid not in zmstr): |
---|
254 | badzm.add( i.uid ) |
---|
255 | badzmt.add( i.mipTable ) |
---|
256 | log.error( 'cmv.00121: bad zonal mean variable: %s.%s, %s, %s, %s, %s' % (i.mipTable, i.label,i.uid, i.prov, i.stid, cmlk[i.stid]) ) |
---|
257 | |
---|
258 | ccp[ i.label ].add( i.positive ) |
---|
259 | ccmv[i.label].append(i) |
---|
260 | ccmv4[ (i.frequency,i.label,i.stid)].append(i) |
---|
261 | ccmv2[ (i.mipTable,i.label)].append(i) |
---|
262 | if self.dq.inx.uid[i.stid]._h.label == 'remarks': |
---|
263 | ss.add( i.stid ) |
---|
264 | nms += 1 |
---|
265 | else: |
---|
266 | st = self.dq.inx.uid[i.stid] |
---|
267 | sp = self.dq.inx.uid[st.spid] |
---|
268 | ccmv6[ (i.frequency,i.label,spidmap.get(sp.label,sp.label),st.tmid)].append((i,st)) |
---|
269 | if i.modeling_realm in ['','__unset__']: |
---|
270 | badRealm.add( (i.mipTable, i.label ) ) |
---|
271 | kkp = [k for k in ccp if len(ccp[k]) > 1] |
---|
272 | if len(kkp) > 0: |
---|
273 | log.error( 'cmv.00170: bad positive attributes: %s, %s' % (len( kkp ),str(kkp)) ) |
---|
274 | else: |
---|
275 | log.info( 'cmv.00170: positive attributes OK' ) |
---|
276 | |
---|
277 | if len( badzm ) > 0: |
---|
278 | log.error( 'cmv.00120: bad zonal mean variables: %s, %s' % (len( badzm ),str(badzmt)) ) |
---|
279 | if len( badRealm ) > 0: |
---|
280 | log.error( 'cmv.00030: bad modeling realms set in %s cmv records' % len( badRealm ) ) |
---|
281 | cc1 = collections.defaultdict( list ) |
---|
282 | for tab, lab in badRealm: |
---|
283 | cc1[tab].append(lab) |
---|
284 | for tab in sorted( cc1.keys() ): |
---|
285 | log.error( 'cmv.00031: table %s: %s' % (tab,str(sorted(cc1[tab]))) ) |
---|
286 | else: |
---|
287 | log.info( 'cmv.00030: modeling realms set ok cmv records' ) |
---|
288 | |
---|
289 | if nms > 0: |
---|
290 | log.error( 'cmv.00010: missing variable structures for %s CMORvar records, bad stids: %s' % (nms,len(ss)) ) |
---|
291 | log.error( 'cmv.00011: bad stids: %s' % str( sorted( list( ss ) ) ) ) |
---|
292 | else: |
---|
293 | log.info( 'cmv.00010: variable structures for all CMORvar records present' ) |
---|
294 | |
---|
295 | ii0 = [i for i in self.dq.coll['CMORvar'].items if i.mipTable == ''] |
---|
296 | if len(ii0) > 0: |
---|
297 | log.error( 'cmv.00020: %48s [%s]: %s' % ('Empty MIP table in CMORvar',len(self.dq.coll['CMORvar'].items),len(ii0)) ) |
---|
298 | else: |
---|
299 | log.info( 'cmv.00020: all CMORvar records have mip table defined' ) |
---|
300 | |
---|
301 | ii = [i for i in self.dq.coll['CMORvar'].items if self.dq.inx.uid[i.vid]._h.label == 'remarks'] |
---|
302 | if len(ii) == 0: |
---|
303 | log.info( 'cmv.00200: all CMORvar records have valid vid' ) |
---|
304 | else: |
---|
305 | ii1 = [i for i in ii if 'requestVar' in self.dq.inx.iref_by_sect[i.uid].a] |
---|
306 | log.error( 'cmv.00200: %48s [%s]: %s (requested: %s)' % ('CMORvar records with bad vid',len(self.dq.coll['CMORvar'].items),len(ii),len(ii1)) ) |
---|
307 | |
---|
308 | iix2 = [] |
---|
309 | iix3 = [] |
---|
310 | for i in ii1: |
---|
311 | ok = False |
---|
312 | ok3 = False |
---|
313 | for u in self.dq.inx.iref_by_sect[i.uid].a['requestVar']: |
---|
314 | vg = self.dq.inx.uid[ self.dq.inx.uid[u].vgid ] |
---|
315 | if vg._h.label != 'remarks': |
---|
316 | ok = True |
---|
317 | iix2.append(i) |
---|
318 | if 'requestLink' in self.dq.inx.iref_by_sect[vg.uid].a: |
---|
319 | iix3.append(i) |
---|
320 | log.error( 'cmv.0021: of these: valid requestVarGroup: %s; valid requestLink(s): %s' % (len(iix2),len(iix3)) ) |
---|
321 | rnv = set() |
---|
322 | for i in iix3: |
---|
323 | if i.label in self.draftNewVar: |
---|
324 | log.error( 'cmv.0125: %s, %s, %s' % (i.uid,i.label,i.title) ) |
---|
325 | rnv.add( i.label ) |
---|
326 | else: |
---|
327 | log.error( 'cmv.0122: %s, %s, %s' % (i.uid,i.label,i.title) ) |
---|
328 | |
---|
329 | if len( rnv ) > 0: |
---|
330 | self.saveRevisedNewVar( rnv ) |
---|
331 | |
---|
332 | |
---|
333 | ii2 = [k for k in ccmv2 if len(ccmv2[k]) > 1] |
---|
334 | if len(ii2) == 0: |
---|
335 | log.info( 'cmv.0030: unique CMOR variable names OK' ) |
---|
336 | else: |
---|
337 | log.error( 'cmv.0030: non-unique CMOR variable naming for %s records' % len(ii2) ) |
---|
338 | log.error( 'cmv.0031: %s' % str(ii2) ) |
---|
339 | |
---|
340 | ii4 = [k for k in ccmv4 if len(ccmv4[k]) > 1] |
---|
341 | ii6 = [k for k in ccmv6 if len(ccmv6[k]) > 1] |
---|
342 | ii4b = list() |
---|
343 | ii6b = list() |
---|
344 | for k in ii4: |
---|
345 | if len(ccmv4[k]) == 2 and set( [i.mipTable[-3:] for i in ccmv4[k]] ) == set(['Ant','Gre']): |
---|
346 | pass |
---|
347 | else: |
---|
348 | ii4b.append(k) |
---|
349 | |
---|
350 | for k in ii6: |
---|
351 | if len(ccmv6[k]) == 2 and set( [i.mipTable[-3:] for i,s in ccmv6[k]] ) == set(['Ant','Gre']): |
---|
352 | pass |
---|
353 | else: |
---|
354 | ii6b.append(k) |
---|
355 | |
---|
356 | if len(ii4b) == 0: |
---|
357 | log.info( 'cmv.0040: unique CMOR variable usage [4] OK' ) |
---|
358 | else: |
---|
359 | log.error( 'cmv.0040: non-unique CMOR variable usage possible for %s records' % len(ii4b) ) |
---|
360 | log.error( 'cmv.0041: %s' % str(ii4b) ) |
---|
361 | kk=0 |
---|
362 | oo = open( 'cmv_probDuplication.csv', 'w' ) |
---|
363 | for k in ii4b: |
---|
364 | kk += 1 |
---|
365 | rec = ('NEXT\t%s\t' % kk ) + str( '\t'.join( k ) ) |
---|
366 | print 'INFO.cmvdup.00001: ',rec |
---|
367 | oo.write( rec + '\n' ) |
---|
368 | vid = None |
---|
369 | for i in ccmv4[k]: |
---|
370 | assert vid == None or i.vid == vid, 'Unexpected vid variation ...: %s: %s %s' % (i.label, i.vid, vid) |
---|
371 | vid = i.vid |
---|
372 | rec = '+\t%s\t%s\t%s\t%s\t%s' % (i.uid,i.mipTable,i.title,i.description,i.prov) |
---|
373 | oo.write( rec + '\n' ) |
---|
374 | print 'INFO.cmvdup.00001: ',rec |
---|
375 | oo.close() |
---|
376 | |
---|
377 | if len(ii6b) == 0: |
---|
378 | log.info( 'cmv.0060: unique CMOR variable usage [6] OK' ) |
---|
379 | else: |
---|
380 | log.info( 'cmv.0060: unique CMOR variable issues [6] %s ' % len(ii6b) ) |
---|
381 | kk=0 |
---|
382 | ccx = collections.defaultdict( set ) |
---|
383 | for k in ii6b: |
---|
384 | ss = set() |
---|
385 | freq,label,spid,tmid = k |
---|
386 | for i,st in ccmv6[k]: |
---|
387 | ss.add( (st.uid,st.label,st.title) ) |
---|
388 | tt = tuple( sorted( list( ss ) ) ) |
---|
389 | ccx[tt].add( (freq,label) ) |
---|
390 | log.info( 'cmv.0061: unique CMOR variable issues [6.1] %s ' % len(ccx.keys()) ) |
---|
391 | for tt in ccx: |
---|
392 | ll = [t[2] for t in tt] |
---|
393 | log.info( 'cmv.0062: unique CMOR variable issues [6.2] %s: %s ' % ('; '.join(ll),str(ccx[tt]) ) ) |
---|
394 | |
---|
395 | oo = open( 'cmv_probDuplication6.csv', 'w' ) |
---|
396 | for k in ii6b: |
---|
397 | kk += 1 |
---|
398 | rec = ('NEXT\t%s\t' % kk ) + str( '\t'.join( k ) ) |
---|
399 | oo.write( rec + '\n' ) |
---|
400 | vid = None |
---|
401 | for i,st in ccmv6[k]: |
---|
402 | assert vid == None or i.vid == vid, 'Unexpected vid variation ...: %s: %s %s' % (i.label, i.vid, vid) |
---|
403 | vid = i.vid |
---|
404 | rec = '+\t%s\t%s\t%s\t%s\t%s\t%s' % (i.uid,i.mipTable,i.title,st.title,i.description,i.prov) |
---|
405 | oo.write( rec + '\n' ) |
---|
406 | oo.close() |
---|
407 | |
---|
408 | ccmv3 = collections.defaultdict( list ) |
---|
409 | for t in ccmv2: |
---|
410 | s = set() |
---|
411 | for i in ccmv2[t]: |
---|
412 | id = i.uid |
---|
413 | if 'varRelLnk' in self.dq.inx.iref_by_sect[id].a: |
---|
414 | for x in self.dq.inx.iref_by_sect[id].a['varRelLnk']: |
---|
415 | s.add( x ) |
---|
416 | s = list(s) |
---|
417 | if len(s) > 0: |
---|
418 | ccmv3[t] = s |
---|
419 | |
---|
420 | ii5 = [k for k in ii2 if k not in ccmv3 or len(ccmv2[k]) > len(ccmv3[k])] |
---|
421 | cc4 = collections.defaultdict( list ) |
---|
422 | for t,v in ii2: |
---|
423 | cc4[t].append( v ) |
---|
424 | |
---|
425 | log.warn( '%48s [%s]: %s [%s tables]' % ('Duplicate variables in CMORvar',len(self.dq.coll['CMORvar'].items),len(ii2), len( cc4.keys() ) ) ) |
---|
426 | showAllCmvErrors=False |
---|
427 | showAllCmvErrors=True |
---|
428 | if showAllCmvErrors: |
---|
429 | rats = ['uid','label','title','description','frequency','mipTable','stid','prov','provNote'] |
---|
430 | oocm = open( 'cmvDup.csv', 'w' ) |
---|
431 | c1 = collections.defaultdict( list ) |
---|
432 | ##for i in ii2: |
---|
433 | ##c1[i.prov].append( i.label ) |
---|
434 | ##log.warn( 'No MIP var: %s ... %s' % (str( sorted( [i.label for i in ii if i.label in self.cc] ) ), str(c1))) |
---|
435 | for k in sorted( cc4.keys() ): |
---|
436 | log.warn( '%16s:: %s' % (k,str(sorted(cc4[k]))) ) |
---|
437 | for v in sorted(cc4[k]): |
---|
438 | sv = set() |
---|
439 | for i in ccmv2[ (k,v) ]: |
---|
440 | sv.add(i.vid) |
---|
441 | |
---|
442 | if len(list(sv)) == 1: |
---|
443 | vun = True |
---|
444 | vv = self.dq.inx.uid[ list(sv)[0] ] |
---|
445 | oocm.write( '@VAR:\t\t\t%s\t%s\t%s\n' % (vv.label,vv.title,vv.description) ) |
---|
446 | else: |
---|
447 | vun = False |
---|
448 | |
---|
449 | for i in ccmv2[ (k,v) ]: |
---|
450 | stt = self.dq.inx.uid[i.stid].title |
---|
451 | if 'Temporal mean' in stt: |
---|
452 | if i.frequency == '3hr': |
---|
453 | i.title += ' [3 hour mean]' |
---|
454 | xtra = '' |
---|
455 | oocm.write( '\t\t' + '\t'.join( [str(i.__dict__[x]) for x in rats] ) + '\t%s%s\n' % (stt,xtra) ) |
---|
456 | if not vun: |
---|
457 | vv = self.dq.inx.uid[i.vid] |
---|
458 | xtra = '\t\t\t\t%s\t%s\t%s\n' % (vv.label,vv.title,vv.description) |
---|
459 | oocm.write(xtra) |
---|
460 | oocm.write( '####\n' ) |
---|
461 | oocm.close() |
---|
462 | t2_ii = ii[:] |
---|
463 | self.ccmv = ccmv |
---|
464 | |
---|
465 | def audit_sn(self): |
---|
466 | cc = collections.defaultdict( list ) |
---|
467 | cc2 = collections.defaultdict( int ) |
---|
468 | nrem = 0 |
---|
469 | snm = set() |
---|
470 | snmm = collections.defaultdict( int ) |
---|
471 | for i in self.dq.coll['var'].items: |
---|
472 | if 'CMORvar' in self.dq.inx.iref_by_sect[i.uid].a: |
---|
473 | srq = set() |
---|
474 | for u in self.dq.inx.iref_by_sect[i.uid].a['CMORvar']: |
---|
475 | if 'requestVar' in self.dq.inx.iref_by_sect[u].a: |
---|
476 | for uu in self.dq.inx.iref_by_sect[u].a['requestVar']: |
---|
477 | rqv = self.dq.inx.uid[uu] |
---|
478 | srq.add( rqv.priority ) |
---|
479 | if len(srq) > 0: |
---|
480 | pm = min( srq ) |
---|
481 | |
---|
482 | kk = i.sn |
---|
483 | if self.dq.inx.uid[kk]._h.label == 'remarks': |
---|
484 | nrem += 1 |
---|
485 | snm.add(kk) |
---|
486 | ## if pm == 1: |
---|
487 | print 'MISSING STANDARD NAME: ',i.label, i.title, i.sn, i.provmip |
---|
488 | snmm[pm] += 1 |
---|
489 | cc2[i.provmip] += 1 |
---|
490 | if i.procnote != []: |
---|
491 | ll = sorted( i.procnote ) |
---|
492 | kk += ':' + '|'.join(ll ) |
---|
493 | cc[kk].append( i.uid ) |
---|
494 | |
---|
495 | log.warn( 'sn.001: Number of missing standard names: %s [for %s variables] {%s}' % (len(snm),nrem,str(snmm)) ) |
---|
496 | log.warn( 'sn.002: Number of missing standard names by MIP: %s ' % (str(cc2)) ) |
---|
497 | ll = [] |
---|
498 | for k in cc: |
---|
499 | if len( cc[k] ) > 1: |
---|
500 | ll.append( k ) |
---|
501 | |
---|
502 | log.warn( 'sn.003: Duplicate use of standard names: %s' % len(ll) ) |
---|
503 | oo = open( 'audit_sn_repeats.csv', 'w' ) |
---|
504 | for k in sorted( ll ): |
---|
505 | orc = [k,] |
---|
506 | l1 = cc[k][:] |
---|
507 | l2 = [] |
---|
508 | l1b = [] |
---|
509 | for u in cc[k]: |
---|
510 | i = self.dq.inx.uid[u] |
---|
511 | if i.prov == "CMIP6 endorsement [SIMIP]": |
---|
512 | l2.append(u) |
---|
513 | else: |
---|
514 | l1b.append(u) |
---|
515 | if len( l2) > 1: |
---|
516 | log.error( 'WARN.simip.00001: repeat use of standard name in SIMIP: %s:: %s' % (k,str(l2)) ) |
---|
517 | if len(l2) == 1: |
---|
518 | orc.append( '*' ) |
---|
519 | else: |
---|
520 | orc.append( '' ) |
---|
521 | |
---|
522 | nnn = 0 |
---|
523 | for u in l2 +l1b: |
---|
524 | i = self.dq.inx.uid[u] |
---|
525 | if "CMORvar" not in self.dq.inx.iref_by_sect[u].a: |
---|
526 | orc += [i.label +'**', i.title] |
---|
527 | else: |
---|
528 | nn = 0 |
---|
529 | for uu in self.dq.inx.iref_by_sect[u].a['CMORvar']: |
---|
530 | cmv = self.dq.inx.uid[uu] |
---|
531 | if 'requestVar' in self.dq.inx.iref_by_sect[uu].a: |
---|
532 | nn += 1 |
---|
533 | if nn == 0: |
---|
534 | orc += [i.label + '*', i.title] |
---|
535 | else: |
---|
536 | nnn += 1 |
---|
537 | orc += [i.label, i.title] |
---|
538 | |
---|
539 | if nnn > 1: |
---|
540 | oo.write( '\t'.join( orc ) + '\n' ) |
---|
541 | oo.close() |
---|
542 | |
---|
543 | def audit_rqv(self): |
---|
544 | ixx = [i for i in self.dq.coll['requestVar'].items if i.vid in self.dq.inx.uid] |
---|
545 | if len(ixx) < len( self.dq.inx.uid ): |
---|
546 | log.error( 'SEVERE.var.00301: request vars with bad links ....' ) |
---|
547 | |
---|
548 | ii = [i for i in ixx if self.dq.inx.uid[i.vid]._h.label == 'remarks'] |
---|
549 | ii2 = [i for i in ii if i.label not in self.cc] |
---|
550 | ii3 = [i.vgid for i in ii if self.dq.inx.uid[i.vgid]._h.label != 'remarks'] |
---|
551 | ##ii4: set of invalid requestVarGroup records |
---|
552 | ii4 = [i for i in ii3 if 'requestLink' not in self.dq.inx.iref_by_sect[i].a ] |
---|
553 | ###### |
---|
554 | ii5Info = 'Set of bad requestVar records linking to a valid requestVarGroup records.' |
---|
555 | ii5 = [i for i in ii if i.vgid not in ii4] |
---|
556 | ##ii6: set of bad requestVar records linking to a valid requestVarGroup records and no variable name match. |
---|
557 | ii6 = [i for i in ii5 if i.label not in self.cc] |
---|
558 | log.error( '%48s [%s]: %s [%s, %s, %s; %s]' % ('Bad variable links in requestVar',len(self.dq.coll['requestVar'].items),len(ii),len(ii2), len(ii5), len(ii6), len(ii4)) ) |
---|
559 | log.info ( str( ii5Info )) |
---|
560 | svg = set() |
---|
561 | for i in ii5: |
---|
562 | log.info( 'ii5: %s' % str([i.label, i.title, i.uid, i.mip, self.dq.inx.uid[i.vgid].title] ) ) |
---|
563 | svg.add( i.vgid ) |
---|
564 | for i in ii2: |
---|
565 | if i.vgid in self.dq.inx.uid and self.dq.inx.uid[i.vgid]._h.label != 'remarks': |
---|
566 | xx = '%s [%s]' % (self.dq.inx.uid[i.vgid].title, i.vgid) |
---|
567 | else: |
---|
568 | xx = i.vgid |
---|
569 | log.info( 'ii2: %s' % str([i.label, i.title, i.uid, i.mip, xx] ) ) |
---|
570 | |
---|
571 | log.warn( 'rvg.0010: requestVarGroups with bad request vars:' ) |
---|
572 | for u in sorted( list(svg) ): |
---|
573 | log.warn( 'rvg.0011: %s: %s' % (u, self.dq.inx.uid[u].title )) |
---|
574 | |
---|
575 | log.info( ' [# records]: # broken [var name not known, link to valid group, valid group and no var name; valid request groups' ) |
---|
576 | showAllRqvErrors=False |
---|
577 | if showAllRqvErrors: |
---|
578 | log.info( '%s\n%s' % (str( [i.label for i in ii if i.label in self.cc]), str( [i.label for i in ii2] ) ) ) |
---|
579 | |
---|
580 | ii1 = [i for i in ii if i.label in self.ccmv] |
---|
581 | listBadRequestVar=False |
---|
582 | if listBadRequestVar: |
---|
583 | for i in ii1: |
---|
584 | log.info( str( i.label,i.mip,i.table,[x.frequency for x in self.ccmv[i.label]] ) ) |
---|
585 | |
---|
586 | def audit_rql(self): |
---|
587 | ii = [i for i in self.dq.coll['requestLink'].items if self.dq.inx.uid[i.refid]._h.label == 'remarks'] |
---|
588 | log.info( '%48s: %s (from %s)' % ('ERROR.rql.001: Bad request group links in requestLink',len(ii), len(self.dq.coll['requestLink'].items)) ) |
---|
589 | rql_detail=True |
---|
590 | if rql_detail: |
---|
591 | for i in ii: |
---|
592 | log.info( '%s: %s, %s [%s]' % ('ERROR.rql.002: ',i.mip,i.label,i.uid) ) |
---|
593 | |
---|
594 | def audit_rqg(self): |
---|
595 | ii = [i for i in self.dq.coll['requestVarGroup'].items if len( dq.inx.iref_by_sect[i.uid].a['requestVar']) == 0] |
---|
596 | ii1 = [i for i in ii if len( dq.inx.iref_by_sect[i.uid].a['requestLink']) != 0] |
---|
597 | log.info( '%48s: %s/%s (from %s)' % ('Request groups with no request links:',len(ii) - len(ii1),len(ii1), len(self.dq.coll['requestVarGroup'].items)) ) |
---|
598 | listEmptyRequestVarGroups=True |
---|
599 | if listEmptyRequestVarGroups: |
---|
600 | for i in ii1: |
---|
601 | log.info ( 'INFO.rqgempty.0001: %s: %s, %s [%s]' % (i.label, i.title, i.mip, i.uid) ) |
---|
602 | |
---|
603 | dq = dreq.loadDreq(manifest=None) |
---|
604 | a = auditor( dq ) |
---|
605 | cd = checkDims( dq ) |
---|