| 1 | |
|---|
| 2 | from dreqPy import dreq |
|---|
| 3 | from dreqPy import misc_utils |
|---|
| 4 | import collections, sys, os |
|---|
| 5 | |
|---|
| 6 | |
|---|
| 7 | python2 = True |
|---|
| 8 | if sys.version_info[0] == 3: |
|---|
| 9 | python2 = False |
|---|
| 10 | |
|---|
| 11 | if len(sys.argv) > 1: |
|---|
| 12 | lname = sys.argv[1] |
|---|
| 13 | else: |
|---|
| 14 | lname = 'audit' |
|---|
| 15 | logFarm = misc_utils.dreqLog(dir='logs') |
|---|
| 16 | log = logFarm.getLog(lname) |
|---|
| 17 | |
|---|
| 18 | class c1(object): |
|---|
| 19 | def __init__(self): |
|---|
| 20 | self.a = collections.defaultdict( list ) |
|---|
| 21 | |
|---|
| 22 | class checkUnits(object): |
|---|
| 23 | |
|---|
| 24 | def __init__(self, cc): |
|---|
| 25 | assert python2, 'The checkUnits module requires cf, which is not available in python 3' |
|---|
| 26 | import cf |
|---|
| 27 | for k in sorted( cc.keys() ): |
|---|
| 28 | try: |
|---|
| 29 | a = cf.units.Units( k ) |
|---|
| 30 | except: |
|---|
| 31 | log.warn( 'Bad CF units: %s' % k ) |
|---|
| 32 | a = None |
|---|
| 33 | |
|---|
| 34 | |
|---|
| 35 | if a != None: |
|---|
| 36 | for k2 in cc[k].a: |
|---|
| 37 | try: |
|---|
| 38 | b = cf.units.Units( k2 ) |
|---|
| 39 | except: |
|---|
| 40 | b = None |
|---|
| 41 | log.warn( 'WARN.001.0003: unit invalid : %s .... %s: %s' % (k,k2,str( cc[k].a[k2]) ) ) |
|---|
| 42 | if b != None and not a.equivalent(b): |
|---|
| 43 | log.warn( 'WARN.001.0002: unit mismatch: %s .... %s: %s' % (k,k2,str( cc[k].a[k2]) ) ) |
|---|
| 44 | |
|---|
| 45 | def splitStr( s ): |
|---|
| 46 | if s == '': |
|---|
| 47 | return [''] |
|---|
| 48 | if s.find( ' ') == -1: |
|---|
| 49 | return [s,] |
|---|
| 50 | return s.split() |
|---|
| 51 | |
|---|
| 52 | class checkDims(object): |
|---|
| 53 | |
|---|
| 54 | def __init__(self, dq): |
|---|
| 55 | """Generate list of used dimensions in ss; |
|---|
| 56 | Check against defined dimenions [todo]""" |
|---|
| 57 | ss = set() |
|---|
| 58 | self.cellm = set() |
|---|
| 59 | cc = collections.defaultdict(list) |
|---|
| 60 | duds = ('', '****', '?',) |
|---|
| 61 | for i in dq.coll['spatialShape'].items: |
|---|
| 62 | if i.dimensions != '': |
|---|
| 63 | for x in i.dimensions.split( '|' ): |
|---|
| 64 | if x in duds: |
|---|
| 65 | cc[x].append( i.uid ) |
|---|
| 66 | else: |
|---|
| 67 | ss.add(x) |
|---|
| 68 | for i in dq.coll['temporalShape'].items: |
|---|
| 69 | if i.dimensions != '': |
|---|
| 70 | for x in i.dimensions.split( '|' ): |
|---|
| 71 | if x in duds: |
|---|
| 72 | cc[x].append( i.uid ) |
|---|
| 73 | else: |
|---|
| 74 | ss.add(x) |
|---|
| 75 | for i in dq.coll['structure'].items: |
|---|
| 76 | self.cellm.add( i.cell_methods ) |
|---|
| 77 | if i.odims != '': |
|---|
| 78 | for x in i.odims.split( '|' ): |
|---|
| 79 | if x in duds: |
|---|
| 80 | cc[x].append( i.uid ) |
|---|
| 81 | else: |
|---|
| 82 | ss.add(x) |
|---|
| 83 | if i.coords != '': |
|---|
| 84 | for x in i.coords.split( '|' ): |
|---|
| 85 | if x in duds: |
|---|
| 86 | cc[x].append( i.uid ) |
|---|
| 87 | else: |
|---|
| 88 | ss.add(x) |
|---|
| 89 | |
|---|
| 90 | cellm = sorted( list( self.cellm ) ) |
|---|
| 91 | for c in cellm: |
|---|
| 92 | log.info( 'cell_methods\t %s' % c ) |
|---|
| 93 | log.info( str( sorted( list(ss) ) ) ) |
|---|
| 94 | ll = set( [i.label for i in dq.coll['grids'].items] ) |
|---|
| 95 | for x in ss: |
|---|
| 96 | if x not in ll: |
|---|
| 97 | log.error( 'ERROR: dimension not found: %s' % x ) |
|---|
| 98 | |
|---|
| 99 | class auditor(object): |
|---|
| 100 | def __init__(self,dq): |
|---|
| 101 | self.dq = dq |
|---|
| 102 | self.run() |
|---|
| 103 | |
|---|
| 104 | def run(self): |
|---|
| 105 | self.audit_var() |
|---|
| 106 | self.audit_sn() |
|---|
| 107 | self.audit_cmv() |
|---|
| 108 | self.audit_rqv() |
|---|
| 109 | self.audit_rql() |
|---|
| 110 | self.audit_rqg() |
|---|
| 111 | self.audit_str() |
|---|
| 112 | |
|---|
| 113 | def audit_var(self): |
|---|
| 114 | cc = collections.defaultdict( list ) |
|---|
| 115 | ss = collections.defaultdict( c1 ) |
|---|
| 116 | umap = { 'Wm-2':'W m-2', 'string':''} |
|---|
| 117 | for i in self.dq.coll['var'].items: |
|---|
| 118 | cc[i.label].append(i) |
|---|
| 119 | isn = self.dq.inx.uid[i.sn] |
|---|
| 120 | if isn._h.label != 'remarks': |
|---|
| 121 | u = umap.get( isn.units, isn.units ) |
|---|
| 122 | ss[u].a[i.units].append( i.label ) |
|---|
| 123 | |
|---|
| 124 | ii = [k for k in cc if len(cc[k]) > 1] |
|---|
| 125 | log.warn( 'var.0001: %48s [%s]: %s' % ('Duplicate variable names',len( self.dq.coll['var'].items ),len(ii)) ) |
|---|
| 126 | showDupVar=False |
|---|
| 127 | showDupVar=True |
|---|
| 128 | showDupVarBrief=False |
|---|
| 129 | if showDupVar: |
|---|
| 130 | for i in ii: |
|---|
| 131 | log.info( '----- %s -----' % i ) |
|---|
| 132 | for x in cc[i]: |
|---|
| 133 | log.info( '%s, %s, %s, %s' % ( x.label,x.title,x.sn,x.prov )) |
|---|
| 134 | elif showDupVarBrief: |
|---|
| 135 | lg.info( str( ii )) |
|---|
| 136 | self.cc = cc |
|---|
| 137 | |
|---|
| 138 | if not python2: |
|---|
| 139 | log.warn( 'Skipping the units check .. not available in python3' ) |
|---|
| 140 | else: |
|---|
| 141 | checkUnits( ss ) |
|---|
| 142 | |
|---|
| 143 | def audit_str(self): |
|---|
| 144 | nms = 0 |
|---|
| 145 | ss = set() |
|---|
| 146 | for i in self.dq.coll['structure'].items: |
|---|
| 147 | if self.dq.inx.uid[i.cmid]._h.label == 'remarks' and i.cell_methods != '': |
|---|
| 148 | ss.add(i.cmid) |
|---|
| 149 | nms += 1 |
|---|
| 150 | if nms > 0: |
|---|
| 151 | log.error( 'str.00010: missing cell methods records for %s records (%s)' % (nms, len(ss) ) ) |
|---|
| 152 | log.error( 'str.00011: %s' % str( sorted(list(ss)) ) ) |
|---|
| 153 | else: |
|---|
| 154 | log.info( 'str.00010: all structure methods have valid cmid' ) |
|---|
| 155 | |
|---|
| 156 | def audit_cmv(self): |
|---|
| 157 | ccmv = collections.defaultdict( list ) |
|---|
| 158 | ccmv2 = collections.defaultdict( list ) |
|---|
| 159 | ccmv4 = collections.defaultdict( list ) |
|---|
| 160 | nms = 0 |
|---|
| 161 | ss = set() |
|---|
| 162 | for i in self.dq.coll['CMORvar'].items: |
|---|
| 163 | ccmv[i.label].append(i) |
|---|
| 164 | ccmv4[ (i.frequency,i.label,i.stid)].append(i) |
|---|
| 165 | ccmv2[ (i.mipTable,i.label)].append(i) |
|---|
| 166 | if self.dq.inx.uid[i.stid]._h.label == 'remarks': |
|---|
| 167 | ss.add( i.stid ) |
|---|
| 168 | nms += 1 |
|---|
| 169 | if nms > 0: |
|---|
| 170 | log.error( 'cmv.00010: missing variable structures for %s CMORvar records, bad stids: %s' % (nms,len(ss)) ) |
|---|
| 171 | log.error( 'cmv.00011: bad stids: %s' % str( sorted( list( ss ) ) ) ) |
|---|
| 172 | else: |
|---|
| 173 | log.info( 'cmv.00010: variable structures for all CMORvar records present' ) |
|---|
| 174 | |
|---|
| 175 | ii0 = [i for i in self.dq.coll['CMORvar'].items if i.mipTable == ''] |
|---|
| 176 | if len(ii0) > 0: |
|---|
| 177 | log.error( 'cmv.00020: %48s [%s]: %s' % ('Empty MIP table in CMORvar',len(self.dq.coll['CMORvar'].items),len(ii0)) ) |
|---|
| 178 | else: |
|---|
| 179 | log.info( 'cmv.00020: all CMORvar records have mip table defined' ) |
|---|
| 180 | |
|---|
| 181 | ii = [i for i in self.dq.coll['CMORvar'].items if self.dq.inx.uid[i.vid]._h.label == 'remarks'] |
|---|
| 182 | if len(ii) == 0: |
|---|
| 183 | log.info( 'cmv.00200: all CMORvar records have valid vid' ) |
|---|
| 184 | else: |
|---|
| 185 | ii1 = [i for i in ii if 'requestVar' in self.dq.inx.iref_by_sect[i.uid].a] |
|---|
| 186 | log.error( 'cmv.00200: %48s [%s]: %s (requested: %s)' % ('CMORvar records with bad vid',len(self.dq.coll['CMORvar'].items),len(ii),len(ii1)) ) |
|---|
| 187 | |
|---|
| 188 | iix2 = [] |
|---|
| 189 | iix3 = [] |
|---|
| 190 | for i in ii1: |
|---|
| 191 | ok = False |
|---|
| 192 | ok3 = False |
|---|
| 193 | for u in self.dq.inx.iref_by_sect[i.uid].a['requestVar']: |
|---|
| 194 | vg = self.dq.inx.uid[ self.dq.inx.uid[u].vgid ] |
|---|
| 195 | if vg._h.label != 'remarks': |
|---|
| 196 | ok = True |
|---|
| 197 | iix2.append(i) |
|---|
| 198 | if 'requestLink' in self.dq.inx.iref_by_sect[vg.uid].a: |
|---|
| 199 | iix3.append(i) |
|---|
| 200 | log.error( 'cmv.0021: of these: valid requestVarGroup: %s; valid requestLink(s): %s' % (len(iix2),len(iix3)) ) |
|---|
| 201 | for i in iix3: |
|---|
| 202 | log.error( 'cmv.0022: %s, %s, %s' % (i.uid,i.label,i.title) ) |
|---|
| 203 | |
|---|
| 204 | ii2 = [k for k in ccmv2 if len(ccmv2[k]) > 1] |
|---|
| 205 | if len(ii2) == 0: |
|---|
| 206 | log.info( 'cmv.0030: unique CMOR variable names OK' ) |
|---|
| 207 | else: |
|---|
| 208 | log.error( 'cmv.0030: non-unique CMOR variable naming for %s records' % len(ii2) ) |
|---|
| 209 | log.error( 'cmv.0031: %s' % str(ii2) ) |
|---|
| 210 | |
|---|
| 211 | ii4 = [k for k in ccmv4 if len(ccmv2[k]) > 1] |
|---|
| 212 | if len(ii4) == 0: |
|---|
| 213 | log.info( 'cmv.0040: unique CMOR variable usage OK' ) |
|---|
| 214 | else: |
|---|
| 215 | log.error( 'cmv.0040: non-unique CMOR variable usage possible for %s records' % len(ii4) ) |
|---|
| 216 | log.error( 'cmv.0041: %s' % str(ii4) ) |
|---|
| 217 | |
|---|
| 218 | ccmv3 = collections.defaultdict( list ) |
|---|
| 219 | for t in ccmv2: |
|---|
| 220 | s = set() |
|---|
| 221 | for i in ccmv2[t]: |
|---|
| 222 | id = i.uid |
|---|
| 223 | if 'varRelLnk' in self.dq.inx.iref_by_sect[id].a: |
|---|
| 224 | for x in self.dq.inx.iref_by_sect[id].a['varRelLnk']: |
|---|
| 225 | s.add( x ) |
|---|
| 226 | s = list(s) |
|---|
| 227 | if len(s) > 0: |
|---|
| 228 | ccmv3[t] = s |
|---|
| 229 | |
|---|
| 230 | ii5 = [k for k in ii2 if k not in ccmv3 or len(ccmv2[k]) > len(ccmv3[k])] |
|---|
| 231 | cc4 = collections.defaultdict( list ) |
|---|
| 232 | for t,v in ii2: |
|---|
| 233 | cc4[t].append( v ) |
|---|
| 234 | |
|---|
| 235 | log.warn( '%48s [%s]: %s [%s tables]' % ('Duplicate variables in CMORvar',len(self.dq.coll['CMORvar'].items),len(ii2), len( cc4.keys() ) ) ) |
|---|
| 236 | showAllCmvErrors=False |
|---|
| 237 | showAllCmvErrors=True |
|---|
| 238 | if showAllCmvErrors: |
|---|
| 239 | rats = ['uid','label','title','description','frequency','mipTable','stid','prov','provNote'] |
|---|
| 240 | oocm = open( 'cmvDup.csv', 'w' ) |
|---|
| 241 | c1 = collections.defaultdict( list ) |
|---|
| 242 | ##for i in ii2: |
|---|
| 243 | ##c1[i.prov].append( i.label ) |
|---|
| 244 | ##log.warn( 'No MIP var: %s ... %s' % (str( sorted( [i.label for i in ii if i.label in self.cc] ) ), str(c1))) |
|---|
| 245 | for k in sorted( cc4.keys() ): |
|---|
| 246 | log.warn( '%16s:: %s' % (k,str(sorted(cc4[k]))) ) |
|---|
| 247 | for v in sorted(cc4[k]): |
|---|
| 248 | sv = set() |
|---|
| 249 | for i in ccmv2[ (k,v) ]: |
|---|
| 250 | sv.add(i.vid) |
|---|
| 251 | |
|---|
| 252 | if len(list(sv)) == 1: |
|---|
| 253 | vun = True |
|---|
| 254 | vv = self.dq.inx.uid[ list(sv)[0] ] |
|---|
| 255 | oocm.write( '@VAR:\t\t\t%s\t%s\t%s\n' % (vv.label,vv.title,vv.description) ) |
|---|
| 256 | else: |
|---|
| 257 | vun = False |
|---|
| 258 | |
|---|
| 259 | for i in ccmv2[ (k,v) ]: |
|---|
| 260 | stt = self.dq.inx.uid[i.stid].title |
|---|
| 261 | if 'Temporal mean' in stt: |
|---|
| 262 | if i.frequency == '3hr': |
|---|
| 263 | i.title += ' [3 hour mean]' |
|---|
| 264 | xtra = '' |
|---|
| 265 | oocm.write( '\t\t' + '\t'.join( [i.__dict__[x] for x in rats] ) + '\t%s%s\n' % (stt,xtra) ) |
|---|
| 266 | if not vun: |
|---|
| 267 | vv = self.dq.inx.uid[i.vid] |
|---|
| 268 | xtra = '\t\t\t\t%s\t%s\t%s\n' % (vv.label,vv.title,vv.description) |
|---|
| 269 | oocm.write(xtra) |
|---|
| 270 | oocm.write( '####\n' ) |
|---|
| 271 | oocm.close() |
|---|
| 272 | t2_ii = ii[:] |
|---|
| 273 | self.ccmv = ccmv |
|---|
| 274 | |
|---|
| 275 | def audit_sn(self): |
|---|
| 276 | cc = collections.defaultdict( list ) |
|---|
| 277 | nrem = 0 |
|---|
| 278 | snm = set() |
|---|
| 279 | for i in self.dq.coll['var'].items: |
|---|
| 280 | kk = i.sn |
|---|
| 281 | if self.dq.inx.uid[kk]._h.label == 'remarks': |
|---|
| 282 | nrem += 1 |
|---|
| 283 | snm.add(kk) |
|---|
| 284 | if i.procnote != []: |
|---|
| 285 | ll = sorted( i.procnote ) |
|---|
| 286 | kk += ':' + '|'.join(ll ) |
|---|
| 287 | cc[kk].append( i.uid ) |
|---|
| 288 | |
|---|
| 289 | log.warn( 'sn.001: Number of missing standard names: %s [for %s variables]' % (len(snm),nrem) ) |
|---|
| 290 | ll = [] |
|---|
| 291 | for k in cc: |
|---|
| 292 | if len( cc[k] ) > 1: |
|---|
| 293 | ll.append( k ) |
|---|
| 294 | |
|---|
| 295 | log.warn( 'sn.002: Duplicate use of standard names: %s' % len(ll) ) |
|---|
| 296 | oo = open( 'audit_sn_repeats.csv', 'w' ) |
|---|
| 297 | for k in sorted( ll ): |
|---|
| 298 | orc = [k,] |
|---|
| 299 | l1 = cc[k][:] |
|---|
| 300 | l2 = [] |
|---|
| 301 | l1b = [] |
|---|
| 302 | for u in cc[k]: |
|---|
| 303 | i = self.dq.inx.uid[u] |
|---|
| 304 | if i.prov == "CMIP6 endorsement [SIMIP]": |
|---|
| 305 | l2.append(u) |
|---|
| 306 | else: |
|---|
| 307 | l1b.append(u) |
|---|
| 308 | if len( l2) > 1: |
|---|
| 309 | log.error( 'ERROR: identified multiple priority choices: %s:: %s' % (k,str(l2)) ) |
|---|
| 310 | if len(l2) == 1: |
|---|
| 311 | orc.append( '*' ) |
|---|
| 312 | else: |
|---|
| 313 | orc.append( '' ) |
|---|
| 314 | |
|---|
| 315 | for u in l2 +l1b: |
|---|
| 316 | i = self.dq.inx.uid[u] |
|---|
| 317 | orc += [i.label, i.title] |
|---|
| 318 | |
|---|
| 319 | oo.write( '\t'.join( orc ) + '\n' ) |
|---|
| 320 | oo.close() |
|---|
| 321 | |
|---|
| 322 | def audit_rqv(self): |
|---|
| 323 | ii = [i for i in self.dq.coll['requestVar'].items if self.dq.inx.uid[i.vid]._h.label == 'remarks'] |
|---|
| 324 | ii2 = [i for i in ii if i.label not in self.cc] |
|---|
| 325 | ii3 = [i.vgid for i in ii if self.dq.inx.uid[i.vgid]._h.label != 'remarks'] |
|---|
| 326 | ##ii4: set of invalid requestVarGroup records |
|---|
| 327 | ii4 = [i for i in ii3 if 'requestLink' not in self.dq.inx.iref_by_sect[i].a ] |
|---|
| 328 | ###### |
|---|
| 329 | ii5Info = 'Set of bad requestVar records linking to a valid requestVarGroup records.' |
|---|
| 330 | ii5 = [i for i in ii if i.vgid not in ii4] |
|---|
| 331 | ##ii6: set of bad requestVar records linking to a valid requestVarGroup records and no variable name match. |
|---|
| 332 | ii6 = [i for i in ii5 if i.label not in self.cc] |
|---|
| 333 | log.error( '%48s [%s]: %s [%s, %s, %s; %s]' % ('Bad variable links in requestVar',len(self.dq.coll['requestVar'].items),len(ii),len(ii2), len(ii5), len(ii6), len(ii4)) ) |
|---|
| 334 | log.info ( str( ii5Info )) |
|---|
| 335 | svg = set() |
|---|
| 336 | for i in ii5: |
|---|
| 337 | log.info( 'ii5: %s' % str([i.label, i.title, i.uid, i.mip, self.dq.inx.uid[i.vgid].title] ) ) |
|---|
| 338 | svg.add( i.vgid ) |
|---|
| 339 | for i in ii2: |
|---|
| 340 | log.info( 'ii2: %s' % str([i.label, i.title, i.uid] ) ) |
|---|
| 341 | |
|---|
| 342 | log.warn( 'rvg.0010: requestVarGroups with bad request vars:' ) |
|---|
| 343 | for u in sorted( list(svg) ): |
|---|
| 344 | log.warn( 'rvg.0011: %s: %s' % (u, self.dq.inx.uid[u].title )) |
|---|
| 345 | |
|---|
| 346 | log.info( ' [# records]: # broken [var name not known, link to valid group, valid group and no var name; valid request groups' ) |
|---|
| 347 | showAllRqvErrors=False |
|---|
| 348 | if showAllRqvErrors: |
|---|
| 349 | log.info( '%s\n%s' % (str( [i.label for i in ii if i.label in self.cc]), str( [i.label for i in ii2] ) ) ) |
|---|
| 350 | |
|---|
| 351 | ii1 = [i for i in ii if i.label in self.ccmv] |
|---|
| 352 | listBadRequestVar=False |
|---|
| 353 | if listBadRequestVar: |
|---|
| 354 | for i in ii1: |
|---|
| 355 | log.info( str( i.label,i.mip,i.table,[x.frequency for x in self.ccmv[i.label]] ) ) |
|---|
| 356 | |
|---|
| 357 | def audit_rql(self): |
|---|
| 358 | ii = [i for i in self.dq.coll['requestLink'].items if self.dq.inx.uid[i.refid]._h.label == 'remarks'] |
|---|
| 359 | log.info( '%48s: %s (from %s)' % ('Bad request group links in requestLink',len(ii), len(self.dq.coll['requestLink'].items)) ) |
|---|
| 360 | |
|---|
| 361 | def audit_rqg(self): |
|---|
| 362 | ii = [i for i in self.dq.coll['requestVarGroup'].items if len( dq.inx.iref_by_sect[i.uid].a['requestVar']) == 0] |
|---|
| 363 | ii1 = [i for i in ii if len( dq.inx.iref_by_sect[i.uid].a['requestLink']) != 0] |
|---|
| 364 | log.info( '%48s: %s/%s (from %s)' % ('Empty request groups',len(ii) - len(ii1),len(ii1), len(self.dq.coll['requestVarGroup'].items)) ) |
|---|
| 365 | listEmptyRequestVarGroups=False |
|---|
| 366 | if listEmptyRequestVarGroups: |
|---|
| 367 | for i in ii1: |
|---|
| 368 | log.info ( '%s: %s, %s' % (i.label, i.title, i.mip) ) |
|---|
| 369 | |
|---|
| 370 | dq = dreq.loadDreq() |
|---|
| 371 | a = auditor( dq ) |
|---|
| 372 | cd = checkDims( dq ) |
|---|
