| 1 | |
|---|
| 2 | from dreqPy import dreq |
|---|
| 3 | from dreqPy import misc_utils |
|---|
| 4 | from ingest import mipTab |
|---|
| 5 | import cmchecl |
|---|
| 6 | import collections, sys, os |
|---|
| 7 | |
|---|
| 8 | python2 = True |
|---|
| 9 | if sys.version_info[0] == 3: |
|---|
| 10 | python2 = False |
|---|
| 11 | |
|---|
| 12 | if len(sys.argv) > 1: |
|---|
| 13 | lname = sys.argv[1] |
|---|
| 14 | else: |
|---|
| 15 | lname = 'audit' |
|---|
| 16 | logFarm = misc_utils.dreqLog(dir='logs') |
|---|
| 17 | log = logFarm.getLog(lname) |
|---|
| 18 | |
|---|
| 19 | class c1(object): |
|---|
| 20 | def __init__(self): |
|---|
| 21 | self.a = collections.defaultdict( list ) |
|---|
| 22 | |
|---|
| 23 | class checkUnits(object): |
|---|
| 24 | |
|---|
| 25 | def __init__(self, cc, extra=None): |
|---|
| 26 | assert python2, 'The checkUnits module requires cf, which is not available in python 3' |
|---|
| 27 | import cf |
|---|
| 28 | if extra != None: |
|---|
| 29 | for k in sorted( extra.keys() ): |
|---|
| 30 | try: |
|---|
| 31 | a = cf.units.Units( k ) |
|---|
| 32 | except: |
|---|
| 33 | log.warn( 'WARN.001.0004: bad units %s (%s)' % (k,str(extra[k])) ) |
|---|
| 34 | |
|---|
| 35 | for k in sorted( cc.keys() ): |
|---|
| 36 | try: |
|---|
| 37 | a = cf.units.Units( k ) |
|---|
| 38 | except: |
|---|
| 39 | log.warn( 'Bad CF units: %s' % k ) |
|---|
| 40 | a = None |
|---|
| 41 | |
|---|
| 42 | if a != None: |
|---|
| 43 | for k2 in cc[k].a: |
|---|
| 44 | try: |
|---|
| 45 | b = cf.units.Units( k2 ) |
|---|
| 46 | except: |
|---|
| 47 | b = None |
|---|
| 48 | log.warn( 'WARN.001.0003: unit invalid : %s .... %s: %s' % (k,k2,str( cc[k].a[k2]) ) ) |
|---|
| 49 | if b != None and not a.equivalent(b): |
|---|
| 50 | log.warn( 'WARN.001.0002: unit mismatch: %s .... %s: %s' % (k,k2,str( cc[k].a[k2]) ) ) |
|---|
| 51 | |
|---|
| 52 | def splitStr( s ): |
|---|
| 53 | if s == '': |
|---|
| 54 | return [''] |
|---|
| 55 | if s.find( ' ') == -1: |
|---|
| 56 | return [s,] |
|---|
| 57 | return s.split() |
|---|
| 58 | |
|---|
| 59 | class checkDims(object): |
|---|
| 60 | |
|---|
| 61 | def __init__(self, dq): |
|---|
| 62 | """Generate list of used dimensions in ss; |
|---|
| 63 | Check against defined dimenions [todo]""" |
|---|
| 64 | ss = set() |
|---|
| 65 | self.cellm = set() |
|---|
| 66 | cc = collections.defaultdict(list) |
|---|
| 67 | duds = ('', '****', '?',) |
|---|
| 68 | for i in dq.coll['spatialShape'].items: |
|---|
| 69 | if i.dimensions != '': |
|---|
| 70 | for x in i.dimensions.split( '|' ): |
|---|
| 71 | if x in duds: |
|---|
| 72 | cc[x].append( i.uid ) |
|---|
| 73 | else: |
|---|
| 74 | ss.add(x) |
|---|
| 75 | for i in dq.coll['temporalShape'].items: |
|---|
| 76 | if i.dimensions != '': |
|---|
| 77 | for x in i.dimensions.split( '|' ): |
|---|
| 78 | if x in duds: |
|---|
| 79 | cc[x].append( i.uid ) |
|---|
| 80 | else: |
|---|
| 81 | ss.add(x) |
|---|
| 82 | for i in dq.coll['structure'].items: |
|---|
| 83 | self.cellm.add( i.cell_methods ) |
|---|
| 84 | if i.odims != '': |
|---|
| 85 | for x in i.odims.split( '|' ): |
|---|
| 86 | if x in duds: |
|---|
| 87 | cc[x].append( i.uid ) |
|---|
| 88 | else: |
|---|
| 89 | ss.add(x) |
|---|
| 90 | if i.coords != '': |
|---|
| 91 | for x in i.coords.split( '|' ): |
|---|
| 92 | if x in duds: |
|---|
| 93 | cc[x].append( i.uid ) |
|---|
| 94 | else: |
|---|
| 95 | ss.add(x) |
|---|
| 96 | |
|---|
| 97 | cellm = sorted( list( self.cellm ) ) |
|---|
| 98 | for c in cellm: |
|---|
| 99 | log.info( 'cell_methods\t %s' % c ) |
|---|
| 100 | log.info( str( sorted( list(ss) ) ) ) |
|---|
| 101 | ll = set( [i.label for i in dq.coll['grids'].items] ) |
|---|
| 102 | for x in ss: |
|---|
| 103 | if x not in ll: |
|---|
| 104 | log.error( 'ERROR: dimension not found: %s' % x ) |
|---|
| 105 | |
|---|
| 106 | class auditor(object): |
|---|
| 107 | def __init__(self,dq): |
|---|
| 108 | self.dq = dq |
|---|
| 109 | self.run() |
|---|
| 110 | |
|---|
| 111 | def run(self): |
|---|
| 112 | self.loadDraftNewVar() |
|---|
| 113 | self.audit_var() |
|---|
| 114 | self.audit_sn() |
|---|
| 115 | self.audit_cmv() |
|---|
| 116 | self.audit_rqv() |
|---|
| 117 | self.audit_rql() |
|---|
| 118 | self.audit_rqg() |
|---|
| 119 | self.audit_str() |
|---|
| 120 | self.audit_cm() |
|---|
| 121 | |
|---|
| 122 | def audit_cm(self): |
|---|
| 123 | log.info( 'Running Cell Methods Audit' ) |
|---|
| 124 | cc = cmchecl.check_cm() |
|---|
| 125 | nf = 0 |
|---|
| 126 | for i in self.dq.coll['cellMethods'].items: |
|---|
| 127 | res = cc.test( i.cell_methods ) |
|---|
| 128 | if not res: |
|---|
| 129 | log.error( 'ERROR.cm.00001: cannot parse cell methods: "%s" [%s]' % (i.cell_methods,i.label) ) |
|---|
| 130 | nf += 1 |
|---|
| 131 | if nf == 0: |
|---|
| 132 | log.info( 'INFO.cm.00001: all cell methods strings parsed by cmchecl' ) |
|---|
| 133 | |
|---|
| 134 | l1 = [i for i in self.dq.coll['structure'].items if i.cell_methods == ""] |
|---|
| 135 | ss = set() |
|---|
| 136 | if len( l1 ) > 0: |
|---|
| 137 | log.error( 'ERROR.cm.00010: %s blank cell methods in structure records: ' % len(l1) ) |
|---|
| 138 | for i in l1: |
|---|
| 139 | log.error( 'ERROR.cm.00011: %s: %s' % ( i.label, i.title ) ) |
|---|
| 140 | for i in l1: |
|---|
| 141 | if 'CMORvar' in self.dq.inx.iref_by_sect[i.uid].a: |
|---|
| 142 | for u in dq.inx.iref_by_sect[i.uid].a['CMORvar']: |
|---|
| 143 | ss.add(u) |
|---|
| 144 | if len(ss) > 0: |
|---|
| 145 | log.error( 'ERROR.cm.00020: %s CMOR vars affected ' % len(ss) ) |
|---|
| 146 | |
|---|
| 147 | def audit_var(self): |
|---|
| 148 | cc = collections.defaultdict( list ) |
|---|
| 149 | ss = collections.defaultdict( c1 ) |
|---|
| 150 | ssx = collections.defaultdict( set ) |
|---|
| 151 | umap = { 'Wm-2':'W m-2', 'string':''} |
|---|
| 152 | for i in self.dq.coll['var'].items: |
|---|
| 153 | cc[i.label].append(i) |
|---|
| 154 | isn = self.dq.inx.uid[i.sn] |
|---|
| 155 | if isn._h.label != 'remarks': |
|---|
| 156 | u = umap.get( isn.units, isn.units ) |
|---|
| 157 | ss[u].a[i.units].append( i.label ) |
|---|
| 158 | else: |
|---|
| 159 | ssx[i.units].add( i.label ) |
|---|
| 160 | |
|---|
| 161 | ii = [k for k in cc if len(cc[k]) > 1] |
|---|
| 162 | log.warn( 'var.0001: %48s [%s]: %s' % ('Duplicate variable names',len( self.dq.coll['var'].items ),len(ii)) ) |
|---|
| 163 | showDupVar=False |
|---|
| 164 | showDupVar=True |
|---|
| 165 | showDupVarBrief=False |
|---|
| 166 | if showDupVar: |
|---|
| 167 | for i in ii: |
|---|
| 168 | log.info( '----- %s -----' % i ) |
|---|
| 169 | for x in cc[i]: |
|---|
| 170 | log.info( '%s, %s, %s, %s' % ( x.label,x.title,x.sn,x.prov )) |
|---|
| 171 | elif showDupVarBrief: |
|---|
| 172 | lg.info( str( ii )) |
|---|
| 173 | self.cc = cc |
|---|
| 174 | |
|---|
| 175 | if not python2: |
|---|
| 176 | log.warn( 'Skipping the units check .. not available in python3' ) |
|---|
| 177 | else: |
|---|
| 178 | checkUnits( ss, extra=ssx ) |
|---|
| 179 | |
|---|
| 180 | def audit_str(self): |
|---|
| 181 | nms = 0 |
|---|
| 182 | ss = set() |
|---|
| 183 | cc = collections.defaultdict( list ) |
|---|
| 184 | for i in self.dq.coll['structure'].items: |
|---|
| 185 | cc[ (i.spid, i.tmid,i.odims,i.coords,i.cmid) ].append( i ) |
|---|
| 186 | if i.cmid != '__unset__' and self.dq.inx.uid[i.cmid]._h.label == 'remarks' and i.cell_methods != '': |
|---|
| 187 | ss.add(i.cmid) |
|---|
| 188 | nms += 1 |
|---|
| 189 | if nms > 0: |
|---|
| 190 | log.error( 'str.00010: missing cell methods records for %s records (%s)' % (nms, len(ss) ) ) |
|---|
| 191 | log.error( 'str.00011: %s' % str( sorted(list(ss)) ) ) |
|---|
| 192 | else: |
|---|
| 193 | log.info( 'str.00010: all structure methods have valid cmid' ) |
|---|
| 194 | |
|---|
| 195 | ks = [k for k in cc if len(cc[k]) > 1] |
|---|
| 196 | log.info( 'str.00020: count of repeated space/time structure: %s' % len(ks) ) |
|---|
| 197 | if len(ks) > 0: |
|---|
| 198 | oo = open( 'structureRepeats.csv', 'w' ) |
|---|
| 199 | for k in ks: |
|---|
| 200 | oo.write( '\t'.join( ['NEXT', k[0], k[1], k[2], k[3], self.dq.inx.uid[k[0]].label, self.dq.inx.uid[k[1]].label] ) + '\n' ) |
|---|
| 201 | for i in cc[k]: |
|---|
| 202 | oo.write( '\t'.join( ['+',i.uid, i.label, i.title, i.odims, i.coords, i.cell_methods, i.cell_measures, i.flag_meanings, i.spid ] ) + '\n' ) |
|---|
| 203 | oo.close() |
|---|
| 204 | |
|---|
| 205 | def saveRevisedNewVar(self,ll): |
|---|
| 206 | oo = open( 'revised_newvars.csv', 'w' ) |
|---|
| 207 | for v in sorted( list( ll ) ): |
|---|
| 208 | oo.write( self.draftNewVar[v] ) |
|---|
| 209 | oo.close() |
|---|
| 210 | |
|---|
| 211 | def loadDraftNewVar(self): |
|---|
| 212 | self.draftNewVar = {} |
|---|
| 213 | for f in ['draft_newvars.csv','draft_newvars02.csv']: |
|---|
| 214 | ii = open( f, 'r' ).readlines() |
|---|
| 215 | ss = set() |
|---|
| 216 | for l in ii: |
|---|
| 217 | r = l.split( '\t' ) |
|---|
| 218 | vn = r[0].strip() |
|---|
| 219 | assert vn not in ss, 'Duplicate variable name in %s: %s' % (f,vn) |
|---|
| 220 | ss.add(vn) |
|---|
| 221 | if vn in self.draftNewVar: |
|---|
| 222 | log.info( 'INFO.newvar.0001: overwriting new var: %s' % vn ) |
|---|
| 223 | self.draftNewVar[vn] = l |
|---|
| 224 | |
|---|
| 225 | def audit_cmv(self): |
|---|
| 226 | zmtabs = ['AERmonZ', 'E6hrZ', 'EdayZ', 'EmonZ'] |
|---|
| 227 | mipt = mipTab.mipt() |
|---|
| 228 | zmtabsi = [mipt.invmap[x] for x in zmtabs] |
|---|
| 229 | ccmv = collections.defaultdict( list ) |
|---|
| 230 | ccmv2 = collections.defaultdict( list ) |
|---|
| 231 | ccmv4 = collections.defaultdict( list ) |
|---|
| 232 | ccmv6 = collections.defaultdict( list ) |
|---|
| 233 | ccp = collections.defaultdict( set ) |
|---|
| 234 | |
|---|
| 235 | cmlk = {} |
|---|
| 236 | zmstr = set() |
|---|
| 237 | for i in self.dq.coll['structure'].items: |
|---|
| 238 | cmlk[i.uid] = i.cell_methods |
|---|
| 239 | if i.cell_methods.find( 'longitude: mean' ) != -1 or i.cell_methods.find( 'global: mean' ): |
|---|
| 240 | zmstr.add( i.uid ) |
|---|
| 241 | print 'ZMSTR', zmstr |
|---|
| 242 | |
|---|
| 243 | ## |
|---|
| 244 | ## check that vars in AERmonZ, E6hrZ, EdayZ, EmonZ have cell_methods with longitude mean. |
|---|
| 245 | ## |
|---|
| 246 | nms = 0 |
|---|
| 247 | ss = set() |
|---|
| 248 | badRealm = set() |
|---|
| 249 | badzm = set() |
|---|
| 250 | badzmt = set() |
|---|
| 251 | spidmap = {'XY-AH':'XY-A/AH','XY-A':'XY-A/AH'} |
|---|
| 252 | for i in self.dq.coll['CMORvar'].items: |
|---|
| 253 | if i.mipTable in zmtabsi and (i.stid not in zmstr): |
|---|
| 254 | badzm.add( i.uid ) |
|---|
| 255 | badzmt.add( i.mipTable ) |
|---|
| 256 | log.error( 'cmv.00121: bad zonal mean variable: %s.%s, %s, %s, %s, %s' % (i.mipTable, i.label,i.uid, i.prov, i.stid, cmlk[i.stid]) ) |
|---|
| 257 | |
|---|
| 258 | ccp[ i.label ].add( i.positive ) |
|---|
| 259 | ccmv[i.label].append(i) |
|---|
| 260 | ccmv4[ (i.frequency,i.label,i.stid)].append(i) |
|---|
| 261 | ccmv2[ (i.mipTable,i.label)].append(i) |
|---|
| 262 | if self.dq.inx.uid[i.stid]._h.label == 'remarks': |
|---|
| 263 | ss.add( i.stid ) |
|---|
| 264 | nms += 1 |
|---|
| 265 | else: |
|---|
| 266 | st = self.dq.inx.uid[i.stid] |
|---|
| 267 | sp = self.dq.inx.uid[st.spid] |
|---|
| 268 | ccmv6[ (i.frequency,i.label,spidmap.get(sp.label,sp.label),st.tmid)].append((i,st)) |
|---|
| 269 | if i.modeling_realm in ['','__unset__']: |
|---|
| 270 | badRealm.add( (i.mipTable, i.label ) ) |
|---|
| 271 | kkp = [k for k in ccp if len(ccp[k]) > 1] |
|---|
| 272 | if len(kkp) > 0: |
|---|
| 273 | log.error( 'cmv.00170: bad positive attributes: %s, %s' % (len( kkp ),str(kkp)) ) |
|---|
| 274 | else: |
|---|
| 275 | log.info( 'cmv.00170: positive attributes OK' ) |
|---|
| 276 | |
|---|
| 277 | if len( badzm ) > 0: |
|---|
| 278 | log.error( 'cmv.00120: bad zonal mean variables: %s, %s' % (len( badzm ),str(badzmt)) ) |
|---|
| 279 | if len( badRealm ) > 0: |
|---|
| 280 | log.error( 'cmv.00030: bad modeling realms set in %s cmv records' % len( badRealm ) ) |
|---|
| 281 | cc1 = collections.defaultdict( list ) |
|---|
| 282 | for tab, lab in badRealm: |
|---|
| 283 | cc1[tab].append(lab) |
|---|
| 284 | for tab in sorted( cc1.keys() ): |
|---|
| 285 | log.error( 'cmv.00031: table %s: %s' % (tab,str(sorted(cc1[tab]))) ) |
|---|
| 286 | else: |
|---|
| 287 | log.info( 'cmv.00030: modeling realms set ok cmv records' ) |
|---|
| 288 | |
|---|
| 289 | if nms > 0: |
|---|
| 290 | log.error( 'cmv.00010: missing variable structures for %s CMORvar records, bad stids: %s' % (nms,len(ss)) ) |
|---|
| 291 | log.error( 'cmv.00011: bad stids: %s' % str( sorted( list( ss ) ) ) ) |
|---|
| 292 | else: |
|---|
| 293 | log.info( 'cmv.00010: variable structures for all CMORvar records present' ) |
|---|
| 294 | |
|---|
| 295 | ii0 = [i for i in self.dq.coll['CMORvar'].items if i.mipTable == ''] |
|---|
| 296 | if len(ii0) > 0: |
|---|
| 297 | log.error( 'cmv.00020: %48s [%s]: %s' % ('Empty MIP table in CMORvar',len(self.dq.coll['CMORvar'].items),len(ii0)) ) |
|---|
| 298 | else: |
|---|
| 299 | log.info( 'cmv.00020: all CMORvar records have mip table defined' ) |
|---|
| 300 | |
|---|
| 301 | ii = [i for i in self.dq.coll['CMORvar'].items if self.dq.inx.uid[i.vid]._h.label == 'remarks'] |
|---|
| 302 | if len(ii) == 0: |
|---|
| 303 | log.info( 'cmv.00200: all CMORvar records have valid vid' ) |
|---|
| 304 | else: |
|---|
| 305 | ii1 = [i for i in ii if 'requestVar' in self.dq.inx.iref_by_sect[i.uid].a] |
|---|
| 306 | log.error( 'cmv.00200: %48s [%s]: %s (requested: %s)' % ('CMORvar records with bad vid',len(self.dq.coll['CMORvar'].items),len(ii),len(ii1)) ) |
|---|
| 307 | |
|---|
| 308 | iix2 = [] |
|---|
| 309 | iix3 = [] |
|---|
| 310 | for i in ii1: |
|---|
| 311 | ok = False |
|---|
| 312 | ok3 = False |
|---|
| 313 | for u in self.dq.inx.iref_by_sect[i.uid].a['requestVar']: |
|---|
| 314 | vg = self.dq.inx.uid[ self.dq.inx.uid[u].vgid ] |
|---|
| 315 | if vg._h.label != 'remarks': |
|---|
| 316 | ok = True |
|---|
| 317 | iix2.append(i) |
|---|
| 318 | if 'requestLink' in self.dq.inx.iref_by_sect[vg.uid].a: |
|---|
| 319 | iix3.append(i) |
|---|
| 320 | log.error( 'cmv.0021: of these: valid requestVarGroup: %s; valid requestLink(s): %s' % (len(iix2),len(iix3)) ) |
|---|
| 321 | rnv = set() |
|---|
| 322 | for i in iix3: |
|---|
| 323 | if i.label in self.draftNewVar: |
|---|
| 324 | log.error( 'cmv.0125: %s, %s, %s' % (i.uid,i.label,i.title) ) |
|---|
| 325 | rnv.add( i.label ) |
|---|
| 326 | else: |
|---|
| 327 | log.error( 'cmv.0122: %s, %s, %s' % (i.uid,i.label,i.title) ) |
|---|
| 328 | |
|---|
| 329 | if len( rnv ) > 0: |
|---|
| 330 | self.saveRevisedNewVar( rnv ) |
|---|
| 331 | |
|---|
| 332 | |
|---|
| 333 | ii2 = [k for k in ccmv2 if len(ccmv2[k]) > 1] |
|---|
| 334 | if len(ii2) == 0: |
|---|
| 335 | log.info( 'cmv.0030: unique CMOR variable names OK' ) |
|---|
| 336 | else: |
|---|
| 337 | log.error( 'cmv.0030: non-unique CMOR variable naming for %s records' % len(ii2) ) |
|---|
| 338 | log.error( 'cmv.0031: %s' % str(ii2) ) |
|---|
| 339 | |
|---|
| 340 | ii4 = [k for k in ccmv4 if len(ccmv4[k]) > 1] |
|---|
| 341 | ii6 = [k for k in ccmv6 if len(ccmv6[k]) > 1] |
|---|
| 342 | ii4b = list() |
|---|
| 343 | ii6b = list() |
|---|
| 344 | for k in ii4: |
|---|
| 345 | if len(ccmv4[k]) == 2 and set( [i.mipTable[-3:] for i in ccmv4[k]] ) == set(['Ant','Gre']): |
|---|
| 346 | pass |
|---|
| 347 | else: |
|---|
| 348 | ii4b.append(k) |
|---|
| 349 | |
|---|
| 350 | for k in ii6: |
|---|
| 351 | if len(ccmv6[k]) == 2 and set( [i.mipTable[-3:] for i,s in ccmv6[k]] ) == set(['Ant','Gre']): |
|---|
| 352 | pass |
|---|
| 353 | else: |
|---|
| 354 | ii6b.append(k) |
|---|
| 355 | |
|---|
| 356 | if len(ii4b) == 0: |
|---|
| 357 | log.info( 'cmv.0040: unique CMOR variable usage [4] OK' ) |
|---|
| 358 | else: |
|---|
| 359 | log.error( 'cmv.0040: non-unique CMOR variable usage possible for %s records' % len(ii4b) ) |
|---|
| 360 | log.error( 'cmv.0041: %s' % str(ii4b) ) |
|---|
| 361 | kk=0 |
|---|
| 362 | oo = open( 'cmv_probDuplication.csv', 'w' ) |
|---|
| 363 | for k in ii4b: |
|---|
| 364 | kk += 1 |
|---|
| 365 | rec = ('NEXT\t%s\t' % kk ) + str( '\t'.join( k ) ) |
|---|
| 366 | print 'INFO.cmvdup.00001: ',rec |
|---|
| 367 | oo.write( rec + '\n' ) |
|---|
| 368 | vid = None |
|---|
| 369 | for i in ccmv4[k]: |
|---|
| 370 | assert vid == None or i.vid == vid, 'Unexpected vid variation ...: %s: %s %s' % (i.label, i.vid, vid) |
|---|
| 371 | vid = i.vid |
|---|
| 372 | rec = '+\t%s\t%s\t%s\t%s\t%s' % (i.uid,i.mipTable,i.title,i.description,i.prov) |
|---|
| 373 | oo.write( rec + '\n' ) |
|---|
| 374 | print 'INFO.cmvdup.00001: ',rec |
|---|
| 375 | oo.close() |
|---|
| 376 | |
|---|
| 377 | if len(ii6b) == 0: |
|---|
| 378 | log.info( 'cmv.0060: unique CMOR variable usage [6] OK' ) |
|---|
| 379 | else: |
|---|
| 380 | log.info( 'cmv.0060: unique CMOR variable issues [6] %s ' % len(ii6b) ) |
|---|
| 381 | kk=0 |
|---|
| 382 | ccx = collections.defaultdict( set ) |
|---|
| 383 | for k in ii6b: |
|---|
| 384 | ss = set() |
|---|
| 385 | freq,label,spid,tmid = k |
|---|
| 386 | for i,st in ccmv6[k]: |
|---|
| 387 | ss.add( (st.uid,st.label,st.title) ) |
|---|
| 388 | tt = tuple( sorted( list( ss ) ) ) |
|---|
| 389 | ccx[tt].add( (freq,label) ) |
|---|
| 390 | log.info( 'cmv.0061: unique CMOR variable issues [6.1] %s ' % len(ccx.keys()) ) |
|---|
| 391 | for tt in ccx: |
|---|
| 392 | ll = [t[2] for t in tt] |
|---|
| 393 | log.info( 'cmv.0062: unique CMOR variable issues [6.2] %s: %s ' % ('; '.join(ll),str(ccx[tt]) ) ) |
|---|
| 394 | |
|---|
| 395 | oo = open( 'cmv_probDuplication6.csv', 'w' ) |
|---|
| 396 | for k in ii6b: |
|---|
| 397 | kk += 1 |
|---|
| 398 | rec = ('NEXT\t%s\t' % kk ) + str( '\t'.join( k ) ) |
|---|
| 399 | oo.write( rec + '\n' ) |
|---|
| 400 | vid = None |
|---|
| 401 | for i,st in ccmv6[k]: |
|---|
| 402 | assert vid == None or i.vid == vid, 'Unexpected vid variation ...: %s: %s %s' % (i.label, i.vid, vid) |
|---|
| 403 | vid = i.vid |
|---|
| 404 | rec = '+\t%s\t%s\t%s\t%s\t%s\t%s' % (i.uid,i.mipTable,i.title,st.title,i.description,i.prov) |
|---|
| 405 | oo.write( rec + '\n' ) |
|---|
| 406 | oo.close() |
|---|
| 407 | |
|---|
| 408 | ccmv3 = collections.defaultdict( list ) |
|---|
| 409 | for t in ccmv2: |
|---|
| 410 | s = set() |
|---|
| 411 | for i in ccmv2[t]: |
|---|
| 412 | id = i.uid |
|---|
| 413 | if 'varRelLnk' in self.dq.inx.iref_by_sect[id].a: |
|---|
| 414 | for x in self.dq.inx.iref_by_sect[id].a['varRelLnk']: |
|---|
| 415 | s.add( x ) |
|---|
| 416 | s = list(s) |
|---|
| 417 | if len(s) > 0: |
|---|
| 418 | ccmv3[t] = s |
|---|
| 419 | |
|---|
| 420 | ii5 = [k for k in ii2 if k not in ccmv3 or len(ccmv2[k]) > len(ccmv3[k])] |
|---|
| 421 | cc4 = collections.defaultdict( list ) |
|---|
| 422 | for t,v in ii2: |
|---|
| 423 | cc4[t].append( v ) |
|---|
| 424 | |
|---|
| 425 | log.warn( '%48s [%s]: %s [%s tables]' % ('Duplicate variables in CMORvar',len(self.dq.coll['CMORvar'].items),len(ii2), len( cc4.keys() ) ) ) |
|---|
| 426 | showAllCmvErrors=False |
|---|
| 427 | showAllCmvErrors=True |
|---|
| 428 | if showAllCmvErrors: |
|---|
| 429 | rats = ['uid','label','title','description','frequency','mipTable','stid','prov','provNote'] |
|---|
| 430 | oocm = open( 'cmvDup.csv', 'w' ) |
|---|
| 431 | c1 = collections.defaultdict( list ) |
|---|
| 432 | ##for i in ii2: |
|---|
| 433 | ##c1[i.prov].append( i.label ) |
|---|
| 434 | ##log.warn( 'No MIP var: %s ... %s' % (str( sorted( [i.label for i in ii if i.label in self.cc] ) ), str(c1))) |
|---|
| 435 | for k in sorted( cc4.keys() ): |
|---|
| 436 | log.warn( '%16s:: %s' % (k,str(sorted(cc4[k]))) ) |
|---|
| 437 | for v in sorted(cc4[k]): |
|---|
| 438 | sv = set() |
|---|
| 439 | for i in ccmv2[ (k,v) ]: |
|---|
| 440 | sv.add(i.vid) |
|---|
| 441 | |
|---|
| 442 | if len(list(sv)) == 1: |
|---|
| 443 | vun = True |
|---|
| 444 | vv = self.dq.inx.uid[ list(sv)[0] ] |
|---|
| 445 | oocm.write( '@VAR:\t\t\t%s\t%s\t%s\n' % (vv.label,vv.title,vv.description) ) |
|---|
| 446 | else: |
|---|
| 447 | vun = False |
|---|
| 448 | |
|---|
| 449 | for i in ccmv2[ (k,v) ]: |
|---|
| 450 | stt = self.dq.inx.uid[i.stid].title |
|---|
| 451 | if 'Temporal mean' in stt: |
|---|
| 452 | if i.frequency == '3hr': |
|---|
| 453 | i.title += ' [3 hour mean]' |
|---|
| 454 | xtra = '' |
|---|
| 455 | oocm.write( '\t\t' + '\t'.join( [str(i.__dict__[x]) for x in rats] ) + '\t%s%s\n' % (stt,xtra) ) |
|---|
| 456 | if not vun: |
|---|
| 457 | vv = self.dq.inx.uid[i.vid] |
|---|
| 458 | xtra = '\t\t\t\t%s\t%s\t%s\n' % (vv.label,vv.title,vv.description) |
|---|
| 459 | oocm.write(xtra) |
|---|
| 460 | oocm.write( '####\n' ) |
|---|
| 461 | oocm.close() |
|---|
| 462 | t2_ii = ii[:] |
|---|
| 463 | self.ccmv = ccmv |
|---|
| 464 | |
|---|
| 465 | def audit_sn(self): |
|---|
| 466 | cc = collections.defaultdict( list ) |
|---|
| 467 | cc2 = collections.defaultdict( int ) |
|---|
| 468 | nrem = 0 |
|---|
| 469 | snm = set() |
|---|
| 470 | snmm = collections.defaultdict( int ) |
|---|
| 471 | for i in self.dq.coll['var'].items: |
|---|
| 472 | if 'CMORvar' in self.dq.inx.iref_by_sect[i.uid].a: |
|---|
| 473 | srq = set() |
|---|
| 474 | for u in self.dq.inx.iref_by_sect[i.uid].a['CMORvar']: |
|---|
| 475 | if 'requestVar' in self.dq.inx.iref_by_sect[u].a: |
|---|
| 476 | for uu in self.dq.inx.iref_by_sect[u].a['requestVar']: |
|---|
| 477 | rqv = self.dq.inx.uid[uu] |
|---|
| 478 | srq.add( rqv.priority ) |
|---|
| 479 | if len(srq) > 0: |
|---|
| 480 | pm = min( srq ) |
|---|
| 481 | |
|---|
| 482 | kk = i.sn |
|---|
| 483 | if self.dq.inx.uid[kk]._h.label == 'remarks': |
|---|
| 484 | nrem += 1 |
|---|
| 485 | snm.add(kk) |
|---|
| 486 | ## if pm == 1: |
|---|
| 487 | print 'MISSING STANDARD NAME: ',i.label, i.title, i.sn, i.provmip |
|---|
| 488 | snmm[pm] += 1 |
|---|
| 489 | cc2[i.provmip] += 1 |
|---|
| 490 | if i.procnote != []: |
|---|
| 491 | ll = sorted( i.procnote ) |
|---|
| 492 | kk += ':' + '|'.join(ll ) |
|---|
| 493 | cc[kk].append( i.uid ) |
|---|
| 494 | |
|---|
| 495 | log.warn( 'sn.001: Number of missing standard names: %s [for %s variables] {%s}' % (len(snm),nrem,str(snmm)) ) |
|---|
| 496 | log.warn( 'sn.002: Number of missing standard names by MIP: %s ' % (str(cc2)) ) |
|---|
| 497 | ll = [] |
|---|
| 498 | for k in cc: |
|---|
| 499 | if len( cc[k] ) > 1: |
|---|
| 500 | ll.append( k ) |
|---|
| 501 | |
|---|
| 502 | log.warn( 'sn.003: Duplicate use of standard names: %s' % len(ll) ) |
|---|
| 503 | oo = open( 'audit_sn_repeats.csv', 'w' ) |
|---|
| 504 | for k in sorted( ll ): |
|---|
| 505 | orc = [k,] |
|---|
| 506 | l1 = cc[k][:] |
|---|
| 507 | l2 = [] |
|---|
| 508 | l1b = [] |
|---|
| 509 | for u in cc[k]: |
|---|
| 510 | i = self.dq.inx.uid[u] |
|---|
| 511 | if i.prov == "CMIP6 endorsement [SIMIP]": |
|---|
| 512 | l2.append(u) |
|---|
| 513 | else: |
|---|
| 514 | l1b.append(u) |
|---|
| 515 | if len( l2) > 1: |
|---|
| 516 | log.error( 'WARN.simip.00001: repeat use of standard name in SIMIP: %s:: %s' % (k,str(l2)) ) |
|---|
| 517 | if len(l2) == 1: |
|---|
| 518 | orc.append( '*' ) |
|---|
| 519 | else: |
|---|
| 520 | orc.append( '' ) |
|---|
| 521 | |
|---|
| 522 | nnn = 0 |
|---|
| 523 | for u in l2 +l1b: |
|---|
| 524 | i = self.dq.inx.uid[u] |
|---|
| 525 | if "CMORvar" not in self.dq.inx.iref_by_sect[u].a: |
|---|
| 526 | orc += [i.label +'**', i.title] |
|---|
| 527 | else: |
|---|
| 528 | nn = 0 |
|---|
| 529 | for uu in self.dq.inx.iref_by_sect[u].a['CMORvar']: |
|---|
| 530 | cmv = self.dq.inx.uid[uu] |
|---|
| 531 | if 'requestVar' in self.dq.inx.iref_by_sect[uu].a: |
|---|
| 532 | nn += 1 |
|---|
| 533 | if nn == 0: |
|---|
| 534 | orc += [i.label + '*', i.title] |
|---|
| 535 | else: |
|---|
| 536 | nnn += 1 |
|---|
| 537 | orc += [i.label, i.title] |
|---|
| 538 | |
|---|
| 539 | if nnn > 1: |
|---|
| 540 | oo.write( '\t'.join( orc ) + '\n' ) |
|---|
| 541 | oo.close() |
|---|
| 542 | |
|---|
| 543 | def audit_rqv(self): |
|---|
| 544 | ixx = [i for i in self.dq.coll['requestVar'].items if i.vid in self.dq.inx.uid] |
|---|
| 545 | if len(ixx) < len( self.dq.inx.uid ): |
|---|
| 546 | log.error( 'SEVERE.var.00301: request vars with bad links ....' ) |
|---|
| 547 | |
|---|
| 548 | ii = [i for i in ixx if self.dq.inx.uid[i.vid]._h.label == 'remarks'] |
|---|
| 549 | ii2 = [i for i in ii if i.label not in self.cc] |
|---|
| 550 | ii3 = [i.vgid for i in ii if self.dq.inx.uid[i.vgid]._h.label != 'remarks'] |
|---|
| 551 | ##ii4: set of invalid requestVarGroup records |
|---|
| 552 | ii4 = [i for i in ii3 if 'requestLink' not in self.dq.inx.iref_by_sect[i].a ] |
|---|
| 553 | ###### |
|---|
| 554 | ii5Info = 'Set of bad requestVar records linking to a valid requestVarGroup records.' |
|---|
| 555 | ii5 = [i for i in ii if i.vgid not in ii4] |
|---|
| 556 | ##ii6: set of bad requestVar records linking to a valid requestVarGroup records and no variable name match. |
|---|
| 557 | ii6 = [i for i in ii5 if i.label not in self.cc] |
|---|
| 558 | log.error( '%48s [%s]: %s [%s, %s, %s; %s]' % ('Bad variable links in requestVar',len(self.dq.coll['requestVar'].items),len(ii),len(ii2), len(ii5), len(ii6), len(ii4)) ) |
|---|
| 559 | log.info ( str( ii5Info )) |
|---|
| 560 | svg = set() |
|---|
| 561 | for i in ii5: |
|---|
| 562 | log.info( 'ii5: %s' % str([i.label, i.title, i.uid, i.mip, self.dq.inx.uid[i.vgid].title] ) ) |
|---|
| 563 | svg.add( i.vgid ) |
|---|
| 564 | for i in ii2: |
|---|
| 565 | if i.vgid in self.dq.inx.uid and self.dq.inx.uid[i.vgid]._h.label != 'remarks': |
|---|
| 566 | xx = '%s [%s]' % (self.dq.inx.uid[i.vgid].title, i.vgid) |
|---|
| 567 | else: |
|---|
| 568 | xx = i.vgid |
|---|
| 569 | log.info( 'ii2: %s' % str([i.label, i.title, i.uid, i.mip, xx] ) ) |
|---|
| 570 | |
|---|
| 571 | log.warn( 'rvg.0010: requestVarGroups with bad request vars:' ) |
|---|
| 572 | for u in sorted( list(svg) ): |
|---|
| 573 | log.warn( 'rvg.0011: %s: %s' % (u, self.dq.inx.uid[u].title )) |
|---|
| 574 | |
|---|
| 575 | log.info( ' [# records]: # broken [var name not known, link to valid group, valid group and no var name; valid request groups' ) |
|---|
| 576 | showAllRqvErrors=False |
|---|
| 577 | if showAllRqvErrors: |
|---|
| 578 | log.info( '%s\n%s' % (str( [i.label for i in ii if i.label in self.cc]), str( [i.label for i in ii2] ) ) ) |
|---|
| 579 | |
|---|
| 580 | ii1 = [i for i in ii if i.label in self.ccmv] |
|---|
| 581 | listBadRequestVar=False |
|---|
| 582 | if listBadRequestVar: |
|---|
| 583 | for i in ii1: |
|---|
| 584 | log.info( str( i.label,i.mip,i.table,[x.frequency for x in self.ccmv[i.label]] ) ) |
|---|
| 585 | |
|---|
| 586 | def audit_rql(self): |
|---|
| 587 | ii = [i for i in self.dq.coll['requestLink'].items if self.dq.inx.uid[i.refid]._h.label == 'remarks'] |
|---|
| 588 | log.info( '%48s: %s (from %s)' % ('ERROR.rql.001: Bad request group links in requestLink',len(ii), len(self.dq.coll['requestLink'].items)) ) |
|---|
| 589 | rql_detail=True |
|---|
| 590 | if rql_detail: |
|---|
| 591 | for i in ii: |
|---|
| 592 | log.info( '%s: %s, %s [%s]' % ('ERROR.rql.002: ',i.mip,i.label,i.uid) ) |
|---|
| 593 | |
|---|
| 594 | def audit_rqg(self): |
|---|
| 595 | ii = [i for i in self.dq.coll['requestVarGroup'].items if len( dq.inx.iref_by_sect[i.uid].a['requestVar']) == 0] |
|---|
| 596 | ii1 = [i for i in ii if len( dq.inx.iref_by_sect[i.uid].a['requestLink']) != 0] |
|---|
| 597 | log.info( '%48s: %s/%s (from %s)' % ('Request groups with no request links:',len(ii) - len(ii1),len(ii1), len(self.dq.coll['requestVarGroup'].items)) ) |
|---|
| 598 | listEmptyRequestVarGroups=True |
|---|
| 599 | if listEmptyRequestVarGroups: |
|---|
| 600 | for i in ii1: |
|---|
| 601 | log.info ( 'INFO.rqgempty.0001: %s: %s, %s [%s]' % (i.label, i.title, i.mip, i.uid) ) |
|---|
| 602 | |
|---|
| 603 | dq = dreq.loadDreq(manifest=None) |
|---|
| 604 | a = auditor( dq ) |
|---|
| 605 | cd = checkDims( dq ) |
|---|
