| 1 | import dreq, collections, string, os, utils_wb |
|---|
| 2 | import htmlTemplates as tmpl |
|---|
| 3 | import xml, re |
|---|
| 4 | import xml.dom, xml.dom.minidom |
|---|
| 5 | |
|---|
| 6 | from utils_wb import uniCleanFunc |
|---|
| 7 | |
|---|
| 8 | empty=re.compile('^$') |
|---|
| 9 | |
|---|
| 10 | dq = dreq.loadDreq() |
|---|
| 11 | inx = dq.inx |
|---|
| 12 | ##inx.makeVarRefs() |
|---|
| 13 | ix_rql_uuid = {} |
|---|
| 14 | ix_rqvg_uuid = {} |
|---|
| 15 | ix_ovar_uuid = {} |
|---|
| 16 | ix_gpi_uuid = {} |
|---|
| 17 | list_gp_ovar = collections.defaultdict( list ) |
|---|
| 18 | xr_var_ovar = collections.defaultdict( list ) |
|---|
| 19 | xr_var_gpi = collections.defaultdict( list ) |
|---|
| 20 | rql_by_name = collections.defaultdict( list ) |
|---|
| 21 | |
|---|
| 22 | def makeVarRefs(uuid, var, iref_by_uuid): |
|---|
| 23 | varRefs = {} |
|---|
| 24 | for thisuuid in var.uuid.keys(): |
|---|
| 25 | if iref_by_uuid.has_key(thisuuid): |
|---|
| 26 | ee1 = collections.defaultdict( list ) |
|---|
| 27 | for k,i in iref_by_uuid[thisuuid]: |
|---|
| 28 | sect,thisi = uuid[i] |
|---|
| 29 | if sect == 'groupItem': |
|---|
| 30 | ee1[sect].append( '%s.%s' % (thisi.mip, thisi.group) ) |
|---|
| 31 | elif sect == 'ovar': |
|---|
| 32 | ee1[sect].append( thisi.mipTable ) |
|---|
| 33 | elif sect == 'revisedTabItem': |
|---|
| 34 | ee1[sect].append( '%s.%s' % (thisi.mip, thisi.table) ) |
|---|
| 35 | varRefs[thisuuid] = ee1 |
|---|
| 36 | return varRefs |
|---|
| 37 | |
|---|
| 38 | varRefs = makeVarRefs( inx.uuid, inx.var, inx.iref_by_uuid) |
|---|
| 39 | |
|---|
| 40 | class updates(object): |
|---|
| 41 | def __init__(self,fndup,fnmult,idir='rev1'): |
|---|
| 42 | assert os.path.isdir( idir ), 'Directory %s not found' % idir |
|---|
| 43 | self.fdup = '%s/%s' % (idir,fndup) |
|---|
| 44 | self.fmult = '%s/%s' % (idir,fnmult) |
|---|
| 45 | for p in [self.fdup,self.fmult]: |
|---|
| 46 | assert os.path.isfile( p ), 'File %s not found' % p |
|---|
| 47 | self.repl = {} |
|---|
| 48 | self.upd = {} |
|---|
| 49 | self.twins = [] |
|---|
| 50 | self.ddel = {} |
|---|
| 51 | |
|---|
| 52 | def scandup(self): |
|---|
| 53 | ii = open( self.fdup ).readlines() |
|---|
| 54 | nn = (len(ii)-1)/2 |
|---|
| 55 | for i in range(nn): |
|---|
| 56 | l1 = string.split( ii[i*2+1], '\t' ) |
|---|
| 57 | l2 = string.split( ii[i*2+2], '\t' ) |
|---|
| 58 | xx = l1[8:10] |
|---|
| 59 | yy = l2[8:10] |
|---|
| 60 | if xx[1] == '' and yy[1] == xx[0]: |
|---|
| 61 | ths = 0 |
|---|
| 62 | assert not self.repl.has_key( yy[0] ), 'duplicate replacement request for %s' % yy[0] |
|---|
| 63 | self.repl[ yy[0] ] = yy[1] |
|---|
| 64 | elif yy[1] == '' and xx[1] == yy[0]: |
|---|
| 65 | ths = 1 |
|---|
| 66 | assert not self.repl.has_key( xx[0] ), 'duplicate replacement request for %s' % xx[0] |
|---|
| 67 | self.repl[ xx[0] ] = xx[1] |
|---|
| 68 | elif l1[10] == 'twin' and l2[10] == 'twin': |
|---|
| 69 | ths = 2 |
|---|
| 70 | self.twins.append( l1[8] ) |
|---|
| 71 | self.twins.append( l2[8] ) |
|---|
| 72 | elif l1[10] == 'inc' and l2[10] == 'inc': |
|---|
| 73 | ths = 3 |
|---|
| 74 | self.ddel[ l1[8] ] = 'inc' |
|---|
| 75 | self.ddel[ l2[8] ] = 'inc' |
|---|
| 76 | else: |
|---|
| 77 | ths = -1 |
|---|
| 78 | print 'ERROR.xxx.0001: Match failed' |
|---|
| 79 | print l1 |
|---|
| 80 | print l2 |
|---|
| 81 | assert False |
|---|
| 82 | |
|---|
| 83 | def scanmult(self): |
|---|
| 84 | ii = open( self.fmult ).readlines() |
|---|
| 85 | nn = (len(ii)-1)/3 |
|---|
| 86 | for i in range(nn): |
|---|
| 87 | l1 = string.split( ii[i*3+1], '\t' ) |
|---|
| 88 | l2 = string.split( ii[i*3+2], '\t' ) |
|---|
| 89 | l3 = string.split( ii[i*3+3], '\t' ) |
|---|
| 90 | yy = [l1[9],l2[9],l3[9]] |
|---|
| 91 | xx = [l1[8],l2[8],l3[8]] |
|---|
| 92 | zz = (l1,l2,l3) |
|---|
| 93 | for j in range(3): |
|---|
| 94 | if yy[j] != '': |
|---|
| 95 | assert yy[j] in xx, 'Invalid replacement option, %s' % yy[j] |
|---|
| 96 | assert not self.repl.has_key( xx[j] ), 'duplicate replacement request for %s' % xx[j] |
|---|
| 97 | self.repl[ xx[j] ] = yy[j] |
|---|
| 98 | elif zz[j][10] == 'twin': |
|---|
| 99 | self.twins.append( zz[j][8] ) |
|---|
| 100 | elif zz[j][11] == 'update': |
|---|
| 101 | tags = map( string.strip, string.split( zz[j][13], ',' ) ) |
|---|
| 102 | self.upd[ xx[j] ] = { 'provNote':zz[j][12], 'tags':tags, 'label':zz[j][0], 'title':zz[j][1] } |
|---|
| 103 | |
|---|
| 104 | up = updates('varDup_20150724.csv', 'varMult_20150725.csv') |
|---|
| 105 | ##up.scandup() |
|---|
| 106 | up.scanmult() |
|---|
| 107 | |
|---|
| 108 | urep = False |
|---|
| 109 | urep = True |
|---|
| 110 | if urep: |
|---|
| 111 | oo = open( 'uuidreplace.csv', 'w' ) |
|---|
| 112 | oo2 = open( 'uuidremove.csv', 'w' ) |
|---|
| 113 | for k in up.repl.keys(): |
|---|
| 114 | if inx.iref_by_uuid.has_key(k): |
|---|
| 115 | kn = up.repl[k] |
|---|
| 116 | for tag,ki in inx.iref_by_uuid[k]: |
|---|
| 117 | oo.write( '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (k,kn,tag,ki, inx.uuid[k][1].label, inx.uuid[kn][1].label, inx.uuid[ki][1].label) ) |
|---|
| 118 | else: |
|---|
| 119 | oo2.write( k + '\n' ) |
|---|
| 120 | oo.close() |
|---|
| 121 | oo2.close() |
|---|
| 122 | oo = open( 'uuidupdate.csv', 'w' ) |
|---|
| 123 | for k in up.upd.keys(): |
|---|
| 124 | ee = up.upd[k] |
|---|
| 125 | oo.write( string.join( [k,ee['provNote'],string.join(ee['tags']),ee['label'], ee['title'] ], '\t') + '\n' ) |
|---|
| 126 | oo.close() |
|---|
| 127 | else: |
|---|
| 128 | oo2 = open( 'uuidremove2.csv', 'w' ) |
|---|
| 129 | for i in dq.coll['var'].items: |
|---|
| 130 | if not inx.iref_by_uuid.has_key(i.uuid): |
|---|
| 131 | oo2.write( string.join( [i.uuid,i.label,i.title,i.prov,i.description], '\t') + '\n' ) |
|---|
| 132 | oo2.close() |
|---|
| 133 | |
|---|
| 134 | ### check back references. |
|---|
| 135 | nbr = 0 |
|---|
| 136 | lbr = [] |
|---|
| 137 | for k in inx.iref_by_uuid.keys(): |
|---|
| 138 | if not inx.uuid.has_key(k): |
|---|
| 139 | nbr += 1 |
|---|
| 140 | lbr.append(k) |
|---|
| 141 | print 'Missing references: ', nbr |
|---|
| 142 | ### can now apply mappings, create updated records and write to new xml? |
|---|
| 143 | |
|---|
| 144 | for i in dq.coll['requestLink'].items: |
|---|
| 145 | rql_by_name[i.label].append( i.uuid ) |
|---|
| 146 | ix_rql_uuid[i.uuid] = i |
|---|
| 147 | |
|---|
| 148 | for i in dq.coll['requestVarGroup'].items: |
|---|
| 149 | ix_rqvg_uuid[i.uuid] = i |
|---|
| 150 | |
|---|
| 151 | |
|---|
| 152 | oo = open( 'uuidinsert.csv', 'w' ) |
|---|
| 153 | for i in dq.coll['revisedTabItem'].items: |
|---|
| 154 | if i.uuid == '__new__': |
|---|
| 155 | if inx.var.label.has_key( i.label ): |
|---|
| 156 | if len( inx.var.label[i.label] ) == 1: |
|---|
| 157 | v = inx.uuid[ inx.var.label[i.label][0] ][1] |
|---|
| 158 | oo.write( string.join( ['unique',i.label,v.label,v.uuid,v.prov,i.mip], '\t' ) + '\n' ) |
|---|
| 159 | else: |
|---|
| 160 | oo.write( string.join( ['ambiguous',i.label,i.mip,str(len(inx.var.label[i.label] ) ) ], '\t' ) + '\n' ) |
|---|
| 161 | oo.close() |
|---|
| 162 | |
|---|
| 163 | oo = open( 'varMult.csv', 'w' ) |
|---|
| 164 | oo2 = open( 'varDup.csv', 'w' ) |
|---|
| 165 | oo3 = open( 'varStar.csv', 'w' ) |
|---|
| 166 | hs = ['label','title','sn','units','description','prov','procnote','procComment','uuid'] |
|---|
| 167 | oo.write( string.join(hs, '\t' ) + '\n' ) |
|---|
| 168 | oo2.write( string.join(hs, '\t' ) + '\n' ) |
|---|
| 169 | oo3.write( string.join(hs, '\t' ) + '\n' ) |
|---|
| 170 | ks = inx.var.label.keys() |
|---|
| 171 | ks.sort() |
|---|
| 172 | for k in ks: |
|---|
| 173 | if len(inx.var.label[k]) == 2: |
|---|
| 174 | v1 = inx.var.uuid[inx.var.label[k][0]] |
|---|
| 175 | v2 = inx.var.uuid[inx.var.label[k][1]] |
|---|
| 176 | cc = map( lambda x: v1.__dict__[x] == v2.__dict__[x], ['title','sn','units','description'] ) |
|---|
| 177 | if all(cc): |
|---|
| 178 | oo2.write( string.join(map( lambda x: v1.__dict__[x], hs) + [v2.uuid,'identical'], '\t' ) + '\n' ) |
|---|
| 179 | else: |
|---|
| 180 | oo2.write( string.join(map( lambda x: v1.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) |
|---|
| 181 | oo2.write( string.join(map( lambda x: v2.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) |
|---|
| 182 | |
|---|
| 183 | elif len(inx.var.label[k]) > 1: |
|---|
| 184 | for i in inx.var.label[k]: |
|---|
| 185 | oo.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) |
|---|
| 186 | |
|---|
| 187 | if k[-2:] == '--': |
|---|
| 188 | for i in (inx.var.label[k] + inx.var.label[k[:-2]]): |
|---|
| 189 | oo3.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) |
|---|
| 190 | oo.close() |
|---|
| 191 | oo2.close() |
|---|
| 192 | oo3.close() |
|---|
| 193 | |
|---|
| 194 | |
|---|
| 195 | for i in dq.coll['groupItem'].items: |
|---|
| 196 | list_gp_ovar[i.gpid].append( i.uuid ) |
|---|
| 197 | |
|---|
| 198 | vns = inx.var.label.keys() |
|---|
| 199 | vns.sort() |
|---|
| 200 | for v in vns: |
|---|
| 201 | if len( inx.var.label[v] ) > 1: |
|---|
| 202 | print 'INFO.001.0001:',v, string.join( map( lambda x: inx.var.uuid[x].sn, inx.var.label[v] ), ';' ) |
|---|
| 203 | |
|---|
| 204 | nok = 0 |
|---|
| 205 | nerr = 0 |
|---|
| 206 | for i in dq.coll['ovar'].items: |
|---|
| 207 | vid = i.vid |
|---|
| 208 | ix_ovar_uuid[i.uuid] = i |
|---|
| 209 | xr_var_ovar[vid].append( i.uuid ) |
|---|
| 210 | if not inx.var.uuid.has_key(vid): |
|---|
| 211 | print 'missing key:',i.__dict__ |
|---|
| 212 | nerr += 1 |
|---|
| 213 | else: |
|---|
| 214 | nok += 1 |
|---|
| 215 | |
|---|
| 216 | nok = 0 |
|---|
| 217 | nerr = 0 |
|---|
| 218 | for i in dq.coll['groupItem'].items: |
|---|
| 219 | vid = i.vid |
|---|
| 220 | ix_gpi_uuid[i.uuid] = i |
|---|
| 221 | xr_var_gpi[vid].append( i.uuid ) |
|---|
| 222 | if not inx.var.uuid.has_key(vid): |
|---|
| 223 | nerr += 1 |
|---|
| 224 | else: |
|---|
| 225 | nok += 1 |
|---|
| 226 | print 'groupItem to var crossref: nok = %s, nerr = %s' % (nok, nerr) |
|---|
| 227 | |
|---|
| 228 | class rqHtml(object): |
|---|
| 229 | |
|---|
| 230 | def __init__(self,odir='./html/'): |
|---|
| 231 | self.odir = odir |
|---|
| 232 | if not os.path.isdir(odir): |
|---|
| 233 | os.mkdir(odir) |
|---|
| 234 | |
|---|
| 235 | def mkRqlHtml(self,name): |
|---|
| 236 | ## [u'comment', u'uuid', u'tab', u'title', u'label', u'grid', 'defaults', u'objective', u'mip', 'globalDefault', u'gridreq'] |
|---|
| 237 | if len( rql_by_name[name] ) == 1: |
|---|
| 238 | self.mkRqlHtml01(rql_by_name[name][0], name ) |
|---|
| 239 | else: |
|---|
| 240 | self.mkRqlHtmlGp(name) |
|---|
| 241 | |
|---|
| 242 | def mkRqlHtmlGp(self,name): |
|---|
| 243 | ee = {} |
|---|
| 244 | ee['title'] = 'CMIP Request Link %s (with multiple definitions)' % name |
|---|
| 245 | self.pageName = 'rql__%s.html' % name |
|---|
| 246 | al =[] |
|---|
| 247 | for i in range( len( rql_by_name[name] ) ): |
|---|
| 248 | this = ix_rql_uuid[rql_by_name[name][i]] |
|---|
| 249 | al.append( tmpl.item % {'item':'<a href="rql__%s__%s.html">[%s]</a>: %s' % (name,i,i,this.title) } ) |
|---|
| 250 | ee['items'] = string.join(al, '\n' ) |
|---|
| 251 | ee['introduction'] = '' |
|---|
| 252 | ee['htmlBody'] = tmpl.indexWrapper % ee |
|---|
| 253 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
|---|
| 254 | self.pageHtml = tmpl.pageWrapper % ee |
|---|
| 255 | self.write() |
|---|
| 256 | for i in range( len( rql_by_name[name] ) ): |
|---|
| 257 | self.mkRqlHtml01(rql_by_name[name][i],i) |
|---|
| 258 | |
|---|
| 259 | def mkRqlHtml01(self,id, tag): |
|---|
| 260 | this = ix_rql_uuid[id] |
|---|
| 261 | ee = {} |
|---|
| 262 | if this.label == tag: |
|---|
| 263 | ee['title'] = 'CMIP Request Link %s' % tag |
|---|
| 264 | self.pageName = 'rql__%s.html' % tag |
|---|
| 265 | else: |
|---|
| 266 | ee['title'] = 'CMIP Request Link %s[%s]' % (this.label,tag) |
|---|
| 267 | self.pageName = 'rql__%s__%s.html' % (this.label,tag) |
|---|
| 268 | atts = this.__dict__.keys() |
|---|
| 269 | atts.sort() |
|---|
| 270 | al =[] |
|---|
| 271 | for a in atts: |
|---|
| 272 | if a not in ['defaults','globalDefault']: |
|---|
| 273 | al.append( tmpl.item % {'item':'<b>%s</b>: %s' % (a,this.__dict__.get(a,'-- Not Set --')) } ) |
|---|
| 274 | ee['items'] = string.join(al, '\n' ) |
|---|
| 275 | ee['introduction'] = '' |
|---|
| 276 | ee['htmlBody'] = tmpl.indexWrapper % ee |
|---|
| 277 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
|---|
| 278 | self.pageHtml = tmpl.pageWrapper % ee |
|---|
| 279 | self.write() |
|---|
| 280 | |
|---|
| 281 | def mkVarHtml(self,name): |
|---|
| 282 | if len( inx.var.label[name] ) == 1: |
|---|
| 283 | self.mkVarHtml01(inx.var.label[name][0], name ) |
|---|
| 284 | else: |
|---|
| 285 | self.mkVarHtmlGp(name) |
|---|
| 286 | |
|---|
| 287 | def mkVarHtmlGp(self,name): |
|---|
| 288 | ee = {} |
|---|
| 289 | ee['title'] = 'CMIP Variable %s (with multiple definitions)' % name |
|---|
| 290 | self.pageName = 'var__%s.html' % name |
|---|
| 291 | al =[] |
|---|
| 292 | for i in range( len( inx.var.label[name] ) ): |
|---|
| 293 | this = inx.var.uuid[inx.var.label[name][i]] |
|---|
| 294 | al.append( tmpl.item % {'item':'<a href="var__%s__%s.html">[%s]</a>: %s' % (name,i,i,this.title) } ) |
|---|
| 295 | ee['items'] = string.join(al, '\n' ) |
|---|
| 296 | ee['introduction'] = '' |
|---|
| 297 | ee['htmlBody'] = tmpl.indexWrapper % ee |
|---|
| 298 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
|---|
| 299 | self.pageHtml = tmpl.pageWrapper % ee |
|---|
| 300 | self.write() |
|---|
| 301 | ##print 'Multi var: %s' % name |
|---|
| 302 | for i in range( len( inx.var.label[name] ) ): |
|---|
| 303 | self.mkVarHtml01(inx.var.label[name][i],i) |
|---|
| 304 | |
|---|
| 305 | def mkVarHtml01(self,id, tag): |
|---|
| 306 | this = inx.var.uuid[id] |
|---|
| 307 | ee = {} |
|---|
| 308 | if this.label == tag: |
|---|
| 309 | ee['title'] = 'CMIP Variable %s' % tag |
|---|
| 310 | self.pageName = 'var__%s.html' % tag |
|---|
| 311 | else: |
|---|
| 312 | ee['title'] = 'CMIP Variable %s[%s]' % (this.label,tag) |
|---|
| 313 | self.pageName = 'var__%s__%s.html' % (this.label,tag) |
|---|
| 314 | atts = this.__dict__.keys() |
|---|
| 315 | atts.sort() |
|---|
| 316 | al =[] |
|---|
| 317 | for a in atts: |
|---|
| 318 | if a not in ['defaults','globalDefault']: |
|---|
| 319 | al.append( tmpl.item % {'item':'<b>%s</b>: %s' % (a,this.__dict__.get(a,'-- Not Set --')) } ) |
|---|
| 320 | |
|---|
| 321 | if inx.iref_by_uuid.has_key(this.uuid): |
|---|
| 322 | assert varRefs.has_key(this.uuid), 'Problem with collected references' |
|---|
| 323 | ee1 = varRefs[this.uuid] |
|---|
| 324 | ks = ee1.keys() |
|---|
| 325 | ks.sort() |
|---|
| 326 | for k in ks: |
|---|
| 327 | al.append( tmpl.item % {'item':'<b>%s</b>: %s' % (k,string.join(ee1[k])) } ) |
|---|
| 328 | ee['items'] = string.join(al, '\n' ) |
|---|
| 329 | ee['introduction'] = '' |
|---|
| 330 | ee['htmlBody'] = tmpl.indexWrapper % ee |
|---|
| 331 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
|---|
| 332 | self.pageHtml = tmpl.pageWrapper % ee |
|---|
| 333 | self.write() |
|---|
| 334 | |
|---|
| 335 | def varHtml(self): |
|---|
| 336 | for k in inx.var.label.keys(): |
|---|
| 337 | self.mkVarHtml(k) |
|---|
| 338 | |
|---|
| 339 | def rqlHtml(self): |
|---|
| 340 | for k in rql_by_name.keys(): |
|---|
| 341 | self.mkRqlHtml(k) |
|---|
| 342 | |
|---|
| 343 | def write(self): |
|---|
| 344 | oo = open( '%s/%s' % (self.odir,self.pageName), 'w' ) |
|---|
| 345 | oo.write( self.pageHtml ) |
|---|
| 346 | oo.close() |
|---|
| 347 | |
|---|
| 348 | |
|---|
| 349 | vh = rqHtml() |
|---|
| 350 | vh.varHtml() |
|---|
| 351 | vh.rqlHtml() |
|---|
| 352 | |
|---|
| 353 | if nerr == 0: |
|---|
| 354 | print 'CHECK 001: %s records checked, no missing references' % nok |
|---|
| 355 | |
|---|
| 356 | ##for k in xr_var_ovar.keys(): |
|---|
| 357 | ##if len( xr_var_ovar[k] ) > 1: |
|---|
| 358 | ##print inx.var.uuid[k].label, map( lambda x: ix_ovar_uuid[x].mipTable, xr_var_ovar[k] ) |
|---|
| 359 | |
|---|
| 360 | shps = {'': 64, 'XYZKT': 13, '4-element vector': 2, 'XYT': 476, '2D vector field ': 2, 'KZT': 4, '2D vector field': 2, 'XYZ': 27, 'XYZT': 204, '2D': 83, 'scalar': 14, 'XY': 88, '?': 21, '2D ': 1, 'XYKT': 3, 'YZT': 16, 'ZST1': 15, 'XKT': 2, 'BasinYT': 1} |
|---|
| 361 | vshpchkMap = {'':'', u'all model levels above 400hPa':'alevStrat', u'all':'Xlev', 3.0:'plev3', '4.0':'plev4', \ |
|---|
| 362 | 36.0:'plev36', u'soil levels':'sdepth', \ |
|---|
| 363 | 1.0:'sfc?', \ |
|---|
| 364 | 16.0:'plev16', 7.0:'plev7', 40.0:'plev40', u'all*':'Xlev', 14.0:'plev14', u'Model levels or 27Plevs':'alev|plev27', \ |
|---|
| 365 | 27.0:'plev27', 17.0:'plev17', u'17 (or23)':'plev17|plev23', 8.0:'plev8', u'all model levels':'alev', 5.0:'plev5'} |
|---|
| 366 | ks = vshpchkMap.keys() |
|---|
| 367 | for k in ks: |
|---|
| 368 | if type(k) == type(0.): |
|---|
| 369 | vshpchkMap[str(k)] = vshpchkMap[k] |
|---|
| 370 | |
|---|
| 371 | print vshpchkMap.keys() |
|---|
| 372 | |
|---|
| 373 | tsmap = { 'mean':[u'daily mean', u'time mean', u'time: day', |
|---|
| 374 | u'Cumulative annual fraction', u'Time mean', u'weighted time mean', u'time: mean', u'mean', u'Mean'], |
|---|
| 375 | '__unknown__':[''], |
|---|
| 376 | 'point':[ u'Instantaneous (end of year)', u'point', u'Synoptic', u'instantaneous', u'time: point', u'synoptic'] } |
|---|
| 377 | tsmap2 = {} |
|---|
| 378 | for k in tsmap.keys(): |
|---|
| 379 | for i in tsmap[k]: |
|---|
| 380 | tsmap2[i] = k |
|---|
| 381 | |
|---|
| 382 | ee = collections.defaultdict( int ) |
|---|
| 383 | for i in dq.coll['groupItem'].items: |
|---|
| 384 | tst = tsmap2[ i.tstyle ] |
|---|
| 385 | dd = '' |
|---|
| 386 | if 'X' in i.shape: |
|---|
| 387 | dd += 'latitude ' |
|---|
| 388 | if 'Y' in i.shape: |
|---|
| 389 | dd += 'longitude ' |
|---|
| 390 | if 'Z' in i.shape: |
|---|
| 391 | if i.levels == '': |
|---|
| 392 | print 'ERROR.001.0001: no levels specified', i.label, i.title |
|---|
| 393 | else: |
|---|
| 394 | zdim = vshpchkMap[i.levels] |
|---|
| 395 | dd += zdim |
|---|
| 396 | ## print '%s::%s::%s|%s' % (i.shape, i.levels, i.tstyle, dd) |
|---|
| 397 | |
|---|
| 398 | class tcmp(object): |
|---|
| 399 | def __init__(self): |
|---|
| 400 | pass |
|---|
| 401 | def cmp(self,x,y): |
|---|
| 402 | return cmp(x.title,y.title) |
|---|
| 403 | |
|---|
| 404 | def atRepr(l,x): |
|---|
| 405 | v = l.__dict__[x] |
|---|
| 406 | if v == '__unset__': |
|---|
| 407 | return '' |
|---|
| 408 | else: |
|---|
| 409 | return v |
|---|
| 410 | |
|---|
| 411 | def dumpcsv( fn, key, atl ): |
|---|
| 412 | oo = open(fn, 'w' ) |
|---|
| 413 | ll = dq.coll[key].items[:] |
|---|
| 414 | ll.sort( tcmp().cmp ) |
|---|
| 415 | oo.write( string.join( atl, '\t' ) + '\n' ) |
|---|
| 416 | for l in ll: |
|---|
| 417 | oo.write( string.join( map( lambda x: atRepr(l,x), atl), '\t' ) + '\n' ) |
|---|
| 418 | oo.close() |
|---|
| 419 | |
|---|
| 420 | def atlSort( ll ): |
|---|
| 421 | oo = [] |
|---|
| 422 | l1 = ['label','title'] |
|---|
| 423 | l2 = ['uuid','defaults','globalDefault'] |
|---|
| 424 | for i in l1: |
|---|
| 425 | if i in ll: |
|---|
| 426 | oo.append(i) |
|---|
| 427 | ll.sort() |
|---|
| 428 | for i in ll: |
|---|
| 429 | if i not in l1 + l2: |
|---|
| 430 | oo.append(i) |
|---|
| 431 | if 'uuid' in ll: |
|---|
| 432 | oo.append( 'uuid' ) |
|---|
| 433 | return oo |
|---|
| 434 | |
|---|
| 435 | for k in dq.coll.keys(): |
|---|
| 436 | if len( dq.coll[k].items ) > 0: |
|---|
| 437 | expl = dq.coll[k].items[0] |
|---|
| 438 | atl = atlSort( expl.__dict__.keys() ) |
|---|
| 439 | print k, atl |
|---|
| 440 | dumpcsv( 'csv2/%s.csv' % k, k, atl ) |
|---|
| 441 | |
|---|
| 442 | oo = open( 'var1.csv', 'w' ) |
|---|
| 443 | ks = ['label','title','sn','units','description','prov','procnote','procComment','uuid'] |
|---|
| 444 | ks2 = [ 'ovar','groupItem','revisedTabItem'] |
|---|
| 445 | oo.write( string.join(ks + ks2, '\t' ) + '\n' ) |
|---|
| 446 | for i in dq.coll['var'].items: |
|---|
| 447 | if i.label[-2:] != '--': |
|---|
| 448 | ee1 = varRefs.get( i.uuid, {} ) |
|---|
| 449 | r2 = map( lambda x: string.join( ee1.get(x, [] ) ), ks2 ) |
|---|
| 450 | oo.write( string.join(map( lambda x: i.__dict__[x], ks) + r2, '\t' ) + '\n' ) |
|---|
| 451 | oo.close() |
|---|
| 452 | |
|---|
| 453 | class annotate(object): |
|---|
| 454 | def __init__(self,src,dreq): |
|---|
| 455 | assert os.path.isfile( src), '%s not found' % src |
|---|
| 456 | self.doc = xml.dom.minidom.parse( src ) |
|---|
| 457 | self.dreq = dreq |
|---|
| 458 | |
|---|
| 459 | def missingRefs(self,mrefs,clear=True): |
|---|
| 460 | this = self.doc.getElementsByTagName('remarks')[0] |
|---|
| 461 | if clear: |
|---|
| 462 | dil = this.getElementsByTagName('item') |
|---|
| 463 | for d in dil: |
|---|
| 464 | this.removeChild(d) |
|---|
| 465 | for k in mrefs.keys(): |
|---|
| 466 | if len( mrefs[k] ) == 1: |
|---|
| 467 | tid = mrefs[k][0][2] |
|---|
| 468 | tattr = mrefs[k][0][1] |
|---|
| 469 | tn = None |
|---|
| 470 | else: |
|---|
| 471 | tid = None |
|---|
| 472 | ee = collections.defaultdict(int) |
|---|
| 473 | tn = str( len( mrefs[k] ) ) |
|---|
| 474 | for t in mrefs[k]: |
|---|
| 475 | s = self.dreq.inx.uuid[t[2]][0] |
|---|
| 476 | ee['%s.%s' % (s,t[1])] += 1 |
|---|
| 477 | if len( ee.keys() ) == 1: |
|---|
| 478 | tattr = ee.keys()[0] |
|---|
| 479 | else: |
|---|
| 480 | tattr = '__multiple__' |
|---|
| 481 | item = self.doc.createElement( 'item' ) |
|---|
| 482 | item.setAttribute( 'uuid', k ) |
|---|
| 483 | item.setAttribute( 'tattr', tattr ) |
|---|
| 484 | if tn != None: |
|---|
| 485 | item.setAttribute( 'techNote', tn ) |
|---|
| 486 | if tid != None: |
|---|
| 487 | item.setAttribute( 'tid', tid ) |
|---|
| 488 | item.setAttribute( 'class', 'missingLink' ) |
|---|
| 489 | item.setAttribute( 'description', 'Missing links detected and marked for fixing' ) |
|---|
| 490 | item.setAttribute( 'prov', 'scanDreq.py:annotate' ) |
|---|
| 491 | this.appendChild( item ) |
|---|
| 492 | |
|---|
| 493 | txt = self.doc.toprettyxml(indent='\t', newl='\n', encoding=None) |
|---|
| 494 | oo = open( 'annotated_20150731.xml', 'w' ) |
|---|
| 495 | lines = string.split( txt, '\n' ) |
|---|
| 496 | for line in lines: |
|---|
| 497 | l = utils_wb.uniCleanFunc( string.strip(line) ) |
|---|
| 498 | if empty.match(l): |
|---|
| 499 | continue |
|---|
| 500 | else: |
|---|
| 501 | oo.write(l + '\n') |
|---|
| 502 | oo.close() |
|---|
| 503 | |
|---|
| 504 | doAnno = True |
|---|
| 505 | if doAnno: |
|---|
| 506 | an = annotate( dq.c.vsamp, dq ) |
|---|
| 507 | an.missingRefs( dq.inx.missingIds ) |
|---|
