1 | import dreq, collections, string, os, utils_wb |
---|
2 | import htmlTemplates as tmpl |
---|
3 | import xml, re |
---|
4 | import xml.dom, xml.dom.minidom |
---|
5 | |
---|
6 | from utils_wb import uniCleanFunc |
---|
7 | |
---|
8 | empty=re.compile('^$') |
---|
9 | |
---|
10 | dq = dreq.loadDreq() |
---|
11 | inx = dq.inx |
---|
12 | ##inx.makeVarRefs() |
---|
13 | ix_rql_uuid = {} |
---|
14 | ix_rqvg_uuid = {} |
---|
15 | ix_ovar_uuid = {} |
---|
16 | ix_gpi_uuid = {} |
---|
17 | list_gp_ovar = collections.defaultdict( list ) |
---|
18 | xr_var_ovar = collections.defaultdict( list ) |
---|
19 | xr_var_gpi = collections.defaultdict( list ) |
---|
20 | rql_by_name = collections.defaultdict( list ) |
---|
21 | |
---|
22 | def makeVarRefs(uuid, var, iref_by_uuid): |
---|
23 | varRefs = {} |
---|
24 | for thisuuid in var.uuid.keys(): |
---|
25 | if iref_by_uuid.has_key(thisuuid): |
---|
26 | ee1 = collections.defaultdict( list ) |
---|
27 | for k,i in iref_by_uuid[thisuuid]: |
---|
28 | sect,thisi = uuid[i] |
---|
29 | if sect == 'groupItem': |
---|
30 | ee1[sect].append( '%s.%s' % (thisi.mip, thisi.group) ) |
---|
31 | elif sect == 'ovar': |
---|
32 | ee1[sect].append( thisi.mipTable ) |
---|
33 | elif sect == 'revisedTabItem': |
---|
34 | ee1[sect].append( '%s.%s' % (thisi.mip, thisi.table) ) |
---|
35 | varRefs[thisuuid] = ee1 |
---|
36 | return varRefs |
---|
37 | |
---|
38 | varRefs = makeVarRefs( inx.uuid, inx.var, inx.iref_by_uuid) |
---|
39 | |
---|
40 | class updates(object): |
---|
41 | def __init__(self,fndup,fnmult,idir='rev1'): |
---|
42 | assert os.path.isdir( idir ), 'Directory %s not found' % idir |
---|
43 | self.fdup = '%s/%s' % (idir,fndup) |
---|
44 | self.fmult = '%s/%s' % (idir,fnmult) |
---|
45 | for p in [self.fdup,self.fmult]: |
---|
46 | assert os.path.isfile( p ), 'File %s not found' % p |
---|
47 | self.repl = {} |
---|
48 | self.upd = {} |
---|
49 | self.twins = [] |
---|
50 | self.ddel = {} |
---|
51 | |
---|
52 | def scandup(self): |
---|
53 | ii = open( self.fdup ).readlines() |
---|
54 | nn = (len(ii)-1)/2 |
---|
55 | for i in range(nn): |
---|
56 | l1 = string.split( ii[i*2+1], '\t' ) |
---|
57 | l2 = string.split( ii[i*2+2], '\t' ) |
---|
58 | xx = l1[8:10] |
---|
59 | yy = l2[8:10] |
---|
60 | if xx[1] == '' and yy[1] == xx[0]: |
---|
61 | ths = 0 |
---|
62 | assert not self.repl.has_key( yy[0] ), 'duplicate replacement request for %s' % yy[0] |
---|
63 | self.repl[ yy[0] ] = yy[1] |
---|
64 | elif yy[1] == '' and xx[1] == yy[0]: |
---|
65 | ths = 1 |
---|
66 | assert not self.repl.has_key( xx[0] ), 'duplicate replacement request for %s' % xx[0] |
---|
67 | self.repl[ xx[0] ] = xx[1] |
---|
68 | elif l1[10] == 'twin' and l2[10] == 'twin': |
---|
69 | ths = 2 |
---|
70 | self.twins.append( l1[8] ) |
---|
71 | self.twins.append( l2[8] ) |
---|
72 | elif l1[10] == 'inc' and l2[10] == 'inc': |
---|
73 | ths = 3 |
---|
74 | self.ddel[ l1[8] ] = 'inc' |
---|
75 | self.ddel[ l2[8] ] = 'inc' |
---|
76 | else: |
---|
77 | ths = -1 |
---|
78 | print 'ERROR.xxx.0001: Match failed' |
---|
79 | print l1 |
---|
80 | print l2 |
---|
81 | assert False |
---|
82 | |
---|
83 | def scanmult(self): |
---|
84 | ii = open( self.fmult ).readlines() |
---|
85 | nn = (len(ii)-1)/3 |
---|
86 | for i in range(nn): |
---|
87 | l1 = string.split( ii[i*3+1], '\t' ) |
---|
88 | l2 = string.split( ii[i*3+2], '\t' ) |
---|
89 | l3 = string.split( ii[i*3+3], '\t' ) |
---|
90 | yy = [l1[9],l2[9],l3[9]] |
---|
91 | xx = [l1[8],l2[8],l3[8]] |
---|
92 | zz = (l1,l2,l3) |
---|
93 | for j in range(3): |
---|
94 | if yy[j] != '': |
---|
95 | assert yy[j] in xx, 'Invalid replacement option, %s' % yy[j] |
---|
96 | assert not self.repl.has_key( xx[j] ), 'duplicate replacement request for %s' % xx[j] |
---|
97 | self.repl[ xx[j] ] = yy[j] |
---|
98 | elif zz[j][10] == 'twin': |
---|
99 | self.twins.append( zz[j][8] ) |
---|
100 | elif zz[j][11] == 'update': |
---|
101 | tags = map( string.strip, string.split( zz[j][13], ',' ) ) |
---|
102 | self.upd[ xx[j] ] = { 'provNote':zz[j][12], 'tags':tags, 'label':zz[j][0], 'title':zz[j][1] } |
---|
103 | |
---|
104 | up = updates('varDup_20150724.csv', 'varMult_20150725.csv') |
---|
105 | ##up.scandup() |
---|
106 | up.scanmult() |
---|
107 | |
---|
108 | urep = False |
---|
109 | urep = True |
---|
110 | if urep: |
---|
111 | oo = open( 'uuidreplace.csv', 'w' ) |
---|
112 | oo2 = open( 'uuidremove.csv', 'w' ) |
---|
113 | for k in up.repl.keys(): |
---|
114 | if inx.iref_by_uuid.has_key(k): |
---|
115 | kn = up.repl[k] |
---|
116 | for tag,ki in inx.iref_by_uuid[k]: |
---|
117 | oo.write( '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (k,kn,tag,ki, inx.uuid[k][1].label, inx.uuid[kn][1].label, inx.uuid[ki][1].label) ) |
---|
118 | else: |
---|
119 | oo2.write( k + '\n' ) |
---|
120 | oo.close() |
---|
121 | oo2.close() |
---|
122 | oo = open( 'uuidupdate.csv', 'w' ) |
---|
123 | for k in up.upd.keys(): |
---|
124 | ee = up.upd[k] |
---|
125 | oo.write( string.join( [k,ee['provNote'],string.join(ee['tags']),ee['label'], ee['title'] ], '\t') + '\n' ) |
---|
126 | oo.close() |
---|
127 | else: |
---|
128 | oo2 = open( 'uuidremove2.csv', 'w' ) |
---|
129 | for i in dq.coll['var'].items: |
---|
130 | if not inx.iref_by_uuid.has_key(i.uuid): |
---|
131 | oo2.write( string.join( [i.uuid,i.label,i.title,i.prov,i.description], '\t') + '\n' ) |
---|
132 | oo2.close() |
---|
133 | |
---|
134 | ### check back references. |
---|
135 | nbr = 0 |
---|
136 | lbr = [] |
---|
137 | for k in inx.iref_by_uuid.keys(): |
---|
138 | if not inx.uuid.has_key(k): |
---|
139 | nbr += 1 |
---|
140 | lbr.append(k) |
---|
141 | print 'Missing references: ', nbr |
---|
142 | ### can now apply mappings, create updated records and write to new xml? |
---|
143 | |
---|
144 | for i in dq.coll['requestLink'].items: |
---|
145 | rql_by_name[i.label].append( i.uuid ) |
---|
146 | ix_rql_uuid[i.uuid] = i |
---|
147 | |
---|
148 | for i in dq.coll['requestVarGroup'].items: |
---|
149 | ix_rqvg_uuid[i.uuid] = i |
---|
150 | |
---|
151 | |
---|
152 | oo = open( 'uuidinsert.csv', 'w' ) |
---|
153 | for i in dq.coll['revisedTabItem'].items: |
---|
154 | if i.uuid == '__new__': |
---|
155 | if inx.var.label.has_key( i.label ): |
---|
156 | if len( inx.var.label[i.label] ) == 1: |
---|
157 | v = inx.uuid[ inx.var.label[i.label][0] ][1] |
---|
158 | oo.write( string.join( ['unique',i.label,v.label,v.uuid,v.prov,i.mip], '\t' ) + '\n' ) |
---|
159 | else: |
---|
160 | oo.write( string.join( ['ambiguous',i.label,i.mip,str(len(inx.var.label[i.label] ) ) ], '\t' ) + '\n' ) |
---|
161 | oo.close() |
---|
162 | |
---|
163 | oo = open( 'varMult.csv', 'w' ) |
---|
164 | oo2 = open( 'varDup.csv', 'w' ) |
---|
165 | oo3 = open( 'varStar.csv', 'w' ) |
---|
166 | hs = ['label','title','sn','units','description','prov','procnote','procComment','uuid'] |
---|
167 | oo.write( string.join(hs, '\t' ) + '\n' ) |
---|
168 | oo2.write( string.join(hs, '\t' ) + '\n' ) |
---|
169 | oo3.write( string.join(hs, '\t' ) + '\n' ) |
---|
170 | ks = inx.var.label.keys() |
---|
171 | ks.sort() |
---|
172 | for k in ks: |
---|
173 | if len(inx.var.label[k]) == 2: |
---|
174 | v1 = inx.var.uuid[inx.var.label[k][0]] |
---|
175 | v2 = inx.var.uuid[inx.var.label[k][1]] |
---|
176 | cc = map( lambda x: v1.__dict__[x] == v2.__dict__[x], ['title','sn','units','description'] ) |
---|
177 | if all(cc): |
---|
178 | oo2.write( string.join(map( lambda x: v1.__dict__[x], hs) + [v2.uuid,'identical'], '\t' ) + '\n' ) |
---|
179 | else: |
---|
180 | oo2.write( string.join(map( lambda x: v1.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) |
---|
181 | oo2.write( string.join(map( lambda x: v2.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) |
---|
182 | |
---|
183 | elif len(inx.var.label[k]) > 1: |
---|
184 | for i in inx.var.label[k]: |
---|
185 | oo.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) |
---|
186 | |
---|
187 | if k[-2:] == '--': |
---|
188 | for i in (inx.var.label[k] + inx.var.label[k[:-2]]): |
---|
189 | oo3.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) |
---|
190 | oo.close() |
---|
191 | oo2.close() |
---|
192 | oo3.close() |
---|
193 | |
---|
194 | |
---|
195 | for i in dq.coll['groupItem'].items: |
---|
196 | list_gp_ovar[i.gpid].append( i.uuid ) |
---|
197 | |
---|
198 | vns = inx.var.label.keys() |
---|
199 | vns.sort() |
---|
200 | for v in vns: |
---|
201 | if len( inx.var.label[v] ) > 1: |
---|
202 | print 'INFO.001.0001:',v, string.join( map( lambda x: inx.var.uuid[x].sn, inx.var.label[v] ), ';' ) |
---|
203 | |
---|
204 | nok = 0 |
---|
205 | nerr = 0 |
---|
206 | for i in dq.coll['ovar'].items: |
---|
207 | vid = i.vid |
---|
208 | ix_ovar_uuid[i.uuid] = i |
---|
209 | xr_var_ovar[vid].append( i.uuid ) |
---|
210 | if not inx.var.uuid.has_key(vid): |
---|
211 | print 'missing key:',i.__dict__ |
---|
212 | nerr += 1 |
---|
213 | else: |
---|
214 | nok += 1 |
---|
215 | |
---|
216 | nok = 0 |
---|
217 | nerr = 0 |
---|
218 | for i in dq.coll['groupItem'].items: |
---|
219 | vid = i.vid |
---|
220 | ix_gpi_uuid[i.uuid] = i |
---|
221 | xr_var_gpi[vid].append( i.uuid ) |
---|
222 | if not inx.var.uuid.has_key(vid): |
---|
223 | nerr += 1 |
---|
224 | else: |
---|
225 | nok += 1 |
---|
226 | print 'groupItem to var crossref: nok = %s, nerr = %s' % (nok, nerr) |
---|
227 | |
---|
228 | class rqHtml(object): |
---|
229 | |
---|
230 | def __init__(self,odir='./html/'): |
---|
231 | self.odir = odir |
---|
232 | if not os.path.isdir(odir): |
---|
233 | os.mkdir(odir) |
---|
234 | |
---|
235 | def mkRqlHtml(self,name): |
---|
236 | ## [u'comment', u'uuid', u'tab', u'title', u'label', u'grid', 'defaults', u'objective', u'mip', 'globalDefault', u'gridreq'] |
---|
237 | if len( rql_by_name[name] ) == 1: |
---|
238 | self.mkRqlHtml01(rql_by_name[name][0], name ) |
---|
239 | else: |
---|
240 | self.mkRqlHtmlGp(name) |
---|
241 | |
---|
242 | def mkRqlHtmlGp(self,name): |
---|
243 | ee = {} |
---|
244 | ee['title'] = 'CMIP Request Link %s (with multiple definitions)' % name |
---|
245 | self.pageName = 'rql__%s.html' % name |
---|
246 | al =[] |
---|
247 | for i in range( len( rql_by_name[name] ) ): |
---|
248 | this = ix_rql_uuid[rql_by_name[name][i]] |
---|
249 | al.append( tmpl.item % {'item':'<a href="rql__%s__%s.html">[%s]</a>: %s' % (name,i,i,this.title) } ) |
---|
250 | ee['items'] = string.join(al, '\n' ) |
---|
251 | ee['introduction'] = '' |
---|
252 | ee['htmlBody'] = tmpl.indexWrapper % ee |
---|
253 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
---|
254 | self.pageHtml = tmpl.pageWrapper % ee |
---|
255 | self.write() |
---|
256 | for i in range( len( rql_by_name[name] ) ): |
---|
257 | self.mkRqlHtml01(rql_by_name[name][i],i) |
---|
258 | |
---|
259 | def mkRqlHtml01(self,id, tag): |
---|
260 | this = ix_rql_uuid[id] |
---|
261 | ee = {} |
---|
262 | if this.label == tag: |
---|
263 | ee['title'] = 'CMIP Request Link %s' % tag |
---|
264 | self.pageName = 'rql__%s.html' % tag |
---|
265 | else: |
---|
266 | ee['title'] = 'CMIP Request Link %s[%s]' % (this.label,tag) |
---|
267 | self.pageName = 'rql__%s__%s.html' % (this.label,tag) |
---|
268 | atts = this.__dict__.keys() |
---|
269 | atts.sort() |
---|
270 | al =[] |
---|
271 | for a in atts: |
---|
272 | if a not in ['defaults','globalDefault']: |
---|
273 | al.append( tmpl.item % {'item':'<b>%s</b>: %s' % (a,this.__dict__.get(a,'-- Not Set --')) } ) |
---|
274 | ee['items'] = string.join(al, '\n' ) |
---|
275 | ee['introduction'] = '' |
---|
276 | ee['htmlBody'] = tmpl.indexWrapper % ee |
---|
277 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
---|
278 | self.pageHtml = tmpl.pageWrapper % ee |
---|
279 | self.write() |
---|
280 | |
---|
281 | def mkVarHtml(self,name): |
---|
282 | if len( inx.var.label[name] ) == 1: |
---|
283 | self.mkVarHtml01(inx.var.label[name][0], name ) |
---|
284 | else: |
---|
285 | self.mkVarHtmlGp(name) |
---|
286 | |
---|
287 | def mkVarHtmlGp(self,name): |
---|
288 | ee = {} |
---|
289 | ee['title'] = 'CMIP Variable %s (with multiple definitions)' % name |
---|
290 | self.pageName = 'var__%s.html' % name |
---|
291 | al =[] |
---|
292 | for i in range( len( inx.var.label[name] ) ): |
---|
293 | this = inx.var.uuid[inx.var.label[name][i]] |
---|
294 | al.append( tmpl.item % {'item':'<a href="var__%s__%s.html">[%s]</a>: %s' % (name,i,i,this.title) } ) |
---|
295 | ee['items'] = string.join(al, '\n' ) |
---|
296 | ee['introduction'] = '' |
---|
297 | ee['htmlBody'] = tmpl.indexWrapper % ee |
---|
298 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
---|
299 | self.pageHtml = tmpl.pageWrapper % ee |
---|
300 | self.write() |
---|
301 | ##print 'Multi var: %s' % name |
---|
302 | for i in range( len( inx.var.label[name] ) ): |
---|
303 | self.mkVarHtml01(inx.var.label[name][i],i) |
---|
304 | |
---|
305 | def mkVarHtml01(self,id, tag): |
---|
306 | this = inx.var.uuid[id] |
---|
307 | ee = {} |
---|
308 | if this.label == tag: |
---|
309 | ee['title'] = 'CMIP Variable %s' % tag |
---|
310 | self.pageName = 'var__%s.html' % tag |
---|
311 | else: |
---|
312 | ee['title'] = 'CMIP Variable %s[%s]' % (this.label,tag) |
---|
313 | self.pageName = 'var__%s__%s.html' % (this.label,tag) |
---|
314 | atts = this.__dict__.keys() |
---|
315 | atts.sort() |
---|
316 | al =[] |
---|
317 | for a in atts: |
---|
318 | if a not in ['defaults','globalDefault']: |
---|
319 | al.append( tmpl.item % {'item':'<b>%s</b>: %s' % (a,this.__dict__.get(a,'-- Not Set --')) } ) |
---|
320 | |
---|
321 | if inx.iref_by_uuid.has_key(this.uuid): |
---|
322 | assert varRefs.has_key(this.uuid), 'Problem with collected references' |
---|
323 | ee1 = varRefs[this.uuid] |
---|
324 | ks = ee1.keys() |
---|
325 | ks.sort() |
---|
326 | for k in ks: |
---|
327 | al.append( tmpl.item % {'item':'<b>%s</b>: %s' % (k,string.join(ee1[k])) } ) |
---|
328 | ee['items'] = string.join(al, '\n' ) |
---|
329 | ee['introduction'] = '' |
---|
330 | ee['htmlBody'] = tmpl.indexWrapper % ee |
---|
331 | ee['htmlHead'] = '''<title>%(title)s</title>''' % ee |
---|
332 | self.pageHtml = tmpl.pageWrapper % ee |
---|
333 | self.write() |
---|
334 | |
---|
335 | def varHtml(self): |
---|
336 | for k in inx.var.label.keys(): |
---|
337 | self.mkVarHtml(k) |
---|
338 | |
---|
339 | def rqlHtml(self): |
---|
340 | for k in rql_by_name.keys(): |
---|
341 | self.mkRqlHtml(k) |
---|
342 | |
---|
343 | def write(self): |
---|
344 | oo = open( '%s/%s' % (self.odir,self.pageName), 'w' ) |
---|
345 | oo.write( self.pageHtml ) |
---|
346 | oo.close() |
---|
347 | |
---|
348 | |
---|
349 | vh = rqHtml() |
---|
350 | vh.varHtml() |
---|
351 | vh.rqlHtml() |
---|
352 | |
---|
353 | if nerr == 0: |
---|
354 | print 'CHECK 001: %s records checked, no missing references' % nok |
---|
355 | |
---|
356 | ##for k in xr_var_ovar.keys(): |
---|
357 | ##if len( xr_var_ovar[k] ) > 1: |
---|
358 | ##print inx.var.uuid[k].label, map( lambda x: ix_ovar_uuid[x].mipTable, xr_var_ovar[k] ) |
---|
359 | |
---|
360 | shps = {'': 64, 'XYZKT': 13, '4-element vector': 2, 'XYT': 476, '2D vector field ': 2, 'KZT': 4, '2D vector field': 2, 'XYZ': 27, 'XYZT': 204, '2D': 83, 'scalar': 14, 'XY': 88, '?': 21, '2D ': 1, 'XYKT': 3, 'YZT': 16, 'ZST1': 15, 'XKT': 2, 'BasinYT': 1} |
---|
361 | vshpchkMap = {'':'', u'all model levels above 400hPa':'alevStrat', u'all':'Xlev', 3.0:'plev3', '4.0':'plev4', \ |
---|
362 | 36.0:'plev36', u'soil levels':'sdepth', \ |
---|
363 | 1.0:'sfc?', \ |
---|
364 | 16.0:'plev16', 7.0:'plev7', 40.0:'plev40', u'all*':'Xlev', 14.0:'plev14', u'Model levels or 27Plevs':'alev|plev27', \ |
---|
365 | 27.0:'plev27', 17.0:'plev17', u'17 (or23)':'plev17|plev23', 8.0:'plev8', u'all model levels':'alev', 5.0:'plev5'} |
---|
366 | ks = vshpchkMap.keys() |
---|
367 | for k in ks: |
---|
368 | if type(k) == type(0.): |
---|
369 | vshpchkMap[str(k)] = vshpchkMap[k] |
---|
370 | |
---|
371 | print vshpchkMap.keys() |
---|
372 | |
---|
373 | tsmap = { 'mean':[u'daily mean', u'time mean', u'time: day', |
---|
374 | u'Cumulative annual fraction', u'Time mean', u'weighted time mean', u'time: mean', u'mean', u'Mean'], |
---|
375 | '__unknown__':[''], |
---|
376 | 'point':[ u'Instantaneous (end of year)', u'point', u'Synoptic', u'instantaneous', u'time: point', u'synoptic'] } |
---|
377 | tsmap2 = {} |
---|
378 | for k in tsmap.keys(): |
---|
379 | for i in tsmap[k]: |
---|
380 | tsmap2[i] = k |
---|
381 | |
---|
382 | ee = collections.defaultdict( int ) |
---|
383 | for i in dq.coll['groupItem'].items: |
---|
384 | tst = tsmap2[ i.tstyle ] |
---|
385 | dd = '' |
---|
386 | if 'X' in i.shape: |
---|
387 | dd += 'latitude ' |
---|
388 | if 'Y' in i.shape: |
---|
389 | dd += 'longitude ' |
---|
390 | if 'Z' in i.shape: |
---|
391 | if i.levels == '': |
---|
392 | print 'ERROR.001.0001: no levels specified', i.label, i.title |
---|
393 | else: |
---|
394 | zdim = vshpchkMap[i.levels] |
---|
395 | dd += zdim |
---|
396 | ## print '%s::%s::%s|%s' % (i.shape, i.levels, i.tstyle, dd) |
---|
397 | |
---|
398 | class tcmp(object): |
---|
399 | def __init__(self): |
---|
400 | pass |
---|
401 | def cmp(self,x,y): |
---|
402 | return cmp(x.title,y.title) |
---|
403 | |
---|
404 | def atRepr(l,x): |
---|
405 | v = l.__dict__[x] |
---|
406 | if v == '__unset__': |
---|
407 | return '' |
---|
408 | else: |
---|
409 | return v |
---|
410 | |
---|
411 | def dumpcsv( fn, key, atl ): |
---|
412 | oo = open(fn, 'w' ) |
---|
413 | ll = dq.coll[key].items[:] |
---|
414 | ll.sort( tcmp().cmp ) |
---|
415 | oo.write( string.join( atl, '\t' ) + '\n' ) |
---|
416 | for l in ll: |
---|
417 | oo.write( string.join( map( lambda x: atRepr(l,x), atl), '\t' ) + '\n' ) |
---|
418 | oo.close() |
---|
419 | |
---|
420 | def atlSort( ll ): |
---|
421 | oo = [] |
---|
422 | l1 = ['label','title'] |
---|
423 | l2 = ['uuid','defaults','globalDefault'] |
---|
424 | for i in l1: |
---|
425 | if i in ll: |
---|
426 | oo.append(i) |
---|
427 | ll.sort() |
---|
428 | for i in ll: |
---|
429 | if i not in l1 + l2: |
---|
430 | oo.append(i) |
---|
431 | if 'uuid' in ll: |
---|
432 | oo.append( 'uuid' ) |
---|
433 | return oo |
---|
434 | |
---|
435 | for k in dq.coll.keys(): |
---|
436 | if len( dq.coll[k].items ) > 0: |
---|
437 | expl = dq.coll[k].items[0] |
---|
438 | atl = atlSort( expl.__dict__.keys() ) |
---|
439 | print k, atl |
---|
440 | dumpcsv( 'csv2/%s.csv' % k, k, atl ) |
---|
441 | |
---|
442 | oo = open( 'var1.csv', 'w' ) |
---|
443 | ks = ['label','title','sn','units','description','prov','procnote','procComment','uuid'] |
---|
444 | ks2 = [ 'ovar','groupItem','revisedTabItem'] |
---|
445 | oo.write( string.join(ks + ks2, '\t' ) + '\n' ) |
---|
446 | for i in dq.coll['var'].items: |
---|
447 | if i.label[-2:] != '--': |
---|
448 | ee1 = varRefs.get( i.uuid, {} ) |
---|
449 | r2 = map( lambda x: string.join( ee1.get(x, [] ) ), ks2 ) |
---|
450 | oo.write( string.join(map( lambda x: i.__dict__[x], ks) + r2, '\t' ) + '\n' ) |
---|
451 | oo.close() |
---|
452 | |
---|
453 | class annotate(object): |
---|
454 | def __init__(self,src,dreq): |
---|
455 | assert os.path.isfile( src), '%s not found' % src |
---|
456 | self.doc = xml.dom.minidom.parse( src ) |
---|
457 | self.dreq = dreq |
---|
458 | |
---|
459 | def missingRefs(self,mrefs,clear=True): |
---|
460 | this = self.doc.getElementsByTagName('remarks')[0] |
---|
461 | if clear: |
---|
462 | dil = this.getElementsByTagName('item') |
---|
463 | for d in dil: |
---|
464 | this.removeChild(d) |
---|
465 | for k in mrefs.keys(): |
---|
466 | if len( mrefs[k] ) == 1: |
---|
467 | tid = mrefs[k][0][2] |
---|
468 | tattr = mrefs[k][0][1] |
---|
469 | tn = None |
---|
470 | else: |
---|
471 | tid = None |
---|
472 | ee = collections.defaultdict(int) |
---|
473 | tn = str( len( mrefs[k] ) ) |
---|
474 | for t in mrefs[k]: |
---|
475 | s = self.dreq.inx.uuid[t[2]][0] |
---|
476 | ee['%s.%s' % (s,t[1])] += 1 |
---|
477 | if len( ee.keys() ) == 1: |
---|
478 | tattr = ee.keys()[0] |
---|
479 | else: |
---|
480 | tattr = '__multiple__' |
---|
481 | item = self.doc.createElement( 'item' ) |
---|
482 | item.setAttribute( 'uuid', k ) |
---|
483 | item.setAttribute( 'tattr', tattr ) |
---|
484 | if tn != None: |
---|
485 | item.setAttribute( 'techNote', tn ) |
---|
486 | if tid != None: |
---|
487 | item.setAttribute( 'tid', tid ) |
---|
488 | item.setAttribute( 'class', 'missingLink' ) |
---|
489 | item.setAttribute( 'description', 'Missing links detected and marked for fixing' ) |
---|
490 | item.setAttribute( 'prov', 'scanDreq.py:annotate' ) |
---|
491 | this.appendChild( item ) |
---|
492 | |
---|
493 | txt = self.doc.toprettyxml(indent='\t', newl='\n', encoding=None) |
---|
494 | oo = open( 'annotated_20150731.xml', 'w' ) |
---|
495 | lines = string.split( txt, '\n' ) |
---|
496 | for line in lines: |
---|
497 | l = utils_wb.uniCleanFunc( string.strip(line) ) |
---|
498 | if empty.match(l): |
---|
499 | continue |
---|
500 | else: |
---|
501 | oo.write(l + '\n') |
---|
502 | oo.close() |
---|
503 | |
---|
504 | doAnno = True |
---|
505 | if doAnno: |
---|
506 | an = annotate( dq.c.vsamp, dq ) |
---|
507 | an.missingRefs( dq.inx.missingIds ) |
---|