Changeset 365
- Timestamp:
- 03/08/15 16:36:00 (5 years ago)
- Location:
- CMIP6dreq/trunk
- Files:
-
- 6 added
- 3 deleted
- 2 edited
- 1 moved
Legend:
- Unmodified
- Added
- Removed
-
CMIP6dreq/trunk/docs/vocabFrameworkSchema_01alpha.xsd
r357 r365 1 1 <?xml version="1.0" encoding="ISO-8859-1" ?> 2 <xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"> 2 <xs:schema 3 xmlns:xs="http://www.w3.org/2001/XMLSchema"> 3 4 <!-- Declarations --> 4 5 … … 11 12 </xs:simpleType> 12 13 14 <xs:simpleType name="st__labUnique_enum"> 15 <xs:restriction base="xs:string"> 16 <xs:enumeration value="Yes"/> 17 <xs:enumeration value="No"/> 18 </xs:restriction> 19 </xs:simpleType> 20 13 21 <xs:complexType name="ct__itemAttribute"> 14 22 <xs:attribute name="label" type="xs:string" use="required"/> 15 23 <xs:attribute name="title" type="xs:string"/> 16 24 <xs:attribute name="type" type="xs:string"/> 25 <xs:attribute name="class" type="xs:string"/> 26 <xs:attribute name="techNote" type="xs:string"/> 17 27 </xs:complexType> 18 28 … … 23 33 <xs:attribute name="label" type="xs:string" use="required"/> 24 34 <xs:attribute name="title" type="xs:string"/> 35 <xs:attribute name="level" type="xs:integer"/> 25 36 <xs:attribute name="id" type="xs:string"/> 37 <xs:attribute name="maxOccurs" type="xs:integer"/> 38 <xs:attribute name="labUnique" type="st__labUnique_enum"/> 26 39 <xs:attribute name="itemLabelMode" type="st__itemLabelMode_enum"/> 27 40 </xs:complexType> -
CMIP6dreq/trunk/src/dreq.py
r357 r365 1 """ Module to read in the data request and make some cross-reference tables to facilitate use. 2 ---------------------------------------------------------------------------------------------- 3 The data request is stored in a flattened format, which nevertheless requires 7 interlinking sections. 1 """This module provides a basic python API to the Data Request. 2 After ingesting the XML documents (configuration and request) the module generates two python objects: 3 1. A collection of records 4 2. Index 4 5 """ 5 6 6 import xml, string, collections 7 7 import xml.dom 8 8 import xml.dom.minidom 9 9 10 class dreqItemBase(object): 11 __doc__ = """A base class used in the definition of records. Designed to be used via a class factory which sets "itemLabelMode" and "attributes" before the class is instantiated: attempting to instantiate the class before setting these will trigger an exception.""" 10 12 def __init__(self,dict=None,xmlMiniDom=None,id='defaultId'): 11 13 dictMode = dict != None … … 13 15 assert not( dictMode and mdMode), 'Mode must be either dictionary of minidom: both assigned' 14 16 assert dictMode or mdMode, 'Mode must be either dictionary of minidom: neither assigned' 15 self.defaults = { }17 self.defaults = { } 16 18 self.globalDefault = '__unset__' 17 19 if dictMode: … … 21 23 22 24 def dictInit( self, dict ): 25 __doc__ = """Initialise from a dictionary.""" 23 26 for a in self.attributes: 24 27 if dict.has_key(a): … … 35 38 else: 36 39 self.__dict__[a] = self.defaults.get( a, self.globalDefault ) 37 38 40 41 39 42 class config(object): 40 41 def __init__(self): 42 self.vdef = '../docs/parVocabDefn.xml' 43 self.vsamp = '../docs/CMIP6DataRequest_v0_01.xml' 44 self.nts = collections.namedtuple( 'sectdef', ['tag','label','title','id','itemLabelMode'] ) 45 self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type'] ) 46 self.ntt = collections.namedtuple( 'sect', ['header','attributes'] ) 47 43 """Read in a vocabulary collection configuration document and a vocabulary document""" 44 45 def __init__(self, configdoc='out/dreqDefn.xml', thisdoc='../workbook/trial_20150724.xml',silent=True): 46 self.silent = silent 47 self.vdef = configdoc 48 self.vsamp = thisdoc 49 self.nts = collections.namedtuple( 'sectdef', ['tag','label','title','id','itemLabelMode','level'] ) 50 self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type','rClass','techNote'] ) 51 self.ntt = collections.namedtuple( 'sectinit', ['header','attributes'] ) 52 self.ntf = collections.namedtuple( 'sect', ['header','attDefn','items'] ) 53 54 self.coll = {} 48 55 doc = xml.dom.minidom.parse( self.vdef ) 49 56 self.contentDoc = xml.dom.minidom.parse( self.vsamp ) 50 ##<vocab label="institute" title="Institute" id="cmip.drv.001" itemLabelMode="def">51 ## <itemAttribute label="label"/>52 57 vl = doc.getElementsByTagName( 'table' ) 53 58 self.slist = [] … … 62 67 self.slist.append( t ) 63 68 69 self.recordAttributeDefn = tables 64 70 for k in tables.keys(): 65 71 vl = self.contentDoc.getElementsByTagName( k ) … … 69 75 i = v.getAttribute( 'id' ) 70 76 il = v.getElementsByTagName( 'item' ) 71 print k, t, i, len(il) 77 self.info( '%s, %s, %s, %s' % ( k, t, i, len(il) ) ) 78 72 79 self.tables[k] = (i,t,len(il)) 73 80 … … 78 85 l1 = [] 79 86 l2 = [] 80 print '#### %s: %s' % (k,len(vl) )81 87 for v in vl: 82 88 t = v.getAttribute( 'title' ) 83 89 i = v.getAttribute( 'id' ) 84 90 il = v.getElementsByTagName( 'item' ) 85 print k, t, i, len(il)91 self.info( '%s, %s, %s, %s' % ( k, t, i, len(il) ) ) 86 92 l1.append( (i,t,len(il)) ) 87 93 … … 93 99 self.tables[k] = l1 94 100 self.tableItems[k] = l2 101 self.coll[k] = self.ntf( self.recordAttributeDefn[k].header, self.recordAttributeDefn[k].attributes, self.tableItems[k] ) 102 103 def info(self,ss): 104 if not self.silent: 105 print ss 106 107 def get(self): 108 return self.coll 95 109 96 110 def itemClassFact(self,itemLabelMode,attributes): … … 103 117 104 118 def parsevcfg(self,v): 119 """Parse a section definition element, including all the record attributes. The results are returned as a namedtuple of attributes for the section and a dictionary of record attribute specifications.""" 105 120 l = v.getAttribute( 'label' ) 106 121 t = v.getAttribute( 'title' ) 107 122 i = v.getAttribute( 'id' ) 108 123 ilm = v.getAttribute( 'itemLabelMode' ) 124 lev = v.getAttribute( 'level' ) 109 125 il = v.getElementsByTagName( 'rowAttribute' ) 110 vtt = self.nts( v.nodeName, l,t,i,ilm ) 111 ll = [] 126 vtt = self.nts( v.nodeName, l,t,i,ilm,lev ) 112 127 idict = {} 113 128 for i in il: 114 129 tt = self.parseicfg(i) 115 130 idict[tt.label] = tt 116 ll.append(tt)117 131 return self.ntt( vtt, idict ) 118 132 119 133 def parseicfg(self,i): 120 l = i.getAttribute( 'label' ) 121 if i.hasAttribute( 'title' ): 122 t = i.getAttribute( 'title' ) 123 self.lastTitle = t 124 else: 125 t = None 126 if i.hasAttribute( 'type' ): 127 ty = i.getAttribute( 'type' ) 128 else: 129 ty = "xs:string" 130 return self.nti( i.nodeName, l,t,ty ) 131 132 c = config() 133 134 """Parse a record attribute specification""" 135 defs = {'type':"xs:string"} 136 ll = [] 137 for k in ['label','title','type','class','techNote']: 138 if i.hasAttribute( k ): 139 ll.append( i.getAttribute( k ) ) 140 else: 141 ll.append( defs.get( k, None ) ) 142 l, t, ty, cls, tn = ll 143 self.lastTitle = t 144 return self.nti( i.nodeName, l,t,ty,cls,tn ) 145 146 class container(object): 147 """Simple container class, to hold a set of dictionaries of lists.""" 148 def __init__(self, atl ): 149 self.uuid = {} 150 for a in atl: 151 self.__dict__[a] = collections.defaultdict( list ) 152 153 class c1(object): 154 def __init__(self): 155 self.a = collections.defaultdict( list ) 134 156 class index(object): 157 """Create an index of the document. Cross-references are generated from attributes with class 'internalLink'. 158 This version assumes that each record is identified by an "uuid" attribute and that there is a "var" section. 159 Invalid internal links are recorded in tme "missingIds" dictionary. 160 For any record, with identifier u, iref_by_uuid[u] gives a list of the section and identifier of records linking to that record. 161 """ 162 135 163 def __init__(self, dreq): 164 self.silent = True 136 165 self.uuid = {} 166 nativeAtts = ['uuid','iref_by_uuid','iref_by_sect','missingIds'] 167 naok = map( lambda x: not dreq.has_key(x), nativeAtts ) 168 assert all(naok), 'This version cannot index collections containing sections with names: %s' % str( nativeAtts ) 137 169 self.var_uuid = {} 138 170 self.var_by_name = collections.defaultdict( list ) 139 171 self.var_by_sn = collections.defaultdict( list ) 140 172 self.iref_by_uuid = collections.defaultdict( list ) 141 irefdict = {'ovar':['vid'], 'groupItem':['gpid','vid'], 'requestLink':['refid'], 'requestItem':['rlid'], 'revisedTabItem':['vid']} 142 for k in dreq.tableItems.keys(): 143 for i in dreq.tableItems[k]: 173 irefdict = collections.defaultdict( list ) 174 for k in dreq.keys(): 175 if dreq[k].attDefn.has_key('sn'): 176 self.__dict__[k] = container( ['label','sn'] ) 177 else: 178 self.__dict__[k] = container( ['label'] ) 179 ## 180 ## collected names of attributes which carry internal links 181 ## 182 for ka in dreq[k].attDefn.keys(): 183 if dreq[k].attDefn[ka].rClass == 'internalLink': 184 irefdict[k].append( ka ) 185 186 for k in dreq.keys(): 187 for i in dreq[k].items: 144 188 self.uuid[i.uuid] = (k,i) 189 190 self.missingIds = collections.defaultdict( list ) 191 self.iref_by_sect = collections.defaultdict( c1 ) 192 for k in dreq.keys(): 145 193 for k2 in irefdict.get( k, [] ): 146 for i in dreq.tableItems[k]: 147 self.iref_by_uuid[ i.__dict__.get( k2 ) ].append( (k2,i.uuid) ) 148 149 for i in dreq.tableItems['var']: 150 self.var_uuid[i.uuid] = i 151 self.var_by_name[i.label].append( i.uuid ) 152 self.var_by_sn[i.sn].append( i.uuid ) 153 154 def makeVarRefs(self): 155 self.varRefs = {} 156 for thisuuid in self.var_uuid.keys(): 157 if self.iref_by_uuid.has_key(thisuuid): 158 ee1 = collections.defaultdict( list ) 159 for k,i in self.iref_by_uuid[thisuuid]: 160 sect,thisi = self.uuid[i] 161 ### irefdict = {'ovar':['vid'], 'groupItem':['gpid','vid'], 'requestLink':['refid'], 'requestItem':['rlid'], 'revisedTabItem':['vid']} 162 if sect == 'groupItem': 163 ee1[sect].append( '%s.%s' % (thisi.mip, thisi.group) ) 164 elif sect == 'ovar': 165 ee1[sect].append( thisi.mipTable ) 166 elif sect == 'revisedTabItem': 167 ee1[sect].append( '%s.%s' % (thisi.mip, thisi.table) ) 168 self.varRefs[thisuuid] = ee1 169 170 inx = index( c ) 171 inx.makeVarRefs() 194 n1 = 0 195 n2 = 0 196 for i in dreq[k].items: 197 id2 = i.__dict__.get( k2 ) 198 if id2 != '__unset__': 199 self.iref_by_uuid[ id2 ].append( (k2,i.uuid) ) 200 self.iref_by_sect[ id2 ].a[k2].append( i.uuid ) 201 if self.uuid.has_key( id2 ): 202 n1 += 1 203 else: 204 n2 += 1 205 self.missingIds[id2].append( (k,k2,i.uuid) ) 206 self.info( 'INFO:: %s, %s: %s (%s)' % (k,k2,n1,n2) ) 207 208 for k in dreq.keys(): 209 for i in dreq[k].items: 210 self.__dict__[k].uuid[i.uuid] = i 211 self.__dict__[k].label[i.label].append( i.uuid ) 212 if dreq[k].attDefn.has_key('sn'): 213 self.__dict__[k].sn[i.sn].append( i.uuid ) 214 215 def info(self,ss): 216 if not self.silent: 217 print ss 218 219 220 class loadDreq(object): 221 def __init__(self,dreqXML='../docs/dreq.xml',configdoc='../docs/dreqDefn.xml' ): 222 self.c = config( thisdoc=dreqXML, configdoc=configdoc,silent=False) 223 self.coll = self.c.get() 224 self.inx = index(self.coll) 225 226 if __name__ == '__main__': 227 dreq = loadDreq() 228 -
CMIP6dreq/trunk/src/scanDreq.py
r357 r365 1 import dreq, collections, string, os 1 import dreq, collections, string, os, utils_wb 2 2 import htmlTemplates as tmpl 3 4 from dreq import inx 5 3 import xml, re 4 import xml.dom, xml.dom.minidom 5 6 from utils_wb import uniCleanFunc 7 8 empty=re.compile('^$') 9 10 dq = dreq.loadDreq() 11 inx = dq.inx 12 ##inx.makeVarRefs() 6 13 ix_rql_uuid = {} 7 14 ix_rqvg_uuid = {} … … 13 20 rql_by_name = collections.defaultdict( list ) 14 21 22 def makeVarRefs(uuid, var, iref_by_uuid): 23 varRefs = {} 24 for thisuuid in var.uuid.keys(): 25 if iref_by_uuid.has_key(thisuuid): 26 ee1 = collections.defaultdict( list ) 27 for k,i in iref_by_uuid[thisuuid]: 28 sect,thisi = uuid[i] 29 if sect == 'groupItem': 30 ee1[sect].append( '%s.%s' % (thisi.mip, thisi.group) ) 31 elif sect == 'ovar': 32 ee1[sect].append( thisi.mipTable ) 33 elif sect == 'revisedTabItem': 34 ee1[sect].append( '%s.%s' % (thisi.mip, thisi.table) ) 35 varRefs[thisuuid] = ee1 36 return varRefs 37 38 varRefs = makeVarRefs( inx.uuid, inx.var, inx.iref_by_uuid) 39 40 class updates(object): 41 def __init__(self,fndup,fnmult,idir='rev1'): 42 assert os.path.isdir( idir ), 'Directory %s not found' % idir 43 self.fdup = '%s/%s' % (idir,fndup) 44 self.fmult = '%s/%s' % (idir,fnmult) 45 for p in [self.fdup,self.fmult]: 46 assert os.path.isfile( p ), 'File %s not found' % p 47 self.repl = {} 48 self.upd = {} 49 self.twins = [] 50 self.ddel = {} 51 52 def scandup(self): 53 ii = open( self.fdup ).readlines() 54 nn = (len(ii)-1)/2 55 for i in range(nn): 56 l1 = string.split( ii[i*2+1], '\t' ) 57 l2 = string.split( ii[i*2+2], '\t' ) 58 xx = l1[8:10] 59 yy = l2[8:10] 60 if xx[1] == '' and yy[1] == xx[0]: 61 ths = 0 62 assert not self.repl.has_key( yy[0] ), 'duplicate replacement request for %s' % yy[0] 63 self.repl[ yy[0] ] = yy[1] 64 elif yy[1] == '' and xx[1] == yy[0]: 65 ths = 1 66 assert not self.repl.has_key( xx[0] ), 'duplicate replacement request for %s' % xx[0] 67 self.repl[ xx[0] ] = xx[1] 68 elif l1[10] == 'twin' and l2[10] == 'twin': 69 ths = 2 70 self.twins.append( l1[8] ) 71 self.twins.append( l2[8] ) 72 elif l1[10] == 'inc' and l2[10] == 'inc': 73 ths = 3 74 self.ddel[ l1[8] ] = 'inc' 75 self.ddel[ l2[8] ] = 'inc' 76 else: 77 ths = -1 78 print 'ERROR.xxx.0001: Match failed' 79 print l1 80 print l2 81 assert False 82 83 def scanmult(self): 84 ii = open( self.fmult ).readlines() 85 nn = (len(ii)-1)/3 86 for i in range(nn): 87 l1 = string.split( ii[i*3+1], '\t' ) 88 l2 = string.split( ii[i*3+2], '\t' ) 89 l3 = string.split( ii[i*3+3], '\t' ) 90 yy = [l1[9],l2[9],l3[9]] 91 xx = [l1[8],l2[8],l3[8]] 92 zz = (l1,l2,l3) 93 for j in range(3): 94 if yy[j] != '': 95 assert yy[j] in xx, 'Invalid replacement option, %s' % yy[j] 96 assert not self.repl.has_key( xx[j] ), 'duplicate replacement request for %s' % xx[j] 97 self.repl[ xx[j] ] = yy[j] 98 elif zz[j][10] == 'twin': 99 self.twins.append( zz[j][8] ) 100 elif zz[j][11] == 'update': 101 tags = map( string.strip, string.split( zz[j][13], ',' ) ) 102 self.upd[ xx[j] ] = { 'provNote':zz[j][12], 'tags':tags, 'label':zz[j][0], 'title':zz[j][1] } 103 104 up = updates('varDup_20150724.csv', 'varMult_20150725.csv') 105 ##up.scandup() 106 up.scanmult() 107 108 urep = False 109 urep = True 110 if urep: 111 oo = open( 'uuidreplace.csv', 'w' ) 112 oo2 = open( 'uuidremove.csv', 'w' ) 113 for k in up.repl.keys(): 114 if inx.iref_by_uuid.has_key(k): 115 kn = up.repl[k] 116 for tag,ki in inx.iref_by_uuid[k]: 117 oo.write( '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (k,kn,tag,ki, inx.uuid[k][1].label, inx.uuid[kn][1].label, inx.uuid[ki][1].label) ) 118 else: 119 oo2.write( k + '\n' ) 120 oo.close() 121 oo2.close() 122 oo = open( 'uuidupdate.csv', 'w' ) 123 for k in up.upd.keys(): 124 ee = up.upd[k] 125 oo.write( string.join( [k,ee['provNote'],string.join(ee['tags']),ee['label'], ee['title'] ], '\t') + '\n' ) 126 oo.close() 127 else: 128 oo2 = open( 'uuidremove2.csv', 'w' ) 129 for i in dq.coll['var'].items: 130 if not inx.iref_by_uuid.has_key(i.uuid): 131 oo2.write( string.join( [i.uuid,i.label,i.title,i.prov,i.description], '\t') + '\n' ) 132 oo2.close() 15 133 16 134 ### check back references. … … 24 142 ### can now apply mappings, create updated records and write to new xml? 25 143 26 for i in d req.c.tableItems['requestLink']:144 for i in dq.coll['requestLink'].items: 27 145 rql_by_name[i.label].append( i.uuid ) 28 146 ix_rql_uuid[i.uuid] = i 29 147 30 for i in d req.c.tableItems['requestVarGroup']:148 for i in dq.coll['requestVarGroup'].items: 31 149 ix_rqvg_uuid[i.uuid] = i 32 150 33 151 34 35 152 oo = open( 'uuidinsert.csv', 'w' ) 36 for i in d req.c.tableItems['revisedTabItem']:153 for i in dq.coll['revisedTabItem'].items: 37 154 if i.uuid == '__new__': 38 if inx.var _by_name.has_key( i.label ):39 if len( inx.var _by_name[i.label] ) == 1:40 v = inx.uuid[ inx.var _by_name[i.label][0] ][1]155 if inx.var.label.has_key( i.label ): 156 if len( inx.var.label[i.label] ) == 1: 157 v = inx.uuid[ inx.var.label[i.label][0] ][1] 41 158 oo.write( string.join( ['unique',i.label,v.label,v.uuid,v.prov,i.mip], '\t' ) + '\n' ) 42 159 else: 43 oo.write( string.join( ['ambiguous',i.label,i.mip,str(len(inx.var _by_name[i.label] ) ) ], '\t' ) + '\n' )160 oo.write( string.join( ['ambiguous',i.label,i.mip,str(len(inx.var.label[i.label] ) ) ], '\t' ) + '\n' ) 44 161 oo.close() 45 162 … … 51 168 oo2.write( string.join(hs, '\t' ) + '\n' ) 52 169 oo3.write( string.join(hs, '\t' ) + '\n' ) 53 ks = inx.var _by_name.keys()170 ks = inx.var.label.keys() 54 171 ks.sort() 55 172 for k in ks: 56 if len(inx.var _by_name[k]) == 2:57 v1 = inx.var _uuid[inx.var_by_name[k][0]]58 v2 = inx.var _uuid[inx.var_by_name[k][1]]173 if len(inx.var.label[k]) == 2: 174 v1 = inx.var.uuid[inx.var.label[k][0]] 175 v2 = inx.var.uuid[inx.var.label[k][1]] 59 176 cc = map( lambda x: v1.__dict__[x] == v2.__dict__[x], ['title','sn','units','description'] ) 60 177 if all(cc): … … 64 181 oo2.write( string.join(map( lambda x: v2.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) 65 182 66 elif len(inx.var _by_name[k]) > 1:67 for i in inx.var _by_name[k]:68 oo.write( string.join(map( lambda x: inx.var _uuid[i].__dict__[x], hs), '\t' ) + '\n' )183 elif len(inx.var.label[k]) > 1: 184 for i in inx.var.label[k]: 185 oo.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) 69 186 70 187 if k[-2:] == '--': 71 for i in (inx.var _by_name[k] + inx.var_by_name[k[:-2]]):72 oo3.write( string.join(map( lambda x: inx.var _uuid[i].__dict__[x], hs), '\t' ) + '\n' )188 for i in (inx.var.label[k] + inx.var.label[k[:-2]]): 189 oo3.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) 73 190 oo.close() 74 191 oo2.close() … … 76 193 77 194 78 for i in d req.c.tableItems['groupItem']:195 for i in dq.coll['groupItem'].items: 79 196 list_gp_ovar[i.gpid].append( i.uuid ) 197 198 vns = inx.var.label.keys() 199 vns.sort() 200 for v in vns: 201 if len( inx.var.label[v] ) > 1: 202 print 'INFO.001.0001:',v, string.join( map( lambda x: inx.var.uuid[x].sn, inx.var.label[v] ), ';' ) 80 203 81 204 nok = 0 82 205 nerr = 0 83 for i in d req.c.tableItems['ovar']:206 for i in dq.coll['ovar'].items: 84 207 vid = i.vid 85 208 ix_ovar_uuid[i.uuid] = i 86 209 xr_var_ovar[vid].append( i.uuid ) 87 if not inx.var _uuid.has_key(vid):210 if not inx.var.uuid.has_key(vid): 88 211 print 'missing key:',i.__dict__ 89 212 nerr += 1 … … 93 216 nok = 0 94 217 nerr = 0 95 for i in d req.c.tableItems['groupItem']:218 for i in dq.coll['groupItem'].items: 96 219 vid = i.vid 97 220 ix_gpi_uuid[i.uuid] = i 98 221 xr_var_gpi[vid].append( i.uuid ) 99 if not inx.var _uuid.has_key(vid):222 if not inx.var.uuid.has_key(vid): 100 223 nerr += 1 101 224 else: 102 225 nok += 1 103 print 'groupItem to var crossref: nok = %s, nerr = %s' ,nok, nerr226 print 'groupItem to var crossref: nok = %s, nerr = %s' % (nok, nerr) 104 227 105 228 class rqHtml(object): … … 157 280 158 281 def mkVarHtml(self,name): 159 if len( inx.var _by_name[name] ) == 1:160 self.mkVarHtml01(inx.var _by_name[name][0], name )282 if len( inx.var.label[name] ) == 1: 283 self.mkVarHtml01(inx.var.label[name][0], name ) 161 284 else: 162 285 self.mkVarHtmlGp(name) … … 167 290 self.pageName = 'var__%s.html' % name 168 291 al =[] 169 for i in range( len( inx.var _by_name[name] ) ):170 this = inx.var _uuid[inx.var_by_name[name][i]]292 for i in range( len( inx.var.label[name] ) ): 293 this = inx.var.uuid[inx.var.label[name][i]] 171 294 al.append( tmpl.item % {'item':'<a href="var__%s__%s.html">[%s]</a>: %s' % (name,i,i,this.title) } ) 172 295 ee['items'] = string.join(al, '\n' ) … … 176 299 self.pageHtml = tmpl.pageWrapper % ee 177 300 self.write() 178 for i in range( len( inx.var_by_name[name] ) ): 179 self.mkVarHtml01(inx.var_by_name[name][i],i) 301 ##print 'Multi var: %s' % name 302 for i in range( len( inx.var.label[name] ) ): 303 self.mkVarHtml01(inx.var.label[name][i],i) 180 304 181 305 def mkVarHtml01(self,id, tag): 182 this = inx.var _uuid[id]306 this = inx.var.uuid[id] 183 307 ee = {} 184 308 if this.label == tag: … … 196 320 197 321 if inx.iref_by_uuid.has_key(this.uuid): 198 assert inx.varRefs.has_key(this.uuid), 'Problem with collected references'199 ee1 = inx.varRefs[this.uuid]322 assert varRefs.has_key(this.uuid), 'Problem with collected references' 323 ee1 = varRefs[this.uuid] 200 324 ks = ee1.keys() 201 325 ks.sort() … … 210 334 211 335 def varHtml(self): 212 for k in inx.var _by_name.keys():336 for k in inx.var.label.keys(): 213 337 self.mkVarHtml(k) 214 338 … … 232 356 ##for k in xr_var_ovar.keys(): 233 357 ##if len( xr_var_ovar[k] ) > 1: 234 ##print inx.var _uuid[k].label, map( lambda x: ix_ovar_uuid[x].mipTable, xr_var_ovar[k] )358 ##print inx.var.uuid[k].label, map( lambda x: ix_ovar_uuid[x].mipTable, xr_var_ovar[k] ) 235 359 236 360 shps = {'': 64, 'XYZKT': 13, '4-element vector': 2, 'XYT': 476, '2D vector field ': 2, 'KZT': 4, '2D vector field': 2, 'XYZ': 27, 'XYZT': 204, '2D': 83, 'scalar': 14, 'XY': 88, '?': 21, '2D ': 1, 'XYKT': 3, 'YZT': 16, 'ZST1': 15, 'XKT': 2, 'BasinYT': 1} … … 257 381 258 382 ee = collections.defaultdict( int ) 259 for i in d req.c.tableItems['groupItem']:383 for i in dq.coll['groupItem'].items: 260 384 tst = tsmap2[ i.tstyle ] 261 385 dd = '' … … 278 402 return cmp(x.title,y.title) 279 403 404 def atRepr(l,x): 405 v = l.__dict__[x] 406 if v == '__unset__': 407 return '' 408 else: 409 return v 410 280 411 def dumpcsv( fn, key, atl ): 281 412 oo = open(fn, 'w' ) 282 ll = d req.c.tableItems[key][:]413 ll = dq.coll[key].items[:] 283 414 ll.sort( tcmp().cmp ) 284 415 oo.write( string.join( atl, '\t' ) + '\n' ) 285 416 for l in ll: 286 oo.write( string.join( map( lambda x: l.__dict__[x], atl), '\t' ) + '\n' )417 oo.write( string.join( map( lambda x: atRepr(l,x), atl), '\t' ) + '\n' ) 287 418 oo.close() 288 419 … … 302 433 return oo 303 434 304 for k in dreq.c.tableItems.keys(): 305 expl = dreq.c.tableItems[k][0] 306 atl = atlSort( expl.__dict__.keys() ) 307 dumpcsv( 'csv2/%s.csv' % k, k, atl ) 435 for k in dq.coll.keys(): 436 if len( dq.coll[k].items ) > 0: 437 expl = dq.coll[k].items[0] 438 atl = atlSort( expl.__dict__.keys() ) 439 print k, atl 440 dumpcsv( 'csv2/%s.csv' % k, k, atl ) 308 441 309 442 oo = open( 'var1.csv', 'w' ) … … 311 444 ks2 = [ 'ovar','groupItem','revisedTabItem'] 312 445 oo.write( string.join(ks + ks2, '\t' ) + '\n' ) 313 for i in d req.c.tableItems['var']:446 for i in dq.coll['var'].items: 314 447 if i.label[-2:] != '--': 315 ee1 = inx.varRefs.get( i.uuid, {} )448 ee1 = varRefs.get( i.uuid, {} ) 316 449 r2 = map( lambda x: string.join( ee1.get(x, [] ) ), ks2 ) 317 inx.var_by_sn[i.sn].append( i.uuid )318 450 oo.write( string.join(map( lambda x: i.__dict__[x], ks) + r2, '\t' ) + '\n' ) 319 451 oo.close() 452 453 class annotate(object): 454 def __init__(self,src,dreq): 455 assert os.path.isfile( src), '%s not found' % src 456 self.doc = xml.dom.minidom.parse( src ) 457 self.dreq = dreq 458 459 def missingRefs(self,mrefs,clear=True): 460 this = self.doc.getElementsByTagName('remarks')[0] 461 if clear: 462 dil = this.getElementsByTagName('item') 463 for d in dil: 464 this.removeChild(d) 465 for k in mrefs.keys(): 466 if len( mrefs[k] ) == 1: 467 tid = mrefs[k][0][2] 468 tattr = mrefs[k][0][1] 469 tn = None 470 else: 471 tid = None 472 ee = collections.defaultdict(int) 473 tn = str( len( mrefs[k] ) ) 474 for t in mrefs[k]: 475 s = self.dreq.inx.uuid[t[2]][0] 476 ee['%s.%s' % (s,t[1])] += 1 477 if len( ee.keys() ) == 1: 478 tattr = ee.keys()[0] 479 else: 480 tattr = '__multiple__' 481 item = self.doc.createElement( 'item' ) 482 item.setAttribute( 'uuid', k ) 483 item.setAttribute( 'tattr', tattr ) 484 if tn != None: 485 item.setAttribute( 'techNote', tn ) 486 if tid != None: 487 item.setAttribute( 'tid', tid ) 488 item.setAttribute( 'class', 'missingLink' ) 489 item.setAttribute( 'description', 'Missing links detected and marked for fixing' ) 490 item.setAttribute( 'prov', 'scanDreq.py:annotate' ) 491 this.appendChild( item ) 492 493 txt = self.doc.toprettyxml(indent='\t', newl='\n', encoding=None) 494 oo = open( 'annotated_20150731.xml', 'w' ) 495 lines = string.split( txt, '\n' ) 496 for line in lines: 497 l = utils_wb.uniCleanFunc( string.strip(line) ) 498 if empty.match(l): 499 continue 500 else: 501 oo.write(l + '\n') 502 oo.close() 503 504 doAnno = True 505 if doAnno: 506 an = annotate( dq.c.vsamp, dq ) 507 an.missingRefs( dq.inx.missingIds )
Note: See TracChangeset
for help on using the changeset viewer.