1 | """This module provides a basic python API to the Data Request. |
---|
2 | After ingesting the XML documents (configuration and request) the module generates two python objects: |
---|
3 | 1. A collection of records |
---|
4 | 2. Index |
---|
5 | """ |
---|
6 | import xml, string, collections |
---|
7 | import xml.dom |
---|
8 | import xml.dom.minidom |
---|
9 | import re, shelve, os, sys |
---|
10 | try: |
---|
11 | from __init__ import DOC_DIR, version, PACKAGE_DIR |
---|
12 | import utilities |
---|
13 | except: |
---|
14 | from dreqPy.__init__ import DOC_DIR, version, PACKAGE_DIR |
---|
15 | from dreqPy import utilities |
---|
16 | |
---|
17 | python2 = True |
---|
18 | if sys.version_info[0] == 3: |
---|
19 | python2 = False |
---|
20 | pythonPre27 = False |
---|
21 | elif sys.version_info[0] == 2: |
---|
22 | pythonPre27 = sys.version_info[1] < 7 |
---|
23 | |
---|
24 | charmeTempl = """<span title="Using the CHARMe annotation system">Comment on this page:<a href="%s/%s/%s.html" class="charme-metadata-document"></a></span> |
---|
25 | |
---|
26 | <span> |
---|
27 | <div id="charme-placeholder"></div> |
---|
28 | </span> |
---|
29 | <br/> |
---|
30 | <!-- the charme-placeholder-all-targets appears to be required, but can be hidden ... --> |
---|
31 | <span style="display: None;"> |
---|
32 | <div id="charme-placeholder-all-targets"></div> |
---|
33 | </span> |
---|
34 | """ |
---|
35 | |
---|
36 | jsh='''<link type="text/css" href="/css/jquery-ui-1.8.16.custom.css" rel="Stylesheet" /> |
---|
37 | <script src="/js/2013/jquery.min.js" type="text/javascript"></script> |
---|
38 | <script src="/js/2013/jquery-ui.min.js" type="text/javascript"></script> |
---|
39 | <script src="/js/2013/jquery.cookie.js" type="text/javascript"></script> |
---|
40 | |
---|
41 | <link type="text/css" href="/css/dreq.css" rel="Stylesheet" /> |
---|
42 | ''' |
---|
43 | |
---|
44 | def dref(i,x): |
---|
45 | return i._inx.uid[i.__dict__[x]] |
---|
46 | |
---|
47 | blockSchemaFile = '%s/%s' % (DOC_DIR, 'BlockSchema.csv' ) |
---|
48 | |
---|
49 | class lutilsC(object): |
---|
50 | |
---|
51 | def __init__(self): |
---|
52 | pass |
---|
53 | |
---|
54 | def addAttributes( self, thisClass, attrDict ): |
---|
55 | """Add a set of attributes, from a dictionary, to a class""" |
---|
56 | for k in attrDict: |
---|
57 | setattr( thisClass, '%s' % k , attrDict[k] ) |
---|
58 | |
---|
59 | def itemClassFact(self, sectionInfo,ns=None): |
---|
60 | class dreqItem(dreqItemBase): |
---|
61 | """Inherits all methods from dreqItemBase. |
---|
62 | |
---|
63 | USAGE |
---|
64 | ----- |
---|
65 | The instanstiated object contains a single data record. The "_h" attribute links to information about the record and the section it belongs to. |
---|
66 | |
---|
67 | object._a: a python dictionary defining the attributes in each record. The keys in the dictionary correspond to the attribute names and the values are python "named tuples" (from the "collections" module). E.g. object._a['priority'].type contains the type of the 'priority' attribute. Type is expressed using XSD schema language, so "xs:integer" implies integer. The "useClass" attribute carries information about usage. If object._a['xxx'].useClass = u'internalLink' then the record attribute provides a link to another element and object.xxx is the unique identifier of that element. |
---|
68 | |
---|
69 | object._h: a python named tuple describing the section. E.g. object._h.title is the section title (E.g. "CMOR Variables") |
---|
70 | """ |
---|
71 | _base=dreqItemBase |
---|
72 | |
---|
73 | dreqItem.__name__ = 'dreqItem_%s' % str( sectionInfo.header.label ) |
---|
74 | dreqItem._h = sectionInfo.header |
---|
75 | dreqItem._a = sectionInfo.attributes |
---|
76 | dreqItem._d = sectionInfo.defaults |
---|
77 | if sectionInfo.attributes != None: |
---|
78 | self.addAttributes(dreqItem, sectionInfo.attributes ) |
---|
79 | ##dreqItem.itemLabelMode = itemLabelMode |
---|
80 | ##dreqItem.attributes = attributes |
---|
81 | dreqItem._rc = rechecks() |
---|
82 | dreqItem.ns = ns |
---|
83 | return dreqItem |
---|
84 | |
---|
85 | |
---|
86 | |
---|
87 | ## |
---|
88 | ## from http://stackoverflow.com/questions/4474754/how-to-keep-comments-while-parsing-xml-using-python-elementtree |
---|
89 | ## |
---|
90 | ## does not work for python3 ... may not be needed. |
---|
91 | ## |
---|
92 | ## needed in python2.6 to preserve comments in output XML |
---|
93 | ## |
---|
94 | def getParser(): |
---|
95 | import xml.etree.ElementTree as ET |
---|
96 | class PCParser(ET.XMLTreeBuilder): |
---|
97 | |
---|
98 | def __init__(self): |
---|
99 | ET.XMLTreeBuilder.__init__(self) |
---|
100 | # assumes ElementTree 1.2.X |
---|
101 | self._parser.CommentHandler = self.handle_comment |
---|
102 | |
---|
103 | def handle_comment(self, data): |
---|
104 | self._target.start(ET.Comment, {}) |
---|
105 | self._target.data(data) |
---|
106 | self._target.end(ET.Comment) |
---|
107 | |
---|
108 | return PCParser |
---|
109 | |
---|
110 | def loadBS(bsfile): |
---|
111 | """Read in the 'BlockSchema' definitions of the attributes defining attributes""" |
---|
112 | ii = open( bsfile, 'r' ).readlines() |
---|
113 | ll = [] |
---|
114 | for l in ii: |
---|
115 | ll.append( [x for x in l.strip().split('\t') ] ) |
---|
116 | cc = collections.defaultdict( dict ) |
---|
117 | |
---|
118 | for l in ll[3:]: |
---|
119 | l[-1] = l[-1] == '1' |
---|
120 | if len(l) < len(ll[2]): |
---|
121 | l.append( '' ) |
---|
122 | try: |
---|
123 | for i in range( len(ll[2]) ): |
---|
124 | cc[l[0]][ll[2][i]] = l[i] |
---|
125 | except: |
---|
126 | print (l) |
---|
127 | raise |
---|
128 | return cc |
---|
129 | |
---|
130 | class rechecks(object): |
---|
131 | """Checks to be applied to strings""" |
---|
132 | def __init__(self): |
---|
133 | self.__isInt = re.compile( '^-{0,1}[0-9]+$' ) |
---|
134 | |
---|
135 | def isIntStr( self, tv ): |
---|
136 | """Check whether a string is a valid representation of an integer.""" |
---|
137 | if type( tv ) not in [type(''),type(u'')]: |
---|
138 | self.reason = 'NOT STRING' |
---|
139 | return False |
---|
140 | ok = self.__isInt.match( tv ) != None |
---|
141 | if not ok: |
---|
142 | self.reason = 'Failed to match regular expression for integers' |
---|
143 | else: |
---|
144 | self.reason = '' |
---|
145 | return ok |
---|
146 | |
---|
147 | class dreqItemBase(object): |
---|
148 | __doc__ = """A base class used in the definition of records. Designed to be used via a class factory which sets "itemLabelMode" and "attributes" before the class is instantiated: attempting to instantiate the class before setting these will trigger an exception.""" |
---|
149 | _indexInitialised = False |
---|
150 | _inx = None |
---|
151 | _urlBase = '' |
---|
152 | _htmlStyle = {} |
---|
153 | _linkAttrStyle = {} |
---|
154 | __charmeEnable__ = {} |
---|
155 | _extraHtml = {} |
---|
156 | |
---|
157 | def __init__(self,idict=None,xmlMiniDom=None,id='defaultId',etree=False): |
---|
158 | self._strictRead = True |
---|
159 | dictMode = idict != None |
---|
160 | mdMode = xmlMiniDom != None |
---|
161 | self._htmlTtl = None |
---|
162 | assert not( dictMode and mdMode), 'Mode must be either dictionary of minidom: both assigned' |
---|
163 | assert dictMode or mdMode, 'Mode must be either dictionary of minidom: neither assigned' |
---|
164 | ##self._defaults = { } |
---|
165 | ##self._globalDefault = '__unset__' |
---|
166 | self._contentInitialised = False |
---|
167 | self._greenIcon = '<img height="12pt" src="/images/154g.png" alt="[i]"/>' |
---|
168 | if dictMode: |
---|
169 | self.dictInit( idict ) |
---|
170 | elif mdMode: |
---|
171 | self.mdInit( xmlMiniDom, etree=etree ) |
---|
172 | |
---|
173 | def hasattr(self,tag): |
---|
174 | """Return True id the argument passed is the name of an attribute of this instance.""" |
---|
175 | return tag in self.__dict__ |
---|
176 | |
---|
177 | def __repr__(self): |
---|
178 | """Provide a one line summary of identifying the object.""" |
---|
179 | if self._contentInitialised: |
---|
180 | return 'Item <%s>: [%s] %s' % (self._h.title,self.label,self.title) |
---|
181 | else: |
---|
182 | return 'Item <%s>: uninitialised' % self._h.title |
---|
183 | |
---|
184 | def __info__(self,full=False): |
---|
185 | """Print a summary of the data held in the object as a list of key/value pairs""" |
---|
186 | if self._contentInitialised: |
---|
187 | print ( 'Item <%s>: [%s] %s' % (self._h.title,self.label,self.title) ) |
---|
188 | for a in self.__dict__.keys(): |
---|
189 | if a[0] != '_' or full: |
---|
190 | if hasattr( self._a[a], 'useClass') and self._a[a].useClass == 'internalLink' and self._base._indexInitialised: |
---|
191 | if self.__dict__[a] in self._base._inx.uid: |
---|
192 | targ = self._base._inx.uid[ self.__dict__[a] ] |
---|
193 | print ( ' %s: [%s]%s [%s]' % ( a, targ._h.label, targ.label, self.__dict__[a] ) ) |
---|
194 | else: |
---|
195 | print ( ' %s: [ERROR: key not found] [%s]' % ( a, self.__dict__[a] ) ) |
---|
196 | else: |
---|
197 | print ( 'INFO: %s: %s' % ( a, self.__dict__[a] ) ) |
---|
198 | else: |
---|
199 | print ( 'Item <%s>: uninitialised' % self.sectionLabel ) |
---|
200 | |
---|
201 | def __href__(self,odir="",label=None,title=None): |
---|
202 | """Generate html text for a link to this item.""" |
---|
203 | igns = ['','__unset__'] |
---|
204 | if title == None: |
---|
205 | if self._htmlTtl == None: |
---|
206 | if 'description' in self.__dict__ and self.description != None and self.description.strip( ) not in igns: |
---|
207 | ttl = self.description |
---|
208 | elif 'title' in self.__dict__ and self.title != None and self.title.strip( ) not in igns: |
---|
209 | ttl = self.title |
---|
210 | else: |
---|
211 | ttl = self.label |
---|
212 | ttl = ttl.replace( '"', '"' ) |
---|
213 | ttl = ttl.replace( '<', '<' ) |
---|
214 | self._htmlTtl = ttl.replace( '>', '>' ) |
---|
215 | title=self._htmlTtl |
---|
216 | if label == None: |
---|
217 | label = self.uid |
---|
218 | if odir == '': |
---|
219 | odir = './' |
---|
220 | |
---|
221 | return '<span title="%s"><a href="%s%s.html">%s</a></span>' % (title,odir,self.uid,label) |
---|
222 | |
---|
223 | def htmlLinkAttrListStyle(self,a,targ,frm=None): |
---|
224 | xx = '; '.join( ['%s [%s]' % (x.label, x.__href__()) for x in targ]) |
---|
225 | return '<li>%s: [%s] %s</li>' % ( a, targ[0]._h.label, xx ) |
---|
226 | |
---|
227 | def getHtmlLinkAttrStyle(self,a): |
---|
228 | """Return a string containing a html fragment for a link to an attribute.""" |
---|
229 | if a in self.__class__._linkAttrStyle: |
---|
230 | return self.__class__._linkAttrStyle[a] |
---|
231 | else: |
---|
232 | return lambda a,targ, frm='': '<li>%s: [%s] %s [%s]</li>' % ( a, targ._h.label, targ.label, targ.__href__() ) |
---|
233 | |
---|
234 | def __htmlLink__(self,a, sect,app): |
---|
235 | """Create html line for a link or list of links""" |
---|
236 | if self._a[a].useClass == 'internalLink': |
---|
237 | lst = self.getHtmlLinkAttrStyle(a) |
---|
238 | try: |
---|
239 | targ = self._base._inx.uid[ self.__dict__[a] ] |
---|
240 | m = lst( app, targ, frm=sect ) |
---|
241 | except: |
---|
242 | print ( 'INFO.cex.00001:',a, self.__dict__[a], sect, self.label ) |
---|
243 | m = '<li>%s: %s .... broken link</li>' % ( app, self.__dict__[a] ) |
---|
244 | raise |
---|
245 | else: |
---|
246 | assert self._a[a].useClass == 'internalLinkList', 'Unrecognised useClass in __htmlLink__: %s' % self._a[a].useClass |
---|
247 | m = self.htmlLinkAttrListStyle( app, [self._base._inx.uid[u] for u in self.__dict__[a] ], frm=sect ) |
---|
248 | return m |
---|
249 | return m |
---|
250 | ##m = '<li>%s, %s: [%s] %s [%s]</li>' % ( a, self.__class__.__dict__[a].__href__(label=self._greenIcon), targ._h.label, targ.label, targ.__href__() ) |
---|
251 | def __html__(self,ghis=None): |
---|
252 | """Create html view""" |
---|
253 | msg = [] |
---|
254 | if self._contentInitialised: |
---|
255 | sect = self._h.label |
---|
256 | msg.append( '<h1>%s: [%s] %s</h1>' % (self._h.title,self.label,self.title) ) |
---|
257 | if sect in self.__charmeEnable__: |
---|
258 | msg.append( charmeTempl % (self.__charmeEnable__[sect].site, 'u', self.uid) ) |
---|
259 | msg.append( '<a href="../index.html">Home</a> → <a href="../index/%s.html">%s section index</a><br/>\n' % (sect, self._h.title) ) |
---|
260 | msg.append( '<ul>' ) |
---|
261 | for a in self.__dict__.keys(): |
---|
262 | if a[0] != '_': |
---|
263 | app = '%s%s' % (a, self.__class__.__dict__[a].__href__(label=self._greenIcon) ) |
---|
264 | if hasattr( self._a[a], 'useClass') and self._a[a].useClass in ['internalLink','internalLinkList'] and self._base._indexInitialised: |
---|
265 | if self.__dict__[a] == '__unset__': |
---|
266 | m = '<li>%s: %s [missing link]</li>' % ( app, self.__dict__[a] ) |
---|
267 | else: |
---|
268 | m = self.__htmlLink__(a, sect,app) |
---|
269 | elif hasattr( self._a[a], 'useClass') and self._a[a].useClass == 'externalUrl': |
---|
270 | m = '<li>%s: <a href="%s" title="%s">%s</a></li>' % ( app, self.__dict__[a], self._a[a].description, self._a[a].title ) |
---|
271 | else: |
---|
272 | m = '<li>%s: %s</li>' % ( app, self.__dict__[a] ) |
---|
273 | msg.append( m ) |
---|
274 | msg.append( '</ul>' ) |
---|
275 | sect = self._h.label |
---|
276 | if sect in self._extraHtml: |
---|
277 | rc, href, hlab = self._extraHtml[sect](self) |
---|
278 | if rc: |
---|
279 | msg.append( '<p><a href="%s">%s</a></p>' % (href,hlab) ) |
---|
280 | ## |
---|
281 | ## add list of inward references |
---|
282 | ## |
---|
283 | if self._base._indexInitialised: |
---|
284 | msg += self.__irefHtml__(sect,ghis) |
---|
285 | |
---|
286 | if sect in self.__charmeEnable__: |
---|
287 | msg.append( '<script src="/js/dreq/charme/charme.js"></script>' ) |
---|
288 | else: |
---|
289 | msg.append( '<b>Item %s: uninitialised</b>' % self.sectionLabel ) |
---|
290 | return msg |
---|
291 | |
---|
292 | |
---|
293 | def __irefHtml__(self, sect,ghis): |
---|
294 | """Returns html (as a list of text strings) for inward references""" |
---|
295 | if self._htmlStyle.get( sect, {} ).get( 'getIrefs', None ) == None: |
---|
296 | return [] |
---|
297 | |
---|
298 | tl = self._htmlStyle[sect]['getIrefs'] |
---|
299 | doall = '__all__' in tl |
---|
300 | if doall: |
---|
301 | tl = self._inx.iref_by_sect[self.uid].a.keys() |
---|
302 | tl1 = [] |
---|
303 | for t in tl: |
---|
304 | if t in self._inx.iref_by_sect[self.uid].a and len( self._inx.iref_by_sect[self.uid].a[t] ) > 0: |
---|
305 | tl1.append( t ) |
---|
306 | am = [] |
---|
307 | if len(tl1) > 0: |
---|
308 | am.append( '''<div class="demo">\n<div id="tabs">\n<ul>''' ) |
---|
309 | for t in tl1: |
---|
310 | u0 = self._inx.iref_by_sect[self.uid].a[t][0] |
---|
311 | this1 = '<li><a href="#tabs-%s">%s</a></li>' % (t,self._inx.uid[u0]._h.title ) |
---|
312 | am.append( this1 ) |
---|
313 | am.append( '</ul>' ) |
---|
314 | for t in tl1: |
---|
315 | u0 = self._inx.iref_by_sect[self.uid].a[t][0] |
---|
316 | am.append( '<div id="tabs-%s">' % t ) |
---|
317 | am.append( '<h3>%s</h3>' % self._inx.uid[u0]._h.title ) |
---|
318 | am.append( '<ul>' ) |
---|
319 | items = [self._inx.uid[u] for u in self._inx.iref_by_sect[self.uid].a[t] ] |
---|
320 | ##items.sort( ds('label').cmp ) |
---|
321 | items = sorted( items, key=ds('label').key ) |
---|
322 | for targ in items: |
---|
323 | if ghis == None: |
---|
324 | m = '<li>%s:%s [%s]</li>' % ( targ._h.label, targ.label, targ.__href__() ) |
---|
325 | else: |
---|
326 | lst = ghis( targ._h.label ) |
---|
327 | m = lst( targ, frm=sect ) |
---|
328 | am.append( m ) |
---|
329 | am.append( '</ul>' ) |
---|
330 | am.append( '</div>' ) |
---|
331 | if len(am) > 0: |
---|
332 | am.append( '</div>' ) |
---|
333 | am0 = [ '<h2>Links from other sections</h2>' ,] |
---|
334 | am0.append( ''' <script> |
---|
335 | $(function() { |
---|
336 | $( "#tabs" ).tabs({cookie: { expires: 1 } }); |
---|
337 | }); |
---|
338 | </script> |
---|
339 | <!-- how to make tab selection stick: http://stackoverflow.com/questions/5066581/jquery-ui-tabs-wont-save-selected-tab-index-upon-page-reload expiry time in days-->''' ) |
---|
340 | return am0 + am |
---|
341 | else: |
---|
342 | return [] |
---|
343 | |
---|
344 | def dictInit( self, idict ): |
---|
345 | __doc__ = """Initialise from a dictionary.""" |
---|
346 | for a in self._a.keys(): |
---|
347 | vv = False |
---|
348 | if a in idict: |
---|
349 | val = idict[a] |
---|
350 | vv = True |
---|
351 | else: |
---|
352 | if type( self.__class__.__dict__[a] ) not in (type( ''), type( u'' )) and (not self.__class__.__dict__[a].required): |
---|
353 | if hasattr( self, a): |
---|
354 | setattr( self, a, None ) |
---|
355 | ##self.a = None |
---|
356 | delattr( self, a ) |
---|
357 | ##if a in self.__dict__: |
---|
358 | ##self.__dict__.pop(a) |
---|
359 | else: |
---|
360 | print ( 'ERROR: attempt to remove non-existent attribute: %s' % a ) |
---|
361 | else: |
---|
362 | val = self._d.defaults.get( a, self._d.glob ) |
---|
363 | vv = True |
---|
364 | if vv: |
---|
365 | setattr( self, a, val ) |
---|
366 | self._contentInitialised = True |
---|
367 | |
---|
368 | def mdInit( self, el, etree=False ): |
---|
369 | __doc__ = """Initialisation from a mindom XML element. The list of attributes must be set by the class factory before the class is initialised""" |
---|
370 | deferredHandling=False |
---|
371 | nw1 = 0 |
---|
372 | tvtl = [] |
---|
373 | if etree: |
---|
374 | ks = set( el.keys() ) |
---|
375 | for a in self._a.keys(): |
---|
376 | if a in ks: |
---|
377 | aa = '%s%s' % (self.ns,a) |
---|
378 | tvtl.append( (a,True, str( el.get( a ) ) ) ) |
---|
379 | elif self._a[a].__dict__.get( 'required', True ) in [False,'false',u'false']: |
---|
380 | tvtl.append( (a,True,None) ) |
---|
381 | else: |
---|
382 | tvtl.append( (a,False,None) ) |
---|
383 | else: |
---|
384 | for a in self._a.keys(): |
---|
385 | if el.hasAttribute( a ): |
---|
386 | tvtl.append( (a,True, str( el.getAttribute( a ) ) ) ) |
---|
387 | ## |
---|
388 | ## attributes are treated as required unless they have a required attribute set to false |
---|
389 | ## |
---|
390 | elif self._a[a].__dict__.get( 'required', True ) not in [False,'false',u'false']: |
---|
391 | tvtl.append( (a,False,None) ) |
---|
392 | |
---|
393 | for a,tv,v in tvtl: |
---|
394 | if tv: |
---|
395 | erase = False |
---|
396 | if v == None: |
---|
397 | pass |
---|
398 | erase = True |
---|
399 | elif self._a[a].type == u'xs:float': |
---|
400 | if v == '': |
---|
401 | v = None |
---|
402 | else: |
---|
403 | try: |
---|
404 | v = float(v) |
---|
405 | except: |
---|
406 | print ( 'Failed to convert real number: %s,%s,%s' % (a,tv,v) ) |
---|
407 | raise |
---|
408 | elif self._a[a].type in [u'aa:st__floatList', u'aa:st__floatListMonInc']: |
---|
409 | v = [float(x) for x in v.split()] |
---|
410 | elif self._a[a].type in [u'aa:st__integerList', u'aa:st__integerListMonInc']: |
---|
411 | v = [int(x) for x in v.split()] |
---|
412 | if self._a[a].type in [u'aa:st__integerListMonInc'] and self._strictRead: |
---|
413 | for i in range(len(v)-1): |
---|
414 | assert v[i] < v[i+1], 'Attribute %s of type %s with non-monotonic value: %s' % (a,self._a[a].type,str(v)) |
---|
415 | elif self._a[a].type == u'xs:integer': |
---|
416 | if self._rc.isIntStr( v ): |
---|
417 | v = int(v) |
---|
418 | else: |
---|
419 | v = v.strip() |
---|
420 | thissect = '%s [%s]' % (self._h.title,self._h.label) |
---|
421 | if v in [ '',u'',' ', u' ', [], '[]']: |
---|
422 | if nw1 < 20: |
---|
423 | print ( 'WARN.050.0001: input integer non-compliant: %s: %s: "%s" -- set to zero' % (thissect,a,v) ) |
---|
424 | nw1 += 1 |
---|
425 | v = 0 |
---|
426 | else: |
---|
427 | try: |
---|
428 | v = int(float(v)) |
---|
429 | print ( 'WARN: input integer non-compliant: %s: %s: %s' % (thissect,a,v) ) |
---|
430 | except: |
---|
431 | msg = 'ERROR: failed to convert integer: %s: %s: %s, %s' % (thissect,a,v,type(v)) |
---|
432 | deferredHandling=True |
---|
433 | elif self._a[a].type == u'xs:boolean': |
---|
434 | v = v in ['true','1'] |
---|
435 | elif self._a[a].type == u'aa:st__stringList': |
---|
436 | if v.find(' ') != -1: |
---|
437 | v = tuple( v.split() ) |
---|
438 | else: |
---|
439 | v = (v,) |
---|
440 | elif self._a[a].type not in [u'xs:string', u'aa:st__uid']: |
---|
441 | print ('ERROR: Type %s not recognised [%s:%s]' % (self._a[a].type,self._h.label,a) ) |
---|
442 | |
---|
443 | if erase: |
---|
444 | ### need to overwrite attribute (which is inherited from parent class) before deleting it. |
---|
445 | ### this may not be needed in python3 |
---|
446 | self.__dict__[a] = '__tmp__' |
---|
447 | delattr( self, a ) |
---|
448 | else: |
---|
449 | self.__dict__[a] = v |
---|
450 | else: |
---|
451 | if a in ['uid',]: |
---|
452 | thissect = '%s' % (self._h.title) |
---|
453 | print ( 'ERROR.020.0001: missing uid: %s: a,tv,v: %s, %s, %s\n %s' % (thissect,a,tv,v,str( self.__dict__ )) ) |
---|
454 | if etree: |
---|
455 | ##p = self.parent_map( el ) |
---|
456 | print ( ks ) |
---|
457 | raise |
---|
458 | import sys |
---|
459 | sys.exit(0) |
---|
460 | self.__dict__[a] = self._d.defaults.get( a, self._d.glob ) |
---|
461 | |
---|
462 | if deferredHandling: |
---|
463 | print ( msg ) |
---|
464 | |
---|
465 | self._contentInitialised = True |
---|
466 | |
---|
467 | |
---|
468 | class config(object): |
---|
469 | """Read in a vocabulary collection configuration document and a vocabulary document""" |
---|
470 | |
---|
471 | def __init__(self, configdoc='out/dreqDefn.xml', thisdoc='../workbook/trial_20150724.xml', manifest=None, useShelve=False, strings=False,configOnly=False): |
---|
472 | self.rc = rechecks() |
---|
473 | self.lu = lutilsC() |
---|
474 | self.silent = True |
---|
475 | self.configOnly = configOnly |
---|
476 | self.coll = {} |
---|
477 | self.recordAttributeDefn = {} |
---|
478 | |
---|
479 | self.nts = collections.namedtuple( 'sectdef', ['tag','label','title','id','itemLabelMode','level','maxOccurs','labUnique','uid','description'] ) |
---|
480 | self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type','useClass','techNote','required'] ) |
---|
481 | self.ntt = collections.namedtuple( 'sectinit', ['header','attributes','defaults'] ) |
---|
482 | self.nt__default = collections.namedtuple( 'deflt', ['defaults','glob'] ) |
---|
483 | self.ntf = collections.namedtuple( 'sect', ['header','attDefn','items'] ) |
---|
484 | self.bscc = loadBS(blockSchemaFile) |
---|
485 | self.strings = strings |
---|
486 | self.docl = [] |
---|
487 | |
---|
488 | self.tt0 = {} |
---|
489 | self.tt1 = {} |
---|
490 | self.ttl2 = [] |
---|
491 | self.docs = {} |
---|
492 | self.version = None |
---|
493 | |
---|
494 | if manifest != None: |
---|
495 | assert os.path.isfile( manifest ), 'Manifest file not found: %s' % manifest |
---|
496 | ii = open(manifest).readlines() |
---|
497 | docl = [] |
---|
498 | for l in ii[1:]: |
---|
499 | bits = l.strip().split() |
---|
500 | assert len( bits ) > 1, 'Failed to parse line in manifest %s: \n%s' % (manifest,l) |
---|
501 | bb = [] |
---|
502 | for b in bits[:2]: |
---|
503 | if not os.path.isfile( b ): |
---|
504 | b = '%s/%s' % (PACKAGE_DIR,b) |
---|
505 | assert os.path.isfile( b ), 'File %s not found (listed in %s)' % (b,manifest ) |
---|
506 | bb.append( b ) |
---|
507 | docl.append( tuple( bb ) ) |
---|
508 | for d,c in docl: |
---|
509 | self.__read__(d, c) |
---|
510 | else: |
---|
511 | self.__read__(thisdoc, configdoc) |
---|
512 | |
---|
513 | def getByUid(self,uid): |
---|
514 | mmm = [] |
---|
515 | for fn,rr in self.docs.items(): |
---|
516 | root, doc = rr |
---|
517 | mm = root.findall( ".//*[@uid='%s']" % uid ) |
---|
518 | mmm += mm |
---|
519 | print ('%s: %s' % (fn, str(mm)) ) |
---|
520 | return mmm |
---|
521 | |
---|
522 | def __read__(self, thisdoc, configdoc): |
---|
523 | self.vdef = configdoc |
---|
524 | self.vsamp = thisdoc |
---|
525 | self.docl.append( (thisdoc,configdoc) ) |
---|
526 | fn = thisdoc.split( '/' )[-1] |
---|
527 | |
---|
528 | if self.strings: |
---|
529 | doc = xml.dom.minidom.parseString( self.vdef ) |
---|
530 | else: |
---|
531 | doc = xml.dom.minidom.parse( self.vdef ) |
---|
532 | ## |
---|
533 | ## elementTree parsing implemented for main document |
---|
534 | ## |
---|
535 | self.etree = False |
---|
536 | self.etree = True |
---|
537 | self.ns = None |
---|
538 | if not self.configOnly: |
---|
539 | if self.etree: |
---|
540 | import xml.etree.cElementTree as cel |
---|
541 | |
---|
542 | if not pythonPre27: |
---|
543 | ## this for of namespace registration not available in 2.6 |
---|
544 | ## absence of registration means module cannot write data exactly as read. |
---|
545 | ## |
---|
546 | cel.register_namespace('', "urn:w3id.org:cmip6.dreq.dreq:a") |
---|
547 | cel.register_namespace('pav', "http://purl.org/pav/2.3") |
---|
548 | |
---|
549 | if not self.strings: |
---|
550 | if python2: |
---|
551 | parser = getParser()() |
---|
552 | self.contentDoc = cel.parse( self.vsamp, parser=parser ) |
---|
553 | else: |
---|
554 | self.contentDoc = cel.parse( self.vsamp ) |
---|
555 | |
---|
556 | root = self.contentDoc.getroot() |
---|
557 | else: |
---|
558 | root = cel.fromstring(self.vsamp) |
---|
559 | |
---|
560 | self.docs[fn] = (root, self.contentDoc) |
---|
561 | bs = root.tag.split( '}' ) |
---|
562 | if len( bs ) > 1: |
---|
563 | self.ns = bs[0] + '}' |
---|
564 | else: |
---|
565 | self.ns = None |
---|
566 | vl = root.findall( './/{http://purl.org/pav/2.3}version' ) |
---|
567 | if self.version != None: |
---|
568 | if vl[0].text != self.version: |
---|
569 | print ('WARNING: version difference between %s [%s] and %s [%s]' % (self.docl[0][0],self.version,thisdoc,vl[0].text) ) |
---|
570 | else: |
---|
571 | self.version = vl[0].text |
---|
572 | self.parent_map = dict((c, p) for p in root.getiterator() for c in p) |
---|
573 | else: |
---|
574 | if self.strings: |
---|
575 | self.contentDoc = xml.dom.minidom.parseString( self.vsamp ) |
---|
576 | else: |
---|
577 | self.contentDoc = xml.dom.minidom.parse( self.vsamp ) |
---|
578 | |
---|
579 | vl = self.contentDoc.getElementsByTagName( 'prologue' ) |
---|
580 | v = vl[0].getElementsByTagName( 'pav:version' ) |
---|
581 | self.version = v[0].firstChild.data |
---|
582 | self.ns = None |
---|
583 | |
---|
584 | vl = doc.getElementsByTagName( 'table' ) + doc.getElementsByTagName( 'annextable' ) |
---|
585 | self.tables = {} |
---|
586 | tables = {} |
---|
587 | self.tableClasses = {} |
---|
588 | self.tableItems = collections.defaultdict( list ) |
---|
589 | ## |
---|
590 | ## this loads in some metadata, but not yet in a useful way. |
---|
591 | ## |
---|
592 | self._t0 = self.parsevcfg(None) |
---|
593 | self._t2 = self.parsevcfg('__main__') |
---|
594 | self._tableClass0 = self.lu.itemClassFact( self._t0, ns=self.ns ) |
---|
595 | ## |
---|
596 | ## define a class for the section heading records. |
---|
597 | ## |
---|
598 | self._t1 = self.parsevcfg('__sect__') |
---|
599 | ## |
---|
600 | ## when used with manifest .. need to preserve entries in "__main__" from each document. |
---|
601 | ## |
---|
602 | self._sectClass0 = self.lu.itemClassFact( self._t1, ns=self.ns ) |
---|
603 | |
---|
604 | for k in self.bscc: |
---|
605 | self.tt0[k] = self._tableClass0(idict=self.bscc[k]) |
---|
606 | if k in self._t0.attributes: |
---|
607 | setattr( self._tableClass0, '%s' % k, self.tt0[k] ) |
---|
608 | if k in self._t1.attributes: |
---|
609 | setattr( self._sectClass0, '%s' % k, self.tt0[k] ) |
---|
610 | |
---|
611 | ## |
---|
612 | ## save header information, as for recordAttributeDefn below |
---|
613 | ## |
---|
614 | self._recAtDef = {'__core__':self._t0, '__sect__':self._t1} |
---|
615 | ## |
---|
616 | ## addition of __core__ to coll dictionary .. |
---|
617 | ## |
---|
618 | self.coll['__core__'] = self.ntf( self._t0.header, self._t0.attributes, [self.tt0[k] for k in self.tt0] ) |
---|
619 | |
---|
620 | ec = {} |
---|
621 | for i in self.coll['__core__'].items: |
---|
622 | ec[i.label] = i |
---|
623 | |
---|
624 | for v in vl: |
---|
625 | t = self.parsevcfg(v) |
---|
626 | tables[t[0].label] = t |
---|
627 | self.tableClasses[t[0].label] = self.lu.itemClassFact( t, ns=self.ns ) |
---|
628 | thisc = self.tableClasses[t[0].label] |
---|
629 | self.tt1[t[0].label] = self._sectClass0( idict=t.header._asdict() ) |
---|
630 | self.tt1[t[0].label].maxOccurs = t.header.maxOccurs |
---|
631 | self.tt1[t[0].label].labUnique = t.header.labUnique |
---|
632 | self.tt1[t[0].label].level = t.header.level |
---|
633 | self.tt1[t[0].label].uid = t.header.uid |
---|
634 | self.tt1[t[0].label].itemLabelMode = t.header.itemLabelMode |
---|
635 | self.ttl2 += [thisc.__dict__[a] for a in t.attributes] |
---|
636 | mil = [t[1] for t in self._t2.attributes.items()] |
---|
637 | self.coll['__main__'] = self.ntf( self._t2.header, self._t2.attributes, self.ttl2 ) |
---|
638 | |
---|
639 | self.coll['__sect__'] = self.ntf( self._t1.header, self._t1.attributes, [self.tt1[k] for k in self.tt1] ) |
---|
640 | |
---|
641 | for sct in ['__core__','__main__','__sect__']: |
---|
642 | for k in self.coll[sct].attDefn.keys(): |
---|
643 | assert k in ec, 'Key %s [%s] not found' % (k,sct) |
---|
644 | self.coll[sct].attDefn[k] = ec[k] |
---|
645 | |
---|
646 | for k in tables: |
---|
647 | self.recordAttributeDefn[k] = tables[k] |
---|
648 | |
---|
649 | if self.configOnly: |
---|
650 | return |
---|
651 | for k in tables.keys(): |
---|
652 | if self.etree: |
---|
653 | vl = root.findall( './/%s%s' % (self.ns,k) ) |
---|
654 | if len(vl) == 1: |
---|
655 | v = vl[0] |
---|
656 | t = v.get( 'title' ) |
---|
657 | i = v.get( 'id' ) |
---|
658 | uid = v.get( 'uid' ) |
---|
659 | useclass = v.get( 'useClass' ) |
---|
660 | |
---|
661 | self.tt1[k].label = k |
---|
662 | self.tt1[k].title = t |
---|
663 | self.tt1[k].id = i |
---|
664 | self.tt1[k].uid = uid |
---|
665 | self.tt1[k].useClass = useclass |
---|
666 | self.tableClasses[k]._h = self.tt1[k] |
---|
667 | il = v.findall( '%sitem' % self.ns ) |
---|
668 | self.info( '%s, %s, %s, %s' % ( k, t, i, len(il) ) ) |
---|
669 | |
---|
670 | self.tables[k] = (i,t,len(il)) |
---|
671 | |
---|
672 | for i in il: |
---|
673 | ii = self.tableClasses[k](xmlMiniDom=i, etree=True) |
---|
674 | self.tableItems[k].append( ii ) |
---|
675 | elif len(vl) > 1: |
---|
676 | assert False, 'not able to handle repeat sections with etree yet' |
---|
677 | else: |
---|
678 | vl = self.contentDoc.getElementsByTagName( k ) |
---|
679 | if len(vl) == 1: |
---|
680 | v = vl[0] |
---|
681 | t = v.getAttribute( 'title' ) |
---|
682 | i = v.getAttribute( 'id' ) |
---|
683 | il = v.getElementsByTagName( 'item' ) |
---|
684 | self.info( '%s, %s, %s, %s' % ( k, t, i, len(il) ) ) |
---|
685 | |
---|
686 | self.tables[k] = (i,t,len(il)) |
---|
687 | |
---|
688 | for i in il: |
---|
689 | ii = self.tableClasses[k](xmlMiniDom=i) |
---|
690 | self.tableItems[k].append( ii ) |
---|
691 | elif len(vl) > 1: |
---|
692 | l1 = [] |
---|
693 | l2 = [] |
---|
694 | for v in vl: |
---|
695 | t = v.getAttribute( 'title' ) |
---|
696 | i = v.getAttribute( 'id' ) |
---|
697 | il = v.getElementsByTagName( 'item' ) |
---|
698 | self.info( '%s, %s, %s, %s' % ( k, t, i, len(il) ) ) |
---|
699 | l1.append( (i,t,len(il)) ) |
---|
700 | |
---|
701 | l2i = [] |
---|
702 | for i in il: |
---|
703 | ii = self.tableClasses[k](xmlMiniDom=i) |
---|
704 | l2i.append( ii ) |
---|
705 | l2.append( l2i ) |
---|
706 | self.tables[k] = l1 |
---|
707 | self.tableItems[k] = l2 |
---|
708 | self.coll[k] = self.ntf( self.recordAttributeDefn[k].header, self.recordAttributeDefn[k].attributes, self.tableItems[k] ) |
---|
709 | |
---|
710 | def info(self,ss): |
---|
711 | """Switchable print function ... switch off by setting self.silent=True""" |
---|
712 | if not self.silent: |
---|
713 | print ( ss ) |
---|
714 | |
---|
715 | def parsevcfg(self,v): |
---|
716 | """Parse a section definition element, including all the record attributes. The results are returned as a named tuple of attributes for the section and a dictionary of record attribute specifications.""" |
---|
717 | if v in [ None,'__main__']: |
---|
718 | idict = {'description':'An extended description of the object', 'title':'Record Description', \ |
---|
719 | 'techNote':'', 'useClass':'__core__', 'superclass':'rdf:property',\ |
---|
720 | 'type':'xs:string', 'uid':'__core__:description', 'label':'label', 'required':'required' } |
---|
721 | if v == None: |
---|
722 | vtt = self.nts( '__core__', 'CoreAttributes', 'X.1 Core Attributes', '00000000', 'def', '0', '0', 'false', '__core__', 'The core attributes, used in defining sections and attributes' ) |
---|
723 | else: |
---|
724 | vtt = self.nts( '__main__', 'DataRequestAttributes', 'X.2 Data Request Attributes', '00000001', 'def', '0', '0', 'false', '__main__' , 'The attributes used to define data request records') |
---|
725 | elif v == '__sect__': |
---|
726 | idict = {'title':'Record Description', \ |
---|
727 | 'uid':'__core__:description', 'label':'label', 'useClass':'text', 'id':'id', 'maxOccurs':'', 'itemLabelMode':'', 'level':'', 'labUnique':'' } |
---|
728 | vtt = self.nts( '__sect__', 'sectionAttributes', 'X.3 Section Attributes', '00000000', 'def', '0', '0', 'false', '__sect__', 'The attributes used to define data request sections' ) |
---|
729 | ##<var label="var" uid="SECTION:var" useClass="vocab" title="MIP Variable" id="cmip.drv.001"> |
---|
730 | else: |
---|
731 | l = v.getAttribute( 'label' ) |
---|
732 | t = v.getAttribute( 'title' ) |
---|
733 | i = v.getAttribute( 'id' ) |
---|
734 | u = v.getAttribute( 'uid' ) |
---|
735 | ilm = v.getAttribute( 'itemLabelMode' ) |
---|
736 | lev = v.getAttribute( 'level' ) |
---|
737 | maxo = v.getAttribute( 'maxOccurs' ) |
---|
738 | labu = v.getAttribute( 'labUnique' ) |
---|
739 | des = v.getAttribute( 'description' ) |
---|
740 | il = v.getElementsByTagName( 'rowAttribute' ) |
---|
741 | ##vtt = self.nts( v.nodeName, l,t,i,ilm,lev, maxo, labu, 's__%s' % v.nodeName ) |
---|
742 | vtt = self.nts( v.nodeName, l,t,i,ilm,lev, maxo, labu, u, des ) |
---|
743 | idict = {} |
---|
744 | for i in il: |
---|
745 | tt = self.parseicfg(i) |
---|
746 | idict[tt.label] = tt |
---|
747 | deflt = self.nt__default( {}, '__unset__' ) |
---|
748 | |
---|
749 | ## return a named tuple: (header, attributes, defaults) |
---|
750 | return self.ntt( vtt, idict, deflt ) |
---|
751 | |
---|
752 | def parseicfg(self,i): |
---|
753 | """Parse a record attribute specification""" |
---|
754 | defs = {'type':"xs:string"} |
---|
755 | ll = [] |
---|
756 | ee = {} |
---|
757 | for k in ['label','title','type','useClass','techNote','description','uid','required']: |
---|
758 | if i.hasAttribute( k ): |
---|
759 | ll.append( i.getAttribute( k ) ) |
---|
760 | else: |
---|
761 | ll.append( defs.get( k, None ) ) |
---|
762 | ee[k] = ll[-1] |
---|
763 | l, t, ty, cls, tn, desc, uid, rq = ll |
---|
764 | self.lastTitle = t |
---|
765 | if rq in ['0', 'false']: |
---|
766 | rq = False |
---|
767 | else: |
---|
768 | rq = True |
---|
769 | ee['required'] = rq |
---|
770 | |
---|
771 | returnClass = True |
---|
772 | if returnClass: |
---|
773 | return self._tableClass0( idict=ee ) |
---|
774 | else: |
---|
775 | return self.nti( i.nodeName, l,t,ty,cls,tn,rq ) |
---|
776 | |
---|
777 | class container(object): |
---|
778 | """Simple container class, to hold a set of dictionaries of lists.""" |
---|
779 | def __init__(self, atl ): |
---|
780 | self.uid = {} |
---|
781 | for a in atl: |
---|
782 | self.__dict__[a] = collections.defaultdict( list ) |
---|
783 | |
---|
784 | class c1(object): |
---|
785 | def __init__(self): |
---|
786 | self.a = collections.defaultdict( list ) |
---|
787 | |
---|
788 | class index(object): |
---|
789 | """Create an index of the document. Cross-references are generated from attributes with class 'internalLink'. |
---|
790 | This version assumes that each record is identified by an "uid" attribute and that there is a "var" section. |
---|
791 | Invalid internal links are recorded in tme "missingIds" dictionary. |
---|
792 | For any record, with identifier u, iref_by_uid[u] gives a list of the section and identifier of records linking to that record. |
---|
793 | """ |
---|
794 | |
---|
795 | def __init__(self, dreq,lock=True): |
---|
796 | self.silent = True |
---|
797 | self.uid = {} |
---|
798 | self.uid2 = collections.defaultdict( list ) |
---|
799 | nativeAtts = ['uid','iref_by_uid','iref_by_sect','missingIds'] |
---|
800 | naok = map( lambda x: not x in dreq, nativeAtts ) |
---|
801 | assert all(naok), 'This version cannot index collections containing sections with names: %s' % str( nativeAtts ) |
---|
802 | self.var_uid = {} |
---|
803 | self.var_by_name = collections.defaultdict( list ) |
---|
804 | self.var_by_sn = collections.defaultdict( list ) |
---|
805 | self.iref_by_uid = collections.defaultdict( list ) |
---|
806 | irefdict = collections.defaultdict( list ) |
---|
807 | for k in dreq.keys(): |
---|
808 | if 'sn' in dreq[k].attDefn: |
---|
809 | self.__dict__[k] = container( ['label','sn'] ) |
---|
810 | else: |
---|
811 | self.__dict__[k] = container( ['label'] ) |
---|
812 | ## |
---|
813 | ## collected names of attributes which carry internal links |
---|
814 | ## |
---|
815 | for ka in dreq[k].attDefn.keys(): |
---|
816 | if hasattr( dreq[k].attDefn[ka], 'useClass') and dreq[k].attDefn[ka].useClass in ['internalLink','internalLinkList']: |
---|
817 | irefdict[k].append( ka ) |
---|
818 | |
---|
819 | for k in dreq.keys(): |
---|
820 | for i in dreq[k].items: |
---|
821 | assert 'uid' in i.__dict__, 'uid not found::\n%s\n%s' % (str(i._h),str(i.__dict__) ) |
---|
822 | if 'uid' in self.uid: |
---|
823 | print ( 'ERROR.100.0001: Duplicate uid: %s [%s]' % (i.uid,i._h.title) ) |
---|
824 | self.uid2[i.uid].append( (k,i) ) |
---|
825 | else: |
---|
826 | ### create index bx uid. |
---|
827 | self.uid[i.uid] = i |
---|
828 | |
---|
829 | self.missingIds = collections.defaultdict( list ) |
---|
830 | self.iref_by_sect = collections.defaultdict( c1 ) |
---|
831 | for k in dreq.keys(): |
---|
832 | for k2 in irefdict.get( k, [] ): |
---|
833 | n1 = 0 |
---|
834 | n2 = 0 |
---|
835 | for i in dreq[k].items: |
---|
836 | if k2 in i.__dict__: |
---|
837 | id2 = i.__dict__.get( k2 ) |
---|
838 | if id2 != '__unset__': |
---|
839 | sect = i._h.label |
---|
840 | ## append attribute name and target -- item i.uid, attribute k2 reference item id2 |
---|
841 | if type(id2) not in [type( [] ),type(())]: |
---|
842 | id2 = [id2,] |
---|
843 | for u in id2: |
---|
844 | self.iref_by_uid[ u ].append( (k2,i.uid) ) |
---|
845 | self.iref_by_sect[ u ].a[sect].append( i.uid ) |
---|
846 | if u in self.uid: |
---|
847 | n1 += 1 |
---|
848 | else: |
---|
849 | n2 += 1 |
---|
850 | self.missingIds[u].append( (k,k2,i.uid) ) |
---|
851 | self.info( 'INFO:: %s, %s: %s (%s)' % (k,k2,n1,n2) ) |
---|
852 | |
---|
853 | for k in dreq.keys(): |
---|
854 | for i in dreq[k].items: |
---|
855 | self.__dict__[k].uid[i.uid] = i |
---|
856 | self.__dict__[k].label[i.label].append( i.uid ) |
---|
857 | if 'sn' in dreq[k].attDefn: |
---|
858 | self.__dict__[k].sn[i.sn].append( i.uid ) |
---|
859 | |
---|
860 | if lock: |
---|
861 | for k in self.iref_by_uid: |
---|
862 | self.iref_by_uid[k] = tuple( self.iref_by_uid[k] ) |
---|
863 | for k in self.iref_by_sect: |
---|
864 | for s in self.iref_by_sect[ k ].a: |
---|
865 | self.iref_by_sect[ k ].a[s] = tuple( self.iref_by_sect[ k ].a[s] ) |
---|
866 | |
---|
867 | def info(self,ss): |
---|
868 | if not self.silent: |
---|
869 | print ( ss ) |
---|
870 | |
---|
871 | class ds(object): |
---|
872 | """Comparison object to assist sorting of lists of dictionaries""" |
---|
873 | def __init__(self,k): |
---|
874 | self.k = k |
---|
875 | def cmp(self,x,y): |
---|
876 | return cmp( x.__dict__[self.k], y.__dict__[self.k] ) |
---|
877 | def key(self,x): |
---|
878 | return x.__dict__[self.k] |
---|
879 | |
---|
880 | class kscl(object): |
---|
881 | """Comparison object to assist sorting of dictionaries of class instances""" |
---|
882 | def __init__(self,idict,k): |
---|
883 | self.k = k |
---|
884 | self.idict = idict |
---|
885 | def cmp(self,x,y): |
---|
886 | return cmp( self.idict[x].__dict__[self.k], self.idict[y].__dict__[self.k] ) |
---|
887 | def key(self,x): |
---|
888 | return self.idict[x].__dict__[self.k] |
---|
889 | |
---|
890 | src1 = '../workbook/trial_20150831.xml' |
---|
891 | |
---|
892 | #DEFAULT LOCATION -- changed automatically when building distribution |
---|
893 | defaultDreq = 'annotated_20150731.xml' |
---|
894 | #DEFAULT CONFIG |
---|
895 | defaultConfig = 'dreq2Defn.xml' |
---|
896 | |
---|
897 | defaultDreqPath = '%s/%s' % (DOC_DIR, defaultDreq ) |
---|
898 | defaultConfigPath = '%s/%s' % (DOC_DIR, defaultConfig ) |
---|
899 | defaultManifestPath = '%s/dreqManifest.txt' % DOC_DIR |
---|
900 | |
---|
901 | class loadDreq(object): |
---|
902 | """Load in a vocabulary document. |
---|
903 | dreqXML: full path to the XML document |
---|
904 | configdoc: full path to associated configuration document |
---|
905 | useShelve: flag to specify whether to retrieve data from cache (not implemented) |
---|
906 | htmlStyles: dictionary of styling directives which influence structure of html page generates by the "makeHtml" method |
---|
907 | """ |
---|
908 | |
---|
909 | def __init__(self,dreqXML=None, configdoc=None, useShelve=False, htmlStyles=None, strings=False, manifest=defaultManifestPath , configOnly=False): |
---|
910 | if manifest == None: |
---|
911 | if dreqXML == None: |
---|
912 | dreqXML=defaultDreqPath |
---|
913 | if configdoc==None: |
---|
914 | configdoc=defaultConfigPath |
---|
915 | self.c = config( thisdoc=dreqXML, configdoc=configdoc, useShelve=useShelve,strings=strings,manifest=manifest,configOnly=configOnly) |
---|
916 | self.coll = self.c.coll |
---|
917 | self.version = self.c.version |
---|
918 | self.softwareVersion = version |
---|
919 | self.indexSortBy = {} |
---|
920 | if not configOnly: |
---|
921 | self.inx = index(self.coll) |
---|
922 | self.itemStyles = {} |
---|
923 | self.defaultItemLineStyle = lambda i, frm='', ann='': '<li>%s: %s</li>' % ( i.label, i.__href__(odir='../u/') ) |
---|
924 | ## |
---|
925 | ## add index to Item base class .. so that it can be accessed by item instances |
---|
926 | ## |
---|
927 | dreqItemBase._inx = self.inx |
---|
928 | dreqItemBase._indexInitialised = True |
---|
929 | ## |
---|
930 | ## load in additional styling directives |
---|
931 | ## |
---|
932 | if htmlStyles != None: |
---|
933 | for k in htmlStyles: |
---|
934 | dreqItemBase._htmlStyle[k] = htmlStyles[k] |
---|
935 | |
---|
936 | ## dreqItemBase._htmlStyle['__general__'] = {'addRemarks':True} |
---|
937 | |
---|
938 | self.pageTmpl = """<html><head><title>%s</title> |
---|
939 | %s |
---|
940 | <link rel="stylesheet" type="text/css" href="%scss/dreq.css"> |
---|
941 | </head><body> |
---|
942 | |
---|
943 | <div id="top"> |
---|
944 | <div id="corner"></div> |
---|
945 | CMIP6 Data Request |
---|
946 | </div> |
---|
947 | <div id="nav"><div><a href="%s" title="Home">Home</a></div></div> |
---|
948 | |
---|
949 | <div id="section"> |
---|
950 | %s |
---|
951 | </div> |
---|
952 | </body></html>""" |
---|
953 | |
---|
954 | def getHtmlItemStyle(self, sect): |
---|
955 | """Get the styling method associated with a given section.""" |
---|
956 | if sect in self.itemStyles: |
---|
957 | return self.itemStyles[sect] |
---|
958 | return self.defaultItemLineStyle |
---|
959 | |
---|
960 | def updateByUid( self, uid, dd, delete=[] ): |
---|
961 | typePar={'xs:string':'x', 'xs:integer':0, 'xs:float':1., 'xs:boolean':True, 'xs:duration':'x'} |
---|
962 | listTypePar={ "aa:st__integerList":1,"aa:st__integerListMonInc":1, "aa:st__stringList":'x', "aa:st__floatList":1. } |
---|
963 | |
---|
964 | mmm = self.c.getByUid( uid ) |
---|
965 | assert len(mmm) == 1, 'Expected single uid match, found: %s' % len(mmm) |
---|
966 | thisdoc = mmm[0] |
---|
967 | item = self.inx.uid[uid] |
---|
968 | d1 = [d for d in delete if d not in item._a] |
---|
969 | e1 = len(d1) > 0 |
---|
970 | if e1: |
---|
971 | print ('ERROR.update.0001: request to delete non-existent keys: %s' % d1 ) |
---|
972 | d1 = [d for d in delete if d in item._a and item._a[d].required ] |
---|
973 | e1b = len(d1) > 0 |
---|
974 | if e1b: |
---|
975 | print ('ERROR.update.0002: request to delete required attributes: %s' % d1 ) |
---|
976 | d2 = [d for d in dd if d not in item._a] |
---|
977 | e2 = len(d2) > 0 |
---|
978 | if e2: |
---|
979 | print ('ERROR.update.0003: request to modify non-existent keys: %s' % d2 ) |
---|
980 | e3 = [] |
---|
981 | for k in [d for d in dd if d in item._a]: |
---|
982 | if item._a[k].type in typePar: |
---|
983 | e3.append( type( dd[k] ) != type( typePar[item._a[k].type] ) ) |
---|
984 | if e3[-1]: |
---|
985 | print ('ERROR.update.0004: value has wrong type [%s] %s --- %s' % (item._a[k].type, dd[k], type( dd[k] ) ) ) |
---|
986 | elif item._a[k].type in listTypePar: |
---|
987 | a = type( dd[k] ) not in [type( [] ), type( () )] |
---|
988 | if a: |
---|
989 | print ('ERROR.update.0005: value has wrong type [%s] %s --- should be list or tuple' % (item._a[k].type, dd[k] ) ) |
---|
990 | else: |
---|
991 | a = not all( [type(x) == type(listTypePar[item._a[k].type]) for x in dd[k]] ) |
---|
992 | if a: |
---|
993 | print ('ERROR.update.0005: value has wrong type [%s] %s --- %s' % (item._a[k].type, dd[k], [type(x)for x in dd[k]] ) ) |
---|
994 | |
---|
995 | if not a and item._a[k].type == 'aa:st__integerListMonInc': |
---|
996 | a = not all( [dd[k][i+1] > dd[k][i] for i in range( len(dd[k]) -1 )] ) |
---|
997 | if a: |
---|
998 | print ('ERROR.update.0006: value should be a monotonic increasing integer list: %s' % str( dd[k] ) ) |
---|
999 | e3.append(a) |
---|
1000 | else: |
---|
1001 | print ( 'Type not recognised: %s' % item._a[k].type ) |
---|
1002 | e3.append( False ) |
---|
1003 | eee = any([e1,e1b,e2,any(e3)]) |
---|
1004 | assert not eee, 'STOPPING: validation errors' |
---|
1005 | self.thisdoc = thisdoc |
---|
1006 | for k in dd: |
---|
1007 | thisdoc.set( k, self.__string4xml__( dd[k], item._a[k].type ) ) |
---|
1008 | item.__dict__[k] = dd[k] |
---|
1009 | |
---|
1010 | def saveXml( self, docfn=None, targ=None ): |
---|
1011 | if docfn == None: |
---|
1012 | docfn, tt = self.c.docs.items()[0] |
---|
1013 | else: |
---|
1014 | tt = self.c.docs[docfn] |
---|
1015 | |
---|
1016 | if targ == None: |
---|
1017 | targ = docfn |
---|
1018 | tt[1].write( targ ) |
---|
1019 | print ('XML document saved to %s' % targ) |
---|
1020 | |
---|
1021 | def __string4xml__(self,v,typ): |
---|
1022 | if typ in ['xs:string','xs:duration']: |
---|
1023 | return v |
---|
1024 | elif typ in ['xs:integer', 'xs:float', 'xs:boolean']: |
---|
1025 | return str(v) |
---|
1026 | elif typ == "aa:st__stringList": |
---|
1027 | return ' '.join(v) |
---|
1028 | elif typ in ["aa:st__integerList","aa:st__integerListMonInc", "aa:st__floatList"]: |
---|
1029 | return ' '.join( [str(x) for x in v] ) |
---|
1030 | else: |
---|
1031 | assert False, 'Data type not recognised' |
---|
1032 | |
---|
1033 | def _sectionSortHelper(self,title): |
---|
1034 | |
---|
1035 | ab = title.split( )[0].split('.') |
---|
1036 | if len( ab ) == 2: |
---|
1037 | a,b = ab |
---|
1038 | |
---|
1039 | if self.c.rc.isIntStr(a): |
---|
1040 | a = '%3.3i' % int(a) |
---|
1041 | if self.c.rc.isIntStr(b): |
---|
1042 | b = '%3.3i' % int(b) |
---|
1043 | rv = (a,b) |
---|
1044 | elif len(ab) == 1: |
---|
1045 | rv = (ab[0],0) |
---|
1046 | else: |
---|
1047 | rv = ab |
---|
1048 | return rv |
---|
1049 | |
---|
1050 | def makeHtml(self,odir='./html', ttl0 = 'Data Request Index', annotations=None): |
---|
1051 | """Generate a html view of the vocabularies, using the "__html__" method of the vocabulary item class to generate a |
---|
1052 | page for each item and also generating index pages. |
---|
1053 | odir: directory for html files; |
---|
1054 | ttl0: Title for main index (in odir/index.html)""" |
---|
1055 | markup = utilities.markupHtml( '' ) |
---|
1056 | |
---|
1057 | ks = self.inx.uid.keys() |
---|
1058 | ##ks.sort( kscl( self.inx.uid, 'title' ).cmp ) |
---|
1059 | ks = sorted( ks, key=kscl( self.inx.uid, 'title' ).key ) |
---|
1060 | for k in ks: |
---|
1061 | i = self.inx.uid[k] |
---|
1062 | ttl = 'Data Request Record: [%s]%s' % (i._h.label,i.label) |
---|
1063 | bdy = '\n'.join( i.__html__( ghis=self.getHtmlItemStyle ) ) |
---|
1064 | oo = open( '%s/u/%s.html' % (odir,i.uid), 'w' ) |
---|
1065 | oo.write( self.pageTmpl % (ttl, jsh, '../', '../index.html', bdy ) ) |
---|
1066 | oo.close() |
---|
1067 | |
---|
1068 | subttl1 = 'Overview tables and search' |
---|
1069 | subttl2 = 'Sections of the data request' |
---|
1070 | msg0 = ['<h1>%s</h1>' % ttl0, '<h2>%s</h2>' % subttl1, '<ul>',] |
---|
1071 | msg0.append( '<li><a href="tab01_3_3.html">Overview: all variables and experiments</a></li>' ) |
---|
1072 | msg0.append( '<li><a href="tab01_1_1.html">Overview: priority 1 variables, tier 1 experiments</a></li>' ) |
---|
1073 | msg0.append( '<li><a href="tab01_1_1_dn.html">Overview: priority 1 variables, tier 1 experiments (grid default to native)</a></li>' ) |
---|
1074 | msg0.append( '<li><a href="mipVars.html">Search for variables</a></li>' ) |
---|
1075 | msg0 += ['</ul>', '<h2>%s</h2>' % subttl2, '<ul>',] |
---|
1076 | ks = sorted( self.coll.keys() ) |
---|
1077 | ee = {} |
---|
1078 | for k in ks: |
---|
1079 | ee[self.coll[k].header.title] = k |
---|
1080 | kks = sorted( ee.keys(), key = self._sectionSortHelper ) |
---|
1081 | for kt in kks: |
---|
1082 | k = ee[kt] |
---|
1083 | ## |
---|
1084 | ## sort on item label |
---|
1085 | ## |
---|
1086 | if annotations != None and k in annotations: |
---|
1087 | ann = annotations[k] |
---|
1088 | else: |
---|
1089 | ann = {} |
---|
1090 | |
---|
1091 | sortkey = self.indexSortBy.get( k,'title') |
---|
1092 | print 'INFO.dreq.0001: %s, %s' % (k,sortkey) |
---|
1093 | ##self.coll[k].items.sort( ds(self.indexSortBy.get( k,'title') ).cmp ) |
---|
1094 | items = sorted( self.coll[k].items, key=ds(sortkey).key ) |
---|
1095 | ttl = 'Data Request Section: %s' % k |
---|
1096 | msg0.append( '<li><a href="index/%s.html">%s [%s]</a></li>\n' % (k,self.coll[k].header.title,k) ) |
---|
1097 | msg = ['<h1>%s</h1>\n' % ttl, '<ul>',] |
---|
1098 | msg.append( '<a href="../index.html">Home</a><br/>\n' ) |
---|
1099 | msg.append( '<p>%s</p>\n' % markup.parse( self.coll[k].header.description ) ) |
---|
1100 | lst = self.getHtmlItemStyle(k) |
---|
1101 | |
---|
1102 | msg1 = [] |
---|
1103 | for i in items: |
---|
1104 | ##m = '<li>%s: %s</li>' % ( i.label, i.__href__(odir='../u/') ) |
---|
1105 | |
---|
1106 | m = lst( i, ann=ann.get( i.label ) ) |
---|
1107 | msg1.append( m ) |
---|
1108 | if k in self.indexSortBy: |
---|
1109 | msg += msg1 |
---|
1110 | else: |
---|
1111 | msg += sorted( msg1 ) |
---|
1112 | msg.append( '</ul>' ) |
---|
1113 | bdy = '\n'.join( msg ) |
---|
1114 | oo = open( '%s/index/%s.html' % (odir,k), 'w' ) |
---|
1115 | oo.write( self.pageTmpl % (ttl, '', '../', '../index.html', bdy ) ) |
---|
1116 | oo.close() |
---|
1117 | msg0.append( '</ul>' ) |
---|
1118 | bdy = '\n'.join( msg0 ) |
---|
1119 | oo = open( '%s/index.html' % odir, 'w' ) |
---|
1120 | oo.write( self.pageTmpl % (ttl0, '', '', 'index.html', bdy ) ) |
---|
1121 | oo.close() |
---|
1122 | |
---|
1123 | if __name__ == '__main__': |
---|
1124 | dreq = loadDreq(manifest='out/dreqManifest.txt' ) |
---|
1125 | |
---|