Changeset 361
- Timestamp:
- 30/07/15 10:37:13 (5 years ago)
- Location:
- CMIP6dreqbuild/trunk
- Files:
-
- 1 added
- 7 edited
- 1 moved
Legend:
- Unmodified
- Added
- Removed
-
CMIP6dreqbuild/trunk/docs/dreq.txt
r356 r361 1 1 ## 2 2 vocab var; MIP Variable; cmip.drv.001; def 3 - sn; units; description; procnote; procComment; prov; uuid 3 - sn {CF Standard Name} <externalTextRef>; units {Units}; description; 4 - procnote {Processing notes} <list> 5 - procComment {Processing comments} <freeText> 6 - prov {Provenance} <freeText> 7 - uuid <recordId> 4 8 ## label==var 5 9 vocab ovar; MIP Output Variable; cmip.drv.001; def 6 - uuid; comment; deflate_level; shuffle; ok_max_mean_abs; flag_meanings; type; ok_min_mean_abs; sn; deflate; valid_min; cell_methods; flag_values; cell_measures; out_name; modeling_realm; units; cell_methods_xx; valid_max; positive; mipTable; dimensions; vid; frequency 7 ## label==var, title==long name 10 - uuid <recordId> 11 - title {Long name} 12 - comment {Description} <freeText> 13 - deflate_level; shuffle; ok_max_mean_abs; flag_meanings; type; ok_min_mean_abs; 14 - sn {CF Standard Name} <externalTextRef> 15 - deflate; valid_min; cell_methods; flag_values; cell_measures; out_name; modeling_realm; units; cell_methods_xx; valid_max; positive; mipTable; dimensions; 16 - vid {Identifier for MIP Variable} <internalLink|var> 17 - frequency {Frequency} <drsVocab> 18 ## 8 19 vocab groupItem; Output Group Item; cmip.drv.001; def 9 - group; table; freq; descriptionEx; shape; levels; tstyle; mask; misc; mip; uuid; new; gpid; vkey; vid 10 ## label==var, title==long name 20 - group; table; freq; descriptionEx; shape; levels; tstyle; mask; misc; mip; uuid <recordId>; new; 21 - gpid {Identifier for request link} <internalLink>; vkey; 22 - vid {Identifier for MIP Variable} <internalLink|var> 23 ## 11 24 vocab revisedTabItem; Revised Table Item; cmip.drv.001; def 12 - table; mip; uuid; priority; vid 13 ## label==tab, title??? 25 - table 26 - mip {Endorsed MIP requesting the variable} 27 - uuid <recordId>; priority; 28 - vid {Identifier for MIP Output Variable} <internalLink|ovar> 29 - vgid {Identifier for Variable Group} <internalLink|requestVarGroup> 30 ## 14 31 vocab requestVarGroup; Request link: linking a set of variables; cmip.drv.001; def 15 - uuid; mip; ref; refNote 32 - uuid <recordId> 33 - mip {Endorsed MIP requesting the data} 34 - ref; refNote 16 35 vocab requestLink; Request link: linking a set of variables and a set of experiments.; cmip.drv.001; def 17 - uuid; mip; tab; objective; grid; gridreq; comment; ref; refNote; refid 18 vocab requestItem; Request Item: specifying the nuber of years for an experiment; cmip.drv.001; def 19 - uuid; mip; tab; expt; rlid; ny 36 - uuid <recordId>; mip; tab; objective; grid; gridreq; comment; ref; refNote; 37 - refid {reference to a request Variable Group} <internalLink> 38 vocab requestItem; Request Item: specifying the number of years for an experiment; cmip.drv.001; def 39 - uuid <recordId>; mip; tab; expt; 40 - rlid {Identifier of corresponding requestLink} <internalLink> 41 - ny [xs:integer] 42 vocab remarks; Remarks about other items; cmip.drv.001; def 43 - uuid <recordId>; 44 - tid {Target identifier: the record ID of the item this refers to.} <internalLink> 45 - tattr {Target attribute: an attribute of the target item, or 'ALL'} 46 - description {Free text remarks -- unless there are specific restrictions associated with the class attribute of this remark} 47 - class {Different classes of remark support a range of specialised usages: free (a free text comment), modelTypeReq (a model type requirement), modelTypeExcl (a model type exclusion), varAlt (an alternative variable that my be used instead of this one), varSup (an alternative variable which, if selected makes this one redundant), attChange (a change in the value of an attribute -- old value provided in 'techNote').} 48 - qid {Identifier required for classes modelTypeReq, modelTypeExcl, varAlt, varSup. For the first two classes, the identifier points to a model type specification, for the last two it points to an output variable specification.} 49 - techNote {Optional additional machine readable content (though not restricted by the schema)} -
CMIP6dreqbuild/trunk/docs/vocabFrameworkSchema.xsd
r322 r361 15 15 <xs:attribute name="title" type="xs:string"/> 16 16 <xs:attribute name="type" type="xs:string"/> 17 <xs:attribute name="class" type="xs:string"/> 18 <xs:attribute name="techNote" type="xs:string"/> 17 19 </xs:complexType> 18 20 -
CMIP6dreqbuild/trunk/docs/xlst_xdsSchemaGen.xml
r353 r361 73 73 <xsl:apply-templates mode="main"/> 74 74 </xs:all> 75 </xs:complexType> 75 </xs:complexType><xsl:text>
</xsl:text> 76 <xs:unique name="uniqueIdentifierConstraint"> 77 <xs:annotation> 78 <xs:appinfo>W3Schools Note</xs:appinfo> 79 <xs:documentation xml:lang="en"> 80 The uniqueIdentifierConstraint imposes the condition that the uuid attribute on "item" elements should have a unique value. 81 </xs:documentation> 82 </xs:annotation><xsl:text>
</xsl:text> 83 <xs:selector xpath=".//item"/> 84 <xs:field xpath="@uuid"/> 85 </xs:unique><xsl:text>
</xsl:text> 76 86 </xs:element><xsl:text>
</xsl:text> 77 87 -
CMIP6dreqbuild/trunk/src/framework/dreq.py
r353 r361 15 15 self.mdInit( xmlMiniDom ) 16 16 17 18 17 def dictInit( self, dict ): 19 18 for a in self.attributes: … … 31 30 else: 32 31 self.__dict__[a] = self.defaults.get( a, self.globalDefault ) 33 34 32 35 33 class config(object): 36 34 37 35 def __init__(self): 38 self.vdef = 'out/ parVocabDefn.xml'36 self.vdef = 'out/dreqDefn.xml' 39 37 self.vsamp = 'out/vocabSample.xml' 40 38 self.vsamp = '../workbook/trial.xml' 41 39 self.vsamp = '../workbook/trial_20150724.xml' 42 40 self.nts = collections.namedtuple( 'sectdef', ['tag','label','title','id','itemLabelMode'] ) 43 self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type' ] )41 self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type','rClass','techNote'] ) 44 42 self.ntt = collections.namedtuple( 'sect', ['header','attributes'] ) 45 43 … … 127 125 else: 128 126 ty = "xs:string" 129 return self.nti( i.nodeName, l,t,ty ) 127 if i.hasAttribute( 'class' ): 128 cls = i.getAttribute( 'class' ) 129 else: 130 cls = None 131 if i.hasAttribute( 'techNote' ): 132 tn = i.getAttribute( 'techNote' ) 133 else: 134 tn = None 135 return self.nti( i.nodeName, l,t,ty,cls,tn ) 130 136 131 137 c = config() -
CMIP6dreqbuild/trunk/src/framework/ptxt.py
r360 r361 4 4 ptxt.py [-f srcFile] [samp|defn] 5 5 ''' 6 import string, re, collections, sys 6 import string, re, collections, sys, uuid 7 7 8 8 if len(sys.argv) == 1: … … 27 27 re_atdef = re.compile( '^(?P<n>\S+)\s*((\[(?P<b>.*?)\]|\{(?P<a>.*?)\}|\<(?P<c>.*?)\>)\s*){0,3}$' ) 28 28 29 nt__itematt = collections.namedtuple( 'itematt', ['name','type','title','clss'] ) 29 ### tuble to define attributes of an item 30 nt__itematt = collections.namedtuple( 'itematt', ['name','type','title','clss','techn'] ) 30 31 31 32 vocab_elTmpl = '''<table label="%(label)s" title="%(title)s" id="%(id)s" itemLabelMode="%(ilm)s"> … … 33 34 </table> 34 35 ''' 35 ial_elTmpl = ' <rowAttribute label="%(label)s"%(wrappedType)s%(wrappedTitle)s%(wrappedClass)s />'36 ial_elTmpl = ' <rowAttribute label="%(label)s"%(wrappedType)s%(wrappedTitle)s%(wrappedClass)s%(wrappedTechn)s/>' 36 37 37 38 expl_Tmpl = '''<%(label)s class="vocab" title="%(title)s" id="%(id)s"> … … 54 55 self.kk = kk 55 56 self.msg( '[%s] %s {%s:%s}' % (self.label, self.title, self.id, self.ilt) ) 56 self.itematts = [ nt__itematt( 'label','xs:string',None,None ),57 nt__itematt( 'title','xs:string',None,None ) ]57 self.itematts = [ nt__itematt( 'label','xs:string',None,None,None ), 58 nt__itematt( 'title','xs:string',None,None,None ) ] 58 59 59 60 def tmpl(self,oo=None,mode="defn"): … … 61 62 ss = [] 62 63 for i in self.itematts: 63 print i64 64 if i.clss != None: 65 65 wrappedClass = ' class="%s"' % i.clss 66 66 else: 67 67 wrappedClass = '' 68 69 if i.techn != None: 70 wrappedTechn = ' techNote="%s"' % i.techn 71 else: 72 wrappedTechn = '' 68 73 69 74 if i.title != None: … … 102 107 value = "noType" 103 108 if i.type == "xs:string": 104 value = 'dummyAt' 109 if i.name == 'uuid': 110 value = str( uuid.uuid1() ) 111 else: 112 value = 'dummyAt' 105 113 elif i.type == "xs:integer": 106 114 value = '25' … … 132 140 bits = map( string.strip, string.split(line[3:], ';' ) ) 133 141 for b in bits: 134 bn = string.split(b, ' ')[0] 135 x = self.pb(b) 136 self.itematts.append( nt__itematt( bn, x[0], x[1], x[2] ) ) 137 self.msg( '%s, %s, %s, %s' % (bn, x[0], x[1], x[2]) ) 142 if string.strip(b) != '': 143 bn = string.split(b, ' ')[0] 144 x = self.pb(b) 145 self.itematts.append( nt__itematt( bn, x[0], x[1], x[2], x[3] ) ) 146 self.msg( '%s, %s, %s, %s, %s' % (bn, x[0], x[1], x[2], x[3]) ) 138 147 139 148 def pb(self, b): … … 151 160 itemType = ee['b'] 152 161 itemClass = ee['c'] 162 itemTechNote = None 153 163 if itemType == None: 154 164 itemType = 'xs:string' 165 if type(itemClass) == type('x'): 166 if string.find(itemClass, '|' ) != -1: 167 bits = string.split( itemClass, '|' ) 168 assert len(bits) == 2, 'ERROR.001.0001: failed to parse attribute definition: %s' % b 169 itemClass = bits[0] 170 itemTechNote = bits[1] 155 171 156 172 #if len(x) > 0: … … 163 179 #else: 164 180 #itemType = 'xs:string' 165 return [itemType, itemTitle,itemClass ]181 return [itemType, itemTitle,itemClass,itemTechNote] 166 182 167 183 class main(object): -
CMIP6dreqbuild/trunk/src/workbook/importShDreq.py
r356 r361 1 1 """Import data request from shelves and put onto compliant XML document. 2 2 ------------------------------------------------------------------------ 3 A new uuid is generat d for elements of the "revisedtabitem" section/table (because it is not present in the shelve).3 A new uuid is generated for elements of the "revisedtabitem" section/table (because it is not present in the shelve). 4 4 5 5 NOTES … … 34 34 self.src=src 35 35 self.repl = collections.defaultdict( list ) 36 self.err0010 = collections.defaultdict( int ) 36 37 self.replItems = {} 37 38 self.importRepl() … … 64 65 print xsn 65 66 print ff.keys() 66 67 68 self.eern = collections.defaultdict( list ) 69 self.eern2 = {} 70 ixrn = 5 71 dups = [] 72 for k in rq.rqvg.keys(): 73 if k[0] != '_': 74 kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'ref', 'refNote', 'refid'] 75 kk = ['uuid', 'mip', 'label', 'title', 'ref', 'refNote'] 76 i = rq.rqvg[k] 77 78 self.eern[i[ixrn]].append( i[0] ) 79 k2 = '%s__%s' % (i[ixrn-1],i[ixrn]) 80 if self.eern2.has_key( k2 ): 81 dups.append( k2 ) 82 self.eern2[k2] = i[0] 83 assert i[0] == k, 'Bad key/uuid' 84 85 assert len( dups ) == 0, 'Duplicate refs: %s' % str(dups) 86 67 87 for k in [ 'var','ovar','groupitem', 'revisedtabitem', 'requestlink', 'requestitem','requestvargroup' ]: 68 88 ##for k in [ 'var']: … … 165 185 ## - table; mip; uuid; priority 166 186 kk = ['var', 'table', 'mip', 'vid', 'priority'] 187 kvg = 'rev__%s.%s' % ( ll[2],ll[1] ) 188 if self.eern2.has_key(kvg): 189 vgid = self.eern2[kvg] 190 else: 191 vgid = '__none__' 192 self.err0010[kvg] += 1 167 193 thisuuid = str( uuid.uuid1() ) 168 194 thisl = string.strip(str( ll[0] ) ) … … 172 198 item.setAttribute( 'title', thisl ) 173 199 item.setAttribute( 'uuid', thisuuid ) 200 item.setAttribute( 'vgid', vgid ) 174 201 if self.insert.has_key(thisl): 175 202 thisold = item.getAttribute( 'vid' ) … … 193 220 thisl = string.strip(str( '%s-%s-%s' % (ll[0],ll[1],ll[2]) ) ) 194 221 for j in range(len(kk)): 195 item.setAttribute( kk[j], str( ll[j] ) ) 222 if kk[j] == 'ny': 223 thisv = str( int(ll[j]) ) 224 else: 225 thisv = str( ll[j] ) 226 item.setAttribute( kk[j], thisv ) 196 227 item.setAttribute( 'title', '%s, %s, %s' % (ll[0],ll[1],ll[2]) ) 197 228 item.setAttribute( 'uuid', str(i) ) … … 199 230 elif k == 'requestvargroup': 200 231 kk = ['uuid', 'mip', 'label', 'title', 'ref', 'refNote'] 232 thisl = string.replace( str( ll[2] ), '.', '-' ) 201 233 for j in range(len(kk)): 202 234 item.setAttribute( kk[j], str( ll[j] ) ) … … 270 302 self.insert[bits[1]] = bits[3] 271 303 272 sampleXml = '../framework/out/ parVocabSample.xml'304 sampleXml = '../framework/out/dreqSample.xml' 273 305 from scansh import rq 274 306 m = main(sampleXml, rq) -
CMIP6dreqbuild/trunk/srcMisc/scansh.py
r356 r361 74 74 kmm = 0 75 75 knf = 0 76 nn0 = 0 77 nn1 = 0 76 78 for k in rq.grps.keys(): 77 79 if k[0] != '_': … … 79 81 if r[-3][0] != '_': 80 82 if r[-4] == 0: 83 nn0 += 1 81 84 if rq.vars.has_key(r[-3]): 82 85 nok1 += 1 … … 85 88 knf += 1 86 89 elif r[-4] == 1: 90 nn1 += 1 87 91 if rq.rqvg.has_key(r[-3]): 88 92 nok2 += 1 … … 95 99 ##print '!GROUP: ',r 96 100 km += 1 97 print 'CHECK.001.0004: on groupItems: %s,%s OK, %s missing reference, not found %s, bad "new" key %s ' % (nok1,nok2,km,knf,kmm)101 print 'CHECK.001.0004: on groupItems: %s,%s OK, %s missing reference, not found %s, bad "new" key %s [%s,%s] ' % (nok1,nok2,km,knf,kmm,nn0,nn1) 98 102 99 103 nok1 = 0 -
CMIP6dreqbuild/trunk/srcMisc/sx2.py
r355 r361 419 419 for k in shrvg.keys(): 420 420 if k[0] != '_': 421 assert not eervg.has_key( shrvg[k][2] ), 'Duplicate rvg: %s' % shrvg[k][2] 421 422 eervg[shrvg[k][2]] = k 422 423 … … 469 470 for t in mmmm: 470 471 kmap000[t[0]] = t[1] 472 473 ## 471 474 ## check links from variable output specs to group. 472 475 ## need to identify what is meant by group labels in "Request Scoping" -- now in eervg.keys() 473 476 ## 477 ## 29th July: loosin some links for groups of revised table items .... 478 ## 479 shb = shelve.open( 'dreq_consol_tables_reviewed_b_v20150708', 'r' ) 480 revisedTabKeys = shb.keys() 481 shb.close() 482 gpids = gpids + revisedTabKeys 474 483 ks = eervg.keys() 475 484 ks.sort() … … 526 535 ## 527 536 rvgref2 = collections.defaultdict( list ) 537 lookup1a = collections.defaultdict( d1 ) 538 lookup1b = {} 528 539 lookup1 = collections.defaultdict( d1 ) 529 540 lookup2 = collections.defaultdict( d1 ) … … 532 543 sh['__info__'] = { 'label':'requestVarGroup', 'title':'Identify variable groups' } 533 544 sh['__cols__'] = [ 'uuid', 'mip', 'label','title','ref','refNote'] 545 ## C4MIP-OceanT2 C4MIP: C4MIP-OceanT2 C4MIP new C4MIP.C_Ocean_T2 546 ## label title mip ref refNote 547 534 548 #### 535 549 #### reconfigure to uuid, mip, label (no "."), title, class, refNote … … 541 555 id0 = shrvg[k][2] 542 556 ttt,kkk = kmapk2[id0] 543 lookup1 [ttt].d[kkk].append(k)557 lookup1a[ttt].d[kkk].append(k) 544 558 if ttt == 'new': 545 559 rvgref2[kkk].append( shrvg[k][0] ) 560 561 for k in lookup1a.keys(): 562 for k2 in lookup1a[k].d.keys(): 563 id0 = lookup1a[k].d[k2][0] 564 id = str( uuid.uuid1() ) 565 lookup1[k].d[k2].append( id) 566 lookup1b[id0] = id 567 568 ndup = 0 569 for k in shlnks.keys(): 570 if k[0] != '_': 571 k1 = '%s.%s' % (shlnks[k][1],shlnks[k][2]) 572 assert kmapk2.has_key(k1), 'Key missing %s, %s' % (k,k1) 573 if not kmapk2.has_key(k1): 574 print '###########',shlnks[k] 575 else: 576 ttt,kkk = kmapk2[k1] 577 id = lookup1[ttt].d[kkk][0] 578 if id == k: 579 print '***DUPLICATE:',k,k1, ttt, kkk, lookup1[ttt].d[kkk] 580 ndup += 1 581 582 assert ndup==0, ndup 546 583 ##sh[k] = list( shrvg[k] ) + list( kmapk2[id0] ) 547 584 ##print '>>>>>',k,sh[k] … … 549 586 for k2 in lookup1[k].d.keys(): 550 587 id = lookup1[k].d[k2][0] 551 id0 = shrvg[id][2] 552 mip = shrvg[id][1] 588 idi = lookup1a[k].d[k2][0] 589 id0 = shrvg[idi][2] 590 mip = shrvg[idi][1] 553 591 ttt,kkk = kmapk2[id0] 554 592 cls = k … … 556 594 refNote = k2 557 595 if cls == 'new': 558 lab = sx2e.ngmap[kkk] 596 if kkk in revisedTabKeys: 597 lab = kkk 598 cls = 'rev' 599 else: 600 lab = sx2e.ngmap[kkk] 559 601 elif cls == 'CMIP5': 560 602 lab = string.replace( k2, '_', '-' ) … … 568 610 if sx2e.r1.match( lab ) == None: 569 611 print 'NON_COMPLIANT label', lab 570 print k,k2,shrvg[id]571 assert False612 ## print k,k2,shrvg[id] 613 ## assert False 572 614 sh[id] = [id,mip,lab,'%s: %s' % (mip,lab), cls, refNote] 615 print '>>>>>>>>>',sh[id] 573 616 574 617 … … 584 627 585 628 sh.close() 586 587 629 588 630 err001 = collections.defaultdict( int ) … … 609 651 r[-1] = '__none__' 610 652 t = ee0_xref[k][0] 611 print '>>>>>>>>>>>>>>>>>>>>',t612 653 r += [t[0],t[1][0]] 613 654 sh[k] = r[:] … … 618 659 sh['__info__'] = { 'label':'requestLinks', 'title':'Links from variable groups to a request id' } 619 660 sh['__cols__'] = [ 'uid', 'mip', 'tab','objective','grid','gridreq','comment','ref','refNote','refid'] 661 ndup = 0 620 662 for k in shlnks.keys(): 621 663 if k[0] != '_': … … 628 670 id = lookup1[ttt].d[kkk][0] 629 671 sh[k] = list( shlnks[k] ) + list( kmapk2[k1] ) + [id,] 672 if sh[k][0] == sh[k][-1]: 673 print 'DUPLICATE LINK:',k,sh[k] 674 ndup += 1 675 print 'Numbr of duplicates %s (%s)' % (ndup, len(sh.keys())) 630 676 sh.close() 631 677 ## link to group items is now via mip + group id, using stuff from kmapk2 ....
Note: See TracChangeset
for help on using the changeset viewer.