Changeset 382


Ignore:
Timestamp:
10/09/15 11:10:54 (5 years ago)
Author:
mjuckes
Message:

misc

Location:
CMIP6dreqbuild/trunk
Files:
17 edited

Legend:

Unmodified
Added
Removed
  • CMIP6dreqbuild/trunk/docs/dreq.txt

    r370 r382  
    1616 - vid {Identifier for MIP Variable} <internalLink|var> 
    1717 - frequency {Frequency}  <drsVocab> 
     18 - gpid {Variable group/table} <internalLink> 
     19 - rowIndex {Row index of entry in source sheet} [xs:integer] 
    1820##  
     21vocab tableSection; CMOR Table Sections; cmip.drv.001; def; 1; 1; No 
     22 - uuid <recordId> 
     23 - gpid {Identifier for CMOR Tables} <internalLink|requestVarGroup> 
     24 - mip {Project} 
     25 - ref 
     26 - refNote 
     27###  'uuid', 'gpid', 'mip', 'label','title','ref','refNote' 
     28## 
    1929vocab groupItem; Output Group Item; cmip.drv.001; def; 2; 1; No 
    2030 - group;   
     
    2434 - shape {Shape of variable (dimensions)} 
    2535 - levels {Number of vertical levels} 
    26  - tstyle; mask; misc; mip; uuid <recordId>; new;  
     36 - tstyle; mask 
     37 - priority [xs:integer] 
     38 - mip; uuid <recordId>; new;  
    2739 - gpid {Identifier for request link} <internalLink>; vkey; 
    2840 - vid {Identifier for MIP Variable} <internalLink|var> 
     
    3143 - table 
    3244 - mip {Endorsed MIP requesting the variable} 
    33  - uuid <recordId>; priority;  
     45 - uuid <recordId> 
     46 - priority  [xs:integer] 
    3447 - vid {Identifier for MIP Output Variable} <internalLink|ovar> 
    3548 - vgid {Identifier for Variable Group} <internalLink|requestVarGroup> 
     
    4255 - uuid <recordId>; mip; tab; objective; grid; gridreq; comment; ref; refNote;  
    4356 - refid {reference to a request Variable Group} <internalLink> 
     57 - opt {option for selecting a subset of variables} 
     58 - opar {parameter associated with *opt*} 
    4459vocab requestItem; Request Item: specifying the number of years for an experiment; cmip.drv.001; def; 5; 1; No 
    4560 - uuid <recordId>; mip; tab; expt;  
  • CMIP6dreqbuild/trunk/docs/xlst_xdsSchemaGen.xml

    r370 r382  
    145145        </xs:attribute><xsl:text>&#xa;</xsl:text> 
    146146        <xs:attribute name="class" fixed="vocab"/><xsl:text>&#xa;</xsl:text> 
     147        <xs:attribute name="uuid" type="xs:string"/><xsl:text>&#xa;</xsl:text> 
    147148    </xs:complexType><xsl:text>&#xa;</xsl:text> 
    148149 
  • CMIP6dreqbuild/trunk/src/framework/dreq.py

    r370 r382  
    164164    self.silent = True 
    165165    self.uuid = {} 
     166    self.uuid2 = collections.defaultdict( list ) 
    166167    nativeAtts = ['uuid','iref_by_uuid','iref_by_sect','missingIds'] 
    167168    naok = map( lambda x: not dreq.has_key(x), nativeAtts ) 
     
    186187    for k in dreq.keys(): 
    187188        for i in dreq[k].items: 
    188           self.uuid[i.uuid] = (k,i) 
     189          if self.uuid.has_key(i.uuid): 
     190            print 'ERROR.100.0001: Duplicate uuid: %s' % i.uuid 
     191            self.uuid2[i.uuid].append( (k,i) ) 
     192          else: 
     193            self.uuid[i.uuid] = (k,i) 
    189194 
    190195    self.missingIds = collections.defaultdict( list ) 
     
    218223 
    219224 
    220 src1 = '../workbook/trial_20150724.xml' 
     225src1 = '../workbook/trial_20150831.xml' 
    221226class loadDreq(object): 
    222227  def __init__(self,dreqXML='annotated_20150731.xml',configdoc='out/dreqDefn.xml' ): 
  • CMIP6dreqbuild/trunk/src/framework/out/vocabDefn.xml

    r370 r382  
    22<defDoc  
    33xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
     4xmlns="uri:w3id.org:cmip6.dreq.framework:a" 
    45xsi:schemaLocation="http://w3id.org/cmip6dr/ns vocabFrameworkSchema_v01beta.xsd"> 
    56<table label="institute" title="Institute" id="cmip.drv.001" itemLabelMode="def" level="0" maxOccurs="1" labUnique="Yes"> 
    67  <rowAttribute label="label"/> 
    78  <rowAttribute label="title"/> 
    8   <rowAttribute label="url"/> 
    9   <rowAttribute label="isni"/> 
     9  <rowAttribute label="url" title="Home page"/> 
     10  <rowAttribute label="isni" title="International Standard Name Identifier (ISO 27729)"/> 
    1011  <rowAttribute label="description"/> 
    1112  <rowAttribute label="uuid"/> 
  • CMIP6dreqbuild/trunk/src/framework/out/vocabSample.xml

    r370 r382  
    11<?xml version="1.0" ?> 
    2 <main> 
     2<main 
     3xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
     4xsi:noNamespaceSchemaLocation="out/dreqSchema.xsd" 
     5xmlns="uri:w3id.org:cmip6.dreq.dreq:a"> 
    36<institute class="vocab" title="Institute" id="cmip.drv.001"> 
    47<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    5 <item id="001.001.001" label="example-01" title="dummy title string" url="dummyAt" isni="dummyAt" description="dummyAt" uuid="215e0ce0-3a81-11e5-882e-ac72891c3257"/> 
     8<item id="001.001.001" label="example-01" title="dummy title string" url="dummyAt" isni="dummyAt" description="dummyAt" uuid="bc5512e4-5713-11e5-9f3f-ac72891c3257"/> 
    69</institute> 
    710<model class="vocab" title="Model" id="cmip.drv.002"> 
    811<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    9 <item id="001.002.001" label="example-01" title="dummy title string" description="dummyAt" url="dummyAt" altUrl="dummyAt" source="dummyAt" uuid="215e48ae-3a81-11e5-882e-ac72891c3257"/> 
     12<item id="001.002.001" label="example-01" title="dummy title string" description="dummyAt" url="dummyAt" altUrl="dummyAt" source="dummyAt" uuid="bc5554fc-5713-11e5-9f3f-ac72891c3257"/> 
    1013</model> 
    1114<physicsvers class="vocab" title="Physics Version" id="cmip.drv.003"> 
    1215<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    13 <item id="001.003.001" label="example-01" title="dummy title string" description="dummyAt" uuid="215e7e6e-3a81-11e5-882e-ac72891c3257"/> 
     16<item id="001.003.001" label="example-01" title="dummy title string" description="dummyAt" uuid="bc55925a-5713-11e5-9f3f-ac72891c3257"/> 
    1417</physicsvers> 
    1518<initialisationmethod class="vocab" title="Initialisation Method" id="cmip.drv.004"> 
    1619<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    17 <item id="001.004.001" label="example-01" title="dummy title string" description="dummyAt" uuid="215eb3c0-3a81-11e5-882e-ac72891c3257"/> 
     20<item id="001.004.001" label="example-01" title="dummy title string" description="dummyAt" uuid="bc55cf72-5713-11e5-9f3f-ac72891c3257"/> 
    1821</initialisationmethod> 
    1922<modelComp class="vocab" title="Model Components" id="cmip.drv.005"> 
    2023<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    21 <item id="001.005.001" label="example-01" title="dummy title string" componentType="dummyAt" sizeH="dummyAt" sizeV="dummyAt" uuid="215eec50-3a81-11e5-882e-ac72891c3257"/> 
     24<item id="001.005.001" label="example-01" title="dummy title string" componentType="dummyAt" sizeH="dummyAt" sizeV="dummyAt" uuid="bc561144-5713-11e5-9f3f-ac72891c3257"/> 
    2225</modelComp> 
    2326<activity class="vocab" title="Coordinating activity" id="cmip.drv.006"> 
    2427<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    25 <item id="001.006.001" label="example-01" title="dummy title string" url="dummyAt" status="dummyAt" uuid="215ef65a-3a81-11e5-882e-ac72891c3257"/> 
     28<item id="001.006.001" label="example-01" title="dummy title string" url="dummyAt" status="dummyAt" uuid="bc562030-5713-11e5-9f3f-ac72891c3257"/> 
    2629</activity> 
    2730<product class="vocab" title="Data product" id="cmip.drv.007"> 
    2831<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    29 <item id="001.007.001" label="example-01" title="dummy title string" uuid="215ef9ac-3a81-11e5-882e-ac72891c3257"/> 
     32<item id="001.007.001" label="example-01" title="dummy title string" uuid="bc56251c-5713-11e5-9f3f-ac72891c3257"/> 
    3033</product> 
    3134<experiment class="vocab" title="Numerical Experiment" id="cmip.drv.008"> 
    3235<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    33 <item id="001.008.001" label="example-01" title="dummy title string" altLabel="dummyAt" description="dummyAt" url="dummyAt" sponsor="dummyAt" uuid="215efe70-3a81-11e5-882e-ac72891c3257"/> 
     36<item id="001.008.001" label="example-01" title="dummy title string" altLabel="dummyAt" description="dummyAt" url="dummyAt" sponsor="dummyAt" uuid="bc562cb0-5713-11e5-9f3f-ac72891c3257"/> 
    3437</experiment> 
    3538<frequency class="vocab" title="Temporal frequency of data values" id="cmip.drv.009"> 
    3639<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    37 <item id="001.009.001" label="example-01" title="dummy title string" iso="dummyAt" uuid="215f01ea-3a81-11e5-882e-ac72891c3257"/> 
     40<item id="001.009.001" label="example-01" title="dummy title string" iso="dummyAt" uuid="bc563340-5713-11e5-9f3f-ac72891c3257"/> 
    3841</frequency> 
    3942<realm class="vocab" title="Modeling realm" id="cmip.drv.010"> 
    4043<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    41 <item id="001.010.001" label="example-01" title="dummy title string" uuid="215f04e2-3a81-11e5-882e-ac72891c3257"/> 
     44<item id="001.010.001" label="example-01" title="dummy title string" uuid="bc5637f0-5713-11e5-9f3f-ac72891c3257"/> 
    4245</realm> 
    4346<grid class="vocab" title="Native grid" id="cmip.drv.011"> 
    4447<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    45 <item id="001.011.001" label="example-01" title="dummy title string" uuid="215f07c6-3a81-11e5-882e-ac72891c3257"/> 
     48<item id="001.011.001" label="example-01" title="dummy title string" uuid="bc563ca0-5713-11e5-9f3f-ac72891c3257"/> 
    4649</grid> 
    4750<dimensions class="vocab" title="MIP table variable dimensions" id="cmip.drv.012"> 
    4851<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    49 <item id="001.012.001" label="example01" title="dummy title string" altLabel="dummyAt" sn="dummyAt" size="25" uuid="215f0bea-3a81-11e5-882e-ac72891c3257"/> 
     52<item id="001.012.001" label="example01" title="dummy title string" altLabel="dummyAt" sn="dummyAt" size="25" uuid="bc564394-5713-11e5-9f3f-ac72891c3257"/> 
    5053</dimensions> 
    5154<forcings class="vocab" title="Forcing" id="cmip.drv.013"> 
    5255<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    53 <item id="001.013.001" label="example-01" title="dummy title string" uuid="215f0ed8-3a81-11e5-882e-ac72891c3257"/> 
     56<item id="001.013.001" label="example-01" title="dummy title string" uuid="bc564844-5713-11e5-9f3f-ac72891c3257"/> 
    5457</forcings> 
    5558<table class="vocab" title="Data specification table" id="cmip.drv.014"> 
    5659<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    57 <item id="001.014.001" label="example-01" title="dummy title string" description="dummyAt" comment="dummyAt" frequency="dummyAt" uuid="215f12c0-3a81-11e5-882e-ac72891c3257"/> 
     60<item id="001.014.001" label="example-01" title="dummy title string" description="dummyAt" comment="dummyAt" frequency="dummyAt" uuid="bc564f06-5713-11e5-9f3f-ac72891c3257"/> 
    5861</table> 
    5962<cref class="vocab" title="Coordinate Reference" id="cmip.drv.015"> 
    6063<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    61 <item id="001.015.001" label="example-01" title="dummy title string" uuid="215f170c-3a81-11e5-882e-ac72891c3257"/> 
     64<item id="001.015.001" label="example-01" title="dummy title string" uuid="bc5655a0-5713-11e5-9f3f-ac72891c3257"/> 
    6265</cref> 
    6366<areaType class="vocab" title="CMIP Area Type" id="cmip.drv.016"> 
    6467<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    65 <item id="001.016.001" label="example-01" title="dummy title string" inCF="dummyAt" uuid="215f1a54-3a81-11e5-882e-ac72891c3257"/> 
     68<item id="001.016.001" label="example-01" title="dummy title string" inCF="dummyAt" uuid="bc565b04-5713-11e5-9f3f-ac72891c3257"/> 
    6669</areaType> 
    6770<timeSlice class="vocab" title="Time slices used in data request" id="cmip.drv.017"> 
    6871<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    69 <item id="001.017.001" label="example-01" title="dummy title string" type="dummyAt" specification="dummyAt" uuid="215f1de2-3a81-11e5-882e-ac72891c3257"/> 
     72<item id="001.017.001" label="example-01" title="dummy title string" type="dummyAt" specification="dummyAt" uuid="bc566112-5713-11e5-9f3f-ac72891c3257"/> 
    7073</timeSlice> 
    7174<modelFamily class="vocab" title="Family of model" id="cmip.drv.018"> 
    7275<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    73 <item id="001.018.001" label="example-01" title="dummy title string" uuid="215f20b2-3a81-11e5-882e-ac72891c3257"/> 
     76<item id="001.018.001" label="example-01" title="dummy title string" uuid="bc566702-5713-11e5-9f3f-ac72891c3257"/> 
    7477</modelFamily> 
    7578<priority class="vocab" title="Variable priority" id="cmip.drv.019"> 
    7679<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    77 <item id="001.019.001" label="1" title="dummy title string" description="dummyAt" uuid="215f23e6-3a81-11e5-882e-ac72891c3257"/> 
     80<item id="001.019.001" label="1" title="dummy title string" description="dummyAt" uuid="bc566c7a-5713-11e5-9f3f-ac72891c3257"/> 
    7881</priority> 
    7982<qcLevel class="vocab" title="Quality control level" id="cmip.drv.020"> 
    8083<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    81 <item id="001.020.001" label="1" title="dummy title string" uuid="215f26a2-3a81-11e5-882e-ac72891c3257"/> 
     84<item id="001.020.001" label="1" title="dummy title string" uuid="bc56712a-5713-11e5-9f3f-ac72891c3257"/> 
    8285</qcLevel> 
    8386<complianceCode class="vocab" title="Data specification compliance codes" id="cmip.drv.021"> 
    8487<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    85 <item id="001.021.001" label="example-01" title="dummy title string" uuid="215f295e-3a81-11e5-882e-ac72891c3257"/> 
     88<item id="001.021.001" label="example-01" title="dummy title string" uuid="bc5675c6-5713-11e5-9f3f-ac72891c3257"/> 
    8689</complianceCode> 
    8790<glossary class="vocab" title="Glossary" id="cmip.drv.022"> 
    8891<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    89 <item id="001.022.001" label="example-01" title="dummy title string" fullName="dummyAt" description="dummyAt" url="dummyAt" uuid="215f2d28-3a81-11e5-882e-ac72891c3257"/> 
     92<item id="001.022.001" label="example-01" title="dummy title string" fullName="dummyAt" description="dummyAt" url="dummyAt" uuid="bc567c7e-5713-11e5-9f3f-ac72891c3257"/> 
    9093</glossary> 
    9194<modelCompType class="vocab" title="Model Component Types/Categories [redundant with *Native Grid*]" id="cmip.drv.023"> 
    9295<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    93 <item id="001.023.001" label="example-01" title="dummy title string" uuid="215f3002-3a81-11e5-882e-ac72891c3257"/> 
     96<item id="001.023.001" label="example-01" title="dummy title string" uuid="bc568138-5713-11e5-9f3f-ac72891c3257"/> 
    9497</modelCompType> 
    9598<modelType class="vocab" title="Class of model" id="cmip.drv.024"> 
    9699<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    97 <item id="001.024.001" label="example-01" title="dummy title string" uuid="215f32b4-3a81-11e5-882e-ac72891c3257"/> 
     100<item id="001.024.001" label="example-01" title="dummy title string" uuid="bc5685d4-5713-11e5-9f3f-ac72891c3257"/> 
    98101</modelType> 
    99102<index class="vocab" title="Index of vocabularies" id="cmip.drv.025"> 
    100103<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    101 <item id="001.025.001" label="example-01" title="dummy title string" table="dummyAt" description="dummyAt" uuid="215f367e-3a81-11e5-882e-ac72891c3257"/> 
     104<item id="001.025.001" label="example-01" title="dummy title string" table="dummyAt" description="dummyAt" uuid="bc568c46-5713-11e5-9f3f-ac72891c3257"/> 
    102105</index> 
    103106<cmorDimAttr class="vocab" title="CMOR Dimension Attributes" id="cmip.drv.026"> 
    104107<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    105 <item id="001.026.001" label="example_03" title="dummy title string" description="dummyAt" uuid="215f3a48-3a81-11e5-882e-ac72891c3257" CMORattr="dummyAt"/> 
     108<item id="001.026.001" label="example_03" title="dummy title string" description="dummyAt" uuid="bc5692a4-5713-11e5-9f3f-ac72891c3257" CMORattr="dummyAt"/> 
    106109</cmorDimAttr> 
    107110<processingOptions class="vocab" title="Processing steps required" id="cmip.drv.027"> 
    108111<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    109 <item id="001.027.001" label="example01" title="dummy title string" description="dummyAt" uuid="215f3d90-3a81-11e5-882e-ac72891c3257"/> 
     112<item id="001.027.001" label="example01" title="dummy title string" description="dummyAt" uuid="bc5697fe-5713-11e5-9f3f-ac72891c3257"/> 
    110113</processingOptions> 
    111114</main> 
  • CMIP6dreqbuild/trunk/src/framework/ptxt.py

    r370 r382  
    214214      mainEl = "main" 
    215215      mainElCont = ''' 
     216xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" 
     217xsi:noNamespaceSchemaLocation="out/dreqSchema.xsd" 
    216218xmlns="uri:w3id.org:cmip6.dreq.dreq:a"''' 
    217219    else: 
  • CMIP6dreqbuild/trunk/src/framework/scanDreq.py

    r375 r382  
    33import xml, re 
    44import xml.dom, xml.dom.minidom 
     5import sets 
    56 
    67from utils_wb import uniCleanFunc 
     
    4142 
    4243class updates(object): 
     44  delToks = sets.Set( ['inc','omit'] ) 
    4345  def __init__(self,fndup,fnmult,idir='rev1'): 
    4446    assert os.path.isdir( idir ), 'Directory %s not found' % idir 
     
    7274        self.twins.append( l1[8] ) 
    7375        self.twins.append( l2[8] ) 
    74       elif l1[10] == 'inc' and  l2[10] == 'inc': 
     76      elif l1[10] in self.delToks and l2[10] in self.delToks: 
    7577        ths = 3 
    76         self.ddel[ l1[8] ] = 'inc' 
    77         self.ddel[ l2[8] ] = 'inc' 
     78        self.ddel[ l1[8] ] = (l1[10],l1[11]) 
     79        self.ddel[ l2[8] ] = (l2[10],l2[11]) 
    7880      else: 
    7981        ths = -1 
     
    104106          self.upd[ xx[j] ] = { 'provNote':zz[j][12], 'tags':tags, 'label':zz[j][0], 'title':zz[j][1] } 
    105107 
     108### 
     109### varDup and varMult created in first parse ----- then editted to select options 
     110### 2nd pass through then generates the replace and remove options -- taking into account cross references 
     111### the results of the 2nd pass go back to ../workbook to generate a new set of inputs. 
     112### 
    106113up = updates('varDup_20150724.csv', 'varMult_20150725.csv') 
    107114##up.scandup() 
    108115up.scanmult() 
    109116 
     117urep = True 
    110118urep = False 
    111 urep = True 
    112119if urep: 
    113120  oo = open( 'uuidreplace.csv', 'w' ) 
     
    117124      kn = up.repl[k] 
    118125      for tag,ki  in inx.iref_by_uuid[k]: 
    119          oo.write( '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (k,kn,tag,ki, inx.uuid[k][1].label,  inx.uuid[kn][1].label, inx.uuid[ki][1].label) ) 
     126         try: 
     127           oo.write( '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' % (k,kn,tag,ki, inx.uuid[k][1].label,  inx.uuid[kn][1].label, inx.uuid[ki][1].label) ) 
     128         except: 
     129           print k,kn,ki 
     130           raise 
    120131    else: 
    121132      oo2.write( k + '\n' ) 
     
    172183ks = inx.var.label.keys() 
    173184ks.sort() 
     185emptySet = sets.Set( ['','unset'] ) 
     186def entryEq(a,b): 
     187  return a == b or (a in emptySet and b in emptySet) 
     188 
     189deferredRecs = [] 
    174190for k in ks: 
    175191  if len(inx.var.label[k]) == 2: 
    176192    v1 = inx.var.uuid[inx.var.label[k][0]] 
    177193    v2 = inx.var.uuid[inx.var.label[k][1]] 
    178     cc = map( lambda x: v1.__dict__[x] == v2.__dict__[x], ['title','sn','units','description']  ) 
     194    cc = map( lambda x: entryEq( v1.__dict__[x], v2.__dict__[x]), ['title','sn','units','description']  ) 
    179195    if all(cc): 
    180       oo2.write( string.join(map( lambda x: v1.__dict__[x], hs) + [v2.uuid,'identical'], '\t' ) + '\n' ) 
     196### where duplicates are identical , collect and output at end of file. 
     197      deferredRecs.append( string.join(map( lambda x: v1.__dict__[x], hs) + [v2.uuid,'identical'], '\t' ) + '\n' ) 
     198      deferredRecs.append( string.join(map( lambda x: v2.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) 
    181199    else: 
    182200      oo2.write( string.join(map( lambda x: v1.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) 
    183     oo2.write( string.join(map( lambda x: v2.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) 
     201      oo2.write( string.join(map( lambda x: v2.__dict__[x], hs) + ['',''], '\t' ) + '\n' ) 
    184202       
    185203  elif len(inx.var.label[k]) > 1: 
     
    190208    for i in (inx.var.label[k] + inx.var.label[k[:-2]]): 
    191209      oo3.write( string.join(map( lambda x: inx.var.uuid[i].__dict__[x], hs), '\t' ) + '\n' ) 
     210 
     211## output auto-filled records for identical duplicates at end of varDup file. 
     212for r in deferredRecs: 
     213  oo2.write( r ) 
    192214oo.close() 
    193215oo2.close() 
     
    365387            1.0:'sfc?', \ 
    366388           16.0:'plev16', 7.0:'plev7', 40.0:'plev40', u'all*':'Xlev', 14.0:'plev14', u'Model levels or 27Plevs':'alev|plev27', \ 
     389           u'17 (or 23 )':'plev17|plev23', u'17 (or 23)':'plev17|plev23', \ 
    367390           27.0:'plev27', 17.0:'plev17', u'17 (or23)':'plev17|plev23', 8.0:'plev8', u'all model levels':'alev', 5.0:'plev5'} 
    368391ks = vshpchkMap.keys() 
     
    375398tsmap = { 'mean':[u'daily mean', u'time mean', u'time: day', 
    376399                u'Cumulative annual fraction', u'Time mean', u'weighted time mean', u'time: mean', u'mean', u'Mean'], 
    377           '__unknown__':[''], 
     400          '__unknown__':['','dummyAt'], 
    378401          'point':[ u'Instantaneous (end of year)', u'point', u'Synoptic', u'instantaneous', u'time: point', u'synoptic'] } 
    379402tsmap2 = {} 
  • CMIP6dreqbuild/trunk/src/workbook/importShDreq.py

    r375 r382  
    3131class main(object): 
    3232 
    33   def __init__(self,src,rq): 
     33  def __init__(self,src,rq,repl=False): 
    3434    self.src=src 
    3535    self.repl = collections.defaultdict( list ) 
    3636    self.err0010 = collections.defaultdict( int ) 
    3737    self.replItems = {} 
    38     self.importRepl() 
    39     self.importRemove() 
    40     self.importUpdate() 
    41     self.importInsert() 
     38    self.remo = {} 
     39    self.upda = {} 
     40    self.insert = {} 
     41    if repl: 
     42      self.importRepl() 
     43      self.importRemove() 
     44      self.importUpdate() 
     45      self.importInsert() 
     46    self.e15_10 = 0 
    4247    
    4348    fok = [test(os.path.isfile(src),'%s not found' % src), ] 
     
    4550    assert all( fok), 'Required input file(s) missing' 
    4651 
    47     for sh in [rq.refti, rq.revti, rq.vars, rq.grps, rq.rqvg, rq.rqli, rq.rqit]: 
     52    for sh in [rq.refti, rq.revti, rq.vars, rq.grps, rq.rqvg, rq.rqli, rq.rqit, rq.rqsect]: 
    4853       print sh['__info__'] 
    4954 
     
    6570    print xsn 
    6671    print ff.keys() 
     72 
     73    addex = False 
     74    exrqvg = {} 
     75    if addex: 
     76      ###  
     77      extravg = [['OMIP', u'OMIP-Oyr', u'OMIP: OMIP-Oyr', 'CMIP5Rev', u'OMIP.Oyr'], 
     78               ['OMIP', u'OMIP-Omon', u'OMIP: OMIP-Omon', 'CMIP5Rev', u'OMIP.Omon'], 
     79               ['OMIP', u'OMIP-day', u'OMIP: OMIP-day', 'CMIP5Rev', u'OMIP.day'], 
     80               ['OMIP', u'OMIP-fx', u'OMIP: OMIP-fx', 'CMIP5Rev', u'OMIP.fx'] ] 
     81      for r in extravg: 
     82        thisuuid = str( uuid.uuid1() ) 
     83        exrqvg[thisuuid] = [thisuuid ,] + r[:] 
    6784     
    6885    self.eern = collections.defaultdict( list ) 
     
    7087    ixrn = 5 
    7188    dups = [] 
    72     for k in rq.rqvg.keys(): 
    73       if k[0] != '_': 
     89    ##for k in rq.rqvg.keys(): 
     90      ##if k[0] != '_': 
     91    for i,k in [ (rq.rqvg[k],k) for k in  rq.rqvg.keys() if k[0] != '_'] + [(exrqvg[k],k) for k in exrqvg.keys()]: 
    7492        kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'ref', 'refNote', 'refid'] 
    7593        kk = ['uuid', 'mip', 'label', 'title', 'ref', 'refNote'] 
    76         i = rq.rqvg[k] 
     94        ##i = rq.rqvg[k] 
    7795         
    7896        self.eern[i[ixrn]].append( i[0] ) 
    7997        k2 = '%s__%s' % (i[ixrn-1],i[ixrn]) 
    8098        if self.eern2.has_key( k2 ): 
    81            dups.append( k2 ) 
     99           dups.append( (k2,k,self.eern2[k2]) ) 
    82100        self.eern2[k2] = i[0] 
    83101        assert i[0] == k, 'Bad key/uuid' 
    84102 
    85     assert len( dups ) == 0, 'Duplicate refs: %s' % str(dups) 
     103    assert len( dups ) == 0, 'Duplicate refs: %s, %s' % (str(dups),str(map( lambda x: rq.rqvg[x[1]], dups )) ) 
    86104         
    87     for k in [ 'var','ovar','groupitem', 'revisedtabitem', 'requestlink', 'requestitem','requestvargroup' ]: 
     105    for k in [ 'var','ovar','groupitem', 'revisedtabitem', 'requestlink', 'requestitem','requestvargroup','tablesection' ]: 
    88106    ##for k in [ 'var']: 
    89107      thissh = {'var':rq.vars,'ovar':rq.refti, 'groupitem':rq.grps, \ 
    90108         'revisedtabitem':rq.revti, 'requestlink':rq.rqli, 'requestitem':rq.rqit, \ 
    91          'requestvargroup':rq.rqvg}[k] 
     109         'requestvargroup':rq.rqvg, 'tablesection':rq.rqsect}[k] 
    92110      lll = [] 
    93111      for i in thissh.keys(): 
     
    151169       u'PMIP3_Oclim':'monClim', u'SPECS_Amon':'mon', u'SPECS_Lmon':'mon', u'CMIP5_cf3hr':'3hr', u'CORDEX_3h':'3hr', \ 
    152170       u'CCMI1_annual':'yr', u'CMIP5_Oyr':'yr'} 
    153             kk = ['uuid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid'] 
     171            kk = ['uuid', 'comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'sn', 'deflate', 'title', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', 'cell_methods_xx', 'valid_max', 'positive', 'var', 'mipTable', 'dimensions', 'vid', 'gpid','rowIndex'] 
    154172## uuid; comment; deflate_level; shuffle; ok_max_mean_abs; flag_meanings; type; ok_min_mean_abs; sn; deflate; valid_min; cell_methods; flag_values; cell_measures; out_name; modeling_realm; units; cell_methods_xx; valid_max; positive; var; mipTable; dimensions; vid 
    155             thisl = string.strip(str( ll[21] ) ) 
    156             for j in range(len(kk)): 
     173 
     174            thisl = string.strip( str( ll[21] ) ) 
     175            for j in range( min(len(kk),len(ll)) ): 
    157176              if j != 21: 
    158177                item.setAttribute( kk[j], str( ll[j] ) ) 
    159             item.setAttribute( 'frequency', tab2freq[ll[22]] ) 
     178            if len(ll) < len(kk): 
     179              print 'ERROR.015.0001: record length short %s (%s):: %s' % (len(ll),len(kk),ll) 
     180 
     181            if ll[22][-3:] in ['mon','day','3hr']: 
     182              fr = ll[22][-3:] 
     183            elif ll[22][-2:] in [ 'yr','fx']: 
     184              fr = ll[22][-2:] 
     185            else: 
     186              fr = tab2freq[ll[22]] 
     187    
     188            item.setAttribute( 'frequency', fr ) 
    160189            thisuuid = ll[0] 
    161190            if self.repl.has_key( thisuuid ): 
     
    171200            for j in range(len(kk)): 
    172201              if j != 1: 
    173                 item.setAttribute( kk[j], str( ll[j] ) ) 
     202                ##item.setAttribute( kk[j], str( ll[j] ) ) 
     203                if kk[j] == 'misc': 
     204                  item.setAttribute( 'priority', str( ll[j] ) ) 
     205                else: 
     206                  item.setAttribute( kk[j], str( ll[j] ) ) 
    174207            item.setAttribute( 'title', thisl ) 
    175208            if string.find(thisl, ' ') != -1: 
    176209              thisl = string.split(thisl)[0] 
    177210            thisuuid = ll[11] 
     211            gpid = str(ll[-3]) 
     212            if not rq.rqvg.has_key(gpid): 
     213              print 'ERROR.015.0010: gpid not found %s' % str(ll) 
     214              self.e15_10 += 1 
    178215            if self.repl.has_key( thisuuid ): 
    179216              for tag,old,new in self.repl[thisuuid]: 
     
    183220################################## 
    184221          elif k == 'revisedtabitem': 
     222#### need to fill gaps in variable groups --- or do better job upstream ##########  !!!!!!!!!!!!!!!! 
    185223## - table; mip; uuid; priority 
     224## 
     225## variable group ids pulled through via mapping above .. very messy  
     226## still missing some OMIP matches .... OMIP revised tables are not explicitly requested anywhere. 
     227## 
    186228            kk = ['var', 'table', 'mip', 'vid', 'priority'] 
    187             kvg = 'rev__%s.%s' % ( ll[2],ll[1] ) 
     229            kvg = 'CMIP5Rev__%s.%s' % ( ll[2],ll[1] ) 
     230            kvg3 = 'rev__%s.%s' % ( ll[2],ll[1] ) 
    188231            if self.eern2.has_key(kvg): 
    189232              vgid = self.eern2[kvg] 
     233            elif self.eern2.has_key(kvg3): 
     234              vgid = self.eern2[kvg3] 
     235            elif cmip5GrpLk[ll[1]].has_key(ll[0]): 
     236              g2 = cmip5GrpLk[ll[1]][ll[0]] 
     237              kvg2 = 'CMIP5Rev__%s.%s' % ( ll[2],g2 ) 
     238              if self.eern2.has_key(kvg2): 
     239                vgid = self.eern2[kvg2] 
     240              else: 
     241                vgid = '__vg_not_found_1__' 
     242                self.err0010[kvg] += 1 
    190243            else: 
    191               vgid = '__none__' 
     244              vgid = '__vg_not_found_2__' 
    192245              self.err0010[kvg] += 1 
    193246            thisuuid = str( uuid.uuid1() ) 
     
    207260          elif k == 'requestlink': 
    208261 ## - uuid; mip; tab; objective; grid; gridreq; comment 
    209             kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'ref', 'refNote', 'refid'] 
     262            kk = ['uid', 'mip', 'tab', 'objective', 'grid', 'gridreq', 'comment', 'opt','opar','ref', 'refNote', 'refid'] 
    210263            emap = {'uid':'uuid'} 
    211264            thisl = string.strip(str( ll[2] ) ) 
    212265            for j in range(len(kk)): 
    213266                thiskk = emap.get(  kk[j],  kk[j] ) 
    214                 item.setAttribute( thiskk, str( ll[j] ) ) 
     267                try: 
     268                  item.setAttribute( thiskk, str( ll[j] ) ) 
     269                except: 
     270                  print 'ERROR.099.0001: Failed to set attribute: %s [%s]' % (str(ll),j) 
     271                  item.setAttribute( thiskk, '__error_99_1__' ) 
     272 
    215273            item.setAttribute( 'title', thisl ) 
    216274################################## 
     
    228286            item.setAttribute( 'uuid', str(i) ) 
    229287################################## 
    230           elif k == 'requestvargroup': 
    231             kk = ['uuid', 'mip', 'label', 'title', 'ref', 'refNote'] 
     288          elif k in ['requestvargroup','tablesection']: 
     289            if k == 'requestvargroup': 
     290              kk = ['uuid', 'mip', 'label', 'title', 'ref', 'refNote'] 
     291            else: 
     292              kk = ['uuid', 'gpid', 'mip', 'label', 'title', 'ref', 'refNote'] 
    232293            thisl = string.replace( str( ll[2] ), '.', '-' ) 
    233294            for j in range(len(kk)): 
     
    260321            print '******************************************' 
    261322             
     323    ##for k in rqvg_extra.keys(): 
     324            ##item = doc.createElement( 'item' ) 
     325            ##ff['...' ][0].appendChild( item ) 
     326 
    262327    this = doc.getElementsByTagName('remarks')[0] 
    263328    dil = this.getElementsByTagName('item') 
     
    284349 
    285350  def importRemove(self,rfile='uuidremove.csv'): 
    286     self.remo = {} 
    287351    for l in open(rfile).readlines(): 
    288352      self.remo[ string.strip(l) ] = 1 
    289353 
    290354  def importUpdate(self,rfile='uuidupdate.csv'): 
    291     self.upda = {} 
    292355    for l in open(rfile).readlines(): 
    293356      bits = string.split( string.strip(l), '\t' ) 
     
    300363 
    301364  def importInsert(self,rfile='uuidinsert.csv'): 
    302     self.insert = {} 
    303365    for l in open(rfile).readlines(): 
    304366      bits = string.split( string.strip(l), '\t' ) 
     
    307369 
    308370sampleXml = '../framework/out/dreqSample.xml' 
     371cmip5GrpLk = collections.defaultdict( dict ) 
    309372from scansh import rq 
     373 
     374for k in rq.cmip5Grps.keys(): 
     375  bits = string.split(k,'_') 
     376  tab = bits[0] 
     377  for v in rq.cmip5Grps[k]: 
     378    cmip5GrpLk[tab][v] = k 
    310379m = main(sampleXml, rq) 
    311380if len(m.err0010.keys()) > 0: 
  • CMIP6dreqbuild/trunk/src/workbook/scansh.py

    r372 r382  
    44odir = 'sh20150708' 
    55odir = '/data/tmp/svn3/exarch/CMIP6dreqbuild/trunk/srcMisc/sh20150827' 
     6import scansh_cfg 
     7reload( scansh_cfg ) 
    68from scansh_cfg import odir 
    79def sopen( id ): 
    810  sh = shelve.open( '%s/%s' % (odir,id ) ) 
    9   print 'Openned %s' % str( sh['__info__'] ) 
    10   print sh['__cols__'] 
     11  print 'Openned %s: %s' % ('%s/%s' % (odir,id ),str( sh['__info__'] ) ) 
     12  print sh['__cols__'], len(sh.keys()) 
    1113  return sh 
    1214 
     
    2123    self.rqli = sopen( 'requestLinks' ) 
    2224    self.rqvg = sopen( 'requestVarGroup' ) 
     25    self.rqsect = sopen( 'requestVarSections' ) 
     26 
     27    self.cmip5Grps = shelve.open( '%s/cmip5Groups' % odir, 'r' ) 
    2328 
    2429rq = rqc() 
     
    5964  if k[0] != '_': 
    6065    r = rq.refti[k] 
    61     if r[-1] != '__none__': 
    62       if rq.vars.has_key(r[-1]): 
     66    if r[-3] != '__none__': 
     67      if rq.vars.has_key(r[-3]): 
    6368        nok += 1 
    6469      else: 
  • CMIP6dreqbuild/trunk/src/workbook/scansh_cfg.py

    r356 r382  
    11 
    22 
    3 odir = '/data/tmp/svn3/exarch/CMIP6dreqbuild/trunk/srcMisc/sh20150708' 
     3odir = '/data/tmp/svn3/exarch/CMIP6dreqbuild/trunk/srcMisc/sh20150827' 
  • CMIP6dreqbuild/trunk/srcMisc/dreqSX.py

    r376 r382  
    55from sets import Set 
    66 
    7 vdate = "20150827" 
     7vdate = "20150907" 
    88 
    9 sh = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'r' ) 
     9shvg = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'r' ) 
    1010shnv = shelve.open( 'dreq_consol_tables_nv_shelve_v%s' % vdate, 'r' ) 
    1111## from vlsc5b --- includes new names from OMIP 
    1212shold = shelve.open( 'dreq_consol_tables_old_shelve_v20150713', 'r' ) 
    13 shlist = [sh,shnv,shold] 
     13shlist = [shvg,shnv,shold] 
    1414 
    1515##MIP columns: recordid, ['comment', 'deflate_level', 'shuffle', 'ok_max_mean_abs', 'flag_meanings', 'type', 'ok_min_mean_abs', 'standard_name', 'deflate', 'long_name', 'valid_min', 'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units', '#cell_methods', 'valid_max', 'positive'] , var, mip table, dims 
     
    4545## dictionary to carry short names of old variables. 
    4646eeold = {} 
    47 ###for ee,shv in [(ee0,sh),(eenv,shnv),(eeold,shold)]: 
    48 for ee,shv in [(ee0,sh),(eenv,shnv)]: 
     47###for ee,shv in [(ee0,shvg),(eenv,shnv),(eeold,shold)]: 
     48for ee,shv in [(ee0,shvg),(eenv,shnv)]: 
    4949  for k in shv.keys(): 
    5050    ee[k] = shv[k] 
     
    5353eeoldl = collections.defaultdict( list ) 
    5454if rmod == 'xls': 
    55   for ee,fn in [(ee_xref,'vlsc5_xref_v20150826.xls'),(ee_mip,'vlsc5_mip_v20150826.xls'),(eeold,'vlsc5b_v20150826.xls')]: 
     55  for ee,fn in [(ee_xref,'vlsc5_xref_v%s.xls' % vdate),(ee_mip,'vlsc5_mip_v%s.xls' % vdate),(eeold,'vlsc5b_v%s.xls' % vdate)]: 
    5656    wb = workbook( fn ) 
    5757    sh = wb.book.sheet_by_name( 'Sheet1' ) 
     
    6060      ee[r[0]] = r 
    6161else: 
    62   for ee,fn,sep,xxx in [(ee_xref,'vlsc5_xref_v20150826.csv',',',False),(ee_mip,'vlsc5_mip_v20150826.csv', '\t',False),(eeoldl,'vlsc5b_v20150826.csv', '\t',True)]: 
     62  for ee,fn,sep,xxx,nrmin in [(ee_xref,'vlsc5_xref_v%s.csv' % vdate,',',False,5),(ee_mip,'vlsc5_mip_v%s.csv' % vdate, '\t',False,24),(eeoldl,'vlsc5b_v%s.csv' % vdate, '\t',True,9)]: 
    6363    for i in open(fn).readlines(): 
    6464      r = map( lambda x: string.strip(str(uniCleanFunc(x))), string.split( string.strip(i), sep ) ) 
    65       while len(r) < 9: 
     65      while len(r) < nrmin: 
    6666        r.append( '' ) 
    6767      if xxx: 
     
    7070        ee[r[0]] = r 
    7171 
     72for k in ee_mip.keys(): 
     73 if len( ee_mip[k][:] ) != 24: 
     74   print 'ERROR.015.0003: Bad record length %s: %s' % (k,ee_mip[k]) 
    7275 
    7376for h in eeoldl.keys(): 
  • CMIP6dreqbuild/trunk/srcMisc/dreq_cfg.py

    r323 r382  
    1515    ee = {} 
    1616    ff = collections.defaultdict( list ) 
     17    fff = {}  
    1718    for l in ii: 
    1819      if l[:2] == './': 
     
    2425      elif l[:2] == '--': 
    2526        fn = string.strip( l[3:] ) 
     27        if string.find( fn, ' ' ) != -1: 
     28          fn,extra = string.split( fn, maxsplit=1) 
     29          fff[fn] = extra 
    2630        assert os.path.isfile( '%s%s/%s' % (self.dir0,mip,fn) ), '%s %s not found' % (mip, fn) 
    2731        ff[mip].append( fn ) 
    2832    self.ee = ee 
    2933    self.ff = ff 
     34    self.fff = fff 
  • CMIP6dreqbuild/trunk/srcMisc/dreq_consol.py

    r348 r382  
    100100  return ee 
    101101     
    102 snli = snlist( dir='../inputsOther/config/cf/' ) 
     102snli = snlist( dir='./config/cf/' ) 
    103103sn,sna = snli.gen_sn_list() 
    104104esn = {} 
     
    109109  esna[s] = s 
    110110 
    111 vdict = collections.defaultdict( list ) 
    112 for l in open( '../inputsOther/vlsc3.csv' ): 
    113   v,l,s,c,m = string.split(string.strip(l),'\t')[:5] 
    114   vdict[v].append(( l,s,c,m)) 
     111useVdict = False 
     112if useVdict: 
     113  vdict = collections.defaultdict( list ) 
     114  for l in open( '../inputsOther/vlsc3.csv' ): 
     115    v,l,s,c,m = string.split(string.strip(l),'\t')[:5] 
     116    vdict[v].append(( l,s,c,m)) 
    115117 
    116118dir0 = '/home/martin/2014/wip/dreq/input/' 
     
    194196      omitOld = False 
    195197      omit = False 
    196       if vdict.has_key(v) and omitOld: 
    197         omit = True 
     198      if useVdict: 
     199        if vdict.has_key(v) and omitOld: 
     200          omit = True 
    198201      s = r[1].value 
    199202      if s in ['','?']: 
     
    234237              if len(ee[v0]) != 1: 
    235238                 v += '!' 
    236               if vdict.has_key(v0): 
    237                  v += '**' 
     239              if useVdict: 
     240                if vdict.has_key(v0): 
     241                   v += '**' 
    238242            wk0.putValue2( self.kk, j+jo, v ) 
    239243          else: 
  • CMIP6dreqbuild/trunk/srcMisc/dreq_consol_dreq.py

    r374 r382  
    351351    return rq(mip,path,parent=self) 
    352352 
    353   def addRow( self, mip, tab, obj, grid, gok, comment ): 
     353  def addRow( self, mip, tab, obj, grid, gok, comment,opt='',opar='' ): 
    354354    k = str( uuid.uuid1() ) 
    355     self.rowList.append( (k,mip,tab, obj, grid, gok, comment) ) 
     355    self.rowList.append( (k,mip,tab, obj, grid, gok, comment, opt, opar) ) 
    356356    return k 
    357357 
     
    475475        comment = rv[5] 
    476476        obj = rv[6] 
    477         uid = self.parent.addRow( self.mip, tab, obj, grid, gok, comment ) 
     477        opt = rv[1] 
     478        opar = rv[2] 
     479        uid = self.parent.addRow( self.mip, tab, obj, grid, gok, comment, opt=opt, opar=opar ) 
    478480        self.readDeckColPair( j, self.ixh, 160, rid=uid ) 
    479481    elif self.opt == 'deck': 
     
    485487        comment = rv[5] 
    486488        obj = rv[6] 
    487         uid = self.parent.addRow( self.mip, tab, obj, grid, gok, comment ) 
     489        opt = rv[1] 
     490        opar = rv[2] 
     491        uid = self.parent.addRow( self.mip, tab, obj, grid, gok, comment, opt=opt, opar=opar ) 
    488492        for je in range(5): 
    489493          self.readDeckColPair( j, self.iccntl + je*2, 160, expt=hh[je], rid=uid ) 
     
    496500        comment = rv[5] 
    497501        obj = rv[6] 
    498         uid = self.parent.addRow( self.mip, tab, obj, grid, gok, comment ) 
     502        opt = rv[1] 
     503        opar = rv[2] 
     504        uid = self.parent.addRow( self.mip, tab, obj, grid, gok, comment, opt=opt, opar=opar ) 
    499505        for je in range(5): 
    500506          self.readDeckColPair( j, self.iccntl + je*2, 160, expt=hh[je], rid=uid ) 
     
    703709### need to give temporary labels/titles here and consolidate after all variables have been collated in 
    704710### sx2. 
     711## rows from "request scoping" 
    705712for i in rqs.rowList: 
    706713  kk ='%s.%s' % (i[1],i[2]) 
     
    716723sh = shelve.open( '%s/requestLinks_tmp' % odir, 'n' ) 
    717724sh['__info__'] = { 'label':'requestLinks', 'title':'Links from variable groups to a request id' } 
    718 sh['__cols__'] = [ 'uid', 'mip', 'tab','objective','grid','gridreq','comment'] 
     725sh['__cols__'] = [ 'uid', 'mip', 'tab','objective','grid','gridreq','comment','opt','opar'] 
    719726for i in rqs.rowList: 
    720727  sh[i[0]] = i[:] 
  • CMIP6dreqbuild/trunk/srcMisc/dreq_consol_tables.py

    r374 r382  
    1111from utils_wb import wbcp, workbook, uniCleanFunc 
    1212 
    13 vdate = '20150827' 
     13vdate = '20150907' 
    1414 
    1515nt__deckrq = collections.namedtuple( 'dckrq', ['control','AMIP','abrupt4xCO2','rq_1pctCO2','historical'] ) 
     
    4343  return l0 
    4444 
    45 def getRowValues( ll, minLen=0 ): 
     45def getRowValues( ll, minLen=0, maxLen=0): 
    4646  oo = [] 
    4747  for i in ll: 
     
    5353      return oo 
    5454    oo.append( '' ) 
     55  if maxLen > 0: 
     56    return oo[:maxLen] 
    5557  return oo 
    56  
    5758 
    5859clabs = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" 
     
    136137class mipTable(object): 
    137138 
    138   def __init__(self,id): 
     139  def __init__(self,id,section): 
    139140     self.vars = collections.defaultdict( list ) 
     141     self.id = id 
     142     self.section = section 
    140143 
    141144class s0(object): 
     
    170173          ll = {} 
    171174          for k2 in self.ref1[k].a[k1].a.keys(): 
    172             ee[  self.ref1[k].a[k1].a[k2][2] ] = k2 
    173           ks = ee.keys() 
     175            ll[  self.ref1[k].a[k1].a[k2][2] ] = k2 
     176          ks = ll.keys() 
    174177          ks.sort() 
    175178          self.ref1[k].a[k1].latest = self.ref1[k].a[k1].a[ ks[-1] ] 
     
    177180          self.ref1[k].a[k1].latest  =  self.ref1[k].a[k1].a[ self.ref1[k].a[k1].a.keys()[0] ] 
    178181 
    179   def addTab(self,tab): 
    180     self.t[tab] = mipTable( tab ) 
    181  
    182   def add(self,mip,path,tab,tabmode='uid'): 
     182  def addTab(self,tab,section): 
     183    self.t[tab] = mipTable( tab,section) 
     184 
     185  def add(self,mip,path,tab,tabmode='uid', extra=None): 
    183186    self.tab = tab 
    184187## rq will add a reference to parent 
    185188    ##print '######',mip, path, tab 
    186     return rq(mip,path,parent=self,tab=tab,tabmode=tabmode) 
     189    return rq(mip,path,parent=self,tab=tab,tabmode=tabmode,section=extra) 
    187190 
    188191  def addItem( self, mip, tab, nn ): 
     
    249252            self.oorecs.append(rec) 
    250253           
    251              
    252254  def showXls(self,wb,r0=3,offset=0): 
    253255    r = r0 
     
    313315class rq(object): 
    314316 
    315   def __init__(self, mip,path, parent=None,tab=u'Omon',tabmode='uid' ): 
     317  def __init__(self, mip,path, parent=None,tab=u'Omon',tabmode='uid',section=None ): 
    316318 
    317319    self.mip = mip 
     320    self.section = section 
    318321    self.parent = parent 
    319322    self.nn = 0 
     
    325328    if self.parent.ref1[mip].a.has_key(self.fn): 
    326329       tt = self.parent.ref1[mip].a[self.fn].latest 
    327        if self.ctime == tt[2] and self.fsize == tt[3]: 
    328          print '%s unchanged' % self.fn 
    329          md5 = tt[4] 
     330       if len( tt ) > 3: 
     331         if self.ctime == tt[2] and self.fsize == tt[3]: 
     332           print '%s unchanged' % self.fn 
     333           md5 = tt[4] 
     334         else: 
     335           md5 = md5sum( path ) 
     336           if md5 == tt[4]: 
     337             print 'md5 unchanged: %s' % self.fn 
     338           else: 
     339             shr1[md5] = (self.fn,path,self.ctime, self.fsize, md5,mip ) 
    330340       else: 
     341         print 'ERROR.888.0001: Unexpected ref1 record (input status): %s' % str(tt) 
    331342         md5 = md5sum( path ) 
    332          if md5 == tt[4]: 
    333            print 'md5 unchanged: %s' % self.fn 
    334          else: 
    335            shr1[md5] = (self.fn,path,self.ctime, self.fsize, md5,mip ) 
    336343    else: 
    337344       md5 = md5sum( path ) 
     
    347354      self.wk1.focus( tab) 
    348355      if not self.parent.t.has_key(self.tabid): 
    349         self.parent.addTab(self.tabid) 
     356        self.parent.addTab(self.tabid,self.section) 
    350357      self.ok = self.parse01() 
    351358    else: 
     
    358365      if self.wk1.currentSi.row(j)[0].ctype == 2: 
    359366        v = string.strip( str( self.wk1.currentSi.row(j)[5].value ) ) 
    360         ee1[v].append(j) 
     367        t2 = self.wk1.currentSi.row(j)[17].ctype 
     368        ee1[v].append((j,t2)) 
    361369    for j in range(3,self.wk1.currentSi.nrows): 
    362370      rv = map( lambda x: x.value, self.wk1.currentSi.row(j) ) 
     
    367375      if self.wk1.currentSi.row(j)[0].ctype == 2: 
    368376        v = string.strip( str( self.wk1.currentSi.row(j)[5].value ) ) 
     377        if self.wk1.currentSi.row(j)[17].ctype == 1: 
     378          v2 = string.strip( str( self.wk1.currentSi.row(j)[17].value ) ) 
     379        else: 
     380          v2 = None 
    369381        if v not in ['0.0','']: 
    370382          if len(ee1[v]) > 1: 
    371              v += '__%s' % ['a','b'][ee1[v].index(j)] 
     383             ##v += '__%s' % ['a','b'][ee1[v].index(j)] 
     384             assert not all( map( lambda x: x[1] != 1, ee1[v] ) ), 'duplicate variable name and no cmor name ... %s %s' % (v,self.mip) 
     385             if v2 != None: 
     386               v = v2 
    372387          ixr = len(self.parent.records ) 
    373388## add record 
     
    388403           
    389404          if p != None: 
    390             self.parent.t[self.tabid].vars[v].append( (self.mip,p,ixr) )  
     405            self.parent.t[self.tabid].vars[v].append( (self.mip,p,ixr,self.section,j) )  
    391406            self.nn += 1 
    392407 
     
    406421    file = 'C4MIP/CMIP6DataRequestCompilationTemplate_C4MIP_06March2015_submitted.xls' 
    407422    fileTmpl = 'CMIP6DataRequestCompilationTemplate_20141218.xls' 
    408     self.newVars = 'CMIP6DataRequest_ConsolidatedNewVariables_20150708.xls' 
     423    self.newVars = 'CMIP6DataRequest_ConsolidatedNewVariables_v20150907.xls' 
    409424    self.nvd = {} 
    410425 
     
    499514 
    500515  def groups(self, vl3=None): 
     516    rh1 = ['Short name', 'Standard Name', 'Table', 'Frequency', 'Description extension (optional)', 'Shape', 'Levels', 'Time mean, point or climatology', 'Mask (optional)'] 
     517    rh2 = ['Short name of group', 'Variable short name', 'Table', 'Frequency', 'Description extension (optional)', 'Shape', 'Levels', 'Time mean, point or climatology', 'Mask (optional)'] 
     518    rh3 = ['Short name of group', 'Variable short name', 'Table', 'Frequency', 'Description extension (optional)', 'Shape', 'Levels', 'Time mean, point or climatology', 'Mask (optional)', 'Priority'] 
     519 
    501520    print self.cfg.ee.keys() 
    502521    omit = ['ALL VARIABLES', 'Objectives','Experiments','Experiment Groups','Request scoping','New variables','__lists__'] 
     
    524543               ll.append(i) 
    525544           assert len(ll) in [1,2], 'Could not parse sheet  %s, %s, %s' % (path,k,s) 
     545           withPriority = False 
     546           hr = sh.row( ll[-1] ) 
    526547           if len(ll) == 1: 
    527548## nt__grptbl = collections.namedtuple( 'grptbl', ['grp','var','srcTable','freq','description','shape','levels','timeProc','mask'] ) 
    528549             iv = 1 
    529550             it = 0 
     551             ok = len( hr ) >= 9 and all( map( lambda x: string.strip(hr[x].value) == rh2[x], range(9) ) ) 
     552             assert ok, '001: Sheet heading not recognised: %s' % str(hr) 
     553             if len(hr) > 9 and hr[9] == 'Priority': 
     554               withPriority = True 
     555               thisl = 10 
     556             else: 
     557               thisl = 9 
    530558           else: 
    531559             print '*************',k,s 
     560             ok = len( hr ) >= 9 and all( map( lambda x: string.strip(hr[x].value) == rh1[x], range(9) ) ) 
     561             assert ok, '002: Sheet heading not recognised: %s' % str(hr) 
    532562             iv = 0 
    533563             it = -1 
    534564             tv = sh.row(2)[1].value 
     565             thisl = 9 
     566 
    535567           irsh = 5 
    536568           for i in range(ll[-1]+1,sh.nrows): 
     
    538570             if iv == 1: 
    539571                 ##self.ntr = nt__grptbl._make( getRowValues( thisr, minLen=9 ) + [105,k,''] ) 
    540                 lll = getRowValues( thisr, minLen=9 ) + [105,k,''] 
     572                lll = getRowValues( thisr, minLen=thisl, maxLen=thisl ) 
    541573             else: 
    542                 lll = getRowValues( thisr, minLen=10 ) + [k,''] 
     574                lll = getRowValues( thisr, minLen=thisl, maxLen=thisl ) 
    543575                lll[1] = lll[0] 
    544576                lll[0] = tv 
     577             if thisl == 9: 
     578               lll.append( 105 ) 
     579             lll += [k,''] 
    545580             if it >= 0: 
    546581               tv = thisr[it].value 
     
    562597               v2 = None 
    563598 
     599             pmsg = '' 
    564600             if v == '*': 
    565601               if t in self.cmip5sns: 
     
    585621                   if self.rqs.t[t].vars.has_key(v): 
    586622                     p = self.rqs.t[t].vars[v][0][1] 
     623                     pmsg = 'a: %s: %s' % (v,str( self.rqs.t[t].vars[v][0] ) ) 
    587624                   elif v2 != None and self.rqs.t[t].vars.has_key(v2): 
    588625                     p = self.rqs.t[t].vars[v2][0][1] 
    589626                     ##print '%s old name for %s' % (v2,v) 
     627                     pmsg = 'b: %s: %s' % (v2,str( self.rqs.t[t].vars[v2][0] ) ) 
    590628                   else: 
    591629                     p = 8 
     
    593631               else: 
    594632                 if string.lower(t)[:3] == 'new' or t == '': 
    595                    p = self.nvd.get( v, [0,7] )[1] 
     633                   p = self.nvd.get( v, [0,7] )[0] 
     634                   pmsg = 'c: %s: %s' % (v,str(self.nvd.get( v, [0,7] )) ) 
    596635                   if p == 7: 
    597                      p = self.nvd.get( v + '!', [0,7] )[1] 
     636                     p = self.nvd.get( v + '!', [0,7] )[0] 
     637                     pmsg = 'd: %s: %s' % (v + '!',str(self.nvd.get( v + '!', [0,7] )) ) 
    598638                   if p != 7: 
    599639                     self.idx += 1 
     
    610650 
    611651### complete ntr and save to shelf. 
     652             if type(p) != type(0): 
     653               print 'ERROR.020.0002: attempt to set non integer priority: %s [%s] ... %s' % (p,str(lll),pmsg) 
     654               p = 106 
    612655             lll[9] = p 
    613656             self.ntr = nt__grptbl._make( lll ) 
     
    646689          else: 
    647690            path = '%s%s/%s' % (self.dir0,k,fn) 
    648           thisrq = self.rqs.add( k, path,s,tabmode=tabmode ) 
     691          extra = self.cfg.fff.get( fn, None ) 
     692          thisrq = self.rqs.add( k, path,s,tabmode=tabmode, extra=extra ) 
    649693          if thisrq.ok: 
    650694            print 'INFO.002:',k,path,s,thisrq.ok, thisrq.nn 
     
    687731vl3 = {} 
    688732##wb3 = workbook( 'vlsc5b_v20150714.xls' ) 
    689 wb3 = workbook( 'vlsc5b_v20150826.xls' ) 
     733wb3 = workbook( 'vlsc5b_v20150907.xls' ) 
    690734sh = wb3.book.sheet_by_name( 'Sheet1' ) 
    691735m = main() 
    692736 
    693 opt = 2 
     737opt = 1 
    694738if opt == 1: 
    695739  for i in range(sh.nrows): 
     
    725769    m.rqs.vn = {} 
    726770    for v in vs: 
    727       for mip,p,ix in m.rqs.t[k].vars[v]: 
     771      for mip,p,ix,sect,rowix in m.rqs.t[k].vars[v]: 
    728772        this = m.rqs.records[ix] 
    729773        if this[16] in ['','0.0',0.0]: 
     
    732776          vn = this[16] 
    733777        if ee.has_key(vn): 
    734           print '***********************DUPLICATE',vn 
     778          print 'ERROR.030.0010: DUPLICATE %s' % vn 
    735779          print m.rqs.t[k].vars[v] 
    736780          print m.rqs.records[ee[vn][0]] 
    737781          print m.rqs.records[ix] 
    738         ee[vn] = (ix,p) 
     782        ee[vn] = (ix,p,sect,rowix) 
    739783    k2 = str(k) 
    740784    m.rqs.vn[k2] = ee 
     
    743787  shr1.close() 
    744788     
    745   print m.idx 
    746789elif opt == 2: 
    747790  m.run1(tabmode='tab') 
  • CMIP6dreqbuild/trunk/srcMisc/sumMip_pp.py

    r375 r382  
    3636      for i in range(sh.nrows): 
    3737        r = map( lambda x: x.value, sh.row(i) ) 
     38        rc = map( lambda x: x.ctype, sh.row(i) ) 
    3839        if started: 
     40          omit = False 
    3941          k2 = str( uuid.uuid1() ) 
    4042          this = [k2,] 
     
    4345              this.append( 'OMIP.%s' % s ) 
    4446            elif ktr1.has_key(k): 
    45               this.append( r[ shKeys.index( ktr1[k] ) ] ) 
     47              if k == 'dims': 
     48                if rc[shKeys.index( ktr1[k] )] == 1: 
     49                  d0 = string.strip( r[ shKeys.index( ktr1[k] ) ] ) 
     50                  if string.find( d0, ' ' ): 
     51                    dl = string.split( r[ shKeys.index( ktr1[k] ) ] ) 
     52                    d0 = string.join( dl, '|' ) 
     53                  this.append( d0 ) 
     54                else: 
     55                  this.append( r[ shKeys.index( ktr1[k] ) ] ) 
     56              elif k == 'var': 
     57                if rc[shKeys.index( ktr1[k] )] == 1: 
     58                  v = string.strip( r[ shKeys.index( ktr1[k] ) ] ) 
     59                elif rc[ shKeys.index( ktr1['out_name'] ) ] == 1: 
     60                  v = string.strip( r[ shKeys.index( ktr1['out_name'] ) ] ) 
     61                else: 
     62                  omit = True 
     63                this.append( v ) 
     64              else: 
     65                this.append( r[ shKeys.index( ktr1[k] ) ] ) 
    4666            else: 
    4767              this.append( '' ) 
    48           eeomip.append( this ) 
     68          if not omit: 
     69            eeomip.append( this ) 
    4970        elif r[0] == 'priority': 
    5071          started = True 
     
    209230 
    210231## read 2nd ref version ...... 
    211 wb2 = workbook( 'vlsc5b_v20150714_v1.xls' ) 
     232wb2 = workbook( 'vlsc5b_v20150907_v1.xls' ) 
    212233sh2 = wb2.book.sheet_by_name( 'Sheet1' ) 
    213234ee2 = {} 
     
    233254#### 
    234255#### want to get additional info from ref: ..... from?? 
    235 yyy = '20150826' 
     256yyy = '20150907' 
    236257oo=open( 'vlsc5b_v%s.csv' % yyy,'w') 
    237258idk = 0 
  • CMIP6dreqbuild/trunk/srcMisc/sx2.py

    r376 r382  
    66 
    77import collections, string, hashlib, uuid, os, sys 
     8import shelve 
    89import dreqSX as sx 
     10from sets import Set 
    911odir = 'sh20150827' 
     12 
     13vdate = '20150907' 
    1014if not os.path.isdir( odir ): 
    1115  os.mkdir( odir ) 
     16 
     17sh1 = shelve.open( '%s/cmip5Groups' % odir, 'n' ) 
     18for k in sx.cmip5so.sop.keys(): 
     19  sh1[k] = sx.cmip5so.sop[k][:] 
     20sh1.close() 
     21#### 'uuid', 'mip', 'label','title','ref','refNote' 
     22####  i, "CMIP6",  key, ...., "CMIP5", '' 
     23cmip5tbs = [] 
     24cmip5gps = [] 
     25cmip5gplk = {} 
     26for k in sx.cmip5so.so.keys(): 
     27  u = str( uuid.uuid1() ) 
     28  cmip5gplk[k] = u 
     29  cmip5tbs.append( [u,"CMIP6",k,"CMIP6 CMOR Table: %s" % k, "CMIP5", "CMIP5 MIP Table: %s" % k] ) 
     30for k in sx.cmip5so.sop.keys(): 
     31  u = str( uuid.uuid1() ) 
     32  cmip5gplk[k] = u 
     33  t,s = string.split( k, '_' ) 
     34  assert sx.cmip5so.so.has_key(t), 'Table for group %s not found' % k 
     35  u2 = cmip5gplk[t] 
     36  cmip5gps.append( [u,u2,"CMIP6",k,"CMIP6 Variable Group: %s" % k, "CMIP5", "CMIP5 MIP Table section: %s" % k] ) 
     37  u3 = str( uuid.uuid1() ) 
     38### section link .... variables in sections are linked to a section which carries a link to the group. 
     39### request links can refer to a section or a group ..... 
    1240 
    1341cmip5vgr = ['Omon','fx','Oyr','Oclim','Omon_3d','Omon_oth','Amon','Lmon','LImon','OImon','aero_3d','aero_oth','cfDay_2d','cfDay_3d','cfMon_3dstd','cfMon_3dmod','cfMon_2dmod','cfMon_sim','day_oth','day_ss','cfOff','cfSites','6hrLev','6hrPlev','3hr','cf3hr_grid','cf3hr_sim'] 
     
    2149ee_usedVars = collections.defaultdict( list ) 
    2250 
    23 kmap = { 'cfday':'cfDay', 'Limon':'LImon', 'CMIP5_Llmon':'CMIP5_LImon', 'NEW':'new', 'New':'new'} 
     51kmap = { 'cfday':'cfDay', 'Limon':'LImon', 'CMIP5_Llmon':'CMIP5_LImon', 'CMIP5 day':'CMIP5_day', 'NEW':'new', 'New':'new', \ 
     52         'CCMI monthly mean 3d':'CCMI1_monthly', \ 
     53         'CCMI Daily zonal mean 2d':'CCMI1_daily', \ 
     54         'CCMI monthly mean zonal mean 2d':'CCMI1_monthly', \ 
     55         'PMIP-LImon':'PMIP_LImon', 'CMIP5 Amon':'CMIP5_Amon', 'CCMI monthly':'CCMI1_monthly', 'PMIP-OImon':'PMIP_OImon'} 
    2456 
    2557for k in sx.eenv.keys(): 
     
    5688kmapmap = {} 
    5789for k in ee_tabs.keys(): 
    58   if string.find( k, ' ' ) != -1: 
    59     k2 = string.split(k)[0] 
     90  k1 = kmap.get( k,k) 
     91  if string.find( k1, ' ' ) != -1: 
     92    k2 = string.split(k1)[0] 
    6093  else: 
    6194    k2 = k 
     
    88121  keys.sort() 
    89122  thisd[k] = keys[:] 
    90   print k, '::: ', keys 
     123  print 'ERROR.020.0001: Missing Group: %s :: %s' % (k, keys) 
    91124    
    92125### look for new variables in variable groups. 
     
    114147## look up table, giving uuid from old mip tables for each mip, var combination. 
    115148mvlookup = collections.defaultdict( d1 ) 
     149mvlookup2 = collections.defaultdict( list ) 
    116150 
    117151ee_unusedVars = [] 
     
    130164      vid = v1 
    131165    mvlookup[t].d[vid].append( u ) 
     166 
     167for u in sx.eeold.keys(): 
     168  mvlookup2[sx.eeold[u][1]].append(  u ) 
    132169 
    133170##for k in kfn: 
     
    158195          ok = True 
    159196        else: 
    160           print 'ERROR.001.010: variable %s not found in %s' % (var,tab) 
     197          emsg = '.001.010: variable %s not found in %s' % (var,tab) 
    161198      else: 
    162           print 'ERROR.001.012: [%s] table not found: %s' % (var,tab) 
     199          emsg = '.001.012: [%s] table not found: %s' % (var,tab) 
     200 
     201      if not ok: 
     202        if mvlookup2.has_key( var ): 
     203          if len( mvlookup2[var] ) == 1: 
     204             ee0_xref[u] += ((2,[mvlookup2[var][0],]),) 
     205             ok = True 
     206             print 'WARN' + emsg + ' [alternative found]' 
     207          else: 
     208            print 'ERROR' + emsg 
    163209 
    164210      if not ok: 
    165211        print '-----',tab,sx.ee0[u] 
    166         ee0_xref[u] += ((-1,[None,]),) 
     212        ee0_xref[u] += ((-1,['__var_not_found_012__',]),) 
    167213   
    168214        if ( len(sx.ee0[u][1]) > 0 and sx.ee0[u][1][0] == '#' ) or ( len(sx.ee0[u][0]) > 0 and sx.ee0[u][0][0] == '#' ): 
     
    304350print 'INFO 001 ##################',nnf, nnm 
    305351 
    306 import shelve 
    307352##sh = shelve.open( 'shelve_vars' ) 
    308353sh = shelve.open( '%s/cmipVars' % odir, 'n' ) 
     
    417462 
    418463shlnks = shelve.open( '%s/requestLinks_tmp' % odir, 'r' ) 
     464## 
     465## read in provisional request groups --- based on request scoping sheet 
     466## 
    419467shrvg = shelve.open( '%s/requestVarGroup_tmp' % odir, 'r' ) 
    420468eervg = {} 
     
    427475    eervg[shrvg[k][2]] = k 
    428476 
    429 sh = shelve.open( 'dreq_consol_tables_reviewed_b_v20150708' ) 
    430 revTabIds = sh.keys()[:] 
     477sh = shelve.open( 'dreq_consol_tables_reviewed_b_v%s' % vdate, 'r' ) 
     478revTabIds = Set( sh.keys()[:] ) 
    431479sh.close() 
    432480 
     
    482530## 29th July: loosing some links for groups of revised table items .... 
    483531## 
    484 shb = shelve.open( 'dreq_consol_tables_reviewed_b_v20150708', 'r' ) 
     532## these are the reviewed cmip5 standard output tables. 
     533## 
     534shb = shelve.open( 'dreq_consol_tables_reviewed_b_v%s' % vdate, 'r' ) 
    485535revisedTabKeys = shb.keys() 
    486536revisedTabKeys.sort() 
    487537revisedTabKeysff = [] 
     538revisedTabKeysNM = {} 
     539tabsects = collections.defaultdict( Set ) 
     540tabsectbyvar = collections.defaultdict( dict ) 
     541tabsectuuid = collections.defaultdict( dict ) 
     542tabuuid = {} 
     543tabuuidrefs = collections.defaultdict( int ) 
    488544for r in revisedTabKeys: 
     545  if not tabuuid.has_key( r ): 
     546    tabuuid[r] = str( uuid.uuid1() ) 
     547 
    489548  if string.find( r, '.' ) != -1: 
    490549    mip,tab = string.split(r, '.' ) 
     
    502561        else: 
    503562          lmod.append(v) 
     563## 
     564## index sections of tables (for OMIP) 
     565## 
     566      tup = shb[r][v] 
     567      if tup[2] != None: 
     568        tabsects[mip].add( (tab,tup[2]) ) 
     569        tabsectbyvar[mip][v] = (tup[2],tup[3]) 
     570        if not tabsectuuid[mip].has_key( tup[2] ): 
     571          tabsectuuid[mip][tup[2]] = str( uuid.uuid1() ) 
     572 
    504573    if len(lnf) > 0: 
    505574      print 'ERROR.008.0001: Revised variables not in original: %s %s' % (r,str(lnf) ) 
    506575    if len(lmod) == 0: 
    507576      print 'INFO.008.0001: No modfications in %s' % r 
     577#### save uuid of table ... for use in requestLinks 
     578      revisedTabKeysNM[r] = cmip5gplk[tab] 
    508579    else: 
    509580      revisedTabKeysff.append( r ) 
    510        
     581 
     582revGpIds = Set() 
     583for t in revTabIds: 
     584  if t[-4:] == 'Omon': 
     585    revGpIds.add( t + '_oth' ) 
     586    revGpIds.add( t + '_3d' ) 
     587  elif t[-5:] == 'cfMon': 
     588    revGpIds.add( t + '_3dstd' ) 
     589    revGpIds.add( t + '_3dmod' ) 
     590    revGpIds.add( t + '_2dmod' ) 
     591    revGpIds.add( t + '_sim' ) 
     592     
    511593shb.close() 
    512594gpids = gpids + revisedTabKeysff 
     
    549631        ok = True 
    550632  if not ok: 
    551     if k in revTabIds: 
    552     ## print 'OK2: ',k 
     633## revTabIds has table names, revGpIds has group names 
     634    if k in revTabIds or k in revGpIds: 
     635      if k in revGpIds: 
     636         print 'OK2: ',k 
    553637      kmapk2[k0] = ('CMIP5Rev',k) 
    554638    else: 
     
    596680    lookup1b[id0] = id 
    597681  
     682### shlnks carries the "request scoping rows" ### 
     683### somehow not getting labels for Omon_oth etc 
    598684ndup = 0 
    599685for k in shlnks.keys(): 
     686  thistr = False 
    600687  if k[0] != '_': 
     688    if shlnks[k][2][-4:] == '_oth': 
     689      thistr = True 
    601690    k1 = '%s.%s' % (shlnks[k][1],shlnks[k][2]) 
     691    if thistr: 
     692      print k1 
    602693    assert kmapk2.has_key(k1), 'Key missing  %s, %s' % (k,k1) 
    603694    if not kmapk2.has_key(k1): 
    604       print '###########',shlnks[k] 
     695      pass 
     696      #print '###########',shlnks[k] 
    605697    else: 
    606698      ttt,kkk = kmapk2[k1] 
    607699      id = lookup1[ttt].d[kkk][0] 
     700      if thistr: 
     701         print ttt,kkk,id 
    608702    if id == k: 
    609703      print '***DUPLICATE:',k,k1, ttt, kkk, lookup1[ttt].d[kkk] 
     
    611705 
    612706assert ndup==0, ndup 
    613     ##sh[k] = list( shrvg[k] ) + list( kmapk2[id0] ) 
    614     ##print '>>>>>',k,sh[k] 
    615707 
    616708for k in lookup1.keys(): 
     
    651743    print '>>>>>>>>>',sh[id] 
    652744 
    653  
    654745for k in ['seaiceday', 'seaicemon']: 
    655746    id =  str( uuid.uuid1() ) 
     
    662753    lookup1[cls].d[refNote].append(id) 
    663754       
     755## add CMIP5 groups ......... 
     756for g in cmip5tbs: 
     757  sh[g[0]] = g[:] 
     758sh.close() 
     759sh = shelve.open( '%s/requestVarSections' % odir, 'n' ) 
     760sh['__info__'] = { 'label':'requestVarGroup', 'title':'Identify variable groups' } 
     761sh['__cols__'] = [ 'uuid', 'gpid', 'mip', 'label','title','ref','refNote'] 
     762## add CMIP5 groups ......... 
     763for g in cmip5gps: 
     764  sh[g[0]] = g[:] 
    664765sh.close() 
    665766 
     
    693794sh = shelve.open( '%s/requestLinks' % odir, 'n' ) 
    694795sh['__info__'] = { 'label':'requestLinks', 'title':'Links from variable groups to a request id' } 
    695 sh['__cols__'] = [ 'uid', 'mip', 'tab','objective','grid','gridreq','comment','ref','refNote','refid'] 
     796sh['__cols__'] = [ 'uid', 'mip', 'tab','objective','grid','gridreq','comment','opt','opar','ref','refNote','refid'] 
    696797ndup = 0 
    697798for k in shlnks.keys(): 
     
    700801    assert kmapk2.has_key(k1), 'Key missing  %s, %s' % (k,k1) 
    701802    if not kmapk2.has_key(k1): 
    702       print '###########',shlnks[k] 
     803      #print '###########',shlnks[k] 
     804      pass 
    703805    else: 
    704806      ttt,kkk = kmapk2[k1] 
    705       id = lookup1[ttt].d[kkk][0] 
     807 
     808      if revisedTabKeysNM.has_key(k1): 
     809        print 'INFO.010.0010: Replacing link for %s' % k1 
     810        id = revisedTabKeysNM[k1] 
     811      else: 
     812        id = lookup1[ttt].d[kkk][0] 
     813 
    706814      sh[k] = list( shlnks[k] ) + list( kmapk2[k1] ) + [id,] 
    707815    if sh[k][0] == sh[k][-1]: 
    708816      print 'DUPLICATE LINK:',k,sh[k] 
    709817      ndup += 1 
    710 print 'Numbr of duplicates %s (%s)' % (ndup, len(sh.keys())) 
     818print 'INFO: Number of duplicates %s (%s)' % (ndup, len(sh.keys())) 
    711819sh.close() 
    712820## link to group items is now via mip + group id, using stuff from kmapk2 .... 
     
    739847                          'type', 'ok_min_mean_abs', 'standard_name', 'deflate', 'long_name', 'valid_min',\ 
    740848                           'cell_methods', 'flag_values', 'cell_measures', 'out_name', 'modeling_realm', 'units',\ 
    741                            '#cell_methods', 'valid_max', 'positive', 'var', 'mipTable','dimensions','vid']  
     849                           '#cell_methods', 'valid_max', 'positive', 'var', 'mipTable','dimensions','vid','gpid']  
     850 
     851## added gpid for link to variable group .... not clear at this point how OMIP to be dealt with here. 
     852## for sections, perhaps need new section linking sections to groups .... 
     853 
    742854 vdefex = {} 
     855 rwixkkk = 0 
    743856 for k in sx.ee_mip.keys(): 
     857  gpid = '__group_unknown__' 
     858  rwix = 0 
     859  mipt = sx.ee_mip[k][22] 
     860  if tabuuid.has_key( mipt ): 
     861    gpid = tabuuid[mipt] 
     862  if mipt[:6] == "CMIP5_": 
     863    m,t = string.split( mipt, '_' ) 
     864    assert cmip5gplk.has_key(t), 'ERROR.009.0010: CMIP5 table link not found for %s' % mipt 
     865    gpid = cmip5gplk[t] 
     866  elif mipt[:5] == "OMIP.": 
     867    m,t = string.split( mipt, '.' ) 
     868# output name 
     869    v0 = sx.ee_mip[k][15] 
     870# variable name 
     871    v1 = sx.ee_mip[k][21] 
     872    if type(v0) == type( 'y' ) and v0 not in ['', ' ']: 
     873      v1 = v0 
     874## check to see if sections are defined. 
     875    if tabsectbyvar.has_key(m): 
     876      print m, sx.ee_mip[k][15], v0, tabsectbyvar[m].has_key(v1), sx.ee_mip[k] 
     877      if tabsectbyvar[m].has_key(v1): 
     878        gpid = tabsectuuid[m][ tabsectbyvar[m][v1][0] ] 
     879        rwix = tabsectbyvar[m][v1][1] 
     880        rwixkkk += 1 
     881    ##    tabsects[mip].add( (tab,tup[2]) ) 
     882        ##tabsectbyvar[mip][v] = tup[2] 
     883        ##if not tabsectuuid[mip].has_key( tup[2] ): 
     884          ##tabsectuuid[mip][tup[2]] = str( uuid.uuid1() ) 
     885    if gpid == '__group_unknown__': 
     886      print 'ERROR.040.0010: OMIP group not found %s, %s, %s' % (mipt,v0,)  
    744887  omit = False 
    745888  if not sx.ee_xref_bck.has_key( k ): 
     
    758901    if len(kl) != 1: 
    759902      print 'ERROR.002.0001: multiple back references %s' % str(kl) 
    760       print sx.ee_mip[k] 
     903      #print sx.ee_mip[k] 
    761904    kv = kl[0] 
    762905    r = sx.eeold[kv] 
     
    777920    kl = [kv,] 
    778921  if not omit: 
     922## count refs, for use in determining where to create group records .... 
     923    tabuuidrefs[gpid] += 1 
    779924    if r[1] != vnam0: 
    780925      print 'INFO.0001.001: name change:',r,vnam0 
    781926    vdefex[k] = list(r) + [vnam0,]   
    782     sh[k] = sx.ee_mip[k][:]  + [kl[0],] 
     927    if len( sx.ee_mip[k][:] ) != 24: 
     928      print 'ERROR.015.0002: Bad record length %s: %s' % (k,sx.ee_mip[k]) 
     929    sh[k] = sx.ee_mip[k][:]  + [kl[0],gpid,rwix]  
    783930 sh.close() 
    784931 
    785  sha = shelve.open( 'dreq_consol_tables_reviewed_a_v20150708' ) 
    786  shb = shelve.open( 'dreq_consol_tables_reviewed_b_v20150708' ) 
     932 sha = shelve.open( 'dreq_consol_tables_reviewed_a_v%s' % vdate ) 
     933 shb = shelve.open( 'dreq_consol_tables_reviewed_b_v%s' % vdate ) 
    787934 keys = shb.keys() 
    788935 keys.sort() 
    789936 k0 = [] 
    790  for k in keys: 
    791   bb = string.split(k, '.' ) 
    792   if len(bb) == 2: 
    793     k0.append( tuple(bb) ) 
     937 for k in revisedTabKeysff: 
     938   bb = string.split(k, '.' ) 
     939   if len(bb) == 2: 
     940     k0.append( tuple(bb) ) 
    794941 
    795942 sh = shelve.open( '%s/revTabItems' % odir, 'n' ) 
    796  sh['__info__'] =  {'label':'revTabItems', 'title':'List of items in revised tables', 'description':"uuid (col 3) references an item in 'refTableItems'", 'prov':'from dreq_consol_tables_reviewed_b_v20150708, cross-referenced with sx.ee_miptables_old' } 
     943 sh['__info__'] =  {'label':'revTabItems', 'title':'List of items in revised tables', 'description':"uuid (col 3) references an item in 'refTableItems'", 'prov':'from dreq_consol_tables_reviewed_b_v%s, cross-referenced with sx.ee_miptables_old' % vdate } 
    797944 sh['__cols__'] = ['var','table','mip','uuid','priority'] 
    798945#### 
    799946 dop = dopen('check3') 
    800947 kr = 0 
     948 krn = 0 
    801949 
    802950 for m,t in k0: 
     
    827975        elif v[:7] == 'include': 
    828976          thisu = '__inc__' 
    829         elif m == 'OMIP' and gg.has_key(v): 
    830           print 'INFO ... new OMIP var found',v 
    831           thisu = '__new__' 
    832977        else: 
    833978          thisu = None 
     
    846991               print 'WARN.003.0001 var not found -- alternative table used [%s]: %s (%s, %s)' % (cl0[0],v,m,t) 
    847992               thisu = sx.ee_byvar[v][ix] 
    848                print sx.ee_mip[thisu] 
    849993                
    850994             ##print map( lambda x: sx.ee_mip[x][-2], sx.ee_byvar[v] ) 
     995          elif m == 'OMIP' and gg.has_key(v): 
     996            print 'INFO ... new OMIP var found',v 
     997            thisu = '__new.OMIP__' 
     998            krn += 1 
     999            id =  str( uuid.uuid1() ) 
     1000            dop.a['omip3'].write( string.join( [id,] + map(lambda x: r[x],[4,0,5,2,7]) + ['%s_%s' % (m,t),'mv.006.%4.4i' % krn], '\t' ) + '\n' ) 
    8511001           
    8521002          if thisu == None: 
    8531003            print 'ERROR.003.0001 var not found: %s (%s, %s)' % (v,m,t) 
     1004            thisu = "__var_not_found__" 
    8541005          if m == 'OMIP': 
    8551006            kr += 1 
    8561007            sr =  map(str,r) 
    8571008            dop.a['omip'].write( string.join( map(str,r), '\t' ) + '\n' ) 
    858             dop.a['omip2'].write( string.join( map(lambda x: sr[x],[4,0,5,2,7]) + ['%s_%s' % (m,t),'mv.005.%4.4i' % kr], '\t' ) + '\n' ) 
     1009            dop.a['omip2'].write( string.join( map(lambda x: r[x],[4,0,5,2,7]) + ['%s_%s' % (m,t),'mv.005.%4.4i' % kr], '\t' ) + '\n' ) 
    8591010           
    8601011      else: 
     
    8641015 dop.close('omip',rename='omip.csv') 
    8651016 dop.close('omip2',rename='omip2.csv') 
     1017 dop.close('omip3',rename='omip3.csv') 
    8661018 sh.close() 
     1019 
     1020 sh = shelve.open( '%s/requestVarGroup' % odir ) 
     1021## add OMIP (+ ....?) 
     1022 nomip = 0 
     1023 for k in tabuuid.keys(): 
     1024   if k[0] != '_': 
     1025  ####cmip5tbs.append( [u,"CMIP6",k,"CMIP6 CMOR Table: %s" % k, "CMIP5", "CMIP5 MIP Table: %s" % k] ) 
     1026     if tabuuidrefs[tabuuid[k]]  > 0: 
     1027        mip,tab = string.split( k, '.' ) 
     1028        u = tabuuid[k] 
     1029        sh[u] = [u,mip,tab,"CMOR Table variant: %s" % k, "CMIP5Rev", k ] 
     1030####'OMIP', u'OMIP-fx', u'OMIP: OMIP-fx', 'CMIP5Rev', u'OMIP.fx' 
     1031        print 'INFO: ADDING GROUP: %s' % str(sh[u]) 
     1032        nomip += 1 
     1033 sh.close() 
     1034 print nomip 
     1035 
Note: See TracChangeset for help on using the changeset viewer.