Changeset 442 for CMIP6dreqbuild


Ignore:
Timestamp:
24/10/15 23:05:00 (7 years ago)
Author:
mjuckes
Message:

Preparing to split documents as a separate package

Location:
CMIP6dreqbuild/trunk
Files:
3 added
12 edited

Legend:

Unmodified
Added
Removed

  • CMIP6dreqbuild/trunk/docs/vocab.txt

    r397 r442  
    3737 - iso 
    3838 - uid 
     39 - isClimatology {True if term is a climatology label} [xs:boolean] 
    3940## vsf=f 
    4041vocab realm; Modeling realm; cmip.drv.004; def; 0; 1; Yes 
    … …  
    9798 - url 
    9899 - uid 
     100vocab modelConfig; Model configuration options; cmip.drv.004; an; 0; 1; Yes 
     101 - title 
     102 - MIPs 
     103 - usage 
     104 - type 
     105 - range  {Range of valid values, e.g. xs:boolean} 

  • CMIP6dreqbuild/trunk/docs/vocabFrameworkSchema.xsd

    r397 r442  
    2626  <xs:attribute name="title" type="xs:string"/> 
    2727  <xs:attribute name="type" type="xs:string"/> 
    28   <xs:attribute name="class" type="xs:string"/> 
     28  <xs:attribute name="useClass" type="xs:string"/> 
    2929  <xs:attribute name="techNote" type="xs:string"/> 
     30  <xs:attribute name="superclass" type="xs:string"/> 
    3031</xs:complexType> 
    3132 

  • CMIP6dreqbuild/trunk/src/framework/Makefile

    r429 r442  
    6262        bash seddreq02.sh "\.\.\/out" "\.\.\/docs" dreqPy/__init__.py > /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy/__init__.py 
    6363        cp htmlTemplates.py dreqPy/scope.py dreqPy/example.py dreqPy/utilities.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy   
     64        cp dreqPy/dreqCmdl.py  dreqPy/simpleCheck.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy 
    6465        cp vocabDemo.py /data/tmp/svn3/exarch/CMIP6dreq/trunk/dreqPy     
    6566        cp out/annotated_20150731.xml /data/tmp/svn3/exarch/CMIP6dreq/trunk/docs/dreq.xml 

  • CMIP6dreqbuild/trunk/src/framework/dreqPy/__init__.py

    r433 r442  
    11 
    2 import os 
     2from packageConfig import * 
    33 
    4 HERE = os.path.dirname(os.path.abspath(__file__) ) 
    5  
    6 ##DOC_DEFAULT_DIR 
    7 DOC_DEFAULT_DIR = os.path.join(HERE, '../out') 
    8  
    9 DOC_DIR = os.environ.get('DRQ_CONFIG_DIR', DOC_DEFAULT_DIR) 
    10  
    11 __versionComment__ = "Support for command line interface added" 
    12 __version__ = "01.beta.09" 
    13 __title__ = "dreqPy" 
    14 __description__ = "CMIP6 Data Request Python API" 
    15 __uri__ = "http://proj.badc.rl.ac.uk/svn/exarch/CMIP6dreq/tags/{0}".format(__version__) 
    16 __author__ = "Martin Juckes" 
    17 __email__ = "martin.juckes@stfc.ac.uk" 
    18 __license__ = "BSD" 
    19 __copyright__ = "Copyright (c) 2015 Science & Technology Facilities Council (STFC)" 

  • CMIP6dreqbuild/trunk/src/framework/dreqPy/dreq.py

    r433 r442  
    5858           for a in self.__dict__.keys(): 
    5959             if a[0] != '_' or full: 
    60                if self._a[a].rClass == 'internalLink' and self._base._indexInitialised: 
     60               if self._a[a].useClass == 'internalLink' and self._base._indexInitialised: 
    6161                 if self.__dict__[a] in self._base._inx.uid: 
    6262                   targ = self._base._inx.uid[ self.__dict__[a] ] 
    … …  
    8989           for a in self.__dict__.keys(): 
    9090             if a[0] != '_': 
    91                if self._a[a].rClass == 'internalLink' and self._base._indexInitialised: 
     91               if self._a[a].useClass == 'internalLink' and self._base._indexInitialised: 
    9292                 if self.__dict__[a] == '__unset__': 
    9393                   m = '<li>%s: %s [missing link]</li>' % ( a, self.__dict__[a] ) 
    … …  
    222222    self.vsamp = thisdoc 
    223223    self.nts = collections.namedtuple( 'sectdef', ['tag','label','title','id','itemLabelMode','level'] ) 
    224     self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type','rClass','techNote'] ) 
     224    self.nti = collections.namedtuple( 'itemdef', ['tag','label','title','type','useClass','techNote'] ) 
    225225    self.ntt = collections.namedtuple( 'sectinit', ['header','attributes','defaults'] ) 
    226226    self.nt__default = collections.namedtuple( 'deflt', ['defaults','glob'] ) 
    … …  
    325325The instanstiated object contains a single data record. The "_h" attribute links to information about the record and the section it belongs to.  
    326326 
    327 object._a: a python dictionary defining the attributes in each record. The keys in the dictionary correspond to the attribute names and the values are python "named tuples" (from the "collections" module). E.g. object._a['priority'].type contains the type of the 'priority' attribute. Type is expressed using XSD schema language, so "xs:integer" implies integer.  The "rClass" attribute carries information about usage. If object._a['xxx'].rClass = u'internalLink' then the record attribute provides a link to another element and object.xxx is the unique identifier of that element. 
     327object._a: a python dictionary defining the attributes in each record. The keys in the dictionary correspond to the attribute names and the values are python "named tuples" (from the "collections" module). E.g. object._a['priority'].type contains the type of the 'priority' attribute. Type is expressed using XSD schema language, so "xs:integer" implies integer.  The "useClass" attribute carries information about usage. If object._a['xxx'].useClass = u'internalLink' then the record attribute provides a link to another element and object.xxx is the unique identifier of that element. 
    328328 
    329329object._h: a python named tuple describing the section. E.g. object._h.title is the section title (E.g. "CMOR Variables") 
    … …  
    361361      defs = {'type':"xs:string"} 
    362362      ll = [] 
    363       for k in ['label','title','type','class','techNote']: 
     363      for k in ['label','title','type','useClass','techNote']: 
    364364        if i.hasAttribute( k ): 
    365365          ll.append( i.getAttribute( k ) ) 
    … …  
    408408    ## 
    409409      for ka in dreq[k].attDefn.keys(): 
    410         if dreq[k].attDefn[ka].rClass == 'internalLink': 
     410        if dreq[k].attDefn[ka].useClass == 'internalLink': 
    411411           irefdict[k].append( ka ) 
    412412 

  • CMIP6dreqbuild/trunk/src/framework/out/dreq2Defn.xml

    r424 r442  
    77  <rowAttribute label="label" title="Record Label"/> 
    88  <rowAttribute label="title" title="Record Title"/> 
    9   <rowAttribute label="sn" title="CF Standard Name" class="externalTextRef"/> 
     9  <rowAttribute label="sn" title="CF Standard Name" useClass="internalLink"/> 
    1010  <rowAttribute label="units" title="Units"/> 
    1111  <rowAttribute label="description"/> 
    1212  <rowAttribute label="title" title="Long name"/> 
    13   <rowAttribute label="procnote" title="Processing notes" class="list"/> 
    14   <rowAttribute label="procComment" title="Processing comments" class="freeText"/> 
    15   <rowAttribute label="prov" title="Provenance" class="freeText"/> 
    16   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
     13  <rowAttribute label="procnote" title="Processing notes" useClass="list"/> 
     14  <rowAttribute label="procComment" title="Processing comments" useClass="freeText"/> 
     15  <rowAttribute label="prov" title="Provenance" useClass="freeText"/> 
     16  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
    1717</table> 
    1818<table label="CMORvar" uid="SECTION:CMORvar" title="CMOR Variable" id="cmip.drv.002" itemLabelMode="def" level="1" maxOccurs="1" labUnique="No"> 
    1919  <rowAttribute label="label" title="Record Label"/> 
    2020  <rowAttribute label="title" title="Record Title"/> 
    21   <rowAttribute label="uid" class="recordId"/> 
     21  <rowAttribute label="uid" useClass="recordId"/> 
    2222  <rowAttribute label="title" title="Long name"/> 
    23   <rowAttribute label="stid" title="Link to a record specifying the structure of the variable (dimensions and associated variable attributes)." class="internalLink" techNote="structure"/> 
    24   <rowAttribute label="vid" title="Identifier for MIP Variable" class="internalLink" techNote="var"/> 
     23  <rowAttribute label="stid" title="Link to a record specifying the structure of the variable (dimensions and associated variable attributes)." useClass="internalLink" techNote="structure"/> 
     24  <rowAttribute label="vid" title="Identifier for MIP Variable" useClass="internalLink" techNote="var"/> 
    2525  <rowAttribute label="valid_min" title="Minimum expected value for this variable."/> 
    2626  <rowAttribute label="valid_max" title="Maximum expected value for this variable."/> 
    … …  
    3030  <rowAttribute label="deflate_level" title="Deflate Level: NetCDF compression parameter"/> 
    3131  <rowAttribute label="shuffle" title="Shuffle: NetCDF compression parameter"/> 
     32  <rowAttribute label="defaultPriority" type="xs:integer" title="Indicative priority for this parameter, which is over-ruled by the requestVar priority setting, but provides a reference for organisation of the CMORvariables"/> 
    3233  <rowAttribute label="type"/> 
    33   <rowAttribute label="modeling_realm" class="drsVocab"/> 
    34   <rowAttribute label="positive" class="CMORdirective"/> 
    35   <rowAttribute label="mipTable" title="The MIP table: each table identifies a collection of variables" class="drsVocab"/> 
     34  <rowAttribute label="modeling_realm" useClass="drsVocab"/> 
     35  <rowAttribute label="positive" useClass="CMORdirective"/> 
     36  <rowAttribute label="mipTable" title="The MIP table: each table identifies a collection of variables" useClass="drsVocab"/> 
    3637  <rowAttribute label="prov"/> 
    3738  <rowAttribute label="provNote"/> 
    38   <rowAttribute label="frequency" title="Frequency of time steps to be archived." class="drsVocab"/> 
     39  <rowAttribute label="frequency" title="Frequency of time steps to be archived." useClass="drsVocab"/> 
    3940  <rowAttribute label="rowIndex" type="xs:integer" title="Row index of entry in source sheet"/> 
    4041  <rowAttribute label="description"/> 
    … …  
    4344  <rowAttribute label="label" title="Record Label"/> 
    4445  <rowAttribute label="title" title="Record Title"/> 
    45   <rowAttribute label="uid" class="recordId"/> 
    46   <rowAttribute label="spid" class="internalLink" techNote="spatialShape"/> 
    47   <rowAttribute label="tmid" class="internalLink" techNote="temporalShape"/> 
     46  <rowAttribute label="uid" useClass="recordId"/> 
     47  <rowAttribute label="spid" useClass="internalLink" techNote="spatialShape"/> 
     48  <rowAttribute label="tmid" useClass="internalLink" techNote="temporalShape"/> 
    4849  <rowAttribute label="odims"/> 
    4950  <rowAttribute label="coords"/> 
    … …  
    5960  <rowAttribute label="label" title="Record Label"/> 
    6061  <rowAttribute label="title" title="Record Title"/> 
    61   <rowAttribute label="uid" class="recordId"/> 
     62  <rowAttribute label="uid" useClass="recordId"/> 
    6263  <rowAttribute label="dimensions"/> 
    6364  <rowAttribute label="shape"/> 
    … …  
    6869  <rowAttribute label="label" title="Record Label"/> 
    6970  <rowAttribute label="title" title="Record Title"/> 
    70   <rowAttribute label="uid" class="recordId"/> 
     71  <rowAttribute label="uid" useClass="recordId"/> 
    7172  <rowAttribute label="dimensions"/> 
    7273  <rowAttribute label="label"/> 
    … …  
    7677  <rowAttribute label="label" title="Record Label"/> 
    7778  <rowAttribute label="title" title="Record Title"/> 
    78   <rowAttribute label="uid" class="recordId"/> 
     79  <rowAttribute label="uid" useClass="recordId"/> 
    7980  <rowAttribute label="priority" type="xs:integer"/> 
    80   <rowAttribute label="vid" title="Identifier for MIP Output Variable" class="internalLink" techNote="ovar"/> 
    81   <rowAttribute label="vgid" title="Identifier for Variable Group" class="internalLink" techNote="requestVarGroup"/> 
     81  <rowAttribute label="vid" title="Identifier for MIP Output Variable" useClass="internalLink" techNote="ovar"/> 
     82  <rowAttribute label="vgid" title="Identifier for Variable Group" useClass="internalLink" techNote="requestVarGroup"/> 
    8283  <rowAttribute label="mip"/> 
    8384  <rowAttribute label="table"/> 
    … …  
    8687  <rowAttribute label="label" title="Record Label"/> 
    8788  <rowAttribute label="title" title="Record Title"/> 
    88   <rowAttribute label="uid" class="recordId"/> 
    89   <rowAttribute label="gpid" title="Identifier for CMOR Tables" class="internalLink" techNote="requestVarGroup"/> 
     89  <rowAttribute label="uid" useClass="recordId"/> 
     90  <rowAttribute label="gpid" title="Identifier for CMOR Tables" useClass="internalLink" techNote="requestVarGroup"/> 
    9091  <rowAttribute label="mip" title="Project"/> 
    9192  <rowAttribute label="ref"/> 
    … …  
    9596  <rowAttribute label="label" title="Record Label"/> 
    9697  <rowAttribute label="title" title="Record Title"/> 
    97   <rowAttribute label="uid" class="recordId"/> 
     98  <rowAttribute label="uid" useClass="recordId"/> 
    9899  <rowAttribute label="mip" title="Endorsed MIP requesting the data"/> 
    99100  <rowAttribute label="ref"/> 
    … …  
    103104  <rowAttribute label="label" title="Record Label"/> 
    104105  <rowAttribute label="title" title="Record Title"/> 
    105   <rowAttribute label="uid" class="recordId"/> 
     106  <rowAttribute label="uid" useClass="recordId"/> 
    106107  <rowAttribute label="mip"/> 
    107108  <rowAttribute label="tab"/> 
    … …  
    112113  <rowAttribute label="ref"/> 
    113114  <rowAttribute label="refNote"/> 
    114   <rowAttribute label="refid" title="reference to a request Variable Group" class="internalLink"/> 
     115  <rowAttribute label="refid" title="reference to a request Variable Group" useClass="internalLink"/> 
    115116  <rowAttribute label="opt" title="option for selecting a subset of variables"/> 
    116117  <rowAttribute label="opar" title="parameter associated with *opt*"/> 
    … …  
    120121  <rowAttribute label="label" title="Record Label"/> 
    121122  <rowAttribute label="title" title="Record Title"/> 
    122   <rowAttribute label="uid" class="recordId"/> 
     123  <rowAttribute label="uid" useClass="recordId"/> 
    123124  <rowAttribute label="mip"/> 
    124125  <rowAttribute label="tab"/> 
    125126  <rowAttribute label="expt"/> 
    126   <rowAttribute label="rlid" title="Identifier of corresponding requestLink" class="internalLink"/> 
    127   <rowAttribute label="esid" title="Identifier experiment(s): a link to an experiment, an experiment group or a MIP" class="internalLink"/> 
     127  <rowAttribute label="rlid" title="Identifier of corresponding requestLink" useClass="internalLink"/> 
     128  <rowAttribute label="esid" title="Identifier experiment(s): a link to an experiment, an experiment group or a MIP" useClass="internalLink"/> 
    128129  <rowAttribute label="esidComment"/> 
    129130  <rowAttribute label="ny" type="xs:integer"/> 
    … …  
    135136  <rowAttribute label="label" title="Record Label"/> 
    136137  <rowAttribute label="title" title="Record Title"/> 
    137   <rowAttribute label="uid" class="recordId"/> 
    138   <rowAttribute label="tid" title="Target identifier: the record ID of the item this refers to." class="internalLink"/> 
     138  <rowAttribute label="uid" useClass="recordId"/> 
     139  <rowAttribute label="tid" title="Target identifier: the record ID of the item this refers to." useClass="internalLink"/> 
    139140  <rowAttribute label="tattr" title="Target attribute: an attribute of the target item, or 'ALL'"/> 
    140141  <rowAttribute label="description" title="Free text remarks -- unless there are specific restrictions associated with the class attribute of this remark"/> 
    … …  
    147148  <rowAttribute label="label" title="Record Label"/> 
    148149  <rowAttribute label="title" title="Record Title"/> 
    149   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
    150   <rowAttribute label="description" title="Description" class="freeText"/> 
    151   <rowAttribute label="egid" title="Identifier for experiment group" class="internalLink"/> 
    152   <rowAttribute label="mip" title="MIP defining experiment" class="internalLink"/> 
     150  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
     151  <rowAttribute label="description" title="Description" useClass="freeText"/> 
     152  <rowAttribute label="egid" title="Identifier for experiment group" useClass="internalLink"/> 
     153  <rowAttribute label="mip" title="MIP defining experiment" useClass="internalLink"/> 
    153154  <rowAttribute label="mcfg" title="Model category"/> 
    154155  <rowAttribute label="tier" type="xs:integer" title="Tier of experiment"/> 
    … …  
    164165  <rowAttribute label="label" title="Record Label"/> 
    165166  <rowAttribute label="title" title="Record Title"/> 
    166   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
     167  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
    167168  <rowAttribute label="tierMin" type="xs:integer" title="Minimum tier of experiments in group"/> 
    168169  <rowAttribute label="ntot" type="xs:integer" title="Total number of years"/> 
    … …  
    171172  <rowAttribute label="label" title="Record Label"/> 
    172173  <rowAttribute label="title" title="Record Title"/> 
    173   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
    174   <rowAttribute label="description" title="Description" class="freeText"/> 
     174  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
     175  <rowAttribute label="description" title="Description" useClass="freeText"/> 
    175176  <rowAttribute label="title" title="Long name"/> 
    176177  <rowAttribute label="mip" title="Endorsed MIP"/> 
    … …  
    179180  <rowAttribute label="label" title="Record Label"/> 
    180181  <rowAttribute label="title" title="Record Title"/> 
    181   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
    182   <rowAttribute label="oid" title="Identifier for a scientific objective" class="internalLink" techNote="objective"/> 
    183   <rowAttribute label="rid" title="Identifier for a request link" class="internalLink" techNote="requestLink"/> 
     182  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
     183  <rowAttribute label="oid" title="Identifier for a scientific objective" useClass="internalLink" techNote="objective"/> 
     184  <rowAttribute label="rid" title="Identifier for a request link" useClass="internalLink" techNote="requestLink"/> 
    184185</table> 
    185186<table label="varChoiceLinkR" uid="SECTION:varChoiceLinkR" title="Links a variable to a choice element" id="cmip.drv.016" itemLabelMode="def" level="0" maxOccurs="1" labUnique="Yes"> 
    186187  <rowAttribute label="label" title="Record Label"/> 
    187188  <rowAttribute label="title" title="Record Title"/> 
    188   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
    189   <rowAttribute label="vid" title="Variable" class="internalLink"/> 
    190   <rowAttribute label="cid" title="Choice" class="internalLink"/> 
     189  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
     190  <rowAttribute label="vid" title="Variable" useClass="internalLink"/> 
     191  <rowAttribute label="cid" title="Choice" useClass="internalLink"/> 
    191192  <rowAttribute label="rank" type="xs:integer" title="For ranked choices, the rank of this variable (higher rank makes lower ranks redundant)"/> 
    192193</table> 
    … …  
    194195  <rowAttribute label="label" title="Record Label"/> 
    195196  <rowAttribute label="title" title="Record Title"/> 
    196   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
    197   <rowAttribute label="vid" title="Variable" class="internalLink"/> 
    198   <rowAttribute label="cfgid" title="Configuration Option" class="internalLink"/> 
     197  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
     198  <rowAttribute label="vid" title="Variable" useClass="internalLink"/> 
     199  <rowAttribute label="cfgid" title="Configuration Option" useClass="internalLink"/> 
    199200  <rowAttribute label="cfg" type="xs:boolean" title="Configuration Value"/> 
    200   <rowAttribute label="cid" title="Choice -- can provide a link to related variables" class="internalLink"/> 
     201  <rowAttribute label="cid" title="Choice -- can provide a link to related variables" useClass="internalLink"/> 
    201202</table> 
    202203<table label="varChoice" uid="SECTION:varChoice" title="Indicates variables for which a there is a range of potential CMOR Varibles" id="cmip.drv.018" itemLabelMode="def" level="0" maxOccurs="1" labUnique="Yes"> 
    203204  <rowAttribute label="label" title="Record Label"/> 
    204205  <rowAttribute label="title" title="Record Title"/> 
    205   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
     206  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
    206207  <rowAttribute label="class" title="Class of choice: heirarchy|cfg"/> 
    207208  <rowAttribute label="title"/> 
    … …  
    213214  <rowAttribute label="label" title="Record Label"/> 
    214215  <rowAttribute label="title" title="Record Title"/> 
    215   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
     216  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
    216217  <rowAttribute label="title"/> 
    217218  <rowAttribute label="description"/> 
    … …  
    222223  <rowAttribute label="label" title="Record Label"/> 
    223224  <rowAttribute label="title" title="Record Title"/> 
    224   <rowAttribute label="uid" title="Record identifier" class="recordId"/> 
     225  <rowAttribute label="uid" title="Record identifier" useClass="recordId"/> 
    225226  <rowAttribute label="title"/> 
    226227  <rowAttribute label="MIPs"/> 
    … …  
    229230  <rowAttribute label="range" title="Range of valid values, e.g. xs:boolean"/> 
    230231</table> 
     232<table label="standardname" uid="SECTION:standardname" title="CF Standard Names" id="cmip.drv.021" itemLabelMode="an" level="0" maxOccurs="1" labUnique="Yes"> 
     233  <rowAttribute label="label" title="Record Label"/> 
     234  <rowAttribute label="title" title="Record Title"/> 
     235  <rowAttribute label="uid" title="CF Standard Name" useClass="recordId"/> 
     236  <rowAttribute label="description"/> 
     237  <rowAttribute label="title"/> 
     238  <rowAttribute label="label"/> 
     239  <rowAttribute label="units" title="Canonical Units"/> 
     240</table> 
    231241</defDoc> 

  • CMIP6dreqbuild/trunk/src/framework/ptxt.py

    r431 r442  
    7272ial_elTmpl = '  <rowAttribute label="%(label)s"%(wrappedType)s%(wrappedTitle)s%(wrappedClass)s%(wrappedTechn)s/>' 
    7373 
    74 expl_Tmpl = '''<%(label)s uid="%(uid)s" class="vocab" title="%(title)s" id="%(id)s"> 
     74expl_Tmpl = '''<%(label)s uid="%(uid)s" useClass="vocab" title="%(title)s" id="%(id)s"> 
    7575<!-- <info srcType="dummy" srcRef="ptxt.py">Dummy entries</info> --> 
    7676%(exampleItem)s 
    … …  
    104104       for i in self.itematts: 
    105105         if i.clss != None: 
    106            wrappedClass = ' class="%s"' % i.clss 
     106           wrappedClass = ' useClass="%s"' % i.clss 
    107107         else: 
    108108           wrappedClass = '' 

  • CMIP6dreqbuild/trunk/src/framework/scanDreq.py

    r430 r442  
    253253   xr_var_ovar[vid].append( i.uid ) 
    254254   if not inx.var.uid.has_key(vid): 
    255      print 'missing key:',i.label, i.prov 
     255     print 'missing key:',i.label, i.prov, vid 
    256256     nerr += 1 
    257257   else: 
    … …  
    519519        this.removeChild(d) 
    520520    for k in mrefs.keys(): 
     521      print '>>>>>>>>>>> ',k 
    521522      if len(  mrefs[k] ) == 1: 
    522523        tid = mrefs[k][0][2] 

  • CMIP6dreqbuild/trunk/srcMisc/dreqSX.py

    r382 r442  
    4646eeold = {} 
    4747###for ee,shv in [(ee0,shvg),(eenv,shnv),(eeold,shold)]: 
    48 for ee,shv in [(ee0,shvg),(eenv,shnv)]: 
    49   for k in shv.keys(): 
    50     ee[k] = shv[k] 
     48##for ee,shv in [(ee0,shvg),(eenv,shnv)]: 
     49for k in shnv: 
     50  eenv[k] = shnv[k] 
     51ccl = collections.defaultdict( int ) 
     52for k in shvg: 
     53  ee0[k] = shvg[k] 
     54  ccl[len(ee0[k])] += 1 
     55 
     56print 'tables ... groups ....',ccl 
     57   
    5158 
    5259rmod = 'csv' 

  • CMIP6dreqbuild/trunk/srcMisc/dreq_consol_tables.py

    r382 r442  
    412412    kk=3 
    413413    self.idx = 0 
     414    self.vl3 = None 
    414415    self.sh = shelve.open( 'dreq_consol_tables_shelve_v%s' % vdate, 'n' ) 
    415416    self.shnv = shelve.open( 'dreq_consol_tables_nv_shelve_v%s' % vdate, 'n' ) 
     417    self.shnvo = shelve.open( 'dreq_consol_tables_nvom_shelve_v%s' % vdate, 'n' ) 
    416418    self.shold = shelve.open( 'dreq_consol_tables_old_shelve_v%s' % vdate, 'n' ) 
     419    self.oonvo = open('DepricatedNewVars.txt', 'w' ) 
    417420 
    418421    base = '/home/martin/2014/wip/dreq/' 
    … …  
    422425    fileTmpl = 'CMIP6DataRequestCompilationTemplate_20141218.xls' 
    423426    self.newVars = 'CMIP6DataRequest_ConsolidatedNewVariables_v20150907.xls' 
     427    self.newVars = 'CMIP6DataRequest_ConsolidatedNewVariables.xls' 
    424428    self.nvd = {} 
    425429 
    … …  
    496500            try: 
    497501              if m == 'LUMIP': 
    498                 p = int( r[12] ) 
    499               else: 
    500                 p = int( r[11] ) 
     502                r = r[:5] + r[6:] 
     503              p = int( r[11] ) 
    501504            except: 
    502505              print '!!!!!! Failed to read priority: ',v,m 
    … …  
    509512            self.nvd[v] = [p,] + r  
    510513    for v in self.nvd.keys(): 
    511         k = str( uuid.uuid1() ) 
    512         self.shnv[k] = self.nvd[v][:] + [k,] 
     514        if v in self.vl3: 
     515          print 'WARN.090.08000: "new" variable in old list: %s: %s ' % (v,str(self.vl3[v]) ) 
     516          self.shnvo[v] = ( self.nvd[v][:], self.vl3[v] ) 
     517          self.oonvo.write( '-- %s --\n' % v ) 
     518          self.oonvo.write( str( self.nvd[v][:]) + '\n' ) 
     519          self.oonvo.write( str( self.vl3[v][:] ) + '\n' ) 
     520        else: 
     521          k = str( uuid.uuid1() ) 
     522          self.shnv[k] = self.nvd[v][:] + [k,] 
    513523    oo.close() 
    514524 
    515   def groups(self, vl3=None): 
     525  def groups(self): 
     526    vl3 = self.vl3 
    516527    rh1 = ['Short name', 'Standard Name', 'Table', 'Frequency', 'Description extension (optional)', 'Shape', 'Levels', 'Time mean, point or climatology', 'Mask (optional)'] 
    517528    rh2 = ['Short name of group', 'Variable short name', 'Table', 'Frequency', 'Description extension (optional)', 'Shape', 'Levels', 'Time mean, point or climatology', 'Mask (optional)'] 
    … …  
    557568               thisl = 9 
    558569           else: 
    559              print '*************',k,s 
     570             print 'ERROR.090.090010: duplicate record ....',k,s 
    560571             ok = len( hr ) >= 9 and all( map( lambda x: string.strip(hr[x].value) == rh1[x], range(9) ) ) 
    561572             assert ok, '002: Sheet heading not recognised: %s' % str(hr) 
    … …  
    567578           irsh = 5 
    568579           for i in range(ll[-1]+1,sh.nrows): 
     580             rowIndex = i 
    569581             thisr = sh.row(i) 
    570582             if iv == 1: 
    … …  
    577589             if thisl == 9: 
    578590               lll.append( 105 ) 
     591 
     592             assert len(lll) == 10,'bad record length ....' 
     593### 
     594### add mip name and space ... 
    579595             lll += [k,''] 
    580596             if it >= 0: 
    … …  
    657673             ku = str( uuid.uuid1() ) 
    658674             lll[11] = ku 
    659              self.sh[ku] = tuple( lll ) 
     675             self.sh[ku] = tuple( lll  + [rowIndex,]) 
    660676###   MIP, group, size, dictionary -- # items in each shape. 
    661677           self.glist.append( (k, s, ll, e1) ) 
    … …  
    729745    wbx.write( 'CMIP6_DataRequest_CMIP5changes.xls' ) 
    730746 
    731 vl3 = {} 
    732747##wb3 = workbook( 'vlsc5b_v20150714.xls' ) 
    733748wb3 = workbook( 'vlsc5b_v20150907.xls' ) 
    … …  
    737752opt = 1 
    738753if opt == 1: 
     754  m.vl3 = {} 
    739755  for i in range(sh.nrows): 
    740756    r = map( lambda x: string.strip(str(uniCleanFunc(x.value))), sh.row(i) ) 
    741757    if len(r) < 9 or r[8] == '': 
    742       vl3[r[1]] = tuple( r[:8] ) 
     758      m.vl3[r[1]] = tuple( r[:8] ) 
    743759    m.shold[r[0]] = r 
    744  
    745760 
    746761  m.run1(nmip=1) 
    747762  m.loadNewVars() 
    748   m.groups( vl3=vl3) 
     763  m.groups( ) 
    749764  k = m.sh.keys()[4] 
    750765  print '##############################################' 
    … …  
    753768  m.sh.close() 
    754769  m.shnv.close() 
     770  m.shnvo.close() 
     771  m.oonvo.close() 
    755772  m.shold.close() 
    756773   

  • CMIP6dreqbuild/trunk/srcMisc/fcc_utils2.py

    r355 r442  
    1010class mipTableScan(object): 
    1111 
    12   def __init__(self, vats = ['standard_name','long_name','units','cell_methods'] ): 
     12  def __init__(self, vats = ['standard_name','long_name','units','cell_methods','valid_min', 'valid_max', 'ok_min_mean_abs', 'ok_max_mean_abs'] ): 
    1313    self.al = [] 
    1414    self.vats = vats 

  • CMIP6dreqbuild/trunk/srcMisc/sx2.py

    r397 r442  
    9898for k in sx.ee0.keys(): 
    9999  r = sx.ee0[k] 
    100   mip = r[-2] 
     100  mip = r[10] 
     101  assert mip == r[-3],'Double check failed' 
    101102  tab = r[2] 
    102103  grp = r[0] 
    … …  
    105106  ee_tabs[thistab].append(k) 
    106107  ee_grps[grp].append(k) 
     108 
     109 
     110import sys 
    107111 
    108112kf = [] 
    … …  
    142146  thise = collections.defaultdict( int ) 
    143147  for k2 in ee_tabs[k]: 
    144     thise[sx.ee0[k2][-2]] += 1 
     148    thise[sx.ee0[k2][12-2]] += 1 
    145149  keys = thise.keys() 
    146150  keys.sort() 
    … …  
    234238 
    235239      if not ok: 
    236         print '-----',tab,sx.ee0[u] 
     240        print 'ERROR.090.077000:',tab,sx.ee0[u] 
    237241        ee0_xref[u] += ((-1,['__var_not_found_012__',]),) 
    238242   
    … …  
    314318    ll_ov.append( ee_mv0[k] ) 
    315319print 'Number of old variables: %s [%s]' % (len(ll_ov), len(ee_mv0.keys()) ) 
    316 print  len(sx.eenv.keys()) 
     320print len(sx.eenv.keys()) 
    317321print "Number of new variables: %s" % ( len(sx.eenv.keys()) ) 
    318322## 
    … …  
    378382sh = shelve.open( '%s/cmipVars' % odir, 'n' ) 
    379383sh['__info__'] = { 'label':'cmipVars', 'title':'CMIP variables', 'prov':'sx.eenv and sx.eeold' } 
    380 sh['__cols__'] = ['label','title','sn','units','description','procnote','procComment','prov'] 
     384sh['__cols__'] = ['label','title','sn','units','description','procnote','procComment','prov','priority0'] 
    381385 
    382386ee = {} 
    … …  
    384388  this = nt_vrecix._make( map( lambda x: sx.eenv[k][x], vrecix_new ) ) 
    385389  ##sh[k] = { 'label':this.v, 'title':this.l, 'sn':this.s, 'units':this.u, 'description':this.c, 'procnote':'', 'procComment':'','prov':'CMIP6 endorsement' } 
    386   sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '','CMIP6 endorsement' ] 
     390  prov = 'CMIP6 endorsement [%s]' % sx.eenv[k][1] 
     391  sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',prov, sx.eenv[k][0] ] 
    387392  ee[this.u] = k 
    388393  if this.u == 'atmos': 
    … …  
    396401  this = nt_vrecix._make( map( lambda x: sx.eeold[k][x], vrecix_old ) ) 
    397402  ##sh[k] = { 'label':this.v, 'title':this.l, 'sn':this.s, 'units':this.u, 'description':this.c, 'procnote':'', 'procComment':'','prov':sx.eeold[k][6]} 
    398   sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',sx.eeold[k][6] ] 
     403  sh[k] = [ this.v, this.l, this.s, this.u, this.c, '', '',sx.eeold[k][6], 101 ] 
    399404  nk += 1 
    400405 
    … …  
    506511## parse 1 
    507512ee0_p1 = {} 
    508 ee0_gp = collections.defaultdict( int ) 
    509513ee0_gp = collections.defaultdict( d1 ) 
     514st1 = set() 
    510515for k in sx.ee0.keys(): 
    511516  r = list(sx.ee0[k][:] ) 
    … …  
    520525    ee0_p1[k] = r[:] 
    521526    ee0_gp[r[0]][r[10]] = k 
     527    st1.add( r[10] ) 
    522528    if r[0] == '': 
    523529       print '################' 
    524530       print r 
     531 
     532 
     533assert 'CFMIP' in st1, 'FAILED sanity check 1 ... CFMIP not in st1 %s' % str(st1) 
     534assert len( list(st1) ) < 40, 'FAILED sanity check 2 ... too many items in st1 [should be list of MIPs] %s' % len(list(st1)) 
    525535print '###################   groups referenced from new groups sheets' 
    526536print ee0_gp.keys() 
    … …  
    800810sh['__info__'] =  {'label':'groupItems', 'title':'List of items in new groups', 'description':"New groups" } 
    801811##(u'DYVR_daily', u'utendnogw', '', u'daily', '', '', '', '', '', u'DynVar', 'DynVar', '772da0b6-25b6-11e5-8cf6-ac72891c3257') 
    802 sh['__cols__'] = ['group','var','table','freq','', 'shape', '', '', '','mip','mip?','uuid','new','gpid','vkey','vid'] 
     812sh['__cols__'] = ['group','var','table','freq','', 'shape', '', '', '','mip','mip?','uuid','rowIndex','new','gpid','vkey','vid'] 
    803813for k in ee0_p1.keys(): 
    804814    kkk = '%s.%s' % (ee0_p1[k][10],ee0_p1[k][0]) 
    … …  
    909919    else: 
    910920      p = sx.cmip5so.so[t].a[v1][0] 
     921      rwix = sx.cmip5so.so[t].a[v1][-2] 
     922      si = sx.cmip5so.so[t].a[v1][-1] 
     923      rwix = rwix + 200*si 
    911924 
    912925## the "Oyr" group is for import, not needed in request 
    … …  
    10281041          ff[vdefex[u][1]] = u 
    10291042      ##if sx.ee_xref_bck.has_key(u): 
     1043 
    10301044    for v0 in vars.keys(): 
    10311045      v = string.strip( v0 ) 
Note: See TracChangeset for help on using the changeset viewer.