Changeset 432 for CMIP6dreq/trunk


Ignore:
Timestamp:
23/10/15 08:18:36 (5 years ago)
Author:
mjuckes
Message:

update fro 01.beta.09

Location:
CMIP6dreq/trunk/dreqPy
Files:
1 deleted
3 edited

Legend:

Unmodified
Added
Removed
  • CMIP6dreq/trunk/dreqPy/__init__.py

    r425 r432  
    11 
     2import os 
    23 
    3 __version__ = "01.beta.08" 
     4HERE = os.path.dirname(os.path.abspath(__file__) ) 
     5 
     6##DOC_DEFAULT_DIR 
     7DOC_DEFAULT_DIR = os.path.join(HERE, '../docs') 
     8 
     9DOC_DIR = os.environ.get('DRQ_CONFIG_DIR', DOC_DEFAULT_DIR) 
     10 
     11__versionComment__ = "Support for command line interface added" 
     12__version__ = "01.beta.09" 
    413__title__ = "dreqPy" 
    514__description__ = "CMIP6 Data Request Python API" 
  • CMIP6dreq/trunk/dreqPy/dreq.py

    r425 r432  
    88import xml.dom.minidom 
    99import re, shelve 
     10from __init__ import DOC_DIR 
    1011 
    1112class rechecks(object): 
     
    460461 
    461462#DEFAULT LOCATION -- changed automatically when building distribution 
    462 defaultDreq = '../docs/dreq.xml' 
     463defaultDreq = 'dreq.xml' 
    463464#DEFAULT CONFIG 
    464 defaultConfig = '../docs/dreq2Defn.xml' 
     465defaultConfig = 'dreq2Defn.xml' 
     466 
     467defaultDreqPath = '%s/%s' % (DOC_DIR, defaultDreq ) 
     468defaultConfigPath = '%s/%s' % (DOC_DIR, defaultConfig ) 
    465469 
    466470class loadDreq(object): 
    467   def __init__(self,dreqXML=defaultDreq, configdoc=defaultConfig, useShelve=False ): 
     471  def __init__(self,dreqXML=defaultDreqPath, configdoc=defaultConfigPath, useShelve=False ): 
    468472    self.c = config( thisdoc=dreqXML, configdoc=configdoc, useShelve=useShelve) 
    469473    self.coll = self.c.get() 
  • CMIP6dreq/trunk/dreqPy/scope.py

    r425 r432  
    55import dreq 
    66from utilities import cmvFilter 
    7 import collections, string 
     7import collections, string, operator 
     8import sys 
    89 
    910class baseException(Exception): 
     
    2021 
    2122nt_mcfg = collections.namedtuple( 'mcfg', ['nho','nlo','nha','nla','nlas','nls','nh1'] ) 
     23class cmpd(object): 
     24  def __init__(self,dct): 
     25    self.d = dct 
     26  def cmp(self,x,y,): 
     27    return cmp( self.d[x], self.d[y] ) 
     28 
     29    self.default_mcfg = nt_mcfg._make( [259200,60,64800,40,20,5,100] ) 
    2230 
    2331def filter1( a, b ): 
     
    7684    for i in self.dq.coll['spatialShape'].items: 
    7785      type = 'a' 
    78       if i.levelFlag == 'false': 
     86      if i.levelFlag == False: 
    7987        ds =  i.dimensions.split( '|' ) 
    8088        if ds[-1] in ['site', 'basin']: 
     
    143151    return self.ntot 
    144152 
    145   def volByExpt( self, l1, ex, exptList, pmax=2, cc=None, retainRedundantRank=False ): 
     153  def volByExpt( self, l1, ex, exptList, pmax=2, cc=None, retainRedundantRank=False, intersection=False ): 
    146154    """volByExpt: calculates the total data volume associated with an experiment/experiment group and a list of request items. 
    147155          The calculation has some approximations concerning the number of years in each experiment group.""" 
     
    150158## 
    151159    inx = self.dq.inx 
     160    imips = {i.mip for i in l1} 
    152161## 
    153162## rql is the set of all request links which are associated with a request item for this experiment set 
     
    160169 
    161170## The complete set of variables associated with these requests: 
    162     rqvg = list({inx.uid[i].refid for i in rql}) 
     171    tm = 1 
     172    if tm == 0: 
     173      rqvg = list({inx.uid[i].refid for i in rql}) 
     174    else: 
     175      cc1 = collections.defaultdict( set ) 
     176      for i in rql: 
     177        cc1[inx.uid[i].mip].add( inx.uid[i].refid ) 
     178 
     179      if intersection: 
     180        ccv = {} 
     181# 
     182# set of request variables for each MIP 
     183## 
     184        for k in cc1: 
     185          thisc = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in cc1[k] ] ) 
     186          ccv[k] = {inx.uid[l].vid for l in list(thisc) if inx.uid[l].priority <= pmax} 
     187 
     188        if len( ccv.keys() ) < len( list(imips) ): 
     189          vars = set() 
     190        else: 
     191          vars =  reduce( operator.and_, [ccv[k] for k in ccv] ) 
     192      else: 
     193        rqvg = reduce( operator.or_, [cc1[k] for k in cc1] ) 
    163194 
    164195###To obtain a set of variables associated with this collection of variable groups: 
    165     col1 = set() 
    166     x = {tuple( {col1.add(i) for i in inx.iref_by_sect[vg].a['requestVar']} ) for vg in rqvg} 
     196 
     197        col1 = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in rqvg ] ) 
     198 
    167199###The collector col1 here accumulates all the record uids, resulting in a single collection. These are request variables, to get a set of CMOR variables at priority <= pmax: 
    168     vars = {inx.uid[l].vid for l in list(col1) if inx.uid[l].priority <= pmax} 
     200        vars = {inx.uid[l].vid for l in list(col1) if inx.uid[l].priority <= pmax} 
     201## 
     202## if looking for the union, would have to do a filter here ... after looking up which vars are requested by each MIP ... 
     203## 
     204## possibly some code re-arrangement would help. 
     205## e.g. create a set for each MIP a couple of lines back .... 
    169206 
    170207### filter out cases where the request does not point to a CMOR variable. 
     
    204241      ov.append( self.dq.inx.uid[v] ) 
    205242    ee = self.listIndexDual( ov, 'frequency', 'label', acount=None, alist=None, cdict=szv, cc=cc ) 
    206     self.ngptot = sum( [  self.sz[inx.uid[v].stid]* npy[inx.uid[v].frequency] *nym[v]  for v in vars] ) 
    207     return (self.ngptot, ee ) 
     243    ff = {} 
     244    for v in vars: 
     245      ff[v] = self.sz[ inx.uid[v].stid ] * npy[inx.uid[v].frequency] * nym[v] 
     246    self.ngptot = sum( [  ff[v]  for v in vars] ) 
     247    return (self.ngptot, ee, ff ) 
    208248 
    209249  def esid_to_exptList(self,esid,deref=False): 
     
    353393      od[k] = d2 
    354394    return od 
     395 
     396class dreqUI(object): 
     397  """Data Request Command line. 
     398------------------------- 
     399      -m <mip>:  MIP of list of MIPs (comma separated); 
     400      -h :       help: print help text; 
     401      -t <tier> maxmum tier; 
     402      -p <priority>  maximum priority; 
     403      --printLinesMax <n>: Maximum number of lines to be printed 
     404      --printVars  : If present, a summary of the variables fitting the selection options will be printed 
     405""" 
     406  def __init__(self,args): 
     407    self.adict = {} 
     408    self.knownargs = {'-m':('m',True), '-p':('p',True), '-t':('t',True), '-h':('h',False), '--printLinesMax':('plm',True), '--printVars':('vars',False)}  
     409    aa = args[:] 
     410    while len(aa) > 0: 
     411      a = aa.pop(0) 
     412      if a in self.knownargs: 
     413        b = self.knownargs[a][0] 
     414        if self.knownargs[a][1]: 
     415          v = aa.pop(0) 
     416          self.adict[b] = v 
     417        else: 
     418          self.adict[b] = True 
     419 
     420    if 'm' in self.adict: 
     421      self.adict['m'] = set(self.adict['m'].split(',') ) 
     422 
     423    integerArgs = {'p','t','plm'} 
     424    for i in integerArgs.intersection( self.adict ): 
     425      self.adict[i] = int( self.adict[i] ) 
     426 
     427  def run(self, dq=None): 
     428    if 'h' in self.adict: 
     429      print self.__doc__ 
     430      return 
     431 
     432    if not 'm' in self.adict: 
     433      print 'Current version requires -m argument'  
     434      print self.__doc__ 
     435      sys.exit(0) 
     436 
     437    if dq == None: 
     438      self.dq = dreq.loadDreq() 
     439    else: 
     440      self.dq = None 
     441 
     442    sc = dreqQuery( dq=self.dq ) 
     443 
     444    ok = True 
     445    for i in self.adict['m']: 
     446        if i not in sc.mips: 
     447          ok = False 
     448          print 'NOT FOUND: ',i 
     449    assert ok,'Available MIPs: %s' % str(sc.mips) 
     450 
     451    tierMax = self.adict.get( 't', 2 ) 
     452    sc.setTierMax(  tierMax ) 
     453    pmax = self.adict.get( 'p', 2 ) 
     454    v0 = sc.volByMip( self.adict['m'], pmax=pmax ) 
     455    print '%7.2fTb' % (v0*2.*1.e-12) 
     456    cc = collections.defaultdict( int ) 
     457    for e in sc.volByE: 
     458      for v in sc.volByE[e][2]: 
     459          cc[v] += sc.volByE[e][2][v] 
     460    x = 0 
     461    for v in cc: 
     462      x += cc[v] 
     463     
     464    vl = sorted( cc.keys(), cmp=cmpd(cc).cmp, reverse=True ) 
     465    if self.adict.get( 'vars', False ): 
     466      printLinesMax = self.adict.get( 'plm', 20 ) 
     467      if printLinesMax > 0: 
     468        mx = min( [printLinesMax,len(vl)] ) 
     469      else: 
     470        mx = len(vl) 
     471 
     472      for v in vl[:mx]: 
     473        print self.dq.inx.uid[v].label, '%7.2fTb' % (cc[v]*2.*1.e-12) 
Note: See TracChangeset for help on using the changeset viewer.