Changeset 432

Timestamp:

23/10/15 08:18:36 (4 years ago)

Author:

mjuckes

Message:

update fro 01.beta.09

Location:

CMIP6dreq/trunk/dreqPy

Files:

• README.txt (deleted)
• __init__.py (modified) (1 diff)
• dreq.py (modified) (2 diffs)
• scope.py (modified) (8 diffs)

CMIP6dreq/trunk/dreqPy/__init__.py

@@ -1 +1 @@
 
+import os
 
-__version__ = "01.beta.08"
+HERE = os.path.dirname(os.path.abspath(__file__) )
+
+##DOC_DEFAULT_DIR
+DOC_DEFAULT_DIR = os.path.join(HERE, '../docs')
+
+DOC_DIR = os.environ.get('DRQ_CONFIG_DIR', DOC_DEFAULT_DIR)
+
+__versionComment__ = "Support for command line interface added"
+__version__ = "01.beta.09"
 __title__ = "dreqPy"
 __description__ = "CMIP6 Data Request Python API"

CMIP6dreq/trunk/dreqPy/dreq.py

@@ -8 +8 @@
 import xml.dom.minidom
 import re, shelve
+from __init__ import DOC_DIR
 
 class rechecks(object):
@@ -460 +461 @@
 
 #DEFAULT LOCATION -- changed automatically when building distribution
-defaultDreq = '../docs/dreq.xml'
+defaultDreq = 'dreq.xml'
 #DEFAULT CONFIG
-defaultConfig = '../docs/dreq2Defn.xml'
+defaultConfig = 'dreq2Defn.xml'
+
+defaultDreqPath = '%s/%s' % (DOC_DIR, defaultDreq )
+defaultConfigPath = '%s/%s' % (DOC_DIR, defaultConfig )
 
 class loadDreq(object):
-  def __init__(self,dreqXML=defaultDreq, configdoc=defaultConfig, useShelve=False ):
+  def __init__(self,dreqXML=defaultDreqPath, configdoc=defaultConfigPath, useShelve=False ):
     self.c = config( thisdoc=dreqXML, configdoc=configdoc, useShelve=useShelve)
     self.coll = self.c.get()

CMIP6dreq/trunk/dreqPy/scope.py

@@ -5 +5 @@
 import dreq
 from utilities import cmvFilter
-import collections, string
+import collections, string, operator
+import sys
 
 class baseException(Exception):
@@ -20 +21 @@
 
 nt_mcfg = collections.namedtuple( 'mcfg', ['nho','nlo','nha','nla','nlas','nls','nh1'] )
+class cmpd(object):
+  def __init__(self,dct):
+    self.d = dct
+  def cmp(self,x,y,):
+    return cmp( self.d[x], self.d[y] )
+
+    self.default_mcfg = nt_mcfg._make( [259200,60,64800,40,20,5,100] )
 
 def filter1( a, b ):
@@ -76 +84 @@
     for i in self.dq.coll['spatialShape'].items:
       type = 'a'
-      if i.levelFlag == 'false':
+      if i.levelFlag == False:
         ds =  i.dimensions.split( '|' )
         if ds[-1] in ['site', 'basin']:
@@ -143 +151 @@
     return self.ntot
 
-  def volByExpt( self, l1, ex, exptList, pmax=2, cc=None, retainRedundantRank=False ):
+  def volByExpt( self, l1, ex, exptList, pmax=2, cc=None, retainRedundantRank=False, intersection=False ):
     """volByExpt: calculates the total data volume associated with an experiment/experiment group and a list of request items.
           The calculation has some approximations concerning the number of years in each experiment group."""
@@ -150 +158 @@
 ##
     inx = self.dq.inx
+    imips = {i.mip for i in l1}
 ##
 ## rql is the set of all request links which are associated with a request item for this experiment set
@@ -160 +169 @@
 
 ## The complete set of variables associated with these requests:
-    rqvg = list({inx.uid[i].refid for i in rql})
+    tm = 1
+    if tm == 0:
+      rqvg = list({inx.uid[i].refid for i in rql})
+    else:
+      cc1 = collections.defaultdict( set )
+      for i in rql:
+        cc1[inx.uid[i].mip].add( inx.uid[i].refid )
+
+      if intersection:
+        ccv = {}
+#
+# set of request variables for each MIP
+##
+        for k in cc1:
+          thisc = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in cc1[k] ] )
+          ccv[k] = {inx.uid[l].vid for l in list(thisc) if inx.uid[l].priority <= pmax}
+
+        if len( ccv.keys() ) < len( list(imips) ):
+          vars = set()
+        else:
+          vars =  reduce( operator.and_, [ccv[k] for k in ccv] )
+      else:
+        rqvg = reduce( operator.or_, [cc1[k] for k in cc1] )
 
 ###To obtain a set of variables associated with this collection of variable groups:
-    col1 = set()
-    x = {tuple( {col1.add(i) for i in inx.iref_by_sect[vg].a['requestVar']} ) for vg in rqvg}
+
+        col1 = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in rqvg ] )
+
 ###The collector col1 here accumulates all the record uids, resulting in a single collection. These are request variables, to get a set of CMOR variables at priority <= pmax:
-    vars = {inx.uid[l].vid for l in list(col1) if inx.uid[l].priority <= pmax}
+        vars = {inx.uid[l].vid for l in list(col1) if inx.uid[l].priority <= pmax}
+##
+## if looking for the union, would have to do a filter here ... after looking up which vars are requested by each MIP ...
+##
+## possibly some code re-arrangement would help.
+## e.g. create a set for each MIP a couple of lines back ....
 
 ### filter out cases where the request does not point to a CMOR variable.
@@ -204 +241 @@
       ov.append( self.dq.inx.uid[v] )
     ee = self.listIndexDual( ov, 'frequency', 'label', acount=None, alist=None, cdict=szv, cc=cc )
-    self.ngptot = sum( [  self.sz[inx.uid[v].stid]* npy[inx.uid[v].frequency] *nym[v]  for v in vars] )
-    return (self.ngptot, ee )
+    ff = {}
+    for v in vars:
+      ff[v] = self.sz[ inx.uid[v].stid ] * npy[inx.uid[v].frequency] * nym[v]
+    self.ngptot = sum( [  ff[v]  for v in vars] )
+    return (self.ngptot, ee, ff )
 
   def esid_to_exptList(self,esid,deref=False):
@@ -353 +393 @@
       od[k] = d2
     return od
+
+class dreqUI(object):
+  """Data Request Command line.
+-------------------------
+      -m <mip>:  MIP of list of MIPs (comma separated);
+      -h :       help: print help text;
+      -t <tier> maxmum tier;
+      -p <priority>  maximum priority;
+      --printLinesMax <n>: Maximum number of lines to be printed
+      --printVars  : If present, a summary of the variables fitting the selection options will be printed
+"""
+  def __init__(self,args):
+    self.adict = {}
+    self.knownargs = {'-m':('m',True), '-p':('p',True), '-t':('t',True), '-h':('h',False), '--printLinesMax':('plm',True), '--printVars':('vars',False)} 
+    aa = args[:]
+    while len(aa) > 0:
+      a = aa.pop(0)
+      if a in self.knownargs:
+        b = self.knownargs[a][0]
+        if self.knownargs[a][1]:
+          v = aa.pop(0)
+          self.adict[b] = v
+        else:
+          self.adict[b] = True
+
+    if 'm' in self.adict:
+      self.adict['m'] = set(self.adict['m'].split(',') )
+
+    integerArgs = {'p','t','plm'}
+    for i in integerArgs.intersection( self.adict ):
+      self.adict[i] = int( self.adict[i] )
+
+  def run(self, dq=None):
+    if 'h' in self.adict:
+      print self.__doc__
+      return
+
+    if not 'm' in self.adict:
+      print 'Current version requires -m argument' 
+      print self.__doc__
+      sys.exit(0)
+
+    if dq == None:
+      self.dq = dreq.loadDreq()
+    else:
+      self.dq = None
+
+    sc = dreqQuery( dq=self.dq )
+
+    ok = True
+    for i in self.adict['m']:
+        if i not in sc.mips:
+          ok = False
+          print 'NOT FOUND: ',i
+    assert ok,'Available MIPs: %s' % str(sc.mips)
+
+    tierMax = self.adict.get( 't', 2 )
+    sc.setTierMax(  tierMax )
+    pmax = self.adict.get( 'p', 2 )
+    v0 = sc.volByMip( self.adict['m'], pmax=pmax )
+    print '%7.2fTb' % (v0*2.*1.e-12)
+    cc = collections.defaultdict( int )
+    for e in sc.volByE:
+      for v in sc.volByE[e][2]:
+          cc[v] += sc.volByE[e][2][v]
+    x = 0
+    for v in cc:
+      x += cc[v]
+    
+    vl = sorted( cc.keys(), cmp=cmpd(cc).cmp, reverse=True )
+    if self.adict.get( 'vars', False ):
+      printLinesMax = self.adict.get( 'plm', 20 )
+      if printLinesMax > 0:
+        mx = min( [printLinesMax,len(vl)] )
+      else:
+        mx = len(vl)
+
+      for v in vl[:mx]:
+        print self.dq.inx.uid[v].label, '%7.2fTb' % (cc[v]*2.*1.e-12)