Changeset 491 for CMIP6dreqbuild

Timestamp:

19/11/15 09:59:58 (7 years ago)

Author:

mjuckes

Message:

updates for python 2.6 and 3

Location:

CMIP6dreqbuild/trunk/src/framework/dreqPy

Files:

• dreq.py (modified) (11 diffs)
• scope.py (modified) (15 diffs)
• simpleCheck.py (modified) (1 diff)

CMIP6dreqbuild/trunk/src/framework/dreqPy/dreq.py

@@ -17 +17 @@
   ll = []
   for l in ii:
-    ll.append( [x[1:-1] for x in string.strip(l).split('\t') ] )
+    ##ll.append( [x[1:-1] for x in string.strip(l).split('\t') ] )
+    ll.append( [x[1:-1] for x in l.strip().split('\t') ] )
   cc = collections.defaultdict( dict )
   for l in ll[3:]:
@@ -31 +32 @@
   def isIntStr( self, tv ):
     """Check whether a string is a valid representation of an integer."""
-    if type( tv ) not in {type(''),type(u'')}:
+    if type( tv ) not in [type(''),type(u'')]:
       self.reason = 'NOT STRING'
       return False
@@ -49 +50 @@
        _linkAttrStyle = {}
 
-       def __init__(self,dict=None,xmlMiniDom=None,id='defaultId',etree=False):
-         dictMode = dict != None
+       def __init__(self,idict=None,xmlMiniDom=None,id='defaultId',etree=False):
+         dictMode = idict != None
          mdMode = xmlMiniDom != None
          assert not( dictMode and mdMode), 'Mode must be either dictionary of minidom: both assigned'
@@ -59 +60 @@
          self._greenIcon = '<img height="12pt" src="/images/154g.png" alt="[i]"/>'
          if dictMode:
-           self.dictInit( dict )
+           self.dictInit( idict )
          elif mdMode:
            self.mdInit( xmlMiniDom, etree=etree )
@@ -88 +89 @@
 
        def __href__(self,odir="",label=None):
-         igns =  {'','__unset__'}
+         igns =  ['','__unset__']
          if 'description' in self.__dict__ and self.description != None and string.strip( self.description ) not in igns:
            ttl = self.description
@@ -170 +171 @@
 
 
-       def dictInit( self, dict ):
+       def dictInit( self, idict ):
          __doc__ = """Initialise from a dictionary."""
          for a in self._a.keys():
-           if a in dict:
-             self.__dict__[a] = dict[a]
+           if a in idict:
+             val = idict[a]
            else:
-             self.__dict__[a] = self._d.defaults.get( a, self._d.glob )
+             val = self._d.defaults.get( a, self._d.glob )
+           setattr( self, a, val )
          self._contentInitialised = True
 
@@ -213 +215 @@
                  v = string.strip(v)
                  thissect = '%s [%s]' % (self._h.title,self._h.tag)
-                 if v in { '',u'',' ', u' '}:
+                 if v in [ '',u'',' ', u' ']:
                    if nw1 < 20:
                      print ( 'WARN.050.0001: input integer non-compliant: %s: %s: "%s" -- set to zero' % (thissect,a,v) )
@@ -226 +228 @@
                      deferredHandling=True
              elif self._a[a].type == u'xs:boolean':
-               v = v in {'true','1'}
+               v = v in ['true','1']
              self.__dict__[a] = v
            else:
-             if a in {'uid'}:
+             if a in ['uid',]:
                thissect = '%s [%s]' % (self._h.title,self._h.tag)
                print ( 'ERROR.020.0001: missing uid: %s' % thissect )
@@ -305 +307 @@
     self.tt0 = {}
     for k in self.bscc:
-      self.tt0[k] = self._tableClass0(dict=self.bscc[k])
+      self.tt0[k] = self._tableClass0(idict=self.bscc[k])
       if k in self._t0.attributes:
         setattr( self._tableClass0, '%s' % k, self.tt0[k] )
@@ -328 +330 @@
       self.tableClasses[t[0].label] = self.itemClassFact( t, ns=self.ns )
       thisc = self.tableClasses[t[0].label]
-      self.tt1[t[0].label] = self._sectClass0( dict=t.header.__dict__ )
+      self.tt1[t[0].label] = self._sectClass0( idict=t.header._asdict() )
       self.tt1[t[0].label].maxOccurs = t.header.maxOccurs
       self.tt1[t[0].label].labUnique = t.header.labUnique
@@ -483 +485 @@
       returnClass = True
       if returnClass:
-        return self._tableClass0( dict=ee )
+        return self._tableClass0( idict=ee )
       else:
         return self.nti( i.nodeName, l,t,ty,cls,tn )

CMIP6dreqbuild/trunk/src/framework/dreqPy/scope.py

@@ -9 +9 @@
 
 python2 = True
-if sys.version_info.major == 3:
+if sys.version_info[0] == 3:
   python2 = False
   from functools import reduce
@@ -74 +74 @@
     self.cmvFilter = cmvFilter( self )
     self.tierMax = tierMax
-    self.mips = { i.mip for i in  self.dq.coll['requestItem'].items}
+
+    self.mips = set()
+    for i in  self.dq.coll['requestItem'].items:
+       self.mips.add(i.mip)
     self.mipls = sorted( list( self.mips ) )
 
     self.default_mcfg = nt_mcfg._make( [259200,60,64800,40,20,5,100] )
-    self.mcfg = {}
-    for k in self.default_mcfg.__dict__.keys():
-      self.mcfg[k] = self.default_mcfg.__dict__[k]
+    self.mcfg = self.default_mcfg._asdict()
+    ##for k in self.default_mcfg.__dict__.keys():
+      ##self.mcfg[k] = self.default_mcfg.__dict__[k]
     self.szcfg()
     self.requestItemExpAll(  )
@@ -131 +134 @@
     if type(mipSel) == type(''):
       t1 = lambda x: x == mipSel
-    elif type(mipSel) == type({1,2}):
+    elif type(mipSel) == type(set()):
       t1 = lambda x: x in mipSel
-    self.rqs = list({self.dq.inx.uid[i.rid] for i in self.dq.coll['objectiveLink'].items if t1(i.label) })
+
+    s = set()
+    for i in self.dq.coll['objectiveLink'].items:
+      if t1(i.label):
+        s.add( self.dq.inx.uid[i.rid] )
+    ##self.rqs = list({self.dq.inx.uid[i.rid] for i in self.dq.coll['objectiveLink'].items if t1(i.label) })
+    self.rqs = list( s )
     return self.rqs
 
@@ -140 +149 @@
     if type(objSel) == type(''):
       t1 = lambda x: x == self.rlu[objSel]
-    elif type(objSel) == type({1,2}):
-      t1 = lambda x: x in {self.rlu[i] for i in objSel}
-
-    self.rqs = list({self.dq.inx.uid[i.rid] for i in self.dq.coll['objectiveLink'].items if t1(i.oid) })
+    elif type(objSel) == type(set()):
+      t1 = lambda x: x in [self.rlu[i] for i in objSel]
+
+    s = set()
+    for i in self.dq.coll['objectiveLink'].items:
+      if t1(i.label):
+        s.add( self.dq.inx.uid[i.oid] )
+##
+    self.rqs = list( s )
+    ##self.rqs = list({self.dq.inx.uid[i.rid] for i in self.dq.coll['objectiveLink'].items if t1(i.oid) })
     return self.rqs
 
@@ -149 +164 @@
     """For a list of request links, return a list of variable group IDs for each experiment"""
     self.cc = collections.defaultdict( list )
-    dummy = {self.cc[i.expt].append(i.rlid) for i in self.dq.coll['requestItem'].items if i.rlid in {j.uid for j in rqList} }
+    ## dummy = {self.cc[i.expt].append(i.rlid) for i in self.dq.coll['requestItem'].items if i.rlid in {j.uid for j in rqList} }
     return self.cc
 
@@ -163 +178 @@
 ##
     inx = self.dq.inx
-    imips = {i.mip for i in l1}
+    imips = set()
+    for i in l1:
+      imips.add(i.mip)
+    ##imips = {i.mip for i in l1}
 ##
 ## rql is the set of all request links which are associated with a request item for this experiment set
 ##
-    l1p = {i for i in l1 if (i.esid == ex) and (i.preset < 0 or i.preset <= pmax) }
-
-    rql0 = {i.rlid for i in l1p}
-
-    rqlInv = {u for u in rql0 if inx.uid[u]._h.label == 'remarks' }
+    l1p = set()
+    for i in l1:
+      if (i.esid == ex) and (i.preset < 0 or i.preset <= pmax):
+        l1p.add(i)
+
+    rql0 = set()
+    for i in l1p:
+       rql0.add(i.rlid)
+
+    rqlInv = set()
+    for u in rql0:
+      if inx.uid[u]._h.label == 'remarks':
+        rqlInv.add( u )
     if len(rqlInv) != 0:
       print ( 'WARNING.001.00002: %s invalid request links from request items ...' % len(rqlInv) )
-    rql = {u for u in rql0 if inx.uid[u]._h.label != 'remarks' }
+    rql = set()
+    for u in rql0:
+       if inx.uid[u]._h.label != 'remarks':
+         rql.add( u ) 
+
     if len( rql ) == 0:
       self.vars = set()
@@ -182 +212 @@
     tm = 1
     if tm == 0:
-      rqvg = list({inx.uid[i].refid for i in rql})
+      s = set()
+      for i in rql:
+        s.add( inx.uid[i].refid )
+      rqvg = list( s )
     else:
       cc1 = collections.defaultdict( set )
@@ -195 +228 @@
         for k in cc1:
           thisc = reduce( operator.or_, [set( inx.iref_by_sect[vg].a['requestVar'] ) for vg in cc1[k] ] )
-          ccv[k] = {inx.uid[l].vid for l in list(thisc) if inx.uid[l].priority <= pmax}
+          s = set()
+          for l in list(thisc):
+             if inx.uid[l].priority <= pmax:
+               s.add( inx.uid[l].vid )
+          ccv[k] = s
 
         if len( ccv.keys() ) < len( list(imips) ):
@@ -209 +246 @@
 
 ###The collector col1 here accumulates all the record uids, resulting in a single collection. These are request variables, to get a set of CMOR variables at priority <= pmax:
-        vars = {inx.uid[l].vid for l in list(col1) if inx.uid[l].priority <= pmax}
+        vars = set()
+        for l in list(col1):
+           if inx.uid[l].priority <= pmax:
+             vars.add(inx.uid[l].vid)
 ##
 ## if looking for the union, would have to do a filter here ... after looking up which vars are requested by each MIP ...
@@ -218 +258 @@
 ### filter out cases where the request does not point to a CMOR variable.
     ##vars = {vid for vid in vars if inx.uid[vid][0] == u'CMORvar'}
-    vars = {vid for vid in vars if inx.uid[vid]._h.label == u'CMORvar'}
+    thisvars = set()
+    for vid in vars:
+       if inx.uid[vid]._h.label == u'CMORvar':
+             thisvars.add(vid)
+    vars = thisvars
 ##
 ## filter by configuration option and rank
@@ -234 +278 @@
 ### for request variables which reference the variable group attached to the link, add the associate CMOR variables, subject to priority
       i = inx.uid[u]
-      e[i.uid] = { inx.uid[x].vid for x in inx.iref_by_sect[i.refid].a['requestVar'] if inx.uid[x].priority <= pmax}
+      e[i.uid] = set()
+      for x in inx.iref_by_sect[i.refid].a['requestVar']:
+           if inx.uid[x].priority <= pmax:
+              e[i.uid].add( inx.uid[x].vid )
 #
 # for each variable, calculate the maximum number of years across all the request links which reference that variable.
@@ -242 +289 @@
     nym = {}
     for v in vars:
-      nym[v] = max( {self.rqiExp[i.uid][2] for i in l1p if i.esid == ex and v in e[i.rlid]} )
+      s = set()
+      for i in l1p:
+        if i.esid == ex and v in e[i.rlid]:
+          s.add( self.rqiExp[i.uid][2] )
+      ##nym[v] = max( {self.rqiExp[i.uid][2] for i in l1p if i.esid == ex and v in e[i.rlid]} )
+      nym[v] = max( s )
 
     szv = {}
@@ -336 +388 @@
   def volByMip( self, mip, pmax=2, retainRedundantRank=False):
 
-    if type(mip) in {type( '' ),type( u'') }:
+    if type(mip) in [type( '' ),type( u'') ]:
       if mip not in self.mips:
         print ( self.mips )
@@ -350 +402 @@
       
     #### The set of experiments/experiment groups:
-    exps = {i.esid for i in l1}
+    exps = set()
+    for i in l1:
+      exps.add( i.esid )
     self.volByE = {}
     vtot = 0
@@ -436 +490 @@
       self.adict['m'] = set(self.adict['m'].split(',') )
 
-    integerArgs = {'p','t','plm'}
+    integerArgs = set( ['p','t','plm'] )
     for i in integerArgs.intersection( self.adict ):
       self.adict[i] = int( self.adict[i] )

CMIP6dreqbuild/trunk/src/framework/dreqPy/simpleCheck.py

@@ -115 +115 @@
       self._clear_ch03()
       return
-    if string.find(ii[0],'validates') != -1:
+    if ii[0].find('validates') != -1:
       print ( 'Sample XML validated' )
       self.ok = True