Changeset 1011

Timestamp:

23/05/17 17:43:15 (3 years ago)

Author:

mjuckes

Message:

experiment mapping details

Location:

CMIP6dreqbuild/trunk/src/framework

Files:

• ing02/inputs/exptMaps.py (added)
• ingest/expts.py (modified) (13 diffs)

CMIP6dreqbuild/trunk/src/framework/ingest/expts.py

@@ -4 +4 @@
 from dreqPy import dreq
 
+
+import exptMaps as em
+
 fmip = re.compile( '([a-zA-Z0-9]{1,10}MIP|DCPP)' )
 EXPTFILE='/data/tmp/svn3/exarch/CMIP6dreqbuild/trunk/src/framework/ingest/CMIP6_Experiments_v50.xlsx'
+EXPTFILE='/data/work/tmp/git/esdoc-docs/cmip6/experiments/spreadsheet/experiments.xlsx'
+## experiments sheet copied and pasted to resolve internal references.
+EXPTFILE='CMIP6_experiments_esdoc_190517.xlsx'
 
 class ktEx(object):
-  def __init__(self, file='/data/work/tmp/git/cmip6-cmor-tables/Tables/CMIP6_CV.json' ): 
+  def __init__(self, file='/data/work/tmp/git/CMIP6_CVs/CMIP6_experiment_id.json' ): 
     assert os.path.isfile( file ),'File %s not found' % file
     ex = json.load( open( file ) )
-    self.expts = ex['CV']['experiment_ids']
+    ##self.expts = ex['CV']['experiment_ids']
+    self.expts = ex['experiment_id']
     self.mip = {}
     for k in self.expts:
-      m0 = self.expts[k]['activity_id']
+      m0 = self.expts[k]['activity_id'][0]
       m = fmip.match( m0 )
       assert m != None, 'No activity for experiment %s' % k
@@ -32 +39 @@
         return 68
       else:
-        print 'SEVERE: badly formatted time constraint: ',tcon
+        print 'SEVERE: badly formatted time constraint: ',tcon,self.rown,self.experiment
         return 0
     elif tcon == '2014-present N yrs':
       return 4
     else:
-      print tcon
       t1 = tcon[:-3].strip()
       if t1.find( ' ' ) != -1:
         t1 = t1.split( ' ' )[-1]
+      if t1.find( '.' ) != -1:
+        return float( t1 )
       return int (t1 )
 
@@ -50 +58 @@
     self.mip = {}
     for i in range(2,sht.nrows):
+      self.rown = i
       r = [x.value for x in sht.row(i)]
       if r[0] != '':
        en = r[2]
-       if en != 'N/A':
-        tcon = r[25]
+       self.experiment = en
+       if en.strip() not in [ 'N/A', 'n/a', '', ' ']:
+        tcon = r[30]
         nyrs = self.parseTempCon( tcon )
         m = fmip.match( r[0] )
@@ -62 +72 @@
           mip = '---'
         if en in cc:
-          print 'SEVERE:  DUPLICATE EXPERIMENT NAME: %s::\n %s\n ----- \n%s ' % (en,str(cc[en]),str(r))
+          print 'SEVERE:  DUPLICATE EXPERIMENT NAME (ES-DOC): %s::\n %s\n ----- \n%s ' % (en,str(cc[en]),str(r))
           self.dn.append(en)
         cc[en] = r
@@ -78 +88 @@
     print '%s records scanned' % len(cc.keys() )
     print self.dn
+
+esDcppMap1 = [u'dcppC-amv-extrop-minus', u'dcppC-amv-extrop-plus', u'dcppC-amv-minus', u'dcppC-amv-plus', u'dcppC-amv-trop-minus', u'dcppC-amv-trop-plus', u'dcppC-ipv-minus', u'dcppC-ipv-plus', u'dcppC-pac']
+ktDcppMap1 = [u'dcppC-amv-ExTrop-neg', u'dcppC-amv-ExTrop-pos', u'dcppC-amv-neg', u'dcppC-amv-pos', u'dcppC-amv-Trop-neg', u'dcppC-amv-Trop-pos',  u'dcppC-ipv-neg', u'dcppC-ipv-pos', u'dcppC-pac-control']
 
 lMaps = { 
@@ -90 +103 @@
          u'ssp37-ssp126Lu':u'ssp370-ssp126Lu'}
 
+for i in range( len( esDcppMap1 ) ):
+   lMaps[esDcppMap1[i]] = ktDcppMap1[i]
+
+rqMaps = {}
+for k in em.exptMap:
+  for i in em.exptMap[k]:
+    assert i not in rqMaps
+    rqMaps[i] = em.exptMap[k][i]
+
 class request(object):
   def __init__(self,xc):
     dq = dreq.loadDreq()
     self.mapped = set()
+    self.mappedbymip = collections.defaultdict( set )
     self.missing = set()
+    self.incomplete = set()
+    self.used = set()
     self.ok = set()
+    cc = collections.defaultdict( list )
+    for k in rqMaps:
+      assert k not in xc.mapToKt
+      xc.mapToKt[k] = rqMaps[k]
     for i in dq.coll['experiment'].items:
       if i.label not in xc.mapToKt:
-        print 'NOT FOUND: %s: %s -- %s' % (i.mip,i.label, i.title) 
-        self.missing.add( i.label )
-      elif xc.mapToKt[i.label] == i.label:
-        self.mapped.add(i.label)
+        if i.label in xc.kt.expts:
+          self.incomplete.add( i.label )
+          self.used.add( i.label )
+        else:
+          self.missing.add( i.label )
+          cc[i.mip].append( i.label )
       else:
-        self.ok.add( i.label )
+        self.used.add( xc.mapToKt[i.label] )
+        if xc.mapToKt[i.label] != i.label:
+          self.mapped.add(i.label)
+          self.mappedbymip[i.mip].add((i.label,xc.mapToKt[i.label]))
+        else:
+          self.ok.add( i.label )
     print '%s experiments: %s ok, %s mapped, %s unknown: ' % (len( dq.coll['experiment'].items ), len(self.ok), len( self.mapped), len( self.missing) )
+    print 'Request experiments not found: %s' % (len(self.missing) )
+    ccu = collections.defaultdict( list )
+    for k in xc.kt.expts:
+      if k not in self.used:
+        ccu[xc.kt.mip[k]].append( k )
+    for m in sorted( cc.keys() ):
+      print 'UNMATCHED: %s: %s' % (m, sorted( cc[m] ) )
+      if m in ccu:
+        print '--- UNUSED: %s' % (sorted( ccu[m] ) )
+    for m in sorted( self.mappedbymip.keys() ):
+      print 'MAPPED: %s:: %s' % (m,sorted( list(self.mappedbymip[m])))
+
 
 class checker(object):
@@ -111 +159 @@
     self.kt = ktEx()
   def xcheck(self):
-    es = esdocEx()
-    kt = ktEx()
+    es = self.es
+    kt = self.kt
     nf = []
     matchByOther = {}
@@ -142 +190 @@
         if not done:
             nf.append( k )
-    print '%s experiments not found in PCMDI experiment CV' % len(nf)
+    print '%s ES-DOC experiments not found in PCMDI experiment CV' % len(nf)
     print sorted( nf )
+    self.nf = nf
 
     nf2 = []
@@ -151 +200 @@
     print '%s experiments not found in ES-DOC experiment list' % len(nf2)
     print sorted( nf2 )
+    self.nf2 = nf2
 
     ee = {}
@@ -166 +216 @@
 
     rf = [x for x in nf if x not in lcMatch]
+    print 'Lower case matches: %s' % str( lcMatch )
+    print 'Mapped matches (PCMDI->ES-DOC) [%s]: %s' % (len(mMatch.keys()),str( mMatch ))
     print '%s ESDOC experiments not found in PCMDI experiment CV after lc' % len(rf)
     print '%s straight matches [%s,%s]' % (len(s0),len(es.expts),len(kt.expts))
@@ -173 +225 @@
         indMatch.add( x[k] )
     rf2 = [x for x in nf2 if x not in indMatch]
-    print '%s PCMDI experiments not found in ES-DOC experiment list indirect matches' % len(rf2)
+    print '%s PCMDI experiments not found in ES-DOC experiment list indirect matches [%s]' % (len(rf2), len(set(rf2)))
     cc = collections.defaultdict( set )
+    print ' ----- IN ES-DOC ONLY: '
     for k in rf:
       cc[ es.mip[k] ].add( k )
@@ -183 +236 @@
       print m, '[ES-DOC]', sorted( list( cc[m] ) )
       if m in cc2:
+          print m, '[PCMDI]', sorted( list( cc2[m] ) )
+    for m in cc2:
+      if m not in cc:
           print m, '[PCMDI]', sorted( list( cc2[m] ) )
 
@@ -223 +279 @@
   print '%s mapped matches' % len( mm.keys())
   print mm
+
+ee = {}
+for k in rq.mapped:
+  ##if k not in xc.kt.expts:
+    ee[k] = xc.mapToKt[k]
+for k in mm:
+  ee[k] = mm[k]
+for k in lcm:
+  ee[k] = lcm[k]
+oo = open( 'exptNameMaps.json', 'w' )
+json.dump( {'info':{'title':'Updating experiment names in data request'}, 'maps':ee}, oo, indent=4, sort_keys=True )
+oo.close()
+
+
 ## from PCMDI github ...