Ignore:
Timestamp:
19/12/16 14:12:13 (3 years ago)
Author:
mjuckes
Message:

misc updates

File:
1 edited

Legend:

Unmodified
Added
Removed
  • CMIP6dreqbuild/trunk/src/framework/ingest/util_checkUpd.py

    r895 r903  
    2323class omipGsScan(object): 
    2424  hphys = ['priority', 'long_name', 'units', 'comment', 'questions & notes', 'output variable name', 'CF standard_name', 'CF standard name AP', 'Standard name status AP', 'Comments AP', 'unconfirmed or proposed standard name', 'unformatted units', 'Units AP', 'cell_methods', 'valid min', 'valid max', 'mean absolute min', 'mean absolute max', 'positive', 'type', 'CMOR dimensions', 'CMOR variable name', 'realm', 'frequency', 'cell_measures', 'flag_values', 'flag_meanings'] 
     25  hphys2 = [u'priority', u'long_name', u'units', u'comment', u'questions & notes', u'output variable name', u'CF standard_name', u'CF standard name AP', u'Standard name status AP', u'Comments AP', u'unformatted units', u'Units AP', u'cell_methods', u'valid min', u'valid max', u'mean absolute min', u'mean absolute max', u'positive', u'type', u'CMOR dimensions', u'CMOR variable name', u'realm', u'frequency', u'cell_measures', u'flag_values', u'flag_meanings'] 
     26  hphys3 = [u'priority', u'long_name', u'units', u'comment', u'questions & notes', u'output variable name', u'CF standard_name', u'CF standard name AP', u'Standard name status AP', u'Comments AP', u'unconfirmed or proposed standard name', u'Units AP', u'unformatted units', u'cell_methods', u'valid min', u'valid max', u'mean absolute min', u'mean absolute max', u'positive', u'type', u'CMOR dimensions', u'CMOR variable name', u'realm', u'frequency', u'cell_measures', u'flag_values', u'flag_meanings'] 
     27 
     28  hbgc = ['priority', 'long_name', 'units', 'comment', 'questions & notes', 'output variable name', 'CF standard_name', 'CF standard_name resolved', 'CF standard name AP', 'Standard name status AP', 'Comments AP', 'Comments resolved', 'unconfirmed or proposed standard name', 'unformatted units', 'Units AP', 'cell_methods', 'valid min', 'valid max', 'mean absolute min', 'mean absolute max', 'positive', 'type', 'CMOR dimensions', 'Resolved CMOR dimensions', 'CMOR variable name', 'realm', 'frequency', 'cell_measures', 'flag_values', 'flag_meanings'] 
     29  hso = ['priority', 'long name', 'units', 'comment', 'questions & notes', 'output variable name', 'standard name', 'unconfirmed or proposed standard name', 'unformatted units', 'cell_methods', 'valid min', 'valid max', 'mean absolute min', 'mean absolute max', 'positive type', 'CMOR dimensions', 'CMOR variable name', 'realm', 'frequency', 'cell_measures', 'flag_values', 'flag_meanings'] 
     30 
     31  hbgc2 = [u'priority', u'long_name', u'units', u'comment', u'questions & notes', u'output variable name', u'CF standard_name', u'CF standard name AP', u'Standard name status AP', u'Comments AP', u'Resolution', u'Resolved Comment', u'Resolved CF standard name', u'Unconfirmed or proposed CF standard name', u'unformatted units', u'Units AP', u'cell_methods', u'valid min', u'valid max', u'mean absolute min', u'mean absolute max', u'positive', u'type', u'CMOR dimensions', u'CMOR variable name', u'realm', u'frequency', u'cell_measures', u'flag_values', u'flag_meanings'] 
     32  hbgc3 = [u'priority', u'long_name', u'units', u'comment', u'questions & notes', u'output variable name', u'CF standard_name', u'CF standard name AP', u'Standard name status AP', u'Comments AP', u'Resolution', u'Resolved Comment', u'Resolved CF standard name', 'unconfirmed or proposed standard name', u'unformatted units', u'Units AP', u'cell_methods', u'valid min', u'valid max', u'mean absolute min', u'mean absolute max', u'positive', u'type', u'CMOR dimensions', u'CMOR variable name', u'realm', u'frequency', u'cell_measures', u'flag_values', u'flag_meanings'] 
     33 
     34  hchem = [u'priority', u'long_name', u'units', u'comment', u'questions & notes', u'output variable name', u'CF standard_name', u'CF standard name AP', u'Standard name status AP', u'Comments AP', 'Unconfirmed or proposed CF standard name', u'unformatted units', u'Units AP', u'cell_methods', u'valid min', u'valid max', u'mean absolute min', u'mean absolute max', u'positive', u'type', u'CMOR dimensions', u'CMOR variable name', u'realm', u'frequency', u'cell_measures', u'flag_values', u'flag_meanings'] 
     35 
     36 
     37  hmap = {'unconfirmed or proposed standard name':'Unconfirmed or proposed CF standard name' } 
     38  hmap = {'Unconfirmed or proposed CF standard name':'unconfirmed or proposed standard name' } 
    2539  hphysx = ['CF standard name AP', 'Standard name status AP', 'Comments AP','Units AP'] 
    2640  def __init__(self): 
     
    3549    self.iu = 8 
    3650 
    37   def parseh(self,ih): 
     51  def parseh(self,ih0): 
     52    ih = [self.hmap.get(x,x) for x in ih0] 
    3853    if len(ih) == len(self.hdgs): 
    3954      self.mode, self.data = (0,None) 
     
    4257    if all( [ih[x] == '' for x in range(8)] ): 
    4358      return (0,None) 
    44     if len(ih) == len(self.hphys): 
    45       if all( [self.hphys[i] == ih[i] for i in range( len(ih) )] ): 
    46         ix = [ih.index(x) for x in self.hdgs] 
    47         cfrule = [ ih.index(x) for x in ['CF standard name AP','CF standard_name'] ] 
    48         unrule =  [ ih.index(x) for x in ['Units AP','unformatted units'] ] 
    49         self.mode, self.data = (1,(ix,cfrule,unrule) ) 
    50         log.info( 'INFO.omip.00011: mode 1: %s' % str(ih) ) 
    51         return (1,(ix,cfrule,unrule) ) 
    52       else: 
    53         print 'ERROR: unable to process: ',ih 
    54         assert False, 'Should not be here' 
    55     else: 
    56       log.info( 'WARN.omip.00021: Bad header length: %s' % str(ih) ) 
    57       return (0,None) 
     59    if len(ih) != len(self.hso): 
     60      mf = False 
     61      for tval in [self.hphys, self.hphys2, self.hphys3, self.hbgc, self.hbgc2, self.hbgc3, self.hchem]: 
     62        if len(ih) == len(tval) and all( [tval[i] == ih[i] for i in range( len(ih) )] ): 
     63          ix = [] 
     64          for  x in self.hdgs: 
     65            if x in ih: 
     66              ix.append( ih.index(x) ) 
     67            elif x == 'unconfirmed or proposed standard name': 
     68              print 'WARN.headings.0001: no proposed standadrd name heading in %s' % str(ih) 
     69              ix.append( None ) 
     70            else: 
     71              print 'FAILED TO PARSE: ',ih 
     72              print self.hdgs 
     73              raise 
     74          cfrule = [ ih.index(x) for x in ['CF standard name AP','CF standard_name'] ] 
     75          unrule =  [ ih.index(x) for x in ['Units AP','unformatted units'] ] 
     76          self.mode, self.data = (1,(ix,cfrule,unrule) ) 
     77          log.info( 'INFO.omip.00011: mode 1: %s' % str(ih) ) 
     78          return (1,(ix,cfrule,unrule) ) 
     79      print 'ERROR: unable to process: ',ih 
     80      assert False, 'Should not be here' 
     81    return (0,None) 
    5882 
    5983  def filter(self,ir): 
     
    6286      return ir 
    6387    if self.mode == 1: 
    64       orec = [ir[x] for x in self.data[0] ] 
     88      orec = [] 
     89      for x in self.data[0]: 
     90        if x == None: 
     91          orec.append( '' ) 
     92        else: 
     93          orec.append( ir[x] ) 
     94      ##orec = [ir[x] for x in self.data[0] ] 
    6595      for x in self.data[1]: 
    6696        if ir[x] != '': 
     
    337367               bb = [x.strip() for x in lll[2].split( '(' )] 
    338368               lll[2] = bb[0] 
     369               lll[2] = 'new' 
    339370               oldv = bb[1][:-1] 
    340371             else: 
     
    569600 
    570601      ih = [x.value for x in sht.row( kk-1 )] 
    571       osc =  omipGsScan() 
    572       isc,tt = osc.parseh( ih ) 
     602      if mip == 'OMIP': 
     603        osc =  omipGsScan() 
     604        isc,tt = osc.parseh( ih ) 
     605      else: 
     606        isc = 0 
    573607 
    574608      s1 = set() 
     
    773807    self.sh['__info__'] = self.shInfo 
    774808    dmin = ['minpblz','tasmin'] 
    775     dmax = ['maxpblz','tasmax'] 
     809    dmax = ['maxpblz','tasmax','sfo3max'] 
    776810 
    777811    self.nnss = 0 
     
    783817           
    784818        sn, ln, v0, units, desc, sect, freq, splab, tlab = lc.cc2[tab].a[var][0] 
     819        x = sect.split( '-' )[0] 
     820        mipt = {'aer6hr':'6hrLev', 'aerzonal':'aermonthlyz'}.get( x,x ) 
    785821        assert splab in lsp.uidByLab, 'Spatial shape label %s not found' % splab 
    786822        spid = lsp.uidByLab[splab] 
     
    810846            uid = self.refUid[tab][var] 
    811847        thisrealm = 'unset' 
    812         rec0 = [uid, desc, '', '', '', '', 'float', '', sn, '', ln, '', cmet, '', cmea, var, 'aerosol', units, '', '', '', var, tab, dims, vid, gpid, sect, p] 
     848        rec0 = [uid, desc, '', '', '', '', 'float', '', sn, '', ln, '', cmet, '', cmea, var, 'aerosol', units, '', '', '', var, mipt, dims, vid, gpid, sect, p] 
    813849      ##self.shCols = ['priority', 'long name', 'units', 'comment', 'questions & notes', 'output variable name', 'CF Standard name', 'unconfirmed or proposed standard name', 'unformatted units', 'cell_methods', 'valid min', 'valid max', 'mean absolute min', 'mean absolute max', 'positive', 'type', 'CMOR dimensions', 'CMOR variable name', 'realm', 'frequency', 'cell_measures', 'flag_values', 'flag_meanings','table','section','row'] 
    814850        rec0 = [p,ln, units, desc, '', var, sn, '', units, cmet, '','','','','','float',dims,var,'aerosol',freq,cmea,'','',tab,sect,-1] 
     
    829865          print 'INFO.aerchem.00021: ', rec 
    830866          nss = 'aerdaily-2d' 
     867 
    831868        if nss != None: 
    832869          self.nnss += 1 
    833           self.sh[ str( uuid.uuid1() ) ] = rec 
     870          rrr = rec[:] 
     871          rrr[1] = nss 
     872          self.sh[ str( uuid.uuid1() ) ] = rrr 
    834873          sect = nss 
    835874        if sect in ['AerChemMIP-hr','AerChemMIP-Day2d','fx']: 
     
    838877          print 'INFO.aerchem.00050:  unrecognised section', sect, rec 
    839878 
    840         lset = {'ssect':sect, 'uid':uid, 'rowIndex':-1, 'mipTable':tab} 
     879        lset = {'ssect':sect, 'uid':uid, 'rowIndex':-1, 'mipTable':mipt} 
    841880        ro = self._getRefRec(rec0,lset) 
    842881        self.shref[uid] = ro 
Note: See TracChangeset for help on using the changeset viewer.