Changeset 150


Ignore:
Timestamp:
23/04/14 17:49:11 (5 years ago)
Author:
astephen
Message:

support for CMIP5 added (not all attributes configured)

Location:
CCCC/trunk
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed
  • CCCC/trunk/c4.py

    r147 r150  
    194194    self.recordFile = 'Rec.txt' 
    195195    self.logDir = 'logs_02' 
     196    self.errs = [] 
    196197     
    197198    # Set default project to "CORDEX" 
     
    226227        flist  = [] 
    227228        fdir = args.pop(0) 
    228         for root, dirs, files in os.walk( fdir ): 
     229        for root, dirs, files in os.walk( fdir, followlinks=True ): 
    229230          for f in files: 
    230231            fpath = '%s/%s' % (root,f) 
    231             if os.path.isfile( fpath ) and f[-3:] == '.nc': 
     232            if (os.path.isfile( fpath ) or os.path.islink( fpath )) and f[-3:] == '.nc': 
    232233              flist.append( fpath ) 
    233234      elif next == '-R': 
     
    247248    assert nn==0, 'Aborting because of unused arguments' 
    248249 
     250    if self.project == 'CMIP5': 
     251      fl0 = [] 
     252      for f in flist: 
     253        if string.find( f, '/latest/' ) != -1: 
     254          fl0.append(f) 
     255      flist = fl0 
     256 
    249257    if forceLogOrg != None: 
    250258      if forceLogOrg == 'single': 
     
    265273    flist.sort() 
    266274    fnl = [] 
    267     nd = 0 
    268275    for f in flist: 
    269276      fn = string.split(f, '/')[-1] 
    270       if fn in fnl: 
    271         print 'ERROR: file name duplicated %s' % fn 
    272         nd += 0 
    273       else: 
    274         fnl.append(fn) 
    275     assert nd == 0, 'Duplicate file names encountered' 
     277      fnl.append(fn) 
     278    nd = 0 
     279    dupl = [] 
     280    for k in range(len(fnl)-1): 
     281      if fnl[k] == fnl[k-1]: 
     282        nd += 1 
     283        dupl.append( fnl[k] ) 
     284    self.dupDict = {} 
     285    for f in dupl: 
     286      self.dupDict[f] = 0 
     287    if nd != 0: 
     288      self.errs.append( 'Duplicate file names encountered: %s' % nd ) 
     289      self.errs.append( dupl ) 
    276290    self.flist = flist 
    277291    self.fnl = fnl 
     
    314328    if flf == None: 
    315329      tstring2 = '%4.4i%2.2i%2.2i' % time.gmtime()[0:3] 
    316       self.fileLogfile = '%s/%s__qclog_%s.txt' % (self.logDir,fn[:-3],tstring2) 
     330      if fn in self.dupDict.keys(): 
     331        tag = '__%2.2i' % self.dupDict[fn] 
     332        self.dupDict[fn] += 1 
     333      else: 
     334        tag = '' 
     335      self.fileLogfile = '%s/%s%s__qclog_%s.txt' % (self.logDir,fn[:-3],tag,tstring2) 
    317336      if self.policyFileLogfileMode in ['n','np']: 
    318337        assert not os.path.isfile( self.fileLogfile ), '%s exists and policy set to new file' % self.fileLogfile 
     
    370389    cal = None 
    371390    c4i.logger.info( 'Starting batch -- number of file: %s' % (len(c4i.flist)) ) 
     391    if len( c4i.errs ) > 0: 
     392      for i in range(0,len( c4i.errs ), 2 ): 
     393        c4i.logger.info( c4i.errs[i] ) 
    372394   
    373395    self.cc.info.amapListDraft = [] 
     
    388410      try: 
    389411  ### need to have a unique name, otherwise get mixing of logs despite close statement below. 
     412  ### if duplicate file names are present, this will be recorded in the main log, tag appended to file level log name (not yet tested). 
    390413        if c4i.logByFile: 
    391414          fLogger = c4i.getFileLog( fn ) 
     
    448471    self.cc.info.log = c4i.logger 
    449472     
    450     if c4i.project not in ['SPECS','CCMI']: 
     473    if c4i.project not in ['SPECS','CCMI','CMIP5']: 
    451474       cbv.c4i = c4i 
    452475       cbv.setLogDict( logDict ) 
  • CCCC/trunk/config_c4.py

    r145 r150  
    77 
    88 
    9 validCordexFrequecies = ['fx','sem','mon','day','6hr','3hr'] 
    10 validSpecsFrequecies = ['fx','mon','day','6hr'] 
    11 validCcmiFrequecies = ['fx','yr','mon','day','hr','subhr'] 
     9validCmip5Frequencies = ['fx','yr','monClim','mon','day','6hr','3hr','subhr'] 
     10validCordexFrequencies = ['fx','sem','mon','day','6hr','3hr'] 
     11validSpecsFrequencies = ['fx','mon','day','6hr'] 
     12validCcmiFrequencies = ['fx','yr','mon','day','hr','subhr'] 
    1213validSpecsExptFamilies = map( lambda x: string.split( x )[0], open( 'specs_vocabs/exptFamily.txt' ).readlines() ) 
    1314 
     
    115116    vocabs = { 'variable':utils.mipVocab(pcfg), \ 
    116117               'Conventions':utils.listControl( 'Conventions', ['CF-1.6'] ), \ 
    117                'frequency':utils.listControl( 'frequency', validSpecsFrequecies ), \ 
     118               'frequency':utils.listControl( 'frequency', validSpecsFrequencies ), \ 
    118119               'experiment_id':utils.listControl( 'experiment_id', validSpecsExptFamilies ), \ 
    119120               'initialization_method':utils.patternControl( 'initialization_method', "[0-9]+" ), \ 
     
    125126               'series':utils.listControl( 'series', ['series1','series2'] ), \ 
    126127             } 
     128  elif pcfg.project == 'CMIP5': 
     129               ##'experiment_id':utils.patternControl( 'experiment_id', "(?P<val>.*)[0-9]{4}", list=validSpecsExptFamilies ), \ 
     130    lrdr = readVocab( 'cmip5_vocabs/') 
     131    vocabs = { 'variable':utils.mipVocab(pcfg), \ 
     132               'Conventions':utils.listControl( 'Conventions', ['CF-1.4','CF-1.5'] ), \ 
     133               'experiment_id':utils.listControl( 'experiment_id', lrdr.getSimpleList( 'experiments.txt' ) ), \ 
     134               'frequency':utils.listControl( 'frequency', validCmip5Frequencies ), \ 
     135               'initialization_method':utils.patternControl( 'initialization_method', "[0-9]+" ), \ 
     136               'physics_version':utils.patternControl( 'physics_version', "[0-9]+" ), \ 
     137               'realization':utils.patternControl( 'realization', "[0-9]+" ), \ 
     138               'project_id':utils.listControl( 'project_id', ['CMIP5'] ), \ 
     139               ## 'institution':utils.listControl( 'institution', validSpecsInstitutions ), \ 
     140               'modeling_realm':utils.listControl( 'realm', ['atmos', 'ocean', 'land', 'landIce', 'seaIce', 'aerosol', 'atmosChem', 'ocnBgchem'], split=True ), \ 
     141             } 
    127142  elif pcfg.project == 'CCMI': 
    128143     
    129144    lrdr = readVocab( 'ccmi_vocabs/') 
    130145    vocabs = { 'variable':utils.mipVocab(pcfg), \ 
    131                'frequency':utils.listControl( 'frequency', validCcmiFrequecies ), \ 
     146               'frequency':utils.listControl( 'frequency', validCcmiFrequencies ), \ 
    132147               'experiment_id':utils.listControl( 'experiment_id', lrdr.getSimpleList( 'ccmi_experiments.txt', bit=-1 ) ), \ 
    133148## do not preserve or check relation between model and institution. 
     
    149164           'model_id':utils.listControl( 'model_id',  validRcmNames ), \ 
    150165           'institute_id':utils.listControl( 'institute_id',  validInstNames ), \ 
    151            'frequency':utils.listControl( 'frequency', validCordexFrequecies ) } 
     166           'frequency':utils.listControl( 'frequency', validCordexFrequencies ) } 
    152167 
    153168  return vocabs 
     
    156171 
    157172  def __init__(self, project): 
    158     knownProjects = ['CCMI','CORDEX','SPECS','__dummy'] 
     173    knownProjects = ['CMIP5','CCMI','CORDEX','SPECS','__dummy'] 
    159174    assert project in knownProjects, 'Project %s not in knownProjects %s' % (project, str(knownProjects)) 
    160175 
     
    188203                        'project':'project_id'} 
    189204 
     205    elif project == 'CMIP5': 
     206      lrdr = readVocab( 'cmip5_vocabs/') 
     207      self.requiredGlobalAttributes = [ 'contact', 'product', 'creation_date', 'tracking_id', \ 
     208              'experiment_id'] 
     209      ##self.requiredGlobalAttributes = lrdr.getSimpleList( 'globalAts.txt' ) 
     210      self.controlledGlobalAttributes = [ 'project_id','experiment_id', 'frequency','Conventions','modeling_realm', \ 
     211                       'initialization_method','physics_version','realization'] 
     212      self.globalAttributesInFn = [None,'@mip_id','model_id','experiment_id','@ensemble'] 
     213#sic_Oimon_EC-Earth2_seaIceBestInit_S19910501_series1_r1i1p1_199501-199502.nc  
     214## mip_id derived from global attribute Table_id (CMOR convention); experiment family derived from experiment_id, ensemble derived from rip attributes. 
     215      self.requiredVarAttributes = ['long_name', 'standard_name', 'units'] 
     216      self.drsMappings = {'variable':'@var', 'institute':'institution', 'product':'product', 'experiment':'experiment_id', \ 
     217                        'ensemble':'@ensemble', 'model':'model_id', 'realm':'modeling_realm', \ 
     218                        'frequency':'frequency',  \ 
     219                        'project':'project_id'} 
     220 
    190221    elif project == 'CCMI': 
    191222      lrdr = readVocab( 'ccmi_vocabs/') 
     
    226257      self.domainIndex = 1 
    227258      self.freqIndex = 7 
     259    elif self.project == 'CMIP5': 
     260## cRoot_Lmon_CESM1-WACCM_rcp85_r3i1p1_200601-205512.nc 
     261      self.fnPartsOkLen = [5,6] 
     262      self.fnPartsOkFixedLen = [5,] 
     263      self.fnPartsOkUnfixedLen = [6,] 
     264      self.checkTrangeLen = False 
     265      self.domainIndex = None 
     266      self.freqIndex = None 
    228267    elif self.project == 'SPECS': 
    229268      self.fnPartsOkLen = [7,8] 
     
    255294       self.mipVocabVgmap = {'6h':'6hr','3h':'3hr'} 
    256295       self.mipVocabFnpat = 'CORDEX_%s' 
     296    elif self.project == 'CMIP5': 
     297       self.mipVocabDir = 'cmip5_vocabs/mip/' 
     298       self.mipVocabTl = ['fx','Oyr','Oclim','Omon','Amon','Lmon','OImon','cfMon','aero','cfDay','day','cfOff','cfSites','6hrLev','6hrPlev','3hr','cf3hr'] 
     299       self.mipVocabVgmap = {} 
     300       self.mipVocabFnpat = 'CMIP5_%s' 
    257301    elif self.project == 'SPECS': 
    258302       self.mipVocabDir = 'specs_vocabs/mip/' 
     
    275319    if self.project == 'CORDEX': 
    276320      self.groupIndex = 7 
    277     elif self.project in ['CCMI','SPECS','__dummy']: 
     321    elif self.project in ['CMIP5','CCMI','SPECS','__dummy']: 
    278322      self.groupIndex = 1 
    279323 
  • CCCC/trunk/summary.py

    r134 r150  
    77 
    88ee = {} 
     9print 'Summarising error reports from %s log file' % len(fl) 
     10nne = 0 
    911for f in fl: 
     12  nef = 0 
    1013  for l in open(f).readlines(): 
    1114    fn = string.split(f,'/')[-1] 
    1215    if string.find(l, 'FAILED') != -1 or string.find(l,'CDMSError:') != -1: 
     16      nef += 1 
    1317      if string.find(l, 'FAILED') != -1: 
    1418         kb1 = 3 
     
    2933      else: 
    3034        print bits 
     35  if nef == 0: 
     36    nne += 1 
    3137 
    3238keys = ee.keys() 
     
    4753 
    4854 
     55print 'Number of files with no errors: %s' % nne 
    4956 
    5057 
     58 
Note: See TracChangeset for help on using the changeset viewer.