Changeset 756 for CMIP6dreqbuild
- Timestamp:
- 22/09/16 14:13:08 (4 years ago)
- Location:
- CMIP6dreqbuild/trunk/src/workbook
- Files:
-
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
CMIP6dreqbuild/trunk/src/workbook/importWbMods.py
r745 r756 4 4 5 5 nt__varGroup = collections.namedtuple( 'varGroup', ['uid','label','title','mip','ref','refNote'] ) 6 7 def cellMethodCheck(cm): 8 rv = 0 9 if cm.find( ' ' ) != -1: 10 cm = cm.replace( ' ', ' ' ) 11 rv = 1 12 13 if cm.find( "area: where" ) != -1: 14 cm = cm.replace( "area: where", "area: mean where" ) 15 rv = 1 16 17 if cm.find( "time:mean" ) != -1: 18 cm = cm.replace( "time:mean", "time: mean" ) 19 rv = 1 20 elif cm.find( "weighted b " ) != -1: 21 return ( cm.replace( "weighted b ", "weighted by " ), 1) 22 23 if cm.find( "time: mean" ) == 0 and len(cm) > 10: 24 if cm[10:].strip()[0] == '(': 25 ix = cm.index(')') + 1 26 else: 27 ix = 10 28 cm = cm[ix:].strip( ) + ' ' + cm[:ix] 29 rv = 1 30 31 if cm.find( "time: point" ) == 0 and len(cm) > 11: 32 if cm[11:].strip()[0] == '(': 33 ix = cm.index(')') + 1 34 else: 35 ix = 11 36 cm = cm[ix:].strip( ) + ' ' + cm[:ix] 37 rv = 1 38 return (cm,rv) 6 39 7 40 class loadVarGroupInfo20(object): … … 97 130 print 'SEVERE:loadstr.0001: duplicate lookup tuple: ',t 98 131 print r[1], self.lookup[t] 132 print 'INFO.loadstr.00022: ',r[1],t 99 133 self.lookup[t] = r[1] 100 134 ## Label Title Spatial shape Other dims coords cell methods cell measures flag meanings flag values Spatial shape (title) Spatial dimensions … … 126 160 v0 = string.strip(r[0]) 127 161 if len(v0) == 0 or v0[0] != '#': 162 vv,rv = cellMethodCheck(v0) 128 163 v1 = string.strip(r[3]) 129 164 if r[1] in ['redirect','change']: 130 165 self.cmmap[v0] = v1 166 if r[1] == 'redirect' and vv != v0: 167 self.cmmap[vv] = v1 131 168 v0 = v1 132 169 if r[1] != 'redirect': 133 170 t = string.strip(r[5]) 134 171 l = string.strip(r[4]) 135 self.cminfo[v0] = (t,l,'CellMethods::%s' % l) 172 if vv != v0: 173 self.cminfo[vv] = (t,l,'CellMethods::%s' % l) 174 self.cmmap[v0] = vv 175 else: 176 self.cminfo[v0] = (t,l,'CellMethods::%s' % l) 136 177 137 178 if r[1] in ['ok','ok?']: … … 402 443 s1 = wb.book.sheet_by_name(u'ss_ref') 403 444 self.ss = {} 445 self.ssu = {} 404 446 self.labByUid = {} 447 self.uidByLab = {} 405 448 uu = set() 406 449 for i in range(s1.nrows): … … 413 456 assert u not in uu, 'Duplicate uid in spatialShape_ref.xls: %s' % u 414 457 self.labByUid[u] = rr[0] 458 self.uidByLab[rr[0]] = u 415 459 self.ss[ k ] = (a,b,lf,li,u) 460 self.ssu[ u ] = (a,b,k,lf,li,u) 461 462 def addTime(self): 463 tshp = { 'time':['time-mean','Temporal mean'], 'time1':['time-point','Instantaneous value (i.e. synoptic or time-step value)'], \ 464 'time2':['climatoglogy','Climatological mean'], \ 465 'time3':['diurnal-cycle','Mean Diurnal Cycle'], \ 466 '':['None','No temporal dimensions ... fixed field'] } 467 tuid = ['7a96eb30-8042-11e6-97ee-ac72891c3257', '7a972f78-8042-11e6-97ee-ac72891c3257', '7a976ce0-8042-11e6-97ee-ac72891c3257', '7a97ae58-8042-11e6-97ee-ac72891c3257','cf34c974-80be-11e6-97ee-ac72891c3257'] 468 self.etd = {} 469 self.tsu = {} 470 self.tByLab = {} 471 472 k = 0 473 for s in tshp: 474 label, description = tshp[s] 475 u = tuid[k] 476 self.etd[s] = (u,label,description,s) 477 self.tsu[u] = (label,description,s) 478 self.tByLab[label] = (u,description,s) 479 k+=1 416 480 417 481 class run(object): -
CMIP6dreqbuild/trunk/src/workbook/inputs/strUidMap.txt
r745 r756 190 190 str-a02 5edc273a-75e2-11e6-b907-ac72891c3257 191 191 str-x002 new__x002__ 192 str-x003 new__x003__ 193 str-x004 new__x004__ 194 str-x006 new__x006__ -
CMIP6dreqbuild/trunk/src/workbook/ivg.py
r745 r756 1 1 from scansh import rq 2 2 import string, collections, uuid, shelve 3 import importWbMods 4 from importWbMods import cellMethodCheck 3 5 4 6 cmip5GrpLk = collections.defaultdict( dict ) 5 6 class cellMethodCheck(object):7 def __init__(self):8 pass9 def check(self,cm):10 if cm.find( "area: where" ) != -1:11 return ( cm.replace( "area: where", "area: mean where" ), 1)12 elif cm.find( "time:mean" ) != -1:13 return ( cm.replace( "time:mean", "time: mean" ), 1)14 elif cm.find( "weighted b " ) != -1:15 return ( cm.replace( "weighted b ", "weighted by " ), 1)16 else:17 return (cm,0)18 7 19 8 class tableUtils(object): … … 108 97 class structureList(object): 109 98 def __init__(self): 99 self.force = ['str-a02',] 100 self.spsh = importWbMods.loadSpatial() 101 self.spsh.addTime() 102 self.cellm = importWbMods.loadCellm() 110 103 self.ll = [] 111 104 self.lldict = dict() 112 105 self.nt_dims = collections.namedtuple( 'dims', ['uid','label','spid','tlab','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','description','procNote','prov'] ) 113 106 self.ee3 = {} 114 self.cmch = cellMethodCheck()107 #self.cmch = cellMethodCheck() 115 108 sh = shelve.open( '../framework/inSh/structRef', 'r' ) 116 109 self.tmap = {} 117 110 self.messageDone = set() 111 cc = collections.defaultdict( dict ) 112 dd = dict() 113 self.orecs = set() 114 self.kr = 0 118 115 for k in sh.keys(): 119 116 if k[0] != '_': … … 121 118 print 'INFO.ivg-str.0001: ',sh[k] 122 119 rr = sh[k][:7] 123 rr[0] = sh[k][- 1]120 rr[0] = sh[k][-2] 124 121 t = tuple( rr ) 125 assert t not in self.tmap, 'duplicate lookup tuple for structures ....%s' % str(t) 126 self.tmap[t] = sh[k][7:] + [sh[k][0],] 127 128 def add(self,cand): 129 cm, fl = self.cmch.check( cand[6] ) 130 if fl == 1: 131 cand[6] = cm 132 nt = self.nt_dims._make( cand ) 133 t1 = ( nt.tlab, nt.spid, nt.odims, nt.coords, nt.cell_methods, nt.cell_measures, nt.flag_meanings ) 134 u = nt.uid 135 label = nt.label 136 tt = tuple( [nt.__dict__[k] for k in ['spid','tlab','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','procNote','prov']] ) 122 flg = sh[k][-1] 123 if flg: 124 assert t not in cc[True], 'duplicate lookup tuple for structures ....%s:\n %s\n %s' % ( str(t), str( dd[t] ), str( sh[k] ) ) 125 dd[t] = sh[k] 126 cc[flg][t] = sh[k][7:-1] + [sh[k][0],] 127 128 for t in cc[True]: 129 self.tmap[t] = cc[True][t] 130 for t in cc[False]: 131 if t not in self.tmap: 132 self.tmap[t] = cc[False][t] 133 134 def close(self): 135 if len(self.orecs) != 0: 136 sh = shelve.open( 'sh/extraStructureRecords', 'n' ) 137 for flg,u,r in self.orecs: 138 sh[u] = (flg,r) 139 sh.close() 140 141 def add(self,cand,t1=None): 142 ## cand = [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] 143 ## look-up structure based on time label, spatial id, other dimensions, coordinates, cell methods, cell measures, flag meanings 144 err = False 145 if t1 == None: 146 cm, fl = cellMethodCheck( cand[6] ) 147 if fl == 1: 148 cand[6] = cm 149 assert len(cm) < 11 or cm[:10] != 'time: mean', 'Should not have time mean first .... %s, %s, %s' % ( str(cand), cm.find( 'time: mean' ), fl ) 150 nt = self.nt_dims._make( cand ) 151 cm = self.cellm.cmmap.get( cm,cm) 152 if cm not in self.cellm.cminfo: 153 print 'SEVERE.cellm.0001: Cell method lookup failed: %s' % cm 154 cmttl = cm 155 cmlab = '---' 156 else: 157 cmttl, cmlab = self.cellm.cminfo[cm][:2] 158 159 t1 = ( nt.tlab, nt.spid, nt.odims, nt.coords, nt.cell_methods, nt.cell_measures, nt.flag_meanings ) 160 u = nt.uid 161 label = nt.label 162 assert nt.spid in self.spsh.ssu, 'Spatial id not found in spsh.ssu: %s' % nt.spid 163 164 slab, sttl,sdim = self.spsh.ssu[ nt.spid ][:3] 165 tttl = self.spsh.tByLab[nt.tlab][1] 166 title = '%s, %s [%s]' % (tttl, sttl, slab) 167 if string.strip( nt.coords ) != '' or string.strip( nt.odims ) != '': 168 title += ' {%s:%s}' % (nt.odims,nt.coords) 169 if cmlab != '': 170 title += ' [%s]' % cmlab 171 172 else: 173 assert False, 'Should not be here .....' 174 175 ##tt = tuple( [nt.__dict__[k] for k in ['spid','tlab','odims','coords','cell_methods','cell_measures','flag_values','flag_meanings','procNote','prov']] ) 137 176 if t1 in self.tmap: 138 u, label, title, description, fv, tlab, tmid = self.tmap[t1] 177 ## Label Title Spatial shape Other dims coords cell methods cell measures flag meanings flag values Spatial shape (title) Spatial dimensions 178 179 u, label, title0, description, fv, tlab, tmid = self.tmap[t1] 139 180 if fv != nt.flag_values: 140 181 print 'ERROR.structure.01020: flag_value mismatch for given flag_meanings: ',cand, fv 182 err = True 141 183 if u != nt.uid or label != nt.label: 142 184 cand[0] = u 143 185 cand[1] = label 144 186 nt = self.nt_dims._make( cand ) 187 orec = tuple( cand + [title,sttl,sdim] ) 188 self.orecs.add( (1,u,orec) ) 145 189 else: 146 190 if t1 not in self.messageDone: 147 191 print 'ERROR.structure.01010: tuple lookup not found: ',cand,t1 192 if t1[1] in self.spsh.labByUid: 193 slab = self.spsh.labByUid[t1[1]] 194 ## cand = [u,label,spid,tlab, odim, crd, cmet, cmea, flgv, flgm, desc, proc, prov] 195 ## Label Title Spatial shape Other dims coords cell methods cell measures flag meanings flag values Spatial shape (title) Spatial dimensions 196 self.kr += 1 197 assert self.kr < 100, 'Too many additional structure records required' 198 label = 'str-d%2.2i' % self.kr 199 cand[1] = label 200 orec = tuple( cand + [title,sttl,sdim] ) 201 self.orecs.add( (0,u,orec) ) 202 203 else: 204 slab = '**spatial structure not found** .. %s' % t1[1] 205 assert False, 'Spatial structures must be consistent with spatialShape_ref.xls' 206 print 'EINFO.structure.01000: ',label,t1,slab 207 nnmm = 0 208 for tt in self.tmap: 209 if self.tmap[tt][1] == 'str-195': 210 print 'EINFO.structure.01040: ',tt, self.tmap[tt] 211 nm = sum( [tt[k] == t1[k] for k in range(7)] ) 212 if nm > 5: 213 nnmm += 1 214 slab = self.spsh.labByUid[tt[1]] 215 lab = self.tmap[tt][1] 216 print 'EINFO.structure.01010: ',lab,tt, slab 217 if nnmm == 0: 218 print 'EINFO.structure.01020: No close matches found' 148 219 self.messageDone.add(t1) 149 220 150 if self.ee3.has_key( t t):151 u = self.ee3[t t]221 if self.ee3.has_key( t1 ): 222 u = self.ee3[t1] 152 223 else: 153 224 u = nt.uid 154 self.ee3[t t] = u225 self.ee3[t1] = u 155 226 if u in self.lldict: 156 227 if self.lldict[u] != nt: … … 161 232 self.lldict[u] = nt 162 233 self.ll.append( (u,nt) ) 163 return ( u,label)234 return (err,u,label) 164 235 165 236 freqmap = {'daily':'day', 'Annual':'yr', 'Timestep':'subhr', '1day':'day', '1mon':'mon', 'month':'mon', 'year':'yr', 'monthly':'mon', 'Day':'day', '6h':'6hr', '3 hourly':'3hr', '3 Hourly':'3hr' }
Note: See TracChangeset
for help on using the changeset viewer.