| 1 | """Parsing adjusted CMIP5 tables. |
|---|
| 2 | """ |
|---|
| 3 | import string |
|---|
| 4 | from fcc_utils2 import snlist |
|---|
| 5 | import xlrd, string, shelve, os, re, sys |
|---|
| 6 | import collections |
|---|
| 7 | import xlutils, xlwt |
|---|
| 8 | import xlutils.copy |
|---|
| 9 | #### |
|---|
| 10 | import dreq_cfg |
|---|
| 11 | from utils_wb import wbcp, workbook |
|---|
| 12 | |
|---|
| 13 | nt__deckrq = collections.namedtuple( 'dckrq', ['control','AMIP','abrupt4xCO2','rq_1pctCO2','historical'] ) |
|---|
| 14 | dd_rq = collections.defaultdict( dict ) |
|---|
| 15 | dd_tbl = collections.defaultdict( int ) |
|---|
| 16 | |
|---|
| 17 | class tupsort: |
|---|
| 18 | def __init__(self,k=0): |
|---|
| 19 | self.k = k |
|---|
| 20 | def cmp(self,x,y): |
|---|
| 21 | return cmp( x[self.k], y[self.k] ) |
|---|
| 22 | |
|---|
| 23 | def uniquify( ll ): |
|---|
| 24 | ll.sort() |
|---|
| 25 | l0 = [ll[0],] |
|---|
| 26 | for l in ll[1:]: |
|---|
| 27 | if l != l0[-1]: |
|---|
| 28 | l0.append(l) |
|---|
| 29 | return l0 |
|---|
| 30 | |
|---|
| 31 | |
|---|
| 32 | clabs = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" |
|---|
| 33 | def clab(n): |
|---|
| 34 | i = n/26 |
|---|
| 35 | assert i < 26, 'Not ready for row number greater than 26*26' |
|---|
| 36 | if i == 0: |
|---|
| 37 | return clabs[n] |
|---|
| 38 | else: |
|---|
| 39 | return clabs[i-1] + clabs[ n - i*26] |
|---|
| 40 | |
|---|
| 41 | def getRow(sht): |
|---|
| 42 | ee = {} |
|---|
| 43 | for i in range(sht.nrows): |
|---|
| 44 | if sht.row(i)[0].ctype == 2 and str( sht.row(i)[0].value ) != '0.0': |
|---|
| 45 | l = map( lambda x: str( x.value ), sht.row(i) ) |
|---|
| 46 | k = l[5] |
|---|
| 47 | try: |
|---|
| 48 | ee[k] = l |
|---|
| 49 | except: |
|---|
| 50 | print l |
|---|
| 51 | raise |
|---|
| 52 | return ee |
|---|
| 53 | |
|---|
| 54 | def outvSer( ov ): |
|---|
| 55 | ll = [] |
|---|
| 56 | for i in ov: |
|---|
| 57 | ll.append( '%s|%s|%s' % tuple( map( str, i) ) ) |
|---|
| 58 | return string.join(ll, '; ' ) |
|---|
| 59 | def joinRec( rl, tab='\t', omit=[],lmax=None ): |
|---|
| 60 | l1 = len(rl) |
|---|
| 61 | if lmax != None and l1 > lmax: |
|---|
| 62 | l1 = lmax |
|---|
| 63 | rl0 = [] |
|---|
| 64 | for i in range(l1): |
|---|
| 65 | if i not in omit: |
|---|
| 66 | rl0.append(string.strip(str(rl[i]))) |
|---|
| 67 | return string.join( map( str, rl0), tab ) |
|---|
| 68 | |
|---|
| 69 | |
|---|
| 70 | def lennbl(ll): |
|---|
| 71 | i = 0 |
|---|
| 72 | ii = 0 |
|---|
| 73 | for l in ll: |
|---|
| 74 | i+= 1 |
|---|
| 75 | if l != '': |
|---|
| 76 | ii = i |
|---|
| 77 | return ii |
|---|
| 78 | |
|---|
| 79 | def matchVals( thisv, valset ): |
|---|
| 80 | ll1 = [] |
|---|
| 81 | ll2 = [] |
|---|
| 82 | lk0 = [] |
|---|
| 83 | kk = 0 |
|---|
| 84 | for vals in valset: |
|---|
| 85 | l1 = min( [lennbl(thisv), lennbl(vals)] ) |
|---|
| 86 | l2 = max( [lennbl(thisv), lennbl(vals)] ) |
|---|
| 87 | ll = [] |
|---|
| 88 | for i in range(l1): |
|---|
| 89 | ll.append( vals[i] == thisv[i] ) |
|---|
| 90 | for i in range(l2-l1): |
|---|
| 91 | ll.append(False) |
|---|
| 92 | if all(ll): |
|---|
| 93 | return ll |
|---|
| 94 | k = 0 |
|---|
| 95 | for l in ll: |
|---|
| 96 | if l: |
|---|
| 97 | k+=1 |
|---|
| 98 | if ll[0]: |
|---|
| 99 | lk0.append( kk ) |
|---|
| 100 | |
|---|
| 101 | ll1.append(k) |
|---|
| 102 | ll2.append(ll) |
|---|
| 103 | kk += 1 |
|---|
| 104 | k0 = max( ll1 ) |
|---|
| 105 | if len(lk0) > 0: |
|---|
| 106 | k0 = lk0[0] |
|---|
| 107 | return ll2[ k0 ] |
|---|
| 108 | return ll2[ ll1.index(k0) ] |
|---|
| 109 | |
|---|
| 110 | class mipTable(object): |
|---|
| 111 | |
|---|
| 112 | def __init__(self,id): |
|---|
| 113 | self.vars = collections.defaultdict( list ) |
|---|
| 114 | |
|---|
| 115 | class rqsummary(object): |
|---|
| 116 | __doc__ = """Create a request summary table, by MIP and variable table. |
|---|
| 117 | Creates a list of default dictionaries, one for each table. An entry for each MIP in each dictionary. |
|---|
| 118 | """ |
|---|
| 119 | |
|---|
| 120 | def __init__(self,exptInfo=None): |
|---|
| 121 | self.tablist = [] |
|---|
| 122 | self.tabindx = {} |
|---|
| 123 | self.mips = collections.defaultdict( int ) |
|---|
| 124 | self.vars = collections.defaultdict( list ) |
|---|
| 125 | self.t = {} |
|---|
| 126 | self.records = [] |
|---|
| 127 | self.exptInfo = exptInfo |
|---|
| 128 | |
|---|
| 129 | def addTab(self,tab): |
|---|
| 130 | self.t[tab] = mipTable( tab ) |
|---|
| 131 | |
|---|
| 132 | def add(self,mip,path,tab): |
|---|
| 133 | self.tab = tab |
|---|
| 134 | return rq(mip,path,parent=self,tab=tab) |
|---|
| 135 | |
|---|
| 136 | def addItem( self, mip, tab, nn ): |
|---|
| 137 | if not self.tabindx.has_key( tab ): |
|---|
| 138 | self.tabindx[tab] = len(self.tablist) |
|---|
| 139 | self.tablist.append( collections.defaultdict( int ) ) |
|---|
| 140 | ix = self.tabindx[tab] |
|---|
| 141 | self.tablist[ix][mip] = nn |
|---|
| 142 | self.mips[mip] += 1 |
|---|
| 143 | |
|---|
| 144 | def checkVars(self): |
|---|
| 145 | keys = self.vars.keys() |
|---|
| 146 | keys.sort() |
|---|
| 147 | for k in keys: |
|---|
| 148 | if len( self.vars[k] ) == 1: |
|---|
| 149 | print k, 'singleton' |
|---|
| 150 | else: |
|---|
| 151 | li = self.vars[k] |
|---|
| 152 | ## getting a bit intricate here -- 3rd element is index of record row |
|---|
| 153 | ll = [self.records[li[0][2]], ] |
|---|
| 154 | aa = collections.defaultdict( list ) |
|---|
| 155 | aa[0].append[0] |
|---|
| 156 | |
|---|
| 157 | def prep(self,mode='vars',tab=None): |
|---|
| 158 | if tab != None: |
|---|
| 159 | self.tab = tab |
|---|
| 160 | kk = 0 |
|---|
| 161 | mips = self.mips.keys() |
|---|
| 162 | mips.sort() |
|---|
| 163 | keys = self.tabindx.keys() |
|---|
| 164 | keys.sort() |
|---|
| 165 | self.oorecs = [] |
|---|
| 166 | if mode == 'vars': |
|---|
| 167 | keys = self.t[self.tab].vars.keys() |
|---|
| 168 | keys.sort() |
|---|
| 169 | for k in keys: |
|---|
| 170 | rec = [k,] |
|---|
| 171 | ee1 = collections.defaultdict( list ) |
|---|
| 172 | ee2 = {} |
|---|
| 173 | for t in self.t[self.tab].vars[k]: |
|---|
| 174 | k0 = joinRec( self.records[t[2]], tab=',', omit=[7,16,23] ) |
|---|
| 175 | ee1[k0].append( t ) |
|---|
| 176 | if len(ee1.keys()) > 1: |
|---|
| 177 | for k1 in ee1.keys(): |
|---|
| 178 | for t in ee1[k1]: |
|---|
| 179 | ee2[t[0]] = k1 |
|---|
| 180 | k1 = ee1.keys() |
|---|
| 181 | ee3 = collections.defaultdict( list ) |
|---|
| 182 | for t in self.t[self.tab].vars[k]: |
|---|
| 183 | ee3[k1.index( ee2[t[0]] ) ].append( '%s[%s]' % (t[0],t[1]) ) |
|---|
| 184 | rec.append( '%s[%s]{%s}' % (t[0],t[1],k1.index( ee2[t[0]]) ) ) |
|---|
| 185 | ##self.oorecs.append(rec) |
|---|
| 186 | for kkk in k1: |
|---|
| 187 | ix = ee1[kkk][0][2] |
|---|
| 188 | ik = k1.index(kkk) |
|---|
| 189 | rec = [k, string.join(ee3[ik],';'),] + list(self.records[ix]) |
|---|
| 190 | self.oorecs.append(rec) |
|---|
| 191 | |
|---|
| 192 | def showXls(self,wb,r0=3,offset=0): |
|---|
| 193 | r = r0 |
|---|
| 194 | thisStj = 2 |
|---|
| 195 | for rec in self.oorecs: |
|---|
| 196 | i = offset |
|---|
| 197 | for item in rec: |
|---|
| 198 | wb.putValue2( r,i, item ) |
|---|
| 199 | i+= 1 |
|---|
| 200 | thisStj = 5 - thisStj |
|---|
| 201 | r+= 1 |
|---|
| 202 | |
|---|
| 203 | def show(self,oo,mode='vars',tab=None): |
|---|
| 204 | if tab != None: |
|---|
| 205 | self.tab = tab |
|---|
| 206 | kk = 0 |
|---|
| 207 | mips = self.mips.keys() |
|---|
| 208 | mips.sort() |
|---|
| 209 | keys = self.tabindx.keys() |
|---|
| 210 | keys.sort() |
|---|
| 211 | if mode == 'vars': |
|---|
| 212 | for rec in self.oorecs: |
|---|
| 213 | srec = joinRec( rec) + '\t' |
|---|
| 214 | oo.write( srec + '\n' ) |
|---|
| 215 | |
|---|
| 216 | elif mode == 'oldvars': |
|---|
| 217 | keys = self.t[self.tab].vars.keys() |
|---|
| 218 | keys.sort() |
|---|
| 219 | for k in keys: |
|---|
| 220 | rec = '%s\t' % k |
|---|
| 221 | ee1 = collections.defaultdict( list ) |
|---|
| 222 | ee2 = {} |
|---|
| 223 | for t in self.t[self.tab].vars[k]: |
|---|
| 224 | k0 = joinRec( self.records[t[2]], tab=',', omit=[7,16,23] ) |
|---|
| 225 | ee1[k0].append( t ) |
|---|
| 226 | if len(ee1.keys()) > 1: |
|---|
| 227 | for k1 in ee1.keys(): |
|---|
| 228 | for t in ee1[k1]: |
|---|
| 229 | ee2[t[0]] = k1 |
|---|
| 230 | k1 = ee1.keys() |
|---|
| 231 | for t in self.t[self.tab].vars[k]: |
|---|
| 232 | rec += '%s[%s]{%s}\t' % (t[0],t[1],k1.index( ee2[t[0]]) ) |
|---|
| 233 | oo.write( rec + '\n' ) |
|---|
| 234 | for kkk in k1: |
|---|
| 235 | ix = ee1[kkk][0][2] |
|---|
| 236 | rec = '--%s--\t' % k1.index(kkk) + joinRec( self.records[ix] ) + '\t' |
|---|
| 237 | oo.write( rec + '\n' ) |
|---|
| 238 | else: |
|---|
| 239 | oo.write( ',' + string.join(mips, ',') + ',\n' ) |
|---|
| 240 | for k in keys: |
|---|
| 241 | r = [] |
|---|
| 242 | ix = self.tabindx[k] |
|---|
| 243 | for m in mips: |
|---|
| 244 | r.append( self.tablist[ix][m] ) |
|---|
| 245 | if max(r) > 0: |
|---|
| 246 | rec = '%s,' % k |
|---|
| 247 | for i in r: |
|---|
| 248 | rec += '%s,' % i |
|---|
| 249 | oo.write( rec + '\n' ) |
|---|
| 250 | |
|---|
| 251 | |
|---|
| 252 | #dd_rq = collections.defaultdict( dict ) |
|---|
| 253 | #dd_tbl = collections.defaultdict( int ) |
|---|
| 254 | class rq(object): |
|---|
| 255 | |
|---|
| 256 | def __init__(self, mip,path, parent=None,tab=u'Omon' ): |
|---|
| 257 | |
|---|
| 258 | self.mip = mip |
|---|
| 259 | self.nn = 0 |
|---|
| 260 | self.wk1 = wbcp( inbook=path ) |
|---|
| 261 | if tab in self.wk1.sns: |
|---|
| 262 | self.tab = tab |
|---|
| 263 | self.wk1.focus( tab) |
|---|
| 264 | self.parent = parent |
|---|
| 265 | if not self.parent.t.has_key(tab): |
|---|
| 266 | self.parent.addTab(tab) |
|---|
| 267 | self.ok = self.parse01() |
|---|
| 268 | else: |
|---|
| 269 | self.ok = False |
|---|
| 270 | |
|---|
| 271 | def parse01(self): |
|---|
| 272 | |
|---|
| 273 | ee1 = collections.defaultdict( list ) |
|---|
| 274 | for j in range(3,self.wk1.currentSi.nrows): |
|---|
| 275 | if self.wk1.currentSi.row(j)[0].ctype == 2: |
|---|
| 276 | v = string.strip( str( self.wk1.currentSi.row(j)[5].value ) ) |
|---|
| 277 | ee1[v].append(j) |
|---|
| 278 | for j in range(3,self.wk1.currentSi.nrows): |
|---|
| 279 | rv = map( lambda x: x.value, self.wk1.currentSi.row(j) ) |
|---|
| 280 | lr0 = len(rv) |
|---|
| 281 | for i in range(10): |
|---|
| 282 | if len(rv) < 26: |
|---|
| 283 | rv.append('') |
|---|
| 284 | if self.wk1.currentSi.row(j)[0].ctype == 2: |
|---|
| 285 | v = string.strip( str( self.wk1.currentSi.row(j)[5].value ) ) |
|---|
| 286 | if v not in ['0.0','']: |
|---|
| 287 | if len(ee1[v]) > 1: |
|---|
| 288 | v += '__%s' % ['a','b'][ee1[v].index(j)] |
|---|
| 289 | ixr = len(self.parent.records ) |
|---|
| 290 | self.parent.records.append( tuple(rv[1:]) ) |
|---|
| 291 | p = int( rv[0] + 0.001 ) |
|---|
| 292 | if lr0 > 24: |
|---|
| 293 | if self.wk1.currentSi.row(j)[24].ctype == 2: |
|---|
| 294 | yv = int( rv[24] + 0.001 ) |
|---|
| 295 | assert yv in [0,1,2], 'Invalid value in column Y, j=%s,v=%s' % (j,v) |
|---|
| 296 | if yv == 0: |
|---|
| 297 | p = None |
|---|
| 298 | elif yv == 2: |
|---|
| 299 | p = 1 |
|---|
| 300 | elif rv[24] != '': |
|---|
| 301 | print 'WARNING[1]:', self.mip, v, p, rv[24] |
|---|
| 302 | |
|---|
| 303 | if p != None: |
|---|
| 304 | self.parent.t[self.tab].vars[v].append( (self.mip,p,ixr) ) |
|---|
| 305 | self.nn += 1 |
|---|
| 306 | |
|---|
| 307 | return True |
|---|
| 308 | |
|---|
| 309 | class main(object): |
|---|
| 310 | def __init__(self): |
|---|
| 311 | kk=3 |
|---|
| 312 | |
|---|
| 313 | base = '/home/martin/2014/wip/dreq/' |
|---|
| 314 | self.dir0 = '/home/martin/2014/wip/dreq/input/' |
|---|
| 315 | file = 'PMIP/CMIP6DataRequestCompilationTemplate_20141218_PMIP_v150228_final.xls' |
|---|
| 316 | file = 'C4MIP/CMIP6DataRequestCompilationTemplate_C4MIP_06March2015_submitted.xls' |
|---|
| 317 | fileTmpl = 'CMIP6DataRequestCompilationTemplate_20141218.xls' |
|---|
| 318 | self.newVars = 'CMIP6DataRequest_ConsolidatedNewVariables_20150428.xls' |
|---|
| 319 | self.nvd = {} |
|---|
| 320 | |
|---|
| 321 | wk0 = wbcp( inbook=base+fileTmpl ) |
|---|
| 322 | self.cfg = dreq_cfg.rqcfg() |
|---|
| 323 | wk0.focus( u'Experiments') |
|---|
| 324 | mlist = collections.defaultdict( list ) |
|---|
| 325 | for i in range(2,wk0.currentSi.nrows): |
|---|
| 326 | mip = str(wk0.currentSi.row(i)[3].value) |
|---|
| 327 | vals = map( lambda x: x.value, wk0.currentSi.row(i) ) |
|---|
| 328 | if mip != '': |
|---|
| 329 | mlist[mip].append( vals ) |
|---|
| 330 | |
|---|
| 331 | self.mips = mlist.keys() |
|---|
| 332 | self.mips.sort() |
|---|
| 333 | |
|---|
| 334 | mipTrans ={ 'geoMIP':'GeoMIP' } |
|---|
| 335 | diagMips = ['DynVar','SIMIP'] |
|---|
| 336 | k2 = self.cfg.ff.keys() |
|---|
| 337 | k2.sort() |
|---|
| 338 | nn = 0 |
|---|
| 339 | for k in k2: |
|---|
| 340 | k = mipTrans.get( k,k) |
|---|
| 341 | if k not in self.mips and k not in diagMips: |
|---|
| 342 | print '%s not found' % k |
|---|
| 343 | nn += 1 |
|---|
| 344 | assert nn == 0, 'MIP naming error?' |
|---|
| 345 | |
|---|
| 346 | exptList = '/home/martin/Documents/CMIP6_ExpermentList_draft01April2015.xls' |
|---|
| 347 | wke = wbcp( exptList ) |
|---|
| 348 | wke.focus( 'Experiments' ) |
|---|
| 349 | expgs = collections.defaultdict( int ) |
|---|
| 350 | expgsEns = collections.defaultdict( int ) |
|---|
| 351 | expgsYpe = collections.defaultdict( int ) |
|---|
| 352 | for k in range(2,wke.currentSi.nrows): |
|---|
| 353 | vals = map( lambda x: x.value, wke.currentSi.row(k) ) |
|---|
| 354 | g = string.strip( vals[1] ) |
|---|
| 355 | if g[:5] == 'DAMIP': |
|---|
| 356 | print '######',g,vals[12],vals[13] |
|---|
| 357 | expgs[g] += vals[13] |
|---|
| 358 | expgsEns[g] = vals[12] |
|---|
| 359 | |
|---|
| 360 | for k in expgs.keys(): |
|---|
| 361 | if expgsEns[k] == 0: |
|---|
| 362 | expgsYpe[k] = 0 |
|---|
| 363 | else: |
|---|
| 364 | try: |
|---|
| 365 | expgsYpe[k] = expgs[k]/expgsEns[k] |
|---|
| 366 | except: |
|---|
| 367 | print k, expgs[k], expgsEns[k] |
|---|
| 368 | raise |
|---|
| 369 | |
|---|
| 370 | self.rqs= rqsummary(exptInfo=(expgs,expgsEns,expgsYpe)) |
|---|
| 371 | self.exptInfo=(expgs,expgsEns,expgsYpe) |
|---|
| 372 | |
|---|
| 373 | def loadNewVars(self): |
|---|
| 374 | assert os.path.isfile( self.newVars ), '%s not found' % self.newVars |
|---|
| 375 | wb = workbook(self.newVars) |
|---|
| 376 | sh = wb.book.sheet_by_name( 'New variables' ) |
|---|
| 377 | oo = open( '/tmp/newvars.txt', 'w' ) |
|---|
| 378 | for i in range(3,sh.nrows): |
|---|
| 379 | r = map( lambda x: x.value, sh.row(i) ) |
|---|
| 380 | try: |
|---|
| 381 | oo.write( string.join(map(str,r) ) + '\n' ) |
|---|
| 382 | except: |
|---|
| 383 | oo.write( r[0] + ' --- rest missing --- \n' ) |
|---|
| 384 | if r[0] != '' and r[4] != '': |
|---|
| 385 | v = string.strip( str( r[4] ) ) |
|---|
| 386 | if v[0] != '#': |
|---|
| 387 | bits = string.split(v,' ') |
|---|
| 388 | if len(bits) == 1: |
|---|
| 389 | m = str( r[0] ) |
|---|
| 390 | p = 0 |
|---|
| 391 | try: |
|---|
| 392 | if m == 'LUMIP': |
|---|
| 393 | p = int( r[12] ) |
|---|
| 394 | else: |
|---|
| 395 | p = int( r[11] ) |
|---|
| 396 | except: |
|---|
| 397 | print '!!!!!! Failed to read priority: ',v,m |
|---|
| 398 | if v[-2:] == '**': |
|---|
| 399 | v = v[:-2] |
|---|
| 400 | elif v[-1] == '!': |
|---|
| 401 | v = v[:-1] |
|---|
| 402 | if self.nvd.has_key(v): |
|---|
| 403 | print '#########',v,self.nvd[v],m,p |
|---|
| 404 | self.nvd[v] = (m,p) |
|---|
| 405 | oo.close() |
|---|
| 406 | |
|---|
| 407 | def groups(self): |
|---|
| 408 | print self.cfg.ee.keys() |
|---|
| 409 | omit = ['ALL VARIABLES', 'Objectives','Experiments','Experiment Groups','Request scoping','New variables','__lists__'] |
|---|
| 410 | keys = self.cfg.ee.keys() |
|---|
| 411 | keys.sort() |
|---|
| 412 | ee = {} |
|---|
| 413 | for k in keys: |
|---|
| 414 | fn = self.cfg.ee[k] |
|---|
| 415 | path = '%s%s/%s' % (self.dir0,k,fn) |
|---|
| 416 | wb = workbook( path ) |
|---|
| 417 | ss = [] |
|---|
| 418 | for s in wb.sns: |
|---|
| 419 | if s not in omit: |
|---|
| 420 | ss.append(s) |
|---|
| 421 | for s in ss: |
|---|
| 422 | e1 = collections.defaultdict( int ) |
|---|
| 423 | if s[:5] != 'CMIP5': |
|---|
| 424 | sh = wb.book.sheet_by_name( s ) |
|---|
| 425 | ll = [] |
|---|
| 426 | for i in range(sh.nrows): |
|---|
| 427 | thisr = sh.row(i) |
|---|
| 428 | tv = thisr[0].value |
|---|
| 429 | if tv[:10] == 'Short name': |
|---|
| 430 | ll.append(i) |
|---|
| 431 | assert len(ll) in [1,2], 'Could not parse sheet %s, %s, %s' % (path,k,s) |
|---|
| 432 | if len(ll) == 1: |
|---|
| 433 | iv = 1 |
|---|
| 434 | it = 0 |
|---|
| 435 | else: |
|---|
| 436 | iv = 0 |
|---|
| 437 | it = -1 |
|---|
| 438 | tv = sh.row(2)[1].value |
|---|
| 439 | irsh = 5 |
|---|
| 440 | for i in range(ll[-1]+1,sh.nrows): |
|---|
| 441 | thisr = sh.row(i) |
|---|
| 442 | if it >= 0: |
|---|
| 443 | tv = thisr[it].value |
|---|
| 444 | v = thisr[iv].value |
|---|
| 445 | bits = string.split(v, ' ') |
|---|
| 446 | if len(bits) > 1: |
|---|
| 447 | v = bits[0] |
|---|
| 448 | t = thisr[2].value |
|---|
| 449 | if t[:6] == "CMIP5_": |
|---|
| 450 | t = t[6:] |
|---|
| 451 | if len(string.split(t, ' ')) > 1: |
|---|
| 452 | t = string.split(t, ' ')[0] |
|---|
| 453 | if t in self.cmip5sns: |
|---|
| 454 | p = 9 |
|---|
| 455 | if self.rqs.t.has_key(t): |
|---|
| 456 | if self.rqs.t[t].vars.has_key(v): |
|---|
| 457 | p = self.rqs.t[t].vars[v][0][1] |
|---|
| 458 | else: |
|---|
| 459 | print '!!!!!!!!! %s not found in %s (%s)' % (v,t,k) |
|---|
| 460 | else: |
|---|
| 461 | print '!!!!!!!!! %s not found in CMIP5 (%s, %s)' % (t,k,v) |
|---|
| 462 | else: |
|---|
| 463 | if string.lower(t)[:3] != 'new' and t != '': |
|---|
| 464 | print '*********** %s not recognised as table %s' % (t,k) |
|---|
| 465 | p = self.nvd.get( v, [0,0] )[1] |
|---|
| 466 | if p == 0: |
|---|
| 467 | p = self.nvd.get( v + '!', [0,0] )[1] |
|---|
| 468 | if p == 0: |
|---|
| 469 | print '*********** p=0:: %s, %s, %s' % (v,t,k) |
|---|
| 470 | ts = string.strip(thisr[irsh].value) |
|---|
| 471 | if string.strip(tv) != '': |
|---|
| 472 | e1['%s:%s' % (ts,p)] += 1 |
|---|
| 473 | print k, s, ll, e1 |
|---|
| 474 | |
|---|
| 475 | def run1(self,nmip=0): |
|---|
| 476 | self.cfg.ff['CMIP5'] = ['/data/work/documents/CMIP5_standard_output.xls'] |
|---|
| 477 | wb = workbook( self.cfg.ff['CMIP5'][0] ) |
|---|
| 478 | wb.sns.sort() |
|---|
| 479 | omit1 = [u'dims', u'general', u'other output',u'CFMIP output'] |
|---|
| 480 | sns = [] |
|---|
| 481 | for s in wb.sns: |
|---|
| 482 | if s not in omit1: |
|---|
| 483 | sns.append(s) |
|---|
| 484 | |
|---|
| 485 | self.cmip5sns = sns[:] |
|---|
| 486 | k2 = self.cfg.ff.keys() |
|---|
| 487 | k2.sort() |
|---|
| 488 | k2.append( 'CMIP5' ) |
|---|
| 489 | |
|---|
| 490 | amips = ['CMIP5'] + self.mips |
|---|
| 491 | if nmip > 0 and nmip < len(amips): |
|---|
| 492 | amips = amips[:nmip] |
|---|
| 493 | for s in sns: |
|---|
| 494 | for k in amips: |
|---|
| 495 | if k in k2: |
|---|
| 496 | for fn in self.cfg.ff[k]: |
|---|
| 497 | if k == 'CMIP5': |
|---|
| 498 | path = fn |
|---|
| 499 | else: |
|---|
| 500 | path = '%s%s/%s' % (self.dir0,k,fn) |
|---|
| 501 | thisrq = self.rqs.add( k, path,s ) |
|---|
| 502 | if thisrq.ok: |
|---|
| 503 | print k,path,s,thisrq.ok, thisrq.nn |
|---|
| 504 | else: |
|---|
| 505 | print k,path,s,thisrq.ok |
|---|
| 506 | |
|---|
| 507 | def run2(self): |
|---|
| 508 | init=False |
|---|
| 509 | if init: |
|---|
| 510 | wbx = wbcp( '/home/martin/Documents/MIPTableSheetTemplate.xls' ) |
|---|
| 511 | else: |
|---|
| 512 | wbx = wbcp( 'CMIP6_DataRequest_CMIP5changes_blank.xls' ) |
|---|
| 513 | |
|---|
| 514 | for s in sns: |
|---|
| 515 | keys = self.rqs.t[s].vars.keys() |
|---|
| 516 | keys.sort() |
|---|
| 517 | ##for k in keys: |
|---|
| 518 | ##print k, self.rqs.t[s].vars[k] |
|---|
| 519 | self.rqs.prep( tab=s ) |
|---|
| 520 | print s, len(self.rqs.oorecs) |
|---|
| 521 | if len(self.rqs.oorecs) > 0: |
|---|
| 522 | if init: |
|---|
| 523 | wbx.copy_sheet( 0, s ) |
|---|
| 524 | wbx.focus( s, old='Template' ) |
|---|
| 525 | else: |
|---|
| 526 | wbx.focus( s ) |
|---|
| 527 | self.rqs.showXls(wbx,r0=2,offset=0) |
|---|
| 528 | |
|---|
| 529 | for i in range( min( 5, len( self.rqs.oorecs) ) ): |
|---|
| 530 | print self.rqs.oorecs[i] |
|---|
| 531 | |
|---|
| 532 | oo = open( 'dreq2/request2_%s.csv' % s, 'w' ) |
|---|
| 533 | self.rqs.show( oo, tab=s ) |
|---|
| 534 | oo.close() |
|---|
| 535 | |
|---|
| 536 | wbx.write( 'CMIP6_DataRequest_CMIP5changes.xls' ) |
|---|
| 537 | |
|---|
| 538 | |
|---|
| 539 | |
|---|
| 540 | m = main() |
|---|
| 541 | m.run1(nmip=1) |
|---|
| 542 | m.loadNewVars() |
|---|
| 543 | m.groups() |
|---|
