source: CMIP6dreqbuild/trunk/srcMisc/sx203.py @ 926

Subversion URL: http://proj.badc.rl.ac.uk/svn/exarch/CMIP6dreqbuild/trunk/srcMisc/sx203.py@1157
Revision 926, 2.7 KB checked in by mjuckes, 3 years ago (diff)

stripped sx203.py from sx202.py

Line 
1
2##
3# still missing map from new groups to old table variables.
4## need a key lookup by variable in each table.
5##
6
7import collections, string, hashlib, uuid, os, sys, json
8import shelve
9import dreqSX as sx
10import dreq_utils
11import utils_wb
12
13from dreq_utils import labcoerce
14odir = 'sh20150827'
15
16vdate = '20150907'
17vdate = '20160309'
18if not os.path.isdir( odir ):
19  os.mkdir( odir )
20doexpt = True
21
22if doexpt:
23
24 ##wb = utils_wb.workbook( 'CMIP6DataRequest_ConsolidatedExperiments_20151028.xls' )
25 cvdir = '/data/work/tmp/git/CMIP6_CVs'
26 efn = 'CMIP6DataRequest_ConsolidatedExperiments_20160714.xls'
27 efn = 'CMIP6DataRequest_ConsolidatedExperiments_20161111.xls'
28 efn = 'CMIP6DataRequest_ConsolidatedExperiments_20170117.xls'
29 wb = utils_wb.workbook( efn )
30 sht = wb.book.sheet_by_name( 'Experiments' )
31 ee = json.load( open( '%s/CMIP6_experiment_id.json' % cvdir ))
32 p = dreq_utils.prconsolexpt()
33 p.parse(sht, extra=ee['experiment_id'])
34
35## info about the deck to fill in.
36 wb2 = utils_wb.workbook( 'CMIP6DataRequest_deckInfo.xls' )
37 sht2 = wb2.book.sheet_by_name( 'Experiments' )
38
39## appends records to "p.records"
40 p.parse(sht2)
41
42 sh = shelve.open( '%s/experiments' % odir, 'n' )
43 sh['__info__'] = { 'label':'experiments', 'title':'Experiments', 'prov':'sx202.py, using %s' % efn }
44 sh['__cols__'] = ['uuid','egid','label','description','mip','mcfg','tier','nstart','starty','endy','yps','ensz','ntot','comment']
45 ##['label','group','mip','description','modelClass','tier','nstart','starty','endy','yps','ensz','ntot','comment']
46 eegg = {}
47 eetm = collections.defaultdict( set )
48 eent = collections.defaultdict( int )
49 elabs = {}
50 for r in p.records:
51   u = str( uuid.uuid1() )
52   gp = r[1]
53   if not eegg.has_key(gp):
54     eegg[gp] = str( uuid.uuid1() )
55 
56   if type( r[5] ) == type( 1 ):
57     eetm[gp].add( r[5] )
58   else:
59     eetm[gp].add( r[5][-1] )
60   try:
61     eent[gp] += r[11]
62   except:
63     print 'SEVERE: failed to increment eent[%s]' % gp
64     print r
65     raise
66     print 'SEVERE: failed to increment eent[%s]: % .... %:: %' % (gp,eent[gp],r[11],str(r))
67   label = labcoerce( r[0], nodash=False )
68   assert label not in elabs, 'Duplicate experiment label: %s\n%s\n%s' % (label,str(r),str(sh[elabs[label]]))
69   elabs[label] = u
70   oo = [u,eegg[gp],label]
71   for j in [3,2,4,5,6,7,8,9,10,11,12]:
72     oo.append( r[j] )
73   sh[u] = oo[:]
74 sh.close()
75
76 sh = shelve.open( '%s/exptGroups' % odir, 'n' )
77 sh['__info__'] = { 'label':'exptGroups', 'title':'Experiment groups', 'prov':'ConsolidatedExperiments' }
78 sh['__cols__'] = ['uuid','label','tiermin','ntot']
79 for k in eegg.keys():
80   if not eegg.has_key(gp):
81     eegg[gp] = str( uuid.uuid1() )
82 
83   lab = labcoerce(k,nodash=True)
84   oo = [eegg[k],lab,min(eetm[k]),eent[k]]
85   sh[oo[0]] = oo[:]
86 sh.close()
87
Note: See TracBrowser for help on using the repository browser.