1 | |
---|
2 | import dreq |
---|
3 | import collections, string, os |
---|
4 | import vrev |
---|
5 | try: |
---|
6 | import xlsxwriter |
---|
7 | except: |
---|
8 | print ('No xlsxwrite: will not make tables ...') |
---|
9 | |
---|
10 | class xlsx(object): |
---|
11 | def __init__(self,fn): |
---|
12 | self.wb = xlsxwriter.Workbook(fn) |
---|
13 | |
---|
14 | def newSheet(self,name): |
---|
15 | self.worksheet = self.wb.add_worksheet(name=name) |
---|
16 | return self.worksheet |
---|
17 | |
---|
18 | def close(self): |
---|
19 | self.wb.close() |
---|
20 | |
---|
21 | def vfmt( x ): |
---|
22 | if x < 1.e9: |
---|
23 | s = '%sM' % int( x*1.e-6 ) |
---|
24 | elif x < 1.e12: |
---|
25 | s = '%sG' % int( x*1.e-9 ) |
---|
26 | elif x < 1.e13: |
---|
27 | s = '%3.1fT' % ( x*1.e-12 ) |
---|
28 | elif x < 1.e15: |
---|
29 | s = '%3iT' % int( x*1.e-12 ) |
---|
30 | elif x < 1.e18: |
---|
31 | s = '%3iP' % int( x*1.e-15 ) |
---|
32 | else: |
---|
33 | s = '{:,.2f}'.format( x*1.e-9 ) |
---|
34 | return s |
---|
35 | |
---|
36 | #priority long name units comment questions & notes output variable name standard name unconfirmed or proposed standard name unformatted units cell_methods valid min valid max mean absolute min mean absolute max positive type CMOR dimensions CMOR variable name realm frequency cell_measures flag_values flag_meanings |
---|
37 | |
---|
38 | strkeys = [u'procNote', u'uid', u'odims', u'flag_meanings', u'prov', u'title', u'tmid', u'label', u'cell_methods', u'coords', u'cell_measures', u'spid', u'flag_values', u'description'] |
---|
39 | |
---|
40 | ntstr = collections.namedtuple( 'ntstr', strkeys ) |
---|
41 | |
---|
42 | class cmpd(object): |
---|
43 | def __init__(self,k): |
---|
44 | self.k = k |
---|
45 | def cmp(self,x,y): |
---|
46 | return cmp( x.__dict__[self.k], y.__dict__[self.k] ) |
---|
47 | |
---|
48 | class cmpd2(object): |
---|
49 | def __init__(self,k1,k2): |
---|
50 | self.k1 = k1 |
---|
51 | self.k2 = k2 |
---|
52 | def cmp(self,x,y): |
---|
53 | if x.__dict__[self.k1] == y.__dict__[self.k1]: |
---|
54 | return cmp( x.__dict__[self.k2], y.__dict__[self.k2] ) |
---|
55 | else: |
---|
56 | return cmp( x.__dict__[self.k1], y.__dict__[self.k1] ) |
---|
57 | |
---|
58 | class cmpdn(object): |
---|
59 | def __init__(self,kl): |
---|
60 | self.kl = kl |
---|
61 | def cmp(self,x,y): |
---|
62 | for k in self.kl: |
---|
63 | if x.__dict__[k] != y.__dict__[k]: |
---|
64 | return cmp( x.__dict__[k], y.__dict__[k] ) |
---|
65 | |
---|
66 | return cmp( 0,0 ) |
---|
67 | |
---|
68 | def cmpAnnex( x, y ): |
---|
69 | ax = len(x) > 2 and x[:2] == 'em' |
---|
70 | ay = len(y) > 2 and y[:2] == 'em' |
---|
71 | bx = len(x) > 5 and x[:5] in ['CMIP5','CORDE','SPECS'] |
---|
72 | by = len(y) > 5 and y[:5] in ['CMIP5','CORDE','SPECS'] |
---|
73 | if ax == ay and bx == by: |
---|
74 | return cmp(x,y) |
---|
75 | elif ax: |
---|
76 | if by: |
---|
77 | return cmp(0,1) |
---|
78 | else: |
---|
79 | return cmp(1,0) |
---|
80 | elif ay: |
---|
81 | if bx: |
---|
82 | return cmp(1,0) |
---|
83 | else: |
---|
84 | return cmp(0,1) |
---|
85 | elif bx: |
---|
86 | return cmp(1,0) |
---|
87 | else: |
---|
88 | return cmp(0,1) |
---|
89 | |
---|
90 | import re |
---|
91 | |
---|
92 | |
---|
93 | class makePurl(object): |
---|
94 | def __init__(self): |
---|
95 | c1 = re.compile( '^[a-zA-Z][a-zA-Z0-9]*$' ) |
---|
96 | mv = dq.coll['var'].items |
---|
97 | oo = open( 'htmlRewrite.txt', 'w' ) |
---|
98 | for v in mv: |
---|
99 | if c1.match( v.label ): |
---|
100 | oo.write( 'RewriteRule ^%s$ http://clipc-services.ceda.ac.uk/dreq/u/%s.html\n' % (v.label,v.uid) ) |
---|
101 | else: |
---|
102 | print ('Match failed: %s' % v.label ) |
---|
103 | oo.close() |
---|
104 | |
---|
105 | class makeTab(object): |
---|
106 | def __init__(self, dq, subset=None, dest='tables/test.xlsx', skipped=set(), collected=None): |
---|
107 | if subset != None: |
---|
108 | cmv = [x for x in dq.coll['CMORvar'].items if x.uid in subset] |
---|
109 | else: |
---|
110 | cmv = dq.coll['CMORvar'].items |
---|
111 | tables = sorted( list( set( [i.mipTable for i in cmv] ) ), cmp=cmpAnnex ) |
---|
112 | |
---|
113 | addMips = True |
---|
114 | if addMips: |
---|
115 | c = vrev.checkVar(dq) |
---|
116 | |
---|
117 | wb = xlsx( dest ) |
---|
118 | |
---|
119 | hdr_cell_format = wb.wb.add_format({'text_wrap': True, 'font_size': 14, 'font_color':'#0000ff', 'bold':1, 'fg_color':'#aaaacc'}) |
---|
120 | hdr_cell_format.set_text_wrap() |
---|
121 | sect_cell_format = wb.wb.add_format({'text_wrap': True, 'font_size': 14, 'font_color':'#0000ff', 'bold':1, 'fg_color':'#ccccbb'}) |
---|
122 | sect_cell_format.set_text_wrap() |
---|
123 | cell_format = wb.wb.add_format({'text_wrap': True, 'font_size': 11}) |
---|
124 | cell_format.set_text_wrap() |
---|
125 | |
---|
126 | mode = 'c' |
---|
127 | sht = wb.newSheet( 'Notes' ) |
---|
128 | tableNotes = [('MIPs (...)','The last two columns in each row list MIPs associated with each variable. The first column in this pair lists the MIPs which are requesting the variable in one or more experiments. The second column lists the MIPs proposing experiments in which this variable is requested. E.g. If a variable is requested in a DECK experiment by HighResMIP, then HighResMIP appears in the first column and DECK in the second')] |
---|
129 | sht.write( 0,0, '', hdr_cell_format ) |
---|
130 | sht.write( 0,1, 'Notes on tables', hdr_cell_format ) |
---|
131 | ri = 0 |
---|
132 | sht.set_column(0,0,30) |
---|
133 | sht.set_column(1,1,60) |
---|
134 | for t in tableNotes: |
---|
135 | ri += 1 |
---|
136 | for i in range(2): |
---|
137 | sht.write( ri,i, t[i], cell_format ) |
---|
138 | |
---|
139 | if collected != None: |
---|
140 | ri += 2 |
---|
141 | sht.write( ri, 0, 'Table', sect_cell_format ) |
---|
142 | sht.write( ri, 1, 'Reference Volume (1 deg. atmosphere, 0.5 deg. ocean)', sect_cell_format ) |
---|
143 | for k in sorted( collected.keys() ): |
---|
144 | ri += 1 |
---|
145 | sht.write( ri, 0, k ) |
---|
146 | sht.write( ri, 1, vfmt( collected[k]*2. ) ) |
---|
147 | |
---|
148 | ncga = 'NetCDF Global Attribute' |
---|
149 | withoo = False |
---|
150 | for t in tables: |
---|
151 | if withoo: |
---|
152 | oo = open( 'tables/test_%s.csv' % t, 'w' ) |
---|
153 | sht = wb.newSheet( t ) |
---|
154 | j = 0 |
---|
155 | if mode == 'c': |
---|
156 | hrec = ['Priority','Long name', 'units', 'description', 'comment', 'Variable Name', 'CF Standard Name', 'cell_methods', 'positive', 'type', 'dimensions', 'CMOR Name', 'modeling_realm', 'frequency', 'cell_measures', 'prov', 'provNote','rowIndex'] |
---|
157 | hcmt = ['Default priority (generally overridden by settings in "requestVar" record)',ncga,'','','Name of variable in file','','','CMOR directive','','','CMOR name, unique within table','','','','','','','','',''] |
---|
158 | sht.set_column(1,1,40) |
---|
159 | sht.set_column(1,3,50) |
---|
160 | sht.set_column(1,4,30) |
---|
161 | sht.set_column(1,5,50) |
---|
162 | sht.set_column(1,6,30) |
---|
163 | sht.set_column(1,9,40) |
---|
164 | sht.set_column(1,18,40) |
---|
165 | sht.set_column(1,19,40) |
---|
166 | else: |
---|
167 | hrec = ['','Long name', 'units', 'description', '', 'Variable Name', 'CF Standard Name', '','', 'cell_methods', 'valid_min', 'valid_max', 'ok_min_mean_abs', 'ok_max_mean_abs', 'positive', 'type', 'dimensions', 'CMOR name', 'modeling_realm', 'frequency', 'cell_measures', 'flag_values', 'flag_meanings', 'prov', 'provNote','rowIndex'] |
---|
168 | if addMips: |
---|
169 | hrec.append( 'MIPs (requesting)' ) |
---|
170 | hrec.append( 'MIPs (by experiment)' ) |
---|
171 | |
---|
172 | ##sht.set_row(0,40) |
---|
173 | for i in range(len(hrec)): |
---|
174 | sht.write( j,i, hrec[i], hdr_cell_format ) |
---|
175 | if hcmt[i] != '': |
---|
176 | sht.write_comment( j,i,hcmt[i]) |
---|
177 | thiscmv = sorted( [v for v in cmv if v.mipTable == t], cmp=cmpdn(['prov','rowIndex','label']).cmp ) |
---|
178 | ##print 'INFO.001: Table %s, rows: %s' % (t,len(thiscmv) ) |
---|
179 | |
---|
180 | for v in thiscmv: |
---|
181 | cv = dq.inx.uid[ v.vid ] |
---|
182 | strc = dq.inx.uid[ v.stid ] |
---|
183 | if strc._h.label == 'remarks': |
---|
184 | print ( 'ERROR: structure not found for %s: %s .. %s (%s)' % (v.uid,v.label,v.title,v.mipTable) ) |
---|
185 | ok = False |
---|
186 | else: |
---|
187 | sshp = dq.inx.uid[ strc.spid ] |
---|
188 | tshp = dq.inx.uid[ strc.tmid ] |
---|
189 | ok = all( [i._h.label != 'remarks' for i in [cv,strc,sshp,tshp]] ) |
---|
190 | #[u'shuffle', u'ok_max_mean_abs', u'vid', '_contentInitialised', u'valid_min', u'frequency', u'uid', u'title', u'rowIndex', u'positive', u'stid', u'mipTable', u'label', u'type', u'description', u'deflate_level', u'deflate', u'provNote', u'ok_min_mean_abs', u'modeling_realm', u'prov', u'valid_max'] |
---|
191 | |
---|
192 | if not ok: |
---|
193 | if (t,v.label) not in skipped: |
---|
194 | print ( 'makeTables: skipping %s %s' % (t,v.label) ) |
---|
195 | skipped.add( (t,v.label) ) |
---|
196 | else: |
---|
197 | dims = [] |
---|
198 | dims += string.split( sshp.dimensions, '|' ) |
---|
199 | dims += string.split( tshp.dimensions, '|' ) |
---|
200 | dims += string.split( strc.odims, '|' ) |
---|
201 | dims += string.split( strc.coords, '|' ) |
---|
202 | dims = string.join( dims ) |
---|
203 | if mode == 'c': |
---|
204 | orec = [str(v.defaultPriority),cv.title, cv.units, cv.description, v.description, cv.label, cv.sn, strc.cell_methods, v.positive, v.type, dims, v.label, v.modeling_realm, v.frequency, strc.cell_measures, v.prov,v.provNote,str(v.rowIndex)] |
---|
205 | else: |
---|
206 | orec = ['',cv.title, cv.units, v.description, '', cv.label, cv.sn, '','', strc.cell_methods, v.valid_min, v.valid_max, v.ok_min_mean_abs, v.ok_max_mean_abs, v.positive, v.type, dims, v.label, v.modeling_realm, v.frequency, strc.cell_measures, strc.flag_values, strc.flag_meanings,v.prov,v.provNote,str(v.rowIndex)] |
---|
207 | if addMips: |
---|
208 | thismips = c.chkCmv( v.uid ) |
---|
209 | thismips2 = c.chkCmv( v.uid, byExpt=True ) |
---|
210 | orec.append( string.join( sorted( list( thismips) ),',') ) |
---|
211 | orec.append( string.join( sorted( list( thismips2) ),',') ) |
---|
212 | |
---|
213 | if withoo: |
---|
214 | oo.write( string.join(orec, '\t' ) + '\n' ) |
---|
215 | j+=1 |
---|
216 | for i in range(len(orec)): |
---|
217 | sht.write( j,i, orec[i], cell_format ) |
---|
218 | if withoo: |
---|
219 | oo.close() |
---|
220 | wb.close() |
---|
221 | |
---|
222 | hdr = """ |
---|
223 | var getData = { |
---|
224 | cols: function() { |
---|
225 | var columns = [ {id:0, name:'Variable', field:0, width: 100}, |
---|
226 | {id:1, name:'Standard name', field:1, width: 210 }, |
---|
227 | {id:2, name:'Long name', field:2, width: 180}, |
---|
228 | {id:3, name:'Units', field:3, width: 180}, |
---|
229 | {id:4, name:'uid', field:4, width: 180}]; |
---|
230 | return columns; |
---|
231 | }, |
---|
232 | data: function() { |
---|
233 | var data = []; |
---|
234 | """ |
---|
235 | ftr = """return data; |
---|
236 | } |
---|
237 | }; |
---|
238 | """ |
---|
239 | rtmpl = 'data[%(n)s] = { "id":%(n)s, 0:"%(var)s", 1:"%(sn)s", 2:"%(ln)s", 3:"%(u)s", 4:"%(uid)s" };' |
---|
240 | |
---|
241 | class htmlTrees(object): |
---|
242 | def __init__(self,dq,odir='html/t/'): |
---|
243 | self.dq = dq |
---|
244 | self.odir = odir |
---|
245 | self.c = vrev.checkVar( dq ) |
---|
246 | self.anno = {} |
---|
247 | for v in dq.coll['var'].items: |
---|
248 | self.makeTree( v ) |
---|
249 | |
---|
250 | def makeTree( self, v ): |
---|
251 | ee = self.c.chk2( v.label ) |
---|
252 | if len(ee.keys()) > 0: |
---|
253 | title = 'Usage tree for %s' % v.label |
---|
254 | bdy = ['<h1>%s</h1>' % title, ] |
---|
255 | bdy.append( '<html><head></head><body>\n' ) |
---|
256 | bdy.append( '<ul>\n' ) |
---|
257 | for k in sorted( ee.keys() ): |
---|
258 | l1, xx = ee[k] |
---|
259 | lx = list( xx ) |
---|
260 | if len( lx ) == 0: |
---|
261 | bdy.append( '<li>%s: Empty</li>\n' % l1 ) |
---|
262 | else: |
---|
263 | bdy.append( '<li>%s:\n<ul>' % l1 ) |
---|
264 | for x in lx: |
---|
265 | bdy.append( '<li>%s</li>\n' % x ) |
---|
266 | bdy.append( '</ul></li>\n' ) |
---|
267 | bdy.append( '</ul></body></html>\n' ) |
---|
268 | oo = open( '%s/%s.html' % (self.odir,v.label), 'w' ) |
---|
269 | oo.write( dq.pageTmpl % ( title, '', '../', '../index.html', string.join( bdy, '\n' ) ) ) |
---|
270 | oo.close() |
---|
271 | self.anno[v.label] = '<a href="../t/%s.html">Usage</a>' % v.label |
---|
272 | else: |
---|
273 | self.anno[v.label] = 'Unused' |
---|
274 | |
---|
275 | |
---|
276 | class makeJs(object): |
---|
277 | def __init__(self,dq): |
---|
278 | n = 0 |
---|
279 | rl = [] |
---|
280 | for v in dq.coll['var'].items: |
---|
281 | var = '%s %s' % (v.label,v.uid) |
---|
282 | sn = v.sn |
---|
283 | ln = v.title |
---|
284 | u = v.units |
---|
285 | uid = v.uid |
---|
286 | rr = rtmpl % locals() |
---|
287 | rl.append( rr ) |
---|
288 | n += 1 |
---|
289 | oo = open( 'data2.js', 'w' ) |
---|
290 | oo.write( hdr ) |
---|
291 | for r in rl: |
---|
292 | oo.write( r + '\n' ) |
---|
293 | oo.write( ftr ) |
---|
294 | oo.close() |
---|
295 | |
---|
296 | |
---|
297 | |
---|
298 | class styles(object): |
---|
299 | def __init__(self): |
---|
300 | pass |
---|
301 | |
---|
302 | def rqvLink01(self,targ,frm='',ann=''): |
---|
303 | if targ._h.label == 'remarks': |
---|
304 | return '<li>%s: %s</li>' % ( targ.__href__(odir='../u/', label=targ.title), "Link to request variable broken" ) |
---|
305 | elif frm != "CMORvar": |
---|
306 | cmv = targ._inx.uid[ targ.vid ] |
---|
307 | if targ._h.label == 'remarks': |
---|
308 | return '<li>%s [%s]: %s</li>' % ( cmv.label, targ.__href__(odir='../u/',label=targ.priority) , 'Variable not defined or not found' ) |
---|
309 | else: |
---|
310 | return '<li>%s [%s]: %s</li>' % ( cmv.label, targ.__href__(odir='../u/',label=targ.priority) , cmv.__href__(odir='../u/',label=cmv.title) ) |
---|
311 | else: |
---|
312 | rg = targ._inx.uid[ targ.vgid ] |
---|
313 | if targ._h.label == 'remarks': |
---|
314 | return '<li>%s [%s]: %s</li>' % ( targ.label, targ.__href__(label=targ.priority) , 'Link not defined or not found' ) |
---|
315 | elif rg._h.label == 'remarks': |
---|
316 | return '<li>%s [%s]: %s</li>' % ( rg.label, targ.__href__(label=targ.priority) , 'Group not defined or not found' ) |
---|
317 | else: |
---|
318 | return '<li>%s [%s]: %s</li>' % ( rg.label, targ.__href__(label=targ.priority) , rg.__href__(label=rg.mip) ) |
---|
319 | |
---|
320 | def snLink01(self,a,targ,frm='',ann=''): |
---|
321 | if targ._h.label == 'remarks': |
---|
322 | return '<li>%s: Standard name under review [%s]</li>' % ( a, targ.__href__() ) |
---|
323 | else: |
---|
324 | return '<li>%s [%s]: %s</li>' % ( targ._h.title, a, targ.__href__(label=targ.label) ) |
---|
325 | |
---|
326 | def rqlLink02(self,targ,frm='',ann=''): |
---|
327 | t2 = targ._inx.uid[targ.refid] |
---|
328 | if t2._h.label == 'remarks': |
---|
329 | return '<li>%s: %s</li>' % ( targ.__href__(odir='../u/', label=targ.title), "Link to variable group broken" ) |
---|
330 | elif frm == "requestVarGroup": |
---|
331 | return '<li>%s: %s [%s]</li>' % ( targ.__href__(odir='../u/', label=targ.mip), targ.title, targ.objective ) |
---|
332 | else: |
---|
333 | gpsz = len(t2._inx.iref_by_sect[t2.uid].a['requestVar']) |
---|
334 | return '<li>%s: Link to group: %s [%s]</li>' % ( targ.__href__(odir='../u/', label='%s:%s' % (targ.mip,targ.title)), t2.__href__(odir='../u/', label=t2.title), gpsz ) |
---|
335 | |
---|
336 | def snLink(self,targ,frm='',ann=''): |
---|
337 | return '<li>%s [%s]: %s</li>' % ( targ.title, targ.units, targ.__href__(odir='../u/') ) |
---|
338 | |
---|
339 | def varLink(self,targ,frm='',ann=''): |
---|
340 | return '<li>%s: %s [%s]%s</li>' % ( targ.__href__(odir='../u/', label=targ.label), targ.title, targ.units, ann ) |
---|
341 | |
---|
342 | def mipLink(self,targ,frm='',ann=''): |
---|
343 | if targ.url != '': |
---|
344 | return '<li>%s: %s <a href="%s">[project site]</a></li>' % ( targ.__href__(odir='../u/', label=targ.label), targ.title, targ.url ) |
---|
345 | else: |
---|
346 | return '<li>%s: %s</li>' % ( targ.__href__(odir='../u/', label=targ.label), targ.title ) |
---|
347 | |
---|
348 | def cmvLink(self,targ,frm='',ann=''): |
---|
349 | t2 = targ._inx.uid[targ.stid] |
---|
350 | return '<li>%s {%s}: %s [%s: %s]</li>' % ( targ.__href__(odir='../u/', label=targ.label), targ.mipTable, targ.title, targ.frequency, t2.title ) |
---|
351 | |
---|
352 | def objLink(self,targ,frm='',ann=''): |
---|
353 | return '<li>%s: %s</li>' % ( targ.label, targ.__href__(odir='../u/', label=targ.title,title=targ.description) ) |
---|
354 | |
---|
355 | def strLink(self,targ,frm='',ann=''): |
---|
356 | return '<li>%s: %s</li>' % ( targ.label, targ.__href__(odir='../u/', label=targ.title) ) |
---|
357 | |
---|
358 | def objLnkLink(self,targ,frm='',ann=''): |
---|
359 | if frm == 'objective': |
---|
360 | t2 = targ._inx.uid[targ.rid] |
---|
361 | t3 = targ._inx.uid[t2.refid] |
---|
362 | thislab = '%s (%s)' % (t2.mip,t3.label) |
---|
363 | return '<li>%s: %s</li>' % ( t2.title, t2.__href__(odir='../u/',label=thislab) ) |
---|
364 | else: |
---|
365 | t2 = targ._inx.uid[targ.oid] |
---|
366 | return '<li>%s: %s</li>' % ( t2.label, t2.__href__(odir='../u/',label=t2.title) ) |
---|
367 | |
---|
368 | def labTtl(self,targ,frm='',ann=''): |
---|
369 | return '<li>%s: %s</li>' % ( targ.__href__(odir='../u/', label=targ.label), targ.title ) |
---|
370 | |
---|
371 | def vgrpLink(self,targ,frm='',ann=''): |
---|
372 | gpsz = len(targ._inx.iref_by_sect[targ.uid].a['requestVar']) |
---|
373 | nlnk = len(targ._inx.iref_by_sect[targ.uid].a['requestLink']) |
---|
374 | return '<li>%s {%s}: %s variables, %s request links</li>' % ( targ.__href__(odir='../u/', label=targ.label), targ.mip, gpsz, nlnk ) |
---|
375 | |
---|
376 | class tables(object): |
---|
377 | def __init__(self,sc, mips, odir='xls'): |
---|
378 | self.sc = sc |
---|
379 | self.dq = sc.dq |
---|
380 | self.mips = mips |
---|
381 | self.odir = odir |
---|
382 | |
---|
383 | def setMlab( self, m ): |
---|
384 | if type(m) == type(''): |
---|
385 | mlab = m |
---|
386 | else: |
---|
387 | ll = sorted( list(m) ) |
---|
388 | if len(ll) == 1: |
---|
389 | mlab = list(m)[0] |
---|
390 | else: |
---|
391 | mlab=string.join( [ x[:2].lower() for x in m ], '.' ) |
---|
392 | return mlab |
---|
393 | |
---|
394 | def doTable(self,m,l1,m2,pmax,collector,acc=True, mlab=None): |
---|
395 | """*acc* allows accumulation of values to be switched off when called in single expt mode""" |
---|
396 | |
---|
397 | if mlab == None: |
---|
398 | mlab = self.setMlab( m ) |
---|
399 | |
---|
400 | |
---|
401 | x = self.sc.volByExpt( l1, m2, expFullEx=(m2 in self.mips), pmax=pmax ) |
---|
402 | if x[0] > 0: |
---|
403 | im2 = self.dq.inx.uid[m2] |
---|
404 | collector[mlab].a[im2.label] += x[0] |
---|
405 | # |
---|
406 | # create sum for each table |
---|
407 | # |
---|
408 | xs = 0 |
---|
409 | kkc = '_%s_%s' % (mlab,im2.label) |
---|
410 | for k in x[2].keys(): |
---|
411 | i = self.dq.inx.uid[k] |
---|
412 | xxx = x[2][k] |
---|
413 | xs += xxx |
---|
414 | if xxx > 0: |
---|
415 | collector[kkc].a[i.mipTable] += xxx |
---|
416 | assert x[0] == xs, 'ERROR.0088: consistency problem %s %s %s %s' % (m,m2,x[0],xs) |
---|
417 | if x[0] == 0: |
---|
418 | print ( 'Zero size: %s, %s' % (m,m2) ) |
---|
419 | if len( x[2].keys() ) > 0: |
---|
420 | print ( 'ERROR:zero: %s, %s: %s' % (m,m2,str(x[2].keys()) ) ) |
---|
421 | |
---|
422 | if acc: |
---|
423 | collector[mlab].a['TOTAL'] += x[0] |
---|
424 | |
---|
425 | dd = collections.defaultdict( list ) |
---|
426 | lll = set() |
---|
427 | for v in x[2].keys(): |
---|
428 | vi = self.sc.dq.inx.uid[v] |
---|
429 | if vi._h.label != 'remarks': |
---|
430 | f,t,l,tt,d,u = (vi.frequency,vi.mipTable,vi.label,vi.title,vi.description,vi.uid) |
---|
431 | lll.add(u) |
---|
432 | dd[t].append( (f,t,l,tt,d,u) ) |
---|
433 | if len( dd.keys() ) > 0: |
---|
434 | collector[mlab].dd[im2.label] = dd |
---|
435 | if im2._h.label == 'experiment': |
---|
436 | dothis = self.sc.tierMax >= im2.tier |
---|
437 | ### |
---|
438 | ### BUT ... there is a treset in the request item .... it may be that some variables are excluded ... |
---|
439 | ### need the variable list itself ..... |
---|
440 | ### |
---|
441 | makeTab( self.sc.dq, subset=lll, dest='%s/%s-%s_%s_%s.xlsx' % (self.odir,mlab,im2.label,self.sc.tierMax,pmax), collected=collector[kkc].a ) |
---|
442 | |
---|
443 | styls = styles() |
---|
444 | |
---|
445 | htmlStyle = {} |
---|
446 | htmlStyle['CMORvar'] = {'getIrefs':['requestVar']} |
---|
447 | htmlStyle['requestVarGroup'] = {'getIrefs':['requestVar','requestLink']} |
---|
448 | htmlStyle['var'] = {'getIrefs':['CMORvar']} |
---|
449 | htmlStyle['objective'] = {'getIrefs':['objectiveLink']} |
---|
450 | htmlStyle['requestLink'] = {'getIrefs':['objectiveLink','requestItem']} |
---|
451 | htmlStyle['exptgroup'] = {'getIrefs':['__all__']} |
---|
452 | htmlStyle['requestItem'] = {'getIrefs':['__all__']} |
---|
453 | htmlStyle['experiment'] = {'getIrefs':['__all__']} |
---|
454 | htmlStyle['mip'] = {'getIrefs':['__all__']} |
---|
455 | htmlStyle['remarks'] = {'getIrefs':['__all__']} |
---|
456 | htmlStyle['varChoice'] = {'getIrefs':['__all__']} |
---|
457 | htmlStyle['spatialShape'] = {'getIrefs':['__all__']} |
---|
458 | htmlStyle['temporalShape'] = {'getIrefs':['__all__']} |
---|
459 | htmlStyle['structure'] = {'getIrefs':['__all__']} |
---|
460 | htmlStyle['standardname'] = {'getIrefs':['__all__']} |
---|
461 | |
---|
462 | if __name__ == "__main__": |
---|
463 | assert os.path.isdir( 'html' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml' |
---|
464 | assert os.path.isdir( 'html/u' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below' |
---|
465 | assert os.path.isdir( 'html/index' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below' |
---|
466 | assert os.path.isdir( 'tables' ), 'Before running this script you need to create a "tables" sub-directory, or edit the makeTab class' |
---|
467 | |
---|
468 | dq = dreq.loadDreq( htmlStyles=htmlStyle) |
---|
469 | ## |
---|
470 | ## add special styles to dq object "itemStyle" dictionary. |
---|
471 | ## |
---|
472 | |
---|
473 | dq.itemStyles['standardname'] = styls.snLink |
---|
474 | dq.itemStyles['var'] = styls.varLink |
---|
475 | dq.itemStyles['mip'] = styls.mipLink |
---|
476 | dq.itemStyles['CMORvar'] = styls.cmvLink |
---|
477 | dq.itemStyles['objective'] = styls.objLink |
---|
478 | dq.itemStyles['structure'] = styls.strLink |
---|
479 | dq.itemStyles['objectiveLink'] = styls.objLnkLink |
---|
480 | dq.itemStyles['requestVarGroup'] = styls.vgrpLink |
---|
481 | dq.itemStyles['requestLink'] = styls.rqlLink02 |
---|
482 | dq.itemStyles['spatialShape'] = styls.labTtl |
---|
483 | dq.coll['var'].items[0].__class__._linkAttrStyle['sn'] = styls.snLink01 |
---|
484 | ##dq.coll['requestVarGroup'].items[0].__class__._linkAttrStyle['requestVar'] = styls.rqvLink01 |
---|
485 | dq.itemStyles['requestVar'] = styls.rqvLink01 |
---|
486 | |
---|
487 | ht = htmlTrees(dq) |
---|
488 | dq.makeHtml( annotations={'var':ht.anno} ) |
---|
489 | try: |
---|
490 | import xlsxwriter |
---|
491 | mt = makeTab( dq) |
---|
492 | except: |
---|
493 | print ('Could not make tables ...') |
---|
494 | raise |
---|
495 | mp = makePurl() |
---|
496 | mj = makeJs( dq ) |
---|