1 | #!/usr/bin/env python |
---|
2 | #************************************************************************************** |
---|
3 | #csmlfeaturetypes.py |
---|
4 | #For creating CSML featuretypes |
---|
5 | #v0.5 split off 11th November 2005 |
---|
6 | #Dominic Lowe, BADC |
---|
7 | #************************************************************************************** |
---|
8 | |
---|
9 | import csmllibs |
---|
10 | import sys |
---|
11 | |
---|
12 | class featureBuilder: |
---|
13 | def __init__(self, dataset_element, gml_FeatureCollection_element,extractType, ffmap,fileExtractDictionary, timedim, timestorage,spatialstorage,valuestorage): |
---|
14 | self.ds_element=dataset_element |
---|
15 | self.gml_FeatureCollection_element = gml_FeatureCollection_element |
---|
16 | self.extractType = extractType |
---|
17 | self.ffmap = ffmap |
---|
18 | self.fileExtractDictionary = fileExtractDictionary |
---|
19 | self.timedim = timedim |
---|
20 | self.timestorage=timestorage |
---|
21 | self.spatialstorage=spatialstorage |
---|
22 | self.valuestorage=valuestorage |
---|
23 | |
---|
24 | #empty list to hold featureMembers |
---|
25 | self.fms =[] |
---|
26 | |
---|
27 | #at the moment, only one featuretype per CSML Dataset is supported. |
---|
28 | #get the featuretype of the first representative file in the ffmap object |
---|
29 | self.featuretype= self.ffmap.getRepresentativeFiles()[0].getFeatureType() |
---|
30 | if self.featuretype == 'GridSeries': |
---|
31 | print 'creating grids' |
---|
32 | self.createCSMLGridSeriesFeatures() |
---|
33 | elif self.featuretype == 'PointSeries': |
---|
34 | self.createCSMLPointSeriesFeatures() |
---|
35 | |
---|
36 | #after the features have been generated append all featureMembers to the feature collection |
---|
37 | self.gml_FeatureCollection_element.members=self.fms |
---|
38 | |
---|
39 | def createCSMLGridSeriesFeatures(self): |
---|
40 | #This function assumes that the variables (features) are shared across identically structured files |
---|
41 | #should be supplied with a featurefilemap object (see csmlfiles for FileMapMaker) |
---|
42 | |
---|
43 | representativeFiles=self.ffmap.getRepresentativeFiles() |
---|
44 | #fileid=0 #prefix used to distinguish variable names for similar variables from different files |
---|
45 | #don't use this - same variables should not be in same data granule. |
---|
46 | for repfile in representativeFiles: |
---|
47 | #fileid=fileid+1 |
---|
48 | listOfFiles=[] |
---|
49 | repfilename=repfile.getRepresentativeFileName() |
---|
50 | listOfFiles.append(repfilename) |
---|
51 | relfiles = repfile.getRelatedFiles() |
---|
52 | for f in relfiles: |
---|
53 | fname = f.getRelatedFileName() |
---|
54 | listOfFiles.append(fname) |
---|
55 | |
---|
56 | #THIS IS THE REALLY SLOW FUNCTION CALL!!!!!######################### |
---|
57 | OrderedFileTimeList,caltype,units = csmllibs.csmltime.getFileTimeList(listOfFiles,self.timedim) |
---|
58 | #build strings to hold times/filenames for current gridseriesfeature |
---|
59 | #cal type returns the type of calendar used |
---|
60 | timeString ='' |
---|
61 | filesinDir = '' |
---|
62 | for j in range (0, len(OrderedFileTimeList)): |
---|
63 | t= OrderedFileTimeList[j][0] |
---|
64 | f = OrderedFileTimeList[j][1] |
---|
65 | timeString = timeString + ' ' + str(t) |
---|
66 | filesinDir = filesinDir + ' ' + f |
---|
67 | #Open representative file and create feature members: |
---|
68 | DI = csmllibs.csmldataiface.DataInterface() |
---|
69 | DI=DI.getUnknownInterfaceType(repfilename) |
---|
70 | DI.openFile(repfilename) |
---|
71 | allVarNames=DI.getListofVariables() |
---|
72 | numFeatures=len(allVarNames) |
---|
73 | #Create a GridSeriesFeature for each variable: |
---|
74 | for i in range(0, numFeatures): |
---|
75 | DI.setVariable(allVarNames[i]) |
---|
76 | dimNames=DI.getVariableAxes() |
---|
77 | if len(dimNames) <= 2: |
---|
78 | #it's an axis or bounds not a feature, try next variable |
---|
79 | continue |
---|
80 | |
---|
81 | GridSeriesFeature_element=csmllibs.Parser.GridSeriesFeature() |
---|
82 | #GridSeriesFeature_element.id=str(fileid)+'__'+str(allVarNames[i]) |
---|
83 | GridSeriesFeature_element.id=str(allVarNames[i]) |
---|
84 | #description: need to get the attribute called long_name (?? TODO - is this CF compliant??) |
---|
85 | #use Ag's getbestname from nappy package? |
---|
86 | desc=DI.getVariableAttribute('long_name') |
---|
87 | try: |
---|
88 | desc=DI.getVariableAttribute('long_name') |
---|
89 | except AttributeError: |
---|
90 | desc = "missing name" |
---|
91 | |
---|
92 | desc=desc.replace('&','&') |
---|
93 | GridSeriesFeature_element.description=csmllibs.Parser.Description(desc) |
---|
94 | |
---|
95 | #*********************************************************************** |
---|
96 | #GridSeriesDomain: |
---|
97 | #*********************************************************************** |
---|
98 | gsDomain=csmllibs.Parser.GridSeriesDomain() |
---|
99 | #*********************************************************************** |
---|
100 | # domainReference element |
---|
101 | #*********************************************************************** |
---|
102 | # |
---|
103 | tpl=csmllibs.Parser.TimePositionList() |
---|
104 | if self.timestorage =='inline': |
---|
105 | tpl.timePositions=timeString |
---|
106 | tpl.frame='%s:%s'%(caltype,units) |
---|
107 | else: |
---|
108 | # do something to create a single extract for the times (from the representative file). |
---|
109 | tpl.timePositions = csmllibs.csmlfileextracts.createSingleExtract(self.extractType, repfilename, self.timedim, len(timeString.split())) |
---|
110 | tpl.frame='%s:%s'%(caltype,units) |
---|
111 | |
---|
112 | gsDomain.domainReference=tpl |
---|
113 | grid=csmllibs.Parser.Grid() |
---|
114 | |
---|
115 | #*********************************************************************** |
---|
116 | #coverageFunction |
---|
117 | mr =csmllibs.Parser.MappingRule(csmllibs.csmlextra.getMappingRule(len(dimNames))) |
---|
118 | |
---|
119 | GridSeriesFeature_element.coverageFunction=mr |
---|
120 | |
---|
121 | #*********************************************************************** |
---|
122 | # gml:rangeSet element |
---|
123 | #*********************************************************************** |
---|
124 | arrSz = DI.getArraySizeOfVar() |
---|
125 | |
---|
126 | try: |
---|
127 | strUom = DI.getVariableAttribute('units') |
---|
128 | except AttributeError: |
---|
129 | # if units attribute doesn't exist: |
---|
130 | strUom ="dimensionless or units not determined" |
---|
131 | |
---|
132 | rs=csmllibs.Parser.RangeSet() |
---|
133 | if self.valuestorage=='inline': |
---|
134 | #To do, store the rangeset inline - use Datablock class??? |
---|
135 | pass |
---|
136 | else: |
---|
137 | #store the rangeSet as an aggregatedArray |
---|
138 | aa=csmllibs.Parser.AggregatedArray() |
---|
139 | aa.arraySize=[] |
---|
140 | aa.arraySize.append(arrSz) |
---|
141 | aa.uom=strUom |
---|
142 | aa.aggType='new' #can it be anything else? |
---|
143 | aa.aggIndex='1' |
---|
144 | #FileExtract (fe) element will be NetCDF/GRIB/PPExtract element (As defined earlier in ExtractType) |
---|
145 | self.extractType= DI.extractType |
---|
146 | if self.extractType=='NetCDFExtract': |
---|
147 | fe = csmllibs.Parser.NetCDFExtract() |
---|
148 | if self.extractType=='NASAAmesExtract': |
---|
149 | fe = csmllibs.Parser.NASAAmesExtract() |
---|
150 | if self.extractType=='GRIBExtract': |
---|
151 | fe = csmllibs.Parser.GRIBExtract() |
---|
152 | if self.extractType=='PPExtract': |
---|
153 | fe = csmllibs.Parser.PPExtract() |
---|
154 | varSize=DI.getShapeOfVar() |
---|
155 | fe.arraySize=varSize |
---|
156 | fe.fileName=filesinDir |
---|
157 | fe.variableName=allVarNames[i] |
---|
158 | aa.component=[fe] |
---|
159 | rs.aggregatedArray=aa |
---|
160 | GridSeriesFeature_element.rangeSet=rs |
---|
161 | ###################################################### |
---|
162 | |
---|
163 | |
---|
164 | # #*********************************************************************** |
---|
165 | # # domainComplement element (and sub-elements) |
---|
166 | # #*********************************************************************** |
---|
167 | grid.srsName='urn:EPSG:GeographicCRS:4326' |
---|
168 | numSpDims=len(varSize) -1 |
---|
169 | grid.srsDimension=str(numSpDims) |
---|
170 | grid.dimension=str(numSpDims) |
---|
171 | ge =csmllibs.Parser.GridEnvelope(low=DI.getLowLimits(), high=DI.getHighLimits()) |
---|
172 | grid.limits=ge |
---|
173 | |
---|
174 | |
---|
175 | #add an axisName element(s) for each spatial dimension. |
---|
176 | # and an ordinate element |
---|
177 | axes=[] |
---|
178 | for i in range (1, len(dimNames)): |
---|
179 | #axisNames |
---|
180 | axisname ='dim'+str(i) |
---|
181 | axes.append(axisname) |
---|
182 | |
---|
183 | |
---|
184 | #ordinates |
---|
185 | grid.ordinates=[] |
---|
186 | for i in range (1, len(dimNames)): |
---|
187 | ord=csmllibs.Parser.GridOrdinateDescription() |
---|
188 | ord.gridAxesSpanned='dim' + str(i) |
---|
189 | ord.sequenceRule=csmllibs.csmlextra.getSeqRule(len(dimNames)) |
---|
190 | dimName=dimNames[len(dimNames)-i] |
---|
191 | ord.definesAxis=dimName |
---|
192 | #look up file extract name in dictionary |
---|
193 | #(axisid stored in dictionary = current filename + variable name) |
---|
194 | axisid=repfilename+dimName |
---|
195 | if self.spatialstorage=='fileextract': |
---|
196 | #refer to extract |
---|
197 | ord.axisValues='#'+self.fileExtractDictionary[axisid] |
---|
198 | else: |
---|
199 | #store inline |
---|
200 | DI.setAxis(dimName) |
---|
201 | ord.axisValues=csmllibs.csmlextra.cleanString(str(DI.getDataForAxis())) |
---|
202 | grid.ordinates.append(ord) |
---|
203 | grid.axisNames=axes |
---|
204 | gsDomain.domainComplement=grid |
---|
205 | GridSeriesFeature_element.domain=gsDomain |
---|
206 | self.fms.append(GridSeriesFeature_element) |
---|
207 | DI.closeFile() |
---|
208 | |
---|
209 | #this is now done outside this function |
---|
210 | #self.gml_FeatureCollection_element.members=self.fms |
---|
211 | |
---|
212 | ######################################################################### |
---|
213 | |
---|
214 | |
---|
215 | def createCSMLProfileFeature(csmldoc, dataset_element, gml_FeatureCollection_element, ffmap, timedim): |
---|
216 | representativeFiles=ffmap.getRepresentativeFiles() |
---|
217 | listOfFiles=[] |
---|
218 | for repfile in representativeFiles: |
---|
219 | repfilename=repfile.getRepresentativeFileName() |
---|
220 | listOfFiles.append(repfilename) |
---|
221 | relfiles = repfile.getRelatedFiles() |
---|
222 | for f in relfiles: |
---|
223 | #hopefully there are no related files at the moment! |
---|
224 | fname = f.getRelatedFileName() |
---|
225 | listOfFiles.append(fname) |
---|
226 | #print listOfFiles |
---|
227 | |
---|
228 | for file in listOfFiles: |
---|
229 | DI = csmllibs.csmldataiface.DataInterface() |
---|
230 | DI=DI.getUnknownInterfaceType(file) |
---|
231 | print'opening file' |
---|
232 | DI.openFile(file) |
---|
233 | print 'getting variables' |
---|
234 | allVarNames=DI.getListofVariables() |
---|
235 | print 'getting feature count' |
---|
236 | numFeatures=len(allVarNames) |
---|
237 | |
---|
238 | print "FEATURES" |
---|
239 | print "***********" |
---|
240 | for i in range (0, len(allVarNames)): |
---|
241 | print allVarNames[i] |
---|
242 | |
---|
243 | for i in range (0, numFeatures): |
---|
244 | gml_featureMember_element=csmldoc.createElement("gml:featureMember") |
---|
245 | ProfileFeature_element=csmldoc.createElement("ProfileFeature") |
---|
246 | ProfileFeature_element.setAttribute('gml:id',str(allVarNames[i])) |
---|
247 | gml_description_element = csmldoc.createElement("gml:description") |
---|
248 | gml_featureMember_element.appendChild(ProfileFeature_element) |
---|
249 | #*********************************************************************** |
---|
250 | #PointSeriesDomain: |
---|
251 | #*********************************************************************** |
---|
252 | ProfileDomain_element=csmldoc.createElement("ProfileDomain") |
---|
253 | |
---|
254 | |
---|
255 | #*********************************************************************** |
---|
256 | # domainReference element (and sub-elements) |
---|
257 | #*********************************************************************** |
---|
258 | domainReference_element=csmldoc.createElement("domainReference") |
---|
259 | #orientedPosition_element=csmldoc.createElement("OrientedPosition") |
---|
260 | #locations_element=csmldoc.createElement("locations") |
---|
261 | #times_element=csmldoc.createElement("times") |
---|
262 | #trajectory_element.appendChild(locations_element) |
---|
263 | #trajectory_element.appendChild(times_element) |
---|
264 | #domainReference_element.appendChild(orientedPosition_element) |
---|
265 | |
---|
266 | #gml_timePositionList_element = csmldoc.createElement("gml:TimePositionList") |
---|
267 | #gml_timePositionList_element.appendChild(csmldoc.createTextNode(timeString)) |
---|
268 | #domainReference_element.appendChild(gml_timePositionList_element) |
---|
269 | ProfileDomain_element.appendChild(domainReference_element) |
---|
270 | #*********************************************************************** |
---|
271 | domainComplement_element=csmldoc.createElement("domainComplement") |
---|
272 | ProfileDomain_element.appendChild(domainComplement_element) |
---|
273 | |
---|
274 | #*********************************************************************** |
---|
275 | # gml:rangeSet_element |
---|
276 | #*********************************************************************** |
---|
277 | |
---|
278 | gml_rangeSet_element=csmldoc.createElement("gml:rangeSet") |
---|
279 | |
---|
280 | #*********************************************************************** |
---|
281 | # gml:coverageFunction element (and sub-element MappingRule) |
---|
282 | #*********************************************************************** |
---|
283 | gml_coverageFunction_element=csmldoc.createElement("gml:coverageFunction") |
---|
284 | MappingRule_element=csmldoc.createElement("MappingRule") |
---|
285 | #MappingRule_element.setAttribute('scanOrder',csmllibs.csmlextra.getMappingRule(len(dimNames))) |
---|
286 | MappingRule_element.setAttribute('scanOrder','tba') |
---|
287 | gml_coverageFunction_element.appendChild(MappingRule_element) |
---|
288 | |
---|
289 | |
---|
290 | gml_featureMember_element.appendChild(ProfileDomain_element) |
---|
291 | gml_featureMember_element.appendChild(gml_rangeSet_element) |
---|
292 | gml_featureMember_element.appendChild(gml_coverageFunction_element) |
---|
293 | gml_FeatureCollection_element.appendChild(gml_featureMember_element) |
---|
294 | |
---|
295 | return |
---|
296 | |
---|
297 | |
---|
298 | |
---|
299 | def createCSMLPointSeriesFeatures(self): |
---|
300 | representativeFiles=self.ffmap.getRepresentativeFiles() |
---|
301 | listOfFiles=[] |
---|
302 | for repfile in representativeFiles: |
---|
303 | repfilename=repfile.getRepresentativeFileName() |
---|
304 | listOfFiles.append(repfilename) |
---|
305 | relfiles = repfile.getRelatedFiles() |
---|
306 | for f in relfiles: |
---|
307 | #hopefully there are no related files at the moment! |
---|
308 | fname = f.getRelatedFileName() |
---|
309 | listOfFiles.append(fname) |
---|
310 | file=repfilename |
---|
311 | DI = csmllibs.csmldataiface.DataInterface() |
---|
312 | DI=DI.getUnknownInterfaceType(file) |
---|
313 | DI.openFile(file) |
---|
314 | allVarNames=DI.getListofVariables() |
---|
315 | numFeatures=len(allVarNames) |
---|
316 | try: |
---|
317 | DI.setAxis(self.timedim) |
---|
318 | times=DI.getDataForAxis() |
---|
319 | except: |
---|
320 | times = DI.getTimes() |
---|
321 | #times = ['time axis not determined'] |
---|
322 | |
---|
323 | print "numFeatures" + str(numFeatures) |
---|
324 | for i in range (0, numFeatures): |
---|
325 | PointSeriesFeature_element=csmllibs.Parser.PointSeriesFeature() |
---|
326 | if str(allVarNames[i]).upper() in ['ERROR FLAG', 'ERROR']: #might need to extend this list |
---|
327 | break |
---|
328 | PointSeriesFeature_element.id=str(allVarNames[i]) |
---|
329 | try: |
---|
330 | desc=DI.getVariableAttribute('long_name') |
---|
331 | # print desc |
---|
332 | except AttributeError: |
---|
333 | desc = "missing name" |
---|
334 | PointSeriesFeature_element.description=csmllibs.Parser.Description(desc) |
---|
335 | |
---|
336 | #*********************************************************************** |
---|
337 | #PointSeriesDomain: |
---|
338 | #*********************************************************************** |
---|
339 | psDomain=csmllibs.Parser.PointSeriesDomain() |
---|
340 | |
---|
341 | print len(times) |
---|
342 | print len(listOfFiles) |
---|
343 | print len(times) * len(listOfFiles) |
---|
344 | #*********************************************************************** |
---|
345 | # domainReference element |
---|
346 | #*********************************************************************** |
---|
347 | t=csmllibs.Parser.Trajectory() |
---|
348 | t.srsName='urn:EPSG:geographicCRS:4326' #TO Do |
---|
349 | t.locations =csmllibs.Parser.DirectPositionList(vals='1 1') |
---|
350 | if len(times) > 200: |
---|
351 | pass |
---|
352 | #create a file extract link for long lists |
---|
353 | filenames=csmllibs.csmlextra.cleanString(str(listOfFiles)) |
---|
354 | # print 'times: ' + str(allVarNames[i]) |
---|
355 | # print len(times) |
---|
356 | # print len(listOfFiles) |
---|
357 | # arraySize=len(times) * len(listOfFiles) |
---|
358 | # fextract=csmllibs.csmlfileextracts.createSingleExtract(self.extractType,filenames,self.timedim,arraySize) |
---|
359 | # tplist = csmllibs.Parser.TimePositionList(timePositions=fextract) |
---|
360 | # t.times=tplist |
---|
361 | else: |
---|
362 | #encode short lists inline: |
---|
363 | t.times=csmllibs.Parser.TimePositionList('#RefSysX',str(times)) |
---|
364 | |
---|
365 | psDomain.domainReference=t |
---|
366 | |
---|
367 | #*********************************************************************** |
---|
368 | # gml:rangeSet_element |
---|
369 | #*********************************************************************** |
---|
370 | DI.setVariable(allVarNames[i]) |
---|
371 | try: |
---|
372 | strUom = DI.getVariableAttribute('units') |
---|
373 | except AttributeError: |
---|
374 | #if units attribute doesn't exist: |
---|
375 | strUom ="dimensionless or units not determined" |
---|
376 | |
---|
377 | try: |
---|
378 | measuredvalues = DI.getDataForVar() |
---|
379 | except: |
---|
380 | measuredvalues = ' could not get values ' |
---|
381 | |
---|
382 | rs=csmllibs.Parser.RangeSet() |
---|
383 | if len(measuredvalues) > 200: |
---|
384 | #create a file extract link for long lists |
---|
385 | arraySize=len(measuredvalues)*len(listOfFiles) |
---|
386 | #TODO this needs to be able to handle inline, use VALUESTORAGE to determine which to use: |
---|
387 | fextract=csmllibs.csmlfileextracts.createSingleExtract(self.extractType,filenames,allVarNames[i],arraySize) |
---|
388 | qlist = csmllibs.Parser.MeasureOrNullList(val=fextract) |
---|
389 | rs.quantityList=qlist |
---|
390 | else: |
---|
391 | #encode short lists inline |
---|
392 | rs.quantityList=csmllibs.Parser.MeasureOrNullList(uom=strUom, val=str(measuredvalues)[1:-1]) |
---|
393 | |
---|
394 | PointSeriesFeature_element.rangeSet=rs |
---|
395 | #*********************************************************************** |
---|
396 | # gml:coverageFunction element (and sub-element MappingRule) |
---|
397 | #*********************************************************************** |
---|
398 | |
---|
399 | #need to do parameter element |
---|
400 | |
---|
401 | |
---|
402 | PointSeriesFeature_element.domain=psDomain |
---|
403 | self.fms.append(PointSeriesFeature_element) |
---|
404 | DI.closeFile() |
---|
405 | |
---|
406 | |
---|
407 | |
---|