1 | #!/usr/bin/env python |
---|
2 | #************************************************************************************** |
---|
3 | #csmlfeaturetypes.py |
---|
4 | #For creating CSML featuretypes |
---|
5 | #v0.5 split off 11th November 2005 |
---|
6 | #Dominic Lowe, BADC |
---|
7 | #************************************************************************************** |
---|
8 | |
---|
9 | import csml.csmllibs |
---|
10 | import sys |
---|
11 | |
---|
12 | class featureBuilder(object): |
---|
13 | def __init__(self, dataset_element, gml_FeatureCollection_element, ffmap,fileExtractDictionary, timedim, timestorage,spatialstorage,valuestorage): |
---|
14 | self.ds_element=dataset_element |
---|
15 | self.gml_FeatureCollection_element = gml_FeatureCollection_element |
---|
16 | self.ffmap = ffmap |
---|
17 | self.fileExtractDictionary = fileExtractDictionary |
---|
18 | self.timedim = timedim |
---|
19 | self.timestorage=timestorage |
---|
20 | self.spatialstorage=spatialstorage |
---|
21 | self.valuestorage=valuestorage |
---|
22 | |
---|
23 | #empty list to hold featureMembers |
---|
24 | self.fms =[] |
---|
25 | |
---|
26 | #at the moment, only one featuretype per CSML Dataset is supported. |
---|
27 | #get the featuretype of the first representative file in the ffmap object |
---|
28 | self.featuretype= self.ffmap.getRepresentativeFiles()[0].getFeatureType() |
---|
29 | #and create the features |
---|
30 | print 'determining feature type' |
---|
31 | print self.featuretype |
---|
32 | if self.featuretype == 'GridSeries': |
---|
33 | self.createCSMLGridSeriesFeatures() |
---|
34 | elif self.featuretype == 'PointSeries': |
---|
35 | self.createCSMLPointSeriesFeatures() |
---|
36 | |
---|
37 | #after the features have been generated append all featureMembers to the feature collection |
---|
38 | self.gml_FeatureCollection_element.members=self.fms |
---|
39 | |
---|
40 | |
---|
41 | #Some internal methods that are of use to all feature types: |
---|
42 | def _getDescriptiveName(self,DI): |
---|
43 | #given a data interface class with the variable or axis set, try to get a descriptive name |
---|
44 | #eg. long name |
---|
45 | try: |
---|
46 | descName=DI.getVariableAttribute('long_name') |
---|
47 | except AttributeError: |
---|
48 | descName = "missing name" |
---|
49 | descName=descName.replace('&','&') #remove ampersands TODO- extend this |
---|
50 | return descName |
---|
51 | |
---|
52 | def _populateListOfFiles(self,repfile): |
---|
53 | #given a representative file, get list of files: one representative file + all related files |
---|
54 | listOfFiles=[] |
---|
55 | repfilename=repfile.getRepresentativeFileName() |
---|
56 | listOfFiles.append(repfilename) |
---|
57 | relfiles = repfile.getRelatedFiles() |
---|
58 | for f in relfiles: |
---|
59 | fname = f.getRelatedFileName() |
---|
60 | listOfFiles.append(fname) |
---|
61 | return repfilename,listOfFiles |
---|
62 | |
---|
63 | def _getFilesAndTimes(self): |
---|
64 | #TODO try and speed up csmllibs.csmltime.getFileTimeList |
---|
65 | OrderedFileTimeList,self.caltype,self.units = csml.csmllibs.csmltime.getFileTimeList(self.listOfFiles,self.timedim) |
---|
66 | #build strings to hold times/filenames for current gridseriesfeature |
---|
67 | self.timeString ='' |
---|
68 | self.filesinDir = '' |
---|
69 | for j in range (0, len(OrderedFileTimeList)): |
---|
70 | t= OrderedFileTimeList[j][0] |
---|
71 | f = OrderedFileTimeList[j][1] |
---|
72 | self.timeString = self.timeString + ' ' + str(t) |
---|
73 | self.filesinDir = self.filesinDir + ' ' + f |
---|
74 | |
---|
75 | def _getCorrectExtractType(self): |
---|
76 | #returns an empty parser file extract object of the correct type. |
---|
77 | if self.extractType=='NetCDFExtract': |
---|
78 | fe = csml.parser.NetCDFExtract() |
---|
79 | if self.extractType=='NASAAmesExtract': |
---|
80 | fe = csml.parser.NASAAmesExtract() |
---|
81 | if self.extractType=='GRIBExtract': |
---|
82 | fe = csml.parser.GRIBExtract() |
---|
83 | if self.extractType=='PPExtract': |
---|
84 | fe = csml.parser.PPExtract() |
---|
85 | return fe |
---|
86 | |
---|
87 | def createCSMLGridSeriesFeatures(self): |
---|
88 | #This method assumes that the variables (features) are shared across identically structured files |
---|
89 | #should be supplied with a featurefilemap object (see csmlfiles for FileMapMaker) |
---|
90 | representativeFiles=self.ffmap.getRepresentativeFiles() |
---|
91 | for repfile in representativeFiles: |
---|
92 | self.repfilename,self.listOfFiles=self._populateListOfFiles(repfile) |
---|
93 | self._getFilesAndTimes() |
---|
94 | #Open representative file and create feature members: |
---|
95 | DI = csml.csmllibs.csmldataiface.DataInterface() |
---|
96 | DI=DI.getUnknownInterfaceType(self.repfilename) |
---|
97 | DI.openFile(self.repfilename) |
---|
98 | allVarNames=DI.getListofVariables() |
---|
99 | numFeatures=len(allVarNames) |
---|
100 | #Create a GridSeriesFeature for each variable: |
---|
101 | for i in range(0, numFeatures): |
---|
102 | DI.setVariable(allVarNames[i]) |
---|
103 | dimNames=DI.getVariableAxes() |
---|
104 | if len(dimNames) <= 2: |
---|
105 | #it's an axis or bounds not a feature, try next variable |
---|
106 | continue |
---|
107 | GridSeriesFeature_element=csml.parser.GridSeriesFeature() |
---|
108 | GridSeriesFeature_element.id=str(allVarNames[i]) |
---|
109 | desc = self._getDescriptiveName(DI) |
---|
110 | GridSeriesFeature_element.description=csml.parser.Description(desc) |
---|
111 | #DOMAIN |
---|
112 | gsDomain=csml.parser.GridSeriesDomain() |
---|
113 | #DOMAIN REFERENCE |
---|
114 | tpl=csml.parser.TimePositionList() |
---|
115 | if self.timestorage =='inline': |
---|
116 | tpl.timePositions=self.timeString |
---|
117 | tpl.frame='%s:%s'%(self.caltype,self.units) |
---|
118 | else: |
---|
119 | # do something to create a single extract for the times (from the representative file). |
---|
120 | tpl.timePositions = csml.csmllibs.csmlfileextracts.createSingleExtract(self.extractType, repfilename, self.timedim, len(self.timeString.split())) |
---|
121 | tpl.frame='%s:%s'%(self.caltype,self.units) |
---|
122 | gsDomain.domainReference=tpl |
---|
123 | grid=csml.parser.Grid() |
---|
124 | #COVERAGE FUNCTION |
---|
125 | mr =csml.parser.MappingRule(csml.csmllibs.csmlextra.getMappingRule(len(dimNames))) |
---|
126 | GridSeriesFeature_element.coverageFunction=mr |
---|
127 | #RANGESET |
---|
128 | arrSz = DI.getArraySizeOfVar() |
---|
129 | try: |
---|
130 | strUom = DI.getVariableAttribute('units') |
---|
131 | except AttributeError: |
---|
132 | # if units attribute doesn't exist: |
---|
133 | strUom ="dimensionless or units not determined" |
---|
134 | rs=csml.parser.RangeSet() |
---|
135 | if self.valuestorage=='inline': |
---|
136 | #TO DO, store the rangeset inline - use Datablock class??? |
---|
137 | pass |
---|
138 | else: |
---|
139 | #store the rangeSet as an aggregatedArray |
---|
140 | aa=csml.parser.AggregatedArray() |
---|
141 | aa.arraySize=[] |
---|
142 | aa.arraySize.append(arrSz) |
---|
143 | aa.uom=strUom |
---|
144 | aa.aggType='new' #can it be anything else? |
---|
145 | aa.aggIndex='1' |
---|
146 | #FileExtract (fe) element will be NetCDF/GRIB/PPExtract element (As defined earlier in ExtractType) |
---|
147 | self.extractType= DI.extractType |
---|
148 | fe = self._getCorrectExtractType() |
---|
149 | varSize=DI.getShapeOfVar() |
---|
150 | fe.arraySize=varSize |
---|
151 | fe.fileName=self.filesinDir |
---|
152 | fe.variableName=allVarNames[i] |
---|
153 | aa.component=[fe] |
---|
154 | rs.aggregatedArray=aa |
---|
155 | GridSeriesFeature_element.rangeSet=rs |
---|
156 | #DOMAIN COMPLEMENT |
---|
157 | grid.srsName='urn:EPSG:GeographicCRS:4326' |
---|
158 | numSpDims=len(varSize) -1 |
---|
159 | grid.srsDimension=str(numSpDims) |
---|
160 | grid.dimension=str(numSpDims) |
---|
161 | ge =csml.parser.GridEnvelope(low=DI.getLowLimits(), high=DI.getHighLimits()) |
---|
162 | grid.limits=ge |
---|
163 | #add an axisName element(s) for each spatial dimension. |
---|
164 | # and an ordinate element |
---|
165 | axes=[] |
---|
166 | for i in range (1, len(dimNames)): |
---|
167 | #axisNames |
---|
168 | axisname ='dim'+str(i) |
---|
169 | axes.append(axisname) |
---|
170 | #ordinates |
---|
171 | grid.ordinates=[] |
---|
172 | for i in range (1, len(dimNames)): |
---|
173 | ord=csml.parser.GridOrdinateDescription() |
---|
174 | ord.gridAxesSpanned='dim' + str(i) |
---|
175 | ord.sequenceRule=csml.csmllibs.csmlextra.getSeqRule(len(dimNames)) |
---|
176 | dimName=dimNames[len(dimNames)-i] |
---|
177 | ord.definesAxis=dimName |
---|
178 | #look up file extract name in dictionary |
---|
179 | #(axisid stored in dictionary = current filename + variable name) |
---|
180 | axisid=self.repfilename+dimName |
---|
181 | if self.spatialstorage=='fileextract': |
---|
182 | #refer to extract |
---|
183 | ord.axisValues='#'+self.fileExtractDictionary[axisid] |
---|
184 | else: |
---|
185 | #store inline |
---|
186 | DI.setAxis(dimName) |
---|
187 | ord.axisValues=csml.csmllibs.csmlextra.cleanString(str(DI.getDataForAxis())) |
---|
188 | grid.ordinates.append(ord) |
---|
189 | grid.axisNames=axes |
---|
190 | gsDomain.domainComplement=grid |
---|
191 | GridSeriesFeature_element.domain=gsDomain |
---|
192 | GridSeriesFeature_element.parameter=csml.parser.Phenomenon(href='http://badc.rl.ac.uk/localparams#%s'%allVarNames[i]) |
---|
193 | self.fms.append(GridSeriesFeature_element) |
---|
194 | DI.closeFile() |
---|
195 | ###End of createCSMLGridSeriesFeatures### |
---|
196 | |
---|
197 | def createCSMLPointSeriesFeatures(self): |
---|
198 | representativeFiles=self.ffmap.getRepresentativeFiles() |
---|
199 | for repfile in representativeFiles: |
---|
200 | self.repfilename,self.listOfFiles=self._populateListOfFiles(repfile) |
---|
201 | self._getFilesAndTimes() |
---|
202 | DI = csml.csmllibs.csmldataiface.DataInterface() |
---|
203 | DI=DI.getUnknownInterfaceType(self.repfilename) |
---|
204 | DI.openFile(self.repfilename) |
---|
205 | allVarNames=DI.getListofVariables() |
---|
206 | numFeatures=len(allVarNames) |
---|
207 | try: |
---|
208 | DI.setAxis(self.timedim) |
---|
209 | times=DI.getDataForAxis() |
---|
210 | except: |
---|
211 | times = DI.getTimes() |
---|
212 | #Create features: |
---|
213 | for i in range (0, numFeatures): |
---|
214 | PointSeriesFeature_element=csml.parser.PointSeriesFeature() |
---|
215 | if str(allVarNames[i]).upper() in ['ERROR FLAG', 'ERROR']: #might need to extend this list |
---|
216 | break |
---|
217 | PointSeriesFeature_element.id=str(allVarNames[i]) |
---|
218 | desc=self._getDescriptiveName(DI) |
---|
219 | PointSeriesFeature_element.description=csml.parser.Description(desc) |
---|
220 | #DOMAIN |
---|
221 | psDomain=csml.parser.PointSeriesDomain() |
---|
222 | t=csml.parser.Trajectory() |
---|
223 | t.srsName='urn:EPSG:geographicCRS:4326' #TO Do |
---|
224 | t.locations =csml.parser.DirectPositionList(vals='1 1') |
---|
225 | |
---|
226 | if self.timestorage =='inline': |
---|
227 | tpl =csml.parser.TimePositionList() |
---|
228 | tpl.timePositions=self.timeString |
---|
229 | tpl.frame='%s:%s'%(self.caltype,self.units) |
---|
230 | t.times=tpl |
---|
231 | else: |
---|
232 | # do something to create a single extract for the times (from the representative file). |
---|
233 | tpl.timePositions = csml.csmllibs.csmlfileextracts.createSingleExtract(self.extractType, repfilename, self.timedim, len(self.timeString.split())) |
---|
234 | tpl.frame='%s:%s'%(self.caltype,self.units) |
---|
235 | |
---|
236 | |
---|
237 | # if self.timestorage =='inline': |
---|
238 | # t.times=csmllibs.Parser.TimePositionList('#RefSysX',str(times)) |
---|
239 | # else: |
---|
240 | # #todo: depends on the file mapping??? |
---|
241 | # t.times=csmllibs.Parser.TimePositionList('#RefSysX','blah') #blah = dummy times |
---|
242 | # print 'times: ' + str(allVarNames[i]) |
---|
243 | # print len(times) |
---|
244 | # print len(listOfFiles) |
---|
245 | # arraySize=len(times) * len(listOfFiles) |
---|
246 | # fextract=csmllibs.csmlfileextracts.createSingleExtract(self.extractType,filenames,self.timedim,arraySize) |
---|
247 | # tplist = csmllibs.Parser.TimePositionList(timePositions=fextract) |
---|
248 | # t.times=tplist |
---|
249 | filenames=csml.csmllibs.csmlextra.cleanString(str(self.listOfFiles)) |
---|
250 | psDomain.domainReference=t |
---|
251 | #RANGESET |
---|
252 | DI.setVariable(allVarNames[i]) |
---|
253 | try: |
---|
254 | strUom = DI.getVariableAttribute('units') |
---|
255 | except AttributeError: |
---|
256 | #if units attribute doesn't exist: |
---|
257 | strUom ="dimensionless or units not determined" |
---|
258 | try: |
---|
259 | measuredvalues = DI.getDataForVar() |
---|
260 | except: |
---|
261 | measuredvalues = ' could not get values ' |
---|
262 | rs=csml.parser.RangeSet() |
---|
263 | if self.valuestorage=='inline': |
---|
264 | #encode inline |
---|
265 | rs.quantityList=csml.parser.MeasureOrNullList(uom=strUom, val=str(measuredvalues)[1:-1]) |
---|
266 | else: |
---|
267 | #create a file extract link |
---|
268 | arraySize=len(measuredvalues)*len(self.listOfFiles) |
---|
269 | #TODO this needs to be able to handle inline, use VALUESTORAGE to determine which to use: |
---|
270 | self.extractType=DI.extractType |
---|
271 | fextract=csml.csmllibs.csmlfileextracts.createSingleExtract(self.extractType,filenames,allVarNames[i],arraySize) |
---|
272 | qlist = csml.parser.MeasureOrNullList(val=fextract) |
---|
273 | rs.quantityList=qlist |
---|
274 | PointSeriesFeature_element.rangeSet=rs |
---|
275 | #COVERAGEFUNCTION |
---|
276 | #PARAMETER |
---|
277 | #need to do parameter and coverageFunction elements |
---|
278 | PointSeriesFeature_element.domain=psDomain |
---|
279 | self.fms.append(PointSeriesFeature_element) |
---|
280 | DI.closeFile() |
---|
281 | |
---|
282 | |
---|
283 | #This function needs revising in light of a) csml parser and b) new profile feature types |
---|
284 | def createCSMLProfileFeature(csmldoc, dataset_element, gml_FeatureCollection_element, ffmap, timedim): |
---|
285 | representativeFiles=ffmap.getRepresentativeFiles() |
---|
286 | listOfFiles=[] |
---|
287 | for repfile in representativeFiles: |
---|
288 | repfilename=repfile.getRepresentativeFileName() |
---|
289 | listOfFiles.append(repfilename) |
---|
290 | relfiles = repfile.getRelatedFiles() |
---|
291 | for f in relfiles: |
---|
292 | #hopefully there are no related files at the moment! |
---|
293 | fname = f.getRelatedFileName() |
---|
294 | listOfFiles.append(fname) |
---|
295 | #print listOfFiles |
---|
296 | |
---|
297 | for file in listOfFiles: |
---|
298 | DI = csmllibs.csmldataiface.DataInterface() |
---|
299 | DI=DI.getUnknownInterfaceType(file) |
---|
300 | print'opening file' |
---|
301 | DI.openFile(file) |
---|
302 | print 'getting variables' |
---|
303 | allVarNames=DI.getListofVariables() |
---|
304 | print 'getting feature count' |
---|
305 | numFeatures=len(allVarNames) |
---|
306 | |
---|
307 | print "FEATURES" |
---|
308 | print "***********" |
---|
309 | for i in range (0, len(allVarNames)): |
---|
310 | print allVarNames[i] |
---|
311 | |
---|
312 | for i in range (0, numFeatures): |
---|
313 | gml_featureMember_element=csmldoc.createElement("gml:featureMember") |
---|
314 | ProfileFeature_element=csmldoc.createElement("ProfileFeature") |
---|
315 | ProfileFeature_element.setAttribute('gml:id',str(allVarNames[i])) |
---|
316 | gml_description_element = csmldoc.createElement("gml:description") |
---|
317 | gml_featureMember_element.appendChild(ProfileFeature_element) |
---|
318 | #*********************************************************************** |
---|
319 | #PointSeriesDomain: |
---|
320 | #*********************************************************************** |
---|
321 | ProfileDomain_element=csmldoc.createElement("ProfileDomain") |
---|
322 | |
---|
323 | |
---|
324 | #*********************************************************************** |
---|
325 | # domainReference element (and sub-elements) |
---|
326 | #*********************************************************************** |
---|
327 | domainReference_element=csmldoc.createElement("domainReference") |
---|
328 | #orientedPosition_element=csmldoc.createElement("OrientedPosition") |
---|
329 | #locations_element=csmldoc.createElement("locations") |
---|
330 | #times_element=csmldoc.createElement("times") |
---|
331 | #trajectory_element.appendChild(locations_element) |
---|
332 | #trajectory_element.appendChild(times_element) |
---|
333 | #domainReference_element.appendChild(orientedPosition_element) |
---|
334 | |
---|
335 | #gml_timePositionList_element = csmldoc.createElement("gml:TimePositionList") |
---|
336 | #gml_timePositionList_element.appendChild(csmldoc.createTextNode(self.timeString)) |
---|
337 | #domainReference_element.appendChild(gml_timePositionList_element) |
---|
338 | ProfileDomain_element.appendChild(domainReference_element) |
---|
339 | #*********************************************************************** |
---|
340 | domainComplement_element=csmldoc.createElement("domainComplement") |
---|
341 | ProfileDomain_element.appendChild(domainComplement_element) |
---|
342 | |
---|
343 | #*********************************************************************** |
---|
344 | # gml:rangeSet_element |
---|
345 | #*********************************************************************** |
---|
346 | |
---|
347 | gml_rangeSet_element=csmldoc.createElement("gml:rangeSet") |
---|
348 | |
---|
349 | #*********************************************************************** |
---|
350 | # gml:coverageFunction element (and sub-element MappingRule) |
---|
351 | #*********************************************************************** |
---|
352 | gml_coverageFunction_element=csmldoc.createElement("gml:coverageFunction") |
---|
353 | MappingRule_element=csmldoc.createElement("MappingRule") |
---|
354 | #MappingRule_element.setAttribute('scanOrder',csmllibs.csmlextra.getMappingRule(len(dimNames))) |
---|
355 | MappingRule_element.setAttribute('scanOrder','tba') |
---|
356 | gml_coverageFunction_element.appendChild(MappingRule_element) |
---|
357 | |
---|
358 | |
---|
359 | gml_featureMember_element.appendChild(ProfileDomain_element) |
---|
360 | gml_featureMember_element.appendChild(gml_rangeSet_element) |
---|
361 | gml_featureMember_element.appendChild(gml_coverageFunction_element) |
---|
362 | gml_FeatureCollection_element.appendChild(gml_featureMember_element) |
---|
363 | |
---|
364 | return |
---|
365 | |
---|
366 | |
---|