| 1 | #!/usr/bin/env python |
|---|
| 2 | #************************************************************************************** |
|---|
| 3 | #csmlfeaturetypes.py |
|---|
| 4 | #For creating CSML featuretypes |
|---|
| 5 | #v0.5 split off 11th November 2005 |
|---|
| 6 | #Dominic Lowe, BADC |
|---|
| 7 | #************************************************************************************** |
|---|
| 8 | |
|---|
| 9 | import csml.csmllibs |
|---|
| 10 | import sys |
|---|
| 11 | |
|---|
| 12 | class featureBuilder(object): |
|---|
| 13 | def __init__(self, dataset_element, gml_FeatureCollection_element, ffmap,fileExtractDictionary, timedim, timestorage,spatialstorage,valuestorage): |
|---|
| 14 | self.ds_element=dataset_element |
|---|
| 15 | self.gml_FeatureCollection_element = gml_FeatureCollection_element |
|---|
| 16 | self.ffmap = ffmap |
|---|
| 17 | self.fileExtractDictionary = fileExtractDictionary |
|---|
| 18 | self.timedim = timedim |
|---|
| 19 | self.timestorage=timestorage |
|---|
| 20 | self.spatialstorage=spatialstorage |
|---|
| 21 | self.valuestorage=valuestorage |
|---|
| 22 | |
|---|
| 23 | #empty list to hold featureMembers |
|---|
| 24 | self.fms =[] |
|---|
| 25 | |
|---|
| 26 | #at the moment, only one featuretype per CSML Dataset is supported. |
|---|
| 27 | #get the featuretype of the first representative file in the ffmap object |
|---|
| 28 | self.featuretype= self.ffmap.getRepresentativeFiles()[0].getFeatureType() |
|---|
| 29 | #and create the features |
|---|
| 30 | print 'determining feature type' |
|---|
| 31 | print self.featuretype |
|---|
| 32 | if self.featuretype == 'GridSeries': |
|---|
| 33 | self.createCSMLGridSeriesFeatures() |
|---|
| 34 | elif self.featuretype == 'PointSeries': |
|---|
| 35 | self.createCSMLPointSeriesFeatures() |
|---|
| 36 | |
|---|
| 37 | #after the features have been generated append all featureMembers to the feature collection |
|---|
| 38 | self.gml_FeatureCollection_element.members=self.fms |
|---|
| 39 | |
|---|
| 40 | |
|---|
| 41 | #Some internal methods that are of use to all feature types: |
|---|
| 42 | def _getDescriptiveName(self,DI): |
|---|
| 43 | #given a data interface class with the variable or axis set, try to get a descriptive name |
|---|
| 44 | #eg. long name |
|---|
| 45 | try: |
|---|
| 46 | descName=DI.getVariableAttribute('long_name') |
|---|
| 47 | except AttributeError: |
|---|
| 48 | descName = "missing name" |
|---|
| 49 | descName=descName.replace('&','&') #remove ampersands TODO- extend this |
|---|
| 50 | return descName |
|---|
| 51 | |
|---|
| 52 | def _populateListOfFiles(self,repfile): |
|---|
| 53 | #given a representative file, get list of files: one representative file + all related files |
|---|
| 54 | listOfFiles=[] |
|---|
| 55 | repfilename=repfile.getRepresentativeFileName() |
|---|
| 56 | listOfFiles.append(repfilename) |
|---|
| 57 | relfiles = repfile.getRelatedFiles() |
|---|
| 58 | for f in relfiles: |
|---|
| 59 | fname = f.getRelatedFileName() |
|---|
| 60 | listOfFiles.append(fname) |
|---|
| 61 | return repfilename,listOfFiles |
|---|
| 62 | |
|---|
| 63 | def _getFilesAndTimes(self): |
|---|
| 64 | #TODO try and speed up csmllibs.csmltime.getFileTimeList |
|---|
| 65 | OrderedFileTimeList,self.caltype,self.units = csml.csmllibs.csmltime.getFileTimeList(self.listOfFiles,self.timedim) |
|---|
| 66 | #build strings to hold times/filenames for current gridseriesfeature |
|---|
| 67 | self.timeString ='' |
|---|
| 68 | self.filesinDir = '' |
|---|
| 69 | for j in range (0, len(OrderedFileTimeList)): |
|---|
| 70 | t= OrderedFileTimeList[j][0] |
|---|
| 71 | f = OrderedFileTimeList[j][1] |
|---|
| 72 | self.timeString = self.timeString + ' ' + str(t) |
|---|
| 73 | self.filesinDir = self.filesinDir + ' ' + f |
|---|
| 74 | |
|---|
| 75 | def _getCorrectExtractType(self): |
|---|
| 76 | #returns an empty parser file extract object of the correct type. |
|---|
| 77 | if self.extractType=='NetCDFExtract': |
|---|
| 78 | fe = csml.parser.NetCDFExtract() |
|---|
| 79 | if self.extractType=='NASAAmesExtract': |
|---|
| 80 | fe = csml.parser.NASAAmesExtract() |
|---|
| 81 | if self.extractType=='GRIBExtract': |
|---|
| 82 | fe = csml.parser.GRIBExtract() |
|---|
| 83 | if self.extractType=='PPExtract': |
|---|
| 84 | fe = csml.parser.PPExtract() |
|---|
| 85 | return fe |
|---|
| 86 | |
|---|
| 87 | def createCSMLGridSeriesFeatures(self): |
|---|
| 88 | #This method assumes that the variables (features) are shared across identically structured files |
|---|
| 89 | #should be supplied with a featurefilemap object (see csmlfiles for FileMapMaker) |
|---|
| 90 | representativeFiles=self.ffmap.getRepresentativeFiles() |
|---|
| 91 | for repfile in representativeFiles: |
|---|
| 92 | self.repfilename,self.listOfFiles=self._populateListOfFiles(repfile) |
|---|
| 93 | self._getFilesAndTimes() |
|---|
| 94 | #Open representative file and create feature members: |
|---|
| 95 | DI = csml.csmllibs.csmldataiface.DataInterface() |
|---|
| 96 | DI=DI.getUnknownInterfaceType(self.repfilename) |
|---|
| 97 | DI.openFile(self.repfilename) |
|---|
| 98 | allVarNames=DI.getListofVariables() |
|---|
| 99 | numFeatures=len(allVarNames) |
|---|
| 100 | #Create a GridSeriesFeature for each variable: |
|---|
| 101 | for i in range(0, numFeatures): |
|---|
| 102 | DI.setVariable(allVarNames[i]) |
|---|
| 103 | dimNames=DI.getVariableAxes() |
|---|
| 104 | if len(dimNames) <= 2: |
|---|
| 105 | #it's an axis or bounds not a feature, try next variable |
|---|
| 106 | continue |
|---|
| 107 | GridSeriesFeature_element=csml.parser.GridSeriesFeature() |
|---|
| 108 | GridSeriesFeature_element.id=str(allVarNames[i]) |
|---|
| 109 | desc = self._getDescriptiveName(DI) |
|---|
| 110 | GridSeriesFeature_element.description=csml.parser.Description(desc) |
|---|
| 111 | #DOMAIN |
|---|
| 112 | gsDomain=csml.parser.GridSeriesDomain() |
|---|
| 113 | #DOMAIN REFERENCE |
|---|
| 114 | tpl=csml.parser.TimePositionList() |
|---|
| 115 | if self.timestorage =='inline': |
|---|
| 116 | tpl.timePositions=self.timeString |
|---|
| 117 | tpl.frame='%s:%s'%(self.caltype,self.units) |
|---|
| 118 | else: |
|---|
| 119 | # do something to create a single extract for the times (from the representative file). |
|---|
| 120 | tpl.timePositions = csml.csmllibs.csmlfileextracts.createSingleExtract(self.extractType, repfilename, self.timedim, len(self.timeString.split())) |
|---|
| 121 | tpl.frame='%s:%s'%(self.caltype,self.units) |
|---|
| 122 | gsDomain.domainReference=tpl |
|---|
| 123 | grid=csml.parser.Grid() |
|---|
| 124 | #COVERAGE FUNCTION |
|---|
| 125 | mr =csml.parser.MappingRule(csml.csmllibs.csmlextra.getMappingRule(len(dimNames))) |
|---|
| 126 | GridSeriesFeature_element.coverageFunction=mr |
|---|
| 127 | #RANGESET |
|---|
| 128 | arrSz = DI.getArraySizeOfVar() |
|---|
| 129 | try: |
|---|
| 130 | strUom = DI.getVariableAttribute('units') |
|---|
| 131 | except AttributeError: |
|---|
| 132 | # if units attribute doesn't exist: |
|---|
| 133 | strUom ="dimensionless or units not determined" |
|---|
| 134 | rs=csml.parser.RangeSet() |
|---|
| 135 | if self.valuestorage=='inline': |
|---|
| 136 | #TO DO, store the rangeset inline - use Datablock class??? |
|---|
| 137 | pass |
|---|
| 138 | else: |
|---|
| 139 | #store the rangeSet as an aggregatedArray |
|---|
| 140 | aa=csml.parser.AggregatedArray() |
|---|
| 141 | aa.arraySize=[] |
|---|
| 142 | aa.arraySize.append(arrSz) |
|---|
| 143 | aa.uom=strUom |
|---|
| 144 | aa.aggType='new' #can it be anything else? |
|---|
| 145 | aa.aggIndex='1' |
|---|
| 146 | #FileExtract (fe) element will be NetCDF/GRIB/PPExtract element (As defined earlier in ExtractType) |
|---|
| 147 | self.extractType= DI.extractType |
|---|
| 148 | fe = self._getCorrectExtractType() |
|---|
| 149 | varSize=DI.getShapeOfVar() |
|---|
| 150 | fe.arraySize=varSize |
|---|
| 151 | fe.fileName=self.filesinDir |
|---|
| 152 | fe.variableName=allVarNames[i] |
|---|
| 153 | aa.component=[fe] |
|---|
| 154 | rs.aggregatedArray=aa |
|---|
| 155 | GridSeriesFeature_element.rangeSet=rs |
|---|
| 156 | #DOMAIN COMPLEMENT |
|---|
| 157 | grid.srsName='urn:EPSG:GeographicCRS:4326' |
|---|
| 158 | numSpDims=len(varSize) -1 |
|---|
| 159 | grid.srsDimension=str(numSpDims) |
|---|
| 160 | grid.dimension=str(numSpDims) |
|---|
| 161 | ge =csml.parser.GridEnvelope(low=DI.getLowLimits(), high=DI.getHighLimits()) |
|---|
| 162 | grid.limits=ge |
|---|
| 163 | #add an axisName element(s) for each spatial dimension. |
|---|
| 164 | # and an ordinate element |
|---|
| 165 | axes=[] |
|---|
| 166 | for i in range (1, len(dimNames)): |
|---|
| 167 | #axisNames |
|---|
| 168 | axisname ='dim'+str(i) |
|---|
| 169 | axes.append(axisname) |
|---|
| 170 | #ordinates |
|---|
| 171 | grid.ordinates=[] |
|---|
| 172 | for i in range (1, len(dimNames)): |
|---|
| 173 | ord=csml.parser.GridOrdinateDescription() |
|---|
| 174 | ord.gridAxesSpanned='dim' + str(i) |
|---|
| 175 | ord.sequenceRule=csml.csmllibs.csmlextra.getSeqRule(len(dimNames)) |
|---|
| 176 | dimName=dimNames[len(dimNames)-i] |
|---|
| 177 | ord.definesAxis=dimName |
|---|
| 178 | #look up file extract name in dictionary |
|---|
| 179 | #(axisid stored in dictionary = current filename + variable name) |
|---|
| 180 | axisid=self.repfilename+dimName |
|---|
| 181 | if self.spatialstorage=='fileextract': |
|---|
| 182 | #refer to extract |
|---|
| 183 | ord.axisValues='#'+self.fileExtractDictionary[axisid] |
|---|
| 184 | else: |
|---|
| 185 | #store inline |
|---|
| 186 | DI.setAxis(dimName) |
|---|
| 187 | ord.axisValues=csml.csmllibs.csmlextra.cleanString(str(DI.getDataForAxis())) |
|---|
| 188 | grid.ordinates.append(ord) |
|---|
| 189 | grid.axisNames=axes |
|---|
| 190 | gsDomain.domainComplement=grid |
|---|
| 191 | GridSeriesFeature_element.domain=gsDomain |
|---|
| 192 | GridSeriesFeature_element.parameter=csml.parser.Phenomenon(href='http://badc.rl.ac.uk/localparams#%s'%allVarNames[i]) |
|---|
| 193 | self.fms.append(GridSeriesFeature_element) |
|---|
| 194 | DI.closeFile() |
|---|
| 195 | ###End of createCSMLGridSeriesFeatures### |
|---|
| 196 | |
|---|
| 197 | def createCSMLPointSeriesFeatures(self): |
|---|
| 198 | representativeFiles=self.ffmap.getRepresentativeFiles() |
|---|
| 199 | for repfile in representativeFiles: |
|---|
| 200 | self.repfilename,self.listOfFiles=self._populateListOfFiles(repfile) |
|---|
| 201 | self._getFilesAndTimes() |
|---|
| 202 | DI = csml.csmllibs.csmldataiface.DataInterface() |
|---|
| 203 | DI=DI.getUnknownInterfaceType(self.repfilename) |
|---|
| 204 | DI.openFile(self.repfilename) |
|---|
| 205 | allVarNames=DI.getListofVariables() |
|---|
| 206 | numFeatures=len(allVarNames) |
|---|
| 207 | try: |
|---|
| 208 | DI.setAxis(self.timedim) |
|---|
| 209 | times=DI.getDataForAxis() |
|---|
| 210 | except: |
|---|
| 211 | times = DI.getTimes() |
|---|
| 212 | #Create features: |
|---|
| 213 | for i in range (0, numFeatures): |
|---|
| 214 | PointSeriesFeature_element=csml.parser.PointSeriesFeature() |
|---|
| 215 | if str(allVarNames[i]).upper() in ['ERROR FLAG', 'ERROR']: #might need to extend this list |
|---|
| 216 | break |
|---|
| 217 | PointSeriesFeature_element.id=str(allVarNames[i]) |
|---|
| 218 | desc=self._getDescriptiveName(DI) |
|---|
| 219 | PointSeriesFeature_element.description=csml.parser.Description(desc) |
|---|
| 220 | #DOMAIN |
|---|
| 221 | psDomain=csml.parser.PointSeriesDomain() |
|---|
| 222 | t=csml.parser.Trajectory() |
|---|
| 223 | t.srsName='urn:EPSG:geographicCRS:4326' #TO Do |
|---|
| 224 | t.locations =csml.parser.DirectPositionList(vals='1 1') |
|---|
| 225 | |
|---|
| 226 | if self.timestorage =='inline': |
|---|
| 227 | tpl =csml.parser.TimePositionList() |
|---|
| 228 | tpl.timePositions=self.timeString |
|---|
| 229 | tpl.frame='%s:%s'%(self.caltype,self.units) |
|---|
| 230 | t.times=tpl |
|---|
| 231 | else: |
|---|
| 232 | # do something to create a single extract for the times (from the representative file). |
|---|
| 233 | tpl.timePositions = csml.csmllibs.csmlfileextracts.createSingleExtract(self.extractType, repfilename, self.timedim, len(self.timeString.split())) |
|---|
| 234 | tpl.frame='%s:%s'%(self.caltype,self.units) |
|---|
| 235 | |
|---|
| 236 | |
|---|
| 237 | # if self.timestorage =='inline': |
|---|
| 238 | # t.times=csmllibs.Parser.TimePositionList('#RefSysX',str(times)) |
|---|
| 239 | # else: |
|---|
| 240 | # #todo: depends on the file mapping??? |
|---|
| 241 | # t.times=csmllibs.Parser.TimePositionList('#RefSysX','blah') #blah = dummy times |
|---|
| 242 | # print 'times: ' + str(allVarNames[i]) |
|---|
| 243 | # print len(times) |
|---|
| 244 | # print len(listOfFiles) |
|---|
| 245 | # arraySize=len(times) * len(listOfFiles) |
|---|
| 246 | # fextract=csmllibs.csmlfileextracts.createSingleExtract(self.extractType,filenames,self.timedim,arraySize) |
|---|
| 247 | # tplist = csmllibs.Parser.TimePositionList(timePositions=fextract) |
|---|
| 248 | # t.times=tplist |
|---|
| 249 | filenames=csml.csmllibs.csmlextra.cleanString(str(self.listOfFiles)) |
|---|
| 250 | psDomain.domainReference=t |
|---|
| 251 | #RANGESET |
|---|
| 252 | DI.setVariable(allVarNames[i]) |
|---|
| 253 | try: |
|---|
| 254 | strUom = DI.getVariableAttribute('units') |
|---|
| 255 | except AttributeError: |
|---|
| 256 | #if units attribute doesn't exist: |
|---|
| 257 | strUom ="dimensionless or units not determined" |
|---|
| 258 | try: |
|---|
| 259 | measuredvalues = DI.getDataForVar() |
|---|
| 260 | except: |
|---|
| 261 | measuredvalues = ' could not get values ' |
|---|
| 262 | rs=csml.parser.RangeSet() |
|---|
| 263 | if self.valuestorage=='inline': |
|---|
| 264 | #encode inline |
|---|
| 265 | rs.quantityList=csml.parser.MeasureOrNullList(uom=strUom, val=str(measuredvalues)[1:-1]) |
|---|
| 266 | else: |
|---|
| 267 | #create a file extract link |
|---|
| 268 | arraySize=len(measuredvalues)*len(self.listOfFiles) |
|---|
| 269 | #TODO this needs to be able to handle inline, use VALUESTORAGE to determine which to use: |
|---|
| 270 | self.extractType=DI.extractType |
|---|
| 271 | fextract=csml.csmllibs.csmlfileextracts.createSingleExtract(self.extractType,filenames,allVarNames[i],arraySize) |
|---|
| 272 | qlist = csml.parser.MeasureOrNullList(val=fextract) |
|---|
| 273 | rs.quantityList=qlist |
|---|
| 274 | PointSeriesFeature_element.rangeSet=rs |
|---|
| 275 | #COVERAGEFUNCTION |
|---|
| 276 | #PARAMETER |
|---|
| 277 | #need to do parameter and coverageFunction elements |
|---|
| 278 | PointSeriesFeature_element.domain=psDomain |
|---|
| 279 | self.fms.append(PointSeriesFeature_element) |
|---|
| 280 | DI.closeFile() |
|---|
| 281 | |
|---|
| 282 | |
|---|
| 283 | #This function needs revising in light of a) csml parser and b) new profile feature types |
|---|
| 284 | def createCSMLProfileFeature(csmldoc, dataset_element, gml_FeatureCollection_element, ffmap, timedim): |
|---|
| 285 | representativeFiles=ffmap.getRepresentativeFiles() |
|---|
| 286 | listOfFiles=[] |
|---|
| 287 | for repfile in representativeFiles: |
|---|
| 288 | repfilename=repfile.getRepresentativeFileName() |
|---|
| 289 | listOfFiles.append(repfilename) |
|---|
| 290 | relfiles = repfile.getRelatedFiles() |
|---|
| 291 | for f in relfiles: |
|---|
| 292 | #hopefully there are no related files at the moment! |
|---|
| 293 | fname = f.getRelatedFileName() |
|---|
| 294 | listOfFiles.append(fname) |
|---|
| 295 | #print listOfFiles |
|---|
| 296 | |
|---|
| 297 | for file in listOfFiles: |
|---|
| 298 | DI = csmllibs.csmldataiface.DataInterface() |
|---|
| 299 | DI=DI.getUnknownInterfaceType(file) |
|---|
| 300 | print'opening file' |
|---|
| 301 | DI.openFile(file) |
|---|
| 302 | print 'getting variables' |
|---|
| 303 | allVarNames=DI.getListofVariables() |
|---|
| 304 | print 'getting feature count' |
|---|
| 305 | numFeatures=len(allVarNames) |
|---|
| 306 | |
|---|
| 307 | print "FEATURES" |
|---|
| 308 | print "***********" |
|---|
| 309 | for i in range (0, len(allVarNames)): |
|---|
| 310 | print allVarNames[i] |
|---|
| 311 | |
|---|
| 312 | for i in range (0, numFeatures): |
|---|
| 313 | gml_featureMember_element=csmldoc.createElement("gml:featureMember") |
|---|
| 314 | ProfileFeature_element=csmldoc.createElement("ProfileFeature") |
|---|
| 315 | ProfileFeature_element.setAttribute('gml:id',str(allVarNames[i])) |
|---|
| 316 | gml_description_element = csmldoc.createElement("gml:description") |
|---|
| 317 | gml_featureMember_element.appendChild(ProfileFeature_element) |
|---|
| 318 | #*********************************************************************** |
|---|
| 319 | #PointSeriesDomain: |
|---|
| 320 | #*********************************************************************** |
|---|
| 321 | ProfileDomain_element=csmldoc.createElement("ProfileDomain") |
|---|
| 322 | |
|---|
| 323 | |
|---|
| 324 | #*********************************************************************** |
|---|
| 325 | # domainReference element (and sub-elements) |
|---|
| 326 | #*********************************************************************** |
|---|
| 327 | domainReference_element=csmldoc.createElement("domainReference") |
|---|
| 328 | #orientedPosition_element=csmldoc.createElement("OrientedPosition") |
|---|
| 329 | #locations_element=csmldoc.createElement("locations") |
|---|
| 330 | #times_element=csmldoc.createElement("times") |
|---|
| 331 | #trajectory_element.appendChild(locations_element) |
|---|
| 332 | #trajectory_element.appendChild(times_element) |
|---|
| 333 | #domainReference_element.appendChild(orientedPosition_element) |
|---|
| 334 | |
|---|
| 335 | #gml_timePositionList_element = csmldoc.createElement("gml:TimePositionList") |
|---|
| 336 | #gml_timePositionList_element.appendChild(csmldoc.createTextNode(self.timeString)) |
|---|
| 337 | #domainReference_element.appendChild(gml_timePositionList_element) |
|---|
| 338 | ProfileDomain_element.appendChild(domainReference_element) |
|---|
| 339 | #*********************************************************************** |
|---|
| 340 | domainComplement_element=csmldoc.createElement("domainComplement") |
|---|
| 341 | ProfileDomain_element.appendChild(domainComplement_element) |
|---|
| 342 | |
|---|
| 343 | #*********************************************************************** |
|---|
| 344 | # gml:rangeSet_element |
|---|
| 345 | #*********************************************************************** |
|---|
| 346 | |
|---|
| 347 | gml_rangeSet_element=csmldoc.createElement("gml:rangeSet") |
|---|
| 348 | |
|---|
| 349 | #*********************************************************************** |
|---|
| 350 | # gml:coverageFunction element (and sub-element MappingRule) |
|---|
| 351 | #*********************************************************************** |
|---|
| 352 | gml_coverageFunction_element=csmldoc.createElement("gml:coverageFunction") |
|---|
| 353 | MappingRule_element=csmldoc.createElement("MappingRule") |
|---|
| 354 | #MappingRule_element.setAttribute('scanOrder',csmllibs.csmlextra.getMappingRule(len(dimNames))) |
|---|
| 355 | MappingRule_element.setAttribute('scanOrder','tba') |
|---|
| 356 | gml_coverageFunction_element.appendChild(MappingRule_element) |
|---|
| 357 | |
|---|
| 358 | |
|---|
| 359 | gml_featureMember_element.appendChild(ProfileDomain_element) |
|---|
| 360 | gml_featureMember_element.appendChild(gml_rangeSet_element) |
|---|
| 361 | gml_featureMember_element.appendChild(gml_coverageFunction_element) |
|---|
| 362 | gml_FeatureCollection_element.appendChild(gml_featureMember_element) |
|---|
| 363 | |
|---|
| 364 | return |
|---|
| 365 | |
|---|
| 366 | |
|---|
