source: DPPP/kml/csml2kml/python/csml2kmlpylon/csml2kmlpylon/controllers/csmlGrapher.py @ 3713

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/DPPP/kml/csml2kml/python/csml2kmlpylon/csml2kmlpylon/controllers/csmlGrapher.py@3713
Revision 3713, 20.6 KB checked in by mkochan, 12 years ago (diff)

Added links to underlying CSML in csmlGrapher (can be toggled for each dataset).

Line 
1# Pylons-specific imports
2import logging
3from csml2kmlpylon.lib.base import *
4from paste.httpexceptions import HTTPBadRequest, HTTPNotFound
5
6# Other imports
7import Image
8from pylab import *
9from matplotlib import dates
10import re
11from cStringIO import StringIO
12from tempfile import NamedTemporaryFile
13import urllib
14
15# CSML and csml2kml imports
16import csml
17import csml2kml.Station
18from csml2kml.utils import wget, URLError
19from csml2kml.ET import Element, SubElement, ElementTree, XML
20from string import lower
21
22log = logging.getLogger(__name__)
23
24class CsmlgrapherController(BaseController):
25
26    class Dataset:
27        '''
28        An auxiliary storage class describing a dataset of csml2kml.Station's;
29        this describes what URL the data is I{retrievable} from, as opposed to this class actually storing any data.
30        E.g. MIDAS and ECN are different datasets (each has a WFS URL from which it is being accessible).
31        '''
32        def __init__(self, id, name, geoServerUrl, datetimeQueryFormat, isCsmlLinkVisible):
33            self.id = id
34            self.name = name
35            self.geoServerUrl = geoServerUrl
36            self.datetimeQueryFormat = datetimeQueryFormat
37            self.isCsmlLinkVisible = isCsmlLinkVisible
38
39    def __call__(self, environ, start_response):
40        '''
41        Initialise the web service by overriding a method that always gets called upon controller construction.
42        More specifically, read in the config file.
43        '''
44
45        # The name of the config file is set in the server's "development.ini" file.
46        configFileName = config['app_conf']['csmlGrapher.configfile']
47       
48        # Load the configuration XML element (but only the part pertaining to the web service)
49        testConfig = ElementTree().parse(configFileName)
50        print testConfig
51        grapherConfig = (ElementTree().parse(configFileName))
52
53        # Initialise the datasets from the config file.
54        self.datasets = {}
55        datasetElements = grapherConfig.findall('Dataset')
56        for datasetElement in datasetElements:
57            id = datasetElement.get('id')
58           
59            self.datasets[id] = self.Dataset(
60                id, 
61                datasetElement.get('name'),
62                datasetElement.find('GeoServerURL').text,
63                datasetElement.find('DatetimeQueryFormat').text,
64                datasetElement.find('IsCSMLLinkVisible').text.lower() == 'yes'
65                )
66
67        # Set other configurable variables
68        self.displayIntervalStart = dates.dateutil.parser.parse(grapherConfig.find('DisplayIntervalStart').text)
69        self.displayIntervalEnd = dates.dateutil.parser.parse(grapherConfig.find('DisplayIntervalEnd').text)
70        self.servedFromUrl = grapherConfig.find('ServedFromURL').text
71        print 'Config file parse finished.'
72
73        # Call the __call__ method of the parent class
74        return BaseController.__call__(self, environ, start_response)
75
76    def _datetimeTo1Jan1970(self, datetime):
77        '''
78        Convert datetime to format a'la C{1-JAN-1970}.
79        @param datetime: The datetime to be converted.
80        @type datetime: C{matplotlib.dates.datetime.datetime}
81        @return: A string like "1-JAN-1970" or "31-MAR-2008".
82        @rtype: C{str}
83        '''
84        # Use format e.g. "1-JAN-1970" or "31-MAR-2008".
85        monthCodes = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
86        return repr(datetime.day) + '-' + monthCodes[datetime.month-1] + '-' + repr(datetime.year)
87
88    def _datetimeToGeoServerDate(self, datetime, datetimeQueryFormat):
89        '''
90        Convert datetime to one of the string datetime formats used in URL queries to WFS.
91        @param datetime: The datetime to be converted.
92        @type datetime: C{matplotlib.dates.datetime.datetime}
93        @param datetimeQueryFormat: Either C{UTC_TIMESTAMP} or C{1-JAN-1970}.
94        @type datetimeQueryFormat: C{str}
95        @return: If datetimeQueryFormat is C{UTC_TIMESTAMP}, returns a UTC timestamp;
96        if datetimeQueryFormat is C{1-JAN-1970}, returns a string like "1-JAN-1970" or "31-MAR-2008".
97        @rtype: C{str}
98        '''
99        if datetimeQueryFormat == 'UTC_TIMESTAMP':
100            # Use the UTC timestamp format, generate timestamp.
101            # (would like to do this using the datetime.strftime() method but it only supplies years after 1900).
102            dt = datetime
103            return '%04d-%02d-%02dT%02d:%02d:%02d' % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
104        elif datetimeQueryFormat == '1-JAN-1970':
105            return self._datetimeTo1Jan1970(datetime)
106        else:
107            raise ValueError('Wrong keyword in the <DatetimeQueryFormat> tag in the config file.')
108
109    def _buildCsmlRequestUrl(self, dataset, station_name, feature_id):
110        '''
111        Build a GeoServer query URL that can be used to retrieve a CSML PointSeriesFeature.
112        @param dataset: The dataset used to retrieve from
113        @type: C{Dataset}
114        @param station_name: Unique full name of the station, eg C{FAIR ISLE}.
115        @type station_name: C{str}
116        @param feature_id: Unique id of the CSML feature, eg C{air_temperature}.
117        @type feature_id: C{str}
118        @return: The URL.
119        @rtype: C{str}
120        '''
121        return urllib.quote(dataset.geoServerUrl + '?request=getfeature&service=wfs&version=1.1.0&typename=csml:PointSeriesFeature&filter=<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml"><ogc:And><ogc:PropertyIsEqualTo><ogc:PropertyName>gml:description</ogc:PropertyName><ogc:Literal>'+ station_name +'</ogc:Literal></ogc:PropertyIsEqualTo><ogc:PropertyIsEqualTo><ogc:PropertyName>csml:parameter/swe:Phenomenon/gml:name</ogc:PropertyName><ogc:Literal>'+ feature_id +'</ogc:Literal></ogc:PropertyIsEqualTo><ogc:PropertyIsBetween><ogc:PropertyName>csml:value/csml:PointSeriesCoverage/csml:pointSeriesDomain/csml:TimeSeries/csml:timePositionList</ogc:PropertyName><ogc:LowerBoundary><ogc:Literal>'+ self._datetimeToGeoServerDate(self.displayIntervalStart, dataset.datetimeQueryFormat) +'</ogc:Literal></ogc:LowerBoundary><ogc:UpperBoundary><ogc:Literal>'+ self._datetimeToGeoServerDate(self.displayIntervalEnd, dataset.datetimeQueryFormat) +'</ogc:Literal></ogc:UpperBoundary></ogc:PropertyIsBetween></ogc:And></ogc:Filter>', '/._:?&=')
122
123    def _retrieveCsmlPointSeriesFeature(self, dataset, station_name, feature_id):
124        '''
125        Retrieve CSML from WFS using an URL query.
126        @param dataset: The dataset used to retrieve from
127        @type: C{Dataset}
128        @param station_name: Unique full name of the station, eg C{FAIR ISLE}.
129        @type station_name: C{str}
130        @param feature_id: Unique id of the CSML feature, eg C{air_temperature}.
131        @type feature_id: C{str}
132        @return: A C{csml.PointSeriesFeature} object representing the single CSML feature if the feature was found;
133        C{None} if a feature collection has been returned, but contains no CSML features (this happens when
134        there are no measured time points in the used time interval).
135        @throws: A C{LookupError} exception with a message if there is no response from the server.
136        '''
137        # Acquire the GeoServer response and parse it into a CSMLFeatureCollection object
138        geoServerRequestUrl = self._buildCsmlRequestUrl(dataset, station_name, feature_id)
139        print geoServerRequestUrl
140        try:
141            geoServerResponse = wget(geoServerRequestUrl)
142        except URLError:
143            raise LookupError('Cannot acquire response from server (wrong URL or server down)')
144        csmlFeatureCollection = csml.parser.CSMLFeatureCollection()
145        csmlFeatureCollection.fromXML( XML(geoServerResponse) )
146
147        # If csmlFeatureCollection contains a single CSML feature, the isolate it. If it contains no feature,
148        # probably no data points are available for that interval.
149        try:
150            csmlFeature = csmlFeatureCollection.featureMembers
151        except AttributeError:
152            raise LookupError(
153                'Wrong feature ID or no data points available for the given time interval ' + 
154                '(perhaps the interval is shorter than the frequency of sampling of data points, ' +
155                'or collection of data for this feature was interrupted during the plotting interval).'
156                )
157
158        # Make sure that the feature is a PointSeriesFeature
159        if not isinstance(csmlFeature, csml.parser.PointSeriesFeature):
160            raise LookupError('CSML feature not a PointSeriesFeature')
161        csmlPointSeriesFeature = csmlFeature
162
163        return csmlPointSeriesFeature
164
165    def plot(self):
166        '''
167        Handler for plotting a specific PointSeries feature given by URL parameter "feature_id".
168        Retrieves the data from a web service, which should preferably run on the same machine.
169        Request URL parameters are:
170        dataset_id      Unique identifier of the dataset to use.
171        feature_id      Unique identifier of the requested CSML PointSeries feature.
172        station_name    Name of the requested station.
173                               
174        Response:              An image/png of the time series for the CSML PointSeries feature.
175        '''
176        def _plot_feature(feature, dataset):
177            '''
178            Plot a PointSeries feature using matplotlib, into a temporary file.
179            @return: An file-like object which represents the file, which can be read from and also closed
180            as a normal file.
181            @rtype: C{tempfile.NamedTemporaryFile}
182            '''
183           
184            # Make sure that time is considered to be UTC time. This is because all CSML data is supposed to be in UTC,
185            # but actually pylab is "time zone aware" and will interpret the time as local time zone time.
186            # Not quite sure if this would be a problem, but rather enforce it is UTC.
187            def _enforce_UTC_timezone(datestr):
188                if datestr[-1] == 'Z':
189                    return datestr
190                else:
191                    return datestr + 'Z'
192           
193            # Isolate time points
194            times=feature.value.pointSeriesDomain.timePositionList.CONTENT.split()
195            times = map(_enforce_UTC_timezone, times)
196
197            # Make a list of times as float numbers being days passed since start of epoch (here 01-01-0001).
198            # (We need to use the wrapping function csml2kml.utils.parseTimestamp() because unfortunately the matplotlib's
199            #  datetime parser cannot handle timestamps like "2004-02-17T24:00:00" directly.)
200            elapsed_times = map(date2num,
201                                map(csml2kml.utils.parseTimestamp,
202                                    times
203                                    )
204                                )
205
206            # Get values of the measured quantity
207            ql = feature.value.rangeSet.quantityList
208            vals = map(float, ql.CONTENT.split())
209
210            # Make sure the times are sorted in ascending order (and rearrange the values accordingly),
211            # this is because we get them potentially unsorted from GeoServer. Also have to handle the sad fact,
212            # that sometimes we are getting duplicate time-value samples from the MIDAS dataset!
213            sortTimesExplicitly = True
214            if sortTimesExplicitly:
215                reorder = {}
216                for i in range(len(elapsed_times)):
217                    reorder[elapsed_times[i]] = i
218                elapsed_times = reorder.keys()
219                elapsed_times.sort()
220                times2 = []
221                vals2 = []
222                for et in elapsed_times:
223                    times2.append(times[reorder[et]])
224                    vals2.append(vals[reorder[et]])
225                times = times2
226                vals = vals2
227
228            print '---times (' + str(len(times)) + ' of them):\n' + str(times)
229            print '---vals (' + str(len(times)) + ' of them):\n' + str(vals)
230
231            # Note the start and end date (they are actually date+time), and time span between them
232
233            start_date = dates.num2date(elapsed_times[0])
234            end_date = dates.num2date(elapsed_times[-1])
235            span = elapsed_times[-1] - elapsed_times[0]
236
237            # Generate tick locator and formatter; these determine the time axis: locator determines tick positions
238            # and formatter format of time. There will be a total of eight ticks.
239            print '---start_date:' + str(start_date)
240            print '---end_date:' + str(end_date)
241            print '---span: ' + str(span)
242            (tickLocator, tickFormatter) = dates.date_ticker_factory(span, numticks=8)
243
244            # Determine the units of measurement
245            uom=ql.uom.title()
246            if ql.uom.islower():
247                uom = uom.lower()
248            if ql.uom.isupper():
249                uom = uom.upper()
250
251            # Prepare the plot of the figure (matplotlib does the actual plotting actions are postponed on file save)
252            fig = figure()
253            plot_date(elapsed_times, vals, 'b-', xdate=True, lw=1)
254            plot_date(elapsed_times, vals, 'go', markeredgecolor = 'g', xdate=True, lw=2)
255            ax = gca()
256            ax.xaxis.set_major_locator(tickLocator)       
257            ax.xaxis.set_major_formatter(tickFormatter)
258            fig.autofmt_xdate()  # make sure times are shown with no overlapping timestamps
259            time_format = '%d-%b-%Y %H:%M:%S UTC'
260            xlabel('Times between %s and %s ' % (start_date.strftime(time_format), end_date.strftime(time_format)))
261            ylabel('Values [%s]' % uom)
262            title('"%s:%s" (%s station %s)' % (dataset.id, feature.id, dataset.name, feature.description.CONTENT))
263            grid(True)
264
265            # Save the figure to a temporary file
266            tempFile = NamedTemporaryFile(suffix='.png')
267            savefig(tempFile.name)
268           
269            return tempFile
270
271        def _set_response(tempFile):
272            '''
273            Set the service's response to an image, containing image read from a temporary location.
274            @param tempFile: An object representing a temporary file
275            @type tempFile: C{tempfile.NamedTemporaryFile}
276            '''
277            img = Image.open(tempFile.name)
278            buf = StringIO()
279            img.save(buf, 'PNG')
280            response.content_type = 'image/png'
281            response.content = buf.getvalue()
282           
283        #---------------------- (function main body) -----------------------
284
285        # Get parameters from the request object
286        try:
287            dataset_id = str(request.params['dataset_id'])     # convert back from Unicode
288            feature_id = str(request.params['feature_id'])     # convert back from Unicode
289            station_name = str(request.params['station_name']) # convert back from Unicode
290        except KeyError:
291            raise HTTPBadRequest('Parameters "dataset_id", "feature_id" and "station_name" must be supplied.')
292
293        # Try to retrieve the csmlPointSeriesFeature object from the appropriate dataset
294        try:
295            dataset = self.datasets[dataset_id]
296            csmlPointSeriesFeature = self._retrieveCsmlPointSeriesFeature(dataset, station_name, feature_id)
297        except LookupError, e:
298            errorMessage = e.args[0]
299            httpNotFound = HTTPNotFound()
300            httpNotFound.explanation = ''
301            httpNotFound.detail = errorMessage
302            raise httpNotFound
303
304        # Try to plot the feature into a temporary file, and put the contents of that file into the response
305        try:
306            tempFile = _plot_feature(csmlPointSeriesFeature, dataset)    # plot the feature into a temporary file
307            _set_response(tempFile)                                      # set the response as an image containing the plot
308        finally:
309            try:
310                tempFile.close()
311            except NameError:
312                pass  # tempFile undefined -- no need to close
313
314    def list(self):
315        '''
316        Handler for generating a list of CSML features per station, with dynamic links to the plot service.
317        Request parameters:
318                               dataset_id      Unique identifier of the dataset to use.
319                               station_name    Name of the requested station.
320                               
321        Response:              A text/html with the page containing links to the plot service on the same server.
322        '''
323
324        # Get parameters from the request object
325        try:
326            dataset_id = str(request.params['dataset_id'])      # convert back from Unicode
327            station_name = str(request.params['station_name'])  # convert back from Unicode
328        except KeyError:
329            raise HTTPBadRequest('Parameters "dataset_id" and "station_name" must be supplied.')
330
331        try:
332            # Identify the appropriate dataset.
333            try:
334                dataset = self.datasets[dataset_id]
335            except KeyError:
336                raise LookupError('Wrong dataset identifier.')
337           
338            # Prepare the request for the GeoServer -- to return a collection of np:Station, containing a single np:Station,
339            # which contains a list of CSML features.
340            geoServerRequestUrl = urllib.quote(dataset.geoServerUrl + '?request=getFeature&service=wfs&version=1.1.0&typename=np:Station&filter=<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml"><ogc:PropertyIsEqualTo><ogc:PropertyName>stationName</ogc:PropertyName><ogc:Literal>' + station_name + '</ogc:Literal></ogc:PropertyIsEqualTo></ogc:Filter>', '/._:?&=')
341
342            # Acquire the GeoServer response and parse it into a WFSStationCollection object
343            try:
344                geoServerResponse = wget(geoServerRequestUrl)
345            except URLError:
346                raise LookupError('Cannot access underlying WFS (server error, wrong network connection, or typo in the config file).')
347
348            wfsStationCollection = csml2kml.Station.WFSStationCollection()
349            wfsStationCollection.parseString(geoServerResponse)
350            if len(wfsStationCollection.stations) != 1:
351                raise LookupError('Requested station not found.')
352            wfsStation = wfsStationCollection.stations[0]
353        except LookupError, e:
354            errorMessage = e.args[0]
355            httpNotFound = HTTPNotFound()
356            httpNotFound.explanation = ''
357            httpNotFound.detail = errorMessage
358            raise httpNotFound
359
360        # Generate HTML with a list of CSML features contained in the station.
361        # Currently, this is done simply by directly generating HTML (i.e. no template was used).
362        # The table column with links to CSML will appear only if that is allowed for the dataset in the config file.
363        htmlRows = ''
364        for stationFeature in wfsStation.stationFeatures:
365            if self.displayIntervalStart >= stationFeature.collectBeginDate and self.displayIntervalEnd <= stationFeature.collectEndDate:
366                htmlLinkToGrapher = '%s/plot?dataset_id=%s&station_name=%s&feature_id=%s' % (
367                    self.servedFromUrl, dataset_id, station_name, stationFeature.featureId
368                    )
369                if dataset.isCsmlLinkVisible:
370                    htmlLinkToCsml = self._buildCsmlRequestUrl(dataset, station_name, stationFeature.featureId)
371                    htmlLinkToCsmlDataitem = '<td><a href="%s">%s</a></td>' % (htmlLinkToCsml, 'CSML')
372                else:
373                    htmlLinkToCsmlDataitem = ''
374                htmlRow = '<tr><td>%s</td>%s<td>%s</td><td>%s</td></tr>' % (
375                    '<a href="%s">%s</a>' % (htmlLinkToGrapher, stationFeature.featureId),
376                    htmlLinkToCsmlDataitem,
377                    self._datetimeTo1Jan1970(stationFeature.collectBeginDate),
378                    self._datetimeTo1Jan1970(stationFeature.collectEndDate)
379                    )
380                htmlRows = htmlRows + htmlRow
381        if dataset.isCsmlLinkVisible:
382            underlyingDataHeaderItem = '<th>Underlying data</th>'
383        else:
384            underlyingDataHeaderItem = ''
385        htmlTable = '<table border="1"><tr><th>Feature plot</th>%s<th>Data collected since*</th><th>Data collected until*</th></tr>%s</table>' % (underlyingDataHeaderItem, htmlRows)
386        htmlHeading = 'List of CSML features for station ' + station_name
387        htmlUsage = '* The sampling frequencies vary between features and there may be no data points for the chosen plotting interval.'
388        htmlBody = '<h2>%s</h2>%s<p>%s' % (htmlHeading, htmlTable, htmlUsage)
389        htmlTitle = htmlHeading
390        html = '<html><title>%s</title><body>%s</body></html>' % (htmlTitle, htmlBody)
391
392        print html
393        response.content_type = 'text/html'
394        response.content = html
Note: See TracBrowser for help on using the repository browser.