source: DPPP/kml/csml2kml/python/csml2kmlpylon/csml2kmlpylon/controllers/csmlGrapher.py @ 3707

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/DPPP/kml/csml2kml/python/csml2kmlpylon/csml2kmlpylon/controllers/csmlGrapher.py@3707
Revision 3707, 19.0 KB checked in by mkochan, 11 years ago (diff)

Changed HTML generation in csmlGrapher -- not using ElementTree now.

Line 
1# Pylons-specific imports
2import logging
3from csml2kmlpylon.lib.base import *
4from paste.httpexceptions import HTTPBadRequest, HTTPNotFound
5
6# Other imports
7import Image
8from pylab import *
9from matplotlib import dates
10import re
11from cStringIO import StringIO
12from tempfile import NamedTemporaryFile
13import urllib
14
15# CSML and csml2kml imports
16import csml
17import csml2kml.Station
18from csml2kml.utils import wget, URLError
19from csml2kml.ET import Element, SubElement, ElementTree, XML
20
21log = logging.getLogger(__name__)
22
23class CsmlgrapherController(BaseController):
24
25    class Dataset:
26        '''
27        An auxiliary storage class describing a dataset of csml2kml.Station's;
28        this describes what URL the data is I{retrievable} from, as opposed to this class actually storing any data.
29        E.g. MIDAS and ECN are different datasets (each has a WFS URL from which it is being accessible).
30        '''
31        def __init__(self, id, name, geoServerUrl, datetimeQueryFormat):
32            self.id = id
33            self.name = name
34            self.geoServerUrl = geoServerUrl
35            self.datetimeQueryFormat = datetimeQueryFormat
36
37    def __call__(self, environ, start_response):
38        '''
39        Initialise the web service by overriding a method that always gets called upon controller construction.
40        More specifically, read in the config file.
41        '''
42
43        # The name of the config file is set in the server's "development.ini" file.
44        configFileName = config['app_conf']['csmlGrapher.configfile']
45       
46        # Load the configuration XML element (but only the part pertaining to the web service)
47        testConfig = ElementTree().parse(configFileName)
48        print testConfig
49        grapherConfig = (ElementTree().parse(configFileName))
50
51        # Initialise the datasets from the config file.
52        self.datasets = {}
53        datasetElements = grapherConfig.findall('Dataset')
54        for datasetElement in datasetElements:
55            id = datasetElement.get('id')
56            self.datasets[id] = self.Dataset(
57                id, 
58                datasetElement.get('name'),
59                datasetElement.find('GeoServerURL').text,
60                datasetElement.find('DatetimeQueryFormat').text
61                )
62
63        # Set other configurable variables
64        self.displayIntervalStart = dates.dateutil.parser.parse(grapherConfig.find('DisplayIntervalStart').text)
65        self.displayIntervalEnd = dates.dateutil.parser.parse(grapherConfig.find('DisplayIntervalEnd').text)
66        self.servedFromUrl = grapherConfig.find('ServedFromURL').text
67        print 'Config file parse finished.'
68
69        # Call the __call__ method of the parent class
70        return BaseController.__call__(self, environ, start_response)
71
72    def _datetimeTo1Jan1970(self, datetime):
73        '''
74        Convert datetime to format a'la C{1-JAN-1970}.
75        @param datetime: The datetime to be converted.
76        @type datetime: C{matplotlib.dates.datetime.datetime}
77        @return: A string like "1-JAN-1970" or "31-MAR-2008".
78        @rtype: C{str}
79        '''
80        # Use format e.g. "1-JAN-1970" or "31-MAR-2008".
81        monthCodes = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
82        return repr(datetime.day) + '-' + monthCodes[datetime.month-1] + '-' + repr(datetime.year)
83
84    def _datetimeToGeoServerDate(self, datetime, datetimeQueryFormat):
85        '''
86        Convert datetime to one of the string datetime formats used in URL queries to WFS.
87        @param datetime: The datetime to be converted.
88        @type datetime: C{matplotlib.dates.datetime.datetime}
89        @param datetimeQueryFormat: Either C{UTC_TIMESTAMP} or C{1-JAN-1970}.
90        @type datetimeQueryFormat: C{str}
91        @return: If datetimeQueryFormat is C{UTC_TIMESTAMP}, returns a UTC timestamp;
92        if datetimeQueryFormat is C{1-JAN-1970}, returns a string like "1-JAN-1970" or "31-MAR-2008".
93        @rtype: C{str}
94        '''
95        if datetimeQueryFormat == 'UTC_TIMESTAMP':
96            # Use the UTC timestamp format, generate timestamp.
97            # (would like to do this using the datetime.strftime() method but it only supplies years after 1900).
98            dt = datetime
99            return '%04d-%02d-%02dT%02d:%02d:%02d' % (dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second)
100        elif datetimeQueryFormat == '1-JAN-1970':
101            return self._datetimeTo1Jan1970(datetime)
102        else:
103            raise ValueError('Wrong keyword in the <DatetimeQueryFormat> tag in the config file.')
104
105    def _retrieveCsmlPointSeriesFeature(self, dataset, station_name, feature_id):
106        '''
107        Retrieve CSML from WFS using an URL query.
108        @param dataset: The dataset used to retrieve from
109        @type: C{Dataset}
110        @param station_name: Unique full name of the station, eg C{FAIR ISLE}.
111        @type station_name: C{str}
112        @param feature_id: Unique id of the CSML feature, eg C{air_temperature}.
113        @type feature_id: C{str}
114        @return: A C{csml.PointSeriesFeature} object representing the single CSML feature if the feature was found;
115        C{None} if a feature collection has been returned, but contains no CSML features (this happens when
116        there are no measured time points in the used time interval).
117        @throws: A C{LookupError} exception with a message if there is no response from the server.
118        '''
119        # Prepare the request for the dataset's GeoServer
120        geoServerRequestUrl = urllib.quote(dataset.geoServerUrl + '?request=getfeature&service=wfs&version=1.1.0&typename=csml:PointSeriesFeature&filter=<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml"><ogc:And><ogc:PropertyIsEqualTo><ogc:PropertyName>gml:description</ogc:PropertyName><ogc:Literal>'+ station_name +'</ogc:Literal></ogc:PropertyIsEqualTo><ogc:PropertyIsEqualTo><ogc:PropertyName>csml:parameter/swe:Phenomenon/gml:name</ogc:PropertyName><ogc:Literal>'+ feature_id +'</ogc:Literal></ogc:PropertyIsEqualTo><ogc:PropertyIsBetween><ogc:PropertyName>csml:value/csml:PointSeriesCoverage/csml:pointSeriesDomain/csml:TimeSeries/csml:timePositionList</ogc:PropertyName><ogc:LowerBoundary><ogc:Literal>'+ self._datetimeToGeoServerDate(self.displayIntervalStart, dataset.datetimeQueryFormat) +'</ogc:Literal></ogc:LowerBoundary><ogc:UpperBoundary><ogc:Literal>'+ self._datetimeToGeoServerDate(self.displayIntervalEnd, dataset.datetimeQueryFormat) +'</ogc:Literal></ogc:UpperBoundary></ogc:PropertyIsBetween></ogc:And></ogc:Filter>', '/._:?&=')
121
122        # Acquire the GeoServer response and parse it into a CSMLFeatureCollection object
123        print geoServerRequestUrl
124        try:
125            geoServerResponse = wget(geoServerRequestUrl)
126        except URLError:
127            raise LookupError('Cannot acquire response from server (wrong URL or server down)')
128        csmlFeatureCollection = csml.parser.CSMLFeatureCollection()
129        csmlFeatureCollection.fromXML( XML(geoServerResponse) )
130
131        # If csmlFeatureCollection contains a single CSML feature, the isolate it. If it contains no feature,
132        # probably no data points are available for that interval.
133        try:
134            csmlFeature = csmlFeatureCollection.featureMembers
135        except AttributeError:
136            raise LookupError(
137                'No data points available for the given time interval. ' + 
138                'Perhaps the interval is shorter than the frequency of data points, ' +
139                'or collection of data for this feature was interrupted at the time.'
140                )
141
142        # Make sure that the feature is a PointSeriesFeature
143        if not isinstance(csmlFeature, csml.parser.PointSeriesFeature):
144            raise LookupError('CSML feature not a PointSeriesFeature')
145        csmlPointSeriesFeature = csmlFeature
146
147        return csmlPointSeriesFeature
148
149    def plot(self):
150        '''
151        Handler for plotting a specific PointSeries feature given by URL parameter "feature_id".
152        Retrieves the data from a web service, which should preferably run on the same machine.
153        Request URL parameters are:
154        dataset_id      Unique identifier of the dataset to use.
155        feature_id      Unique identifier of the requested CSML PointSeries feature.
156        station_name    Name of the requested station.
157                               
158        Response:              An image/png of the time series for the CSML PointSeries feature.
159        '''
160        def _plot_feature(feature, dataset):
161            '''
162            Plot a PointSeries feature using matplotlib, into a temporary file.
163            @return: An file-like object which represents the file, which can be read from and also closed
164            as a normal file.
165            @rtype: C{tempfile.NamedTemporaryFile}
166            '''
167           
168            # Make sure that time is considered to be UTC time. This is because all CSML data is supposed to be in UTC,
169            # but actually pylab is "time zone aware" and will interpret the time as local time zone time.
170            # Not quite sure if this would be a problem, but rather enforce it is UTC.
171            def _enforce_UTC_timezone(datestr):
172                if datestr[-1] == 'Z':
173                    return datestr
174                else:
175                    return datestr + 'Z'
176           
177            # Isolate time points
178            times=feature.value.pointSeriesDomain.timePositionList.CONTENT.split()
179            times = map(_enforce_UTC_timezone, times)
180
181            # Make a list of times as float numbers being days passed since start of epoch (here 01-01-0001).
182            # (We need to use the wrapping function csml2kml.utils.parseTimestamp() because unfortunately the matplotlib's
183            #  datetime parser cannot handle timestamps like "2004-02-17T24:00:00" directly.)
184            elapsed_times = map(date2num,
185                                map(csml2kml.utils.parseTimestamp,
186                                    times
187                                    )
188                                )
189
190            # Get values of the measured quantity
191            ql = feature.value.rangeSet.quantityList
192            vals = map(float, ql.CONTENT.split())
193
194            # Make sure the times are sorted in ascending order (and rearrange the values accordingly),
195            # this is because we get them potentially unsorted from GeoServer. Also have to handle the sad fact,
196            # that sometimes we are getting duplicate time-value samples from the MIDAS dataset!
197            sortTimesExplicitly = True
198            if sortTimesExplicitly:
199                reorder = {}
200                for i in range(len(elapsed_times)):
201                    reorder[elapsed_times[i]] = i
202                elapsed_times = reorder.keys()
203                elapsed_times.sort()
204                times2 = []
205                vals2 = []
206                for et in elapsed_times:
207                    times2.append(times[reorder[et]])
208                    vals2.append(vals[reorder[et]])
209                times = times2
210                vals = vals2
211
212            print '---times (' + str(len(times)) + ' of them):\n' + str(times)
213            print '---vals (' + str(len(times)) + ' of them):\n' + str(vals)
214
215            # Note the start and end date (they are actually date+time), and time span between them
216
217            start_date = dates.num2date(elapsed_times[0])
218            end_date = dates.num2date(elapsed_times[-1])
219            span = elapsed_times[-1] - elapsed_times[0]
220
221            # Generate tick locator and formatter; these determine the time axis: locator determines tick positions
222            # and formatter format of time. There will be a total of eight ticks.
223            print '---start_date:' + str(start_date)
224            print '---end_date:' + str(end_date)
225            print '---span: ' + str(span)
226            (tickLocator, tickFormatter) = dates.date_ticker_factory(span, numticks=8)
227
228            # Determine the units of measurement
229            uom=ql.uom.title()
230            if ql.uom.islower():
231                uom = uom.lower()
232            if ql.uom.isupper():
233                uom = uom.upper()
234
235            # Prepare the plot of the figure (matplotlib does the actual plotting actions are postponed on file save)
236            fig = figure()
237            plot_date(elapsed_times, vals, 'b-', xdate=True, lw=1)
238            plot_date(elapsed_times, vals, 'go', markeredgecolor = 'g', xdate=True, lw=2)
239            ax = gca()
240            ax.xaxis.set_major_locator(tickLocator)       
241            ax.xaxis.set_major_formatter(tickFormatter)
242            fig.autofmt_xdate()  # make sure times are shown with no overlapping timestamps
243            time_format = '%d-%b-%Y %H:%M:%S UTC'
244            xlabel('Times between %s and %s ' % (start_date.strftime(time_format), end_date.strftime(time_format)))
245            ylabel('Values [%s]' % uom)
246            title('"%s:%s" (%s station %s)' % (dataset.id, feature.id, dataset.name, feature.description.CONTENT))
247            grid(True)
248
249            # Save the figure to a temporary file
250            tempFile = NamedTemporaryFile(suffix='.png')
251            savefig(tempFile.name)
252           
253            return tempFile
254
255        def _set_response(tempFile):
256            '''
257            Set the service's response to an image, containing image read from a temporary location.
258            @param tempFile: An object representing a temporary file
259            @type tempFile: C{tempfile.NamedTemporaryFile}
260            '''
261            img = Image.open(tempFile.name)
262            buf = StringIO()
263            img.save(buf, 'PNG')
264            response.content_type = 'image/png'
265            response.content = buf.getvalue()
266           
267        #---------------------- (function main body) -----------------------
268
269        # Get parameters from the request object
270        try:
271            dataset_id = str(request.params['dataset_id'])     # convert back from Unicode
272            feature_id = str(request.params['feature_id'])     # convert back from Unicode
273            station_name = str(request.params['station_name']) # convert back from Unicode
274        except KeyError:
275            raise HTTPBadRequest('Parameters "dataset_id", "feature_id" and "station_name" must be supplied.')
276
277        # Try to retrieve the csmlPointSeriesFeature object from the appropriate dataset
278        try:
279            dataset = self.datasets[dataset_id]
280            csmlPointSeriesFeature = self._retrieveCsmlPointSeriesFeature(dataset, station_name, feature_id)
281        except LookupError, e:
282            errorMessage = e.args[0]
283            httpNotFound = HTTPNotFound()
284            httpNotFound.explanation = ''
285            httpNotFound.detail = errorMessage
286            raise httpNotFound
287
288        # Try to plot the feature into a temporary file, and put the contents of that file into the response
289        try:
290            tempFile = _plot_feature(csmlPointSeriesFeature, dataset)    # plot the feature into a temporary file
291            _set_response(tempFile)                                      # set the response as an image containing the plot
292        finally:
293            try:
294                tempFile.close()
295            except NameError:
296                pass  # tempFile undefined -- no need to close
297
298    def list(self):
299        '''
300        Handler for generating a list of CSML features per station, with dynamic links to the plot service.
301        Request parameters:
302                               dataset_id      Unique identifier of the dataset to use.
303                               station_name    Name of the requested station.
304                               
305        Response:              A text/html with the page containing links to the plot service on the same server.
306        '''
307
308        # Get parameters from the request object
309        try:
310            dataset_id = str(request.params['dataset_id'])      # convert back from Unicode
311            station_name = str(request.params['station_name'])  # convert back from Unicode
312        except KeyError:
313            raise HTTPBadRequest('Parameters "dataset_id" and "station_name" must be supplied.')
314
315        try:
316            # Identify the appropriate dataset.
317            try:
318                dataset = self.datasets[dataset_id]
319            except KeyError:
320                raise LookupError('Wrong dataset identifier.')
321           
322            # Prepare the request for the GeoServer -- to return a collection of np:Station, containing a single np:Station,
323            # which contains a list of CSML features.
324            geoServerRequestUrl = urllib.quote(dataset.geoServerUrl + '?request=getFeature&service=wfs&version=1.1.0&typename=np:Station&filter=<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml"><ogc:PropertyIsEqualTo><ogc:PropertyName>stationName</ogc:PropertyName><ogc:Literal>' + station_name + '</ogc:Literal></ogc:PropertyIsEqualTo></ogc:Filter>', '/._:?&=')
325
326            # Acquire the GeoServer response and parse it into a WFSStationCollection object
327            try:
328                geoServerResponse = wget(geoServerRequestUrl)
329            except URLError:
330                raise LookupError('Cannot access underlying WFS (server error, wrong network connection, or typo in the config file).')
331
332            wfsStationCollection = csml2kml.Station.WFSStationCollection()
333            wfsStationCollection.parseString(geoServerResponse)
334            if len(wfsStationCollection.stations) != 1:
335                raise LookupError('Requested station not found.')
336            wfsStation = wfsStationCollection.stations[0]
337        except LookupError, e:
338            errorMessage = e.args[0]
339            httpNotFound = HTTPNotFound()
340            httpNotFound.explanation = ''
341            httpNotFound.detail = errorMessage
342            raise httpNotFound
343
344        # Generate HTML with a list of CSML features contained in the station.
345        # Currently, this is done simply by directly generating HTML (i.e. no template was used).
346        htmlRows = ''
347        for stationFeature in wfsStation.stationFeatures:
348            if self.displayIntervalStart >= stationFeature.collectBeginDate and self.displayIntervalEnd <= stationFeature.collectEndDate:
349                htmlLinkToGrapher = '%s/plot?dataset_id=%s&station_name=%s&feature_id=%s' % (
350                    self.servedFromUrl, dataset_id, station_name, stationFeature.featureId
351                    )
352                htmlRow = '<tr><td>%s</td><td>%s</td><td>%s</td></tr>' % (
353                    '<a href="%s">%s</a>' % (htmlLinkToGrapher, stationFeature.featureId),
354                    self._datetimeTo1Jan1970(stationFeature.collectBeginDate),
355                    self._datetimeTo1Jan1970(stationFeature.collectEndDate)
356                    )
357                htmlRows = htmlRows + htmlRow
358        htmlTable = '<table border="1"><tr><th>Feature name</th><th>Data collected since</th><th>Data collected until</th></tr>%s</table>' % htmlRows
359        htmlHeading = 'List of CSML features for station ' + station_name
360        htmlUsage = ''
361        htmlBody = '<h2>%s</h2>%s<p>%s' % (htmlHeading, htmlUsage, htmlTable)
362        htmlTitle = htmlHeading
363        html = '<html><title>%s</title><body>%s</body></html>' % (htmlTitle, htmlBody)
364
365        print html
366        response.content_type = 'text/html'
367        response.content = html
Note: See TracBrowser for help on using the repository browser.