source: DPPP/kml/csml2kml/python/csml2kmlpylon/csml2kmlpylon/controllers/csmlGrapher.py @ 3702

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/DPPP/kml/csml2kml/python/csml2kmlpylon/csml2kmlpylon/controllers/csmlGrapher.py@3702
Revision 3702, 16.5 KB checked in by mkochan, 12 years ago (diff)

Added proper HTTP exceptions to list() in csmlGrapher.

Line 
1# Pylons-specific imports
2import logging
3from csml2kmlpylon.lib.base import *
4from paste.httpexceptions import HTTPBadRequest, HTTPNotFound
5
6# Other imports
7import Image
8from pylab import *
9from matplotlib import dates
10import re
11from cStringIO import StringIO
12from tempfile import NamedTemporaryFile
13import urllib
14
15# CSML and csml2kml imports
16import csml
17import csml2kml.Station
18from csml2kml.utils import wget, URLError
19from csml2kml.ET import Element, SubElement, ElementTree, XML
20
21log = logging.getLogger(__name__)
22
23class CsmlgrapherController(BaseController):
24
25    class Dataset:
26        '''
27        An auxiliary storage class describing a dataset of csml2kml.Station's;
28        this describes what URL the data is I{retrievable} from, as opposed to this class actually storing any data.
29        E.g. MIDAS and ECN are different datasets (each has a WFS URL from which it is being accessible).
30        '''
31        def __init__(self, id, name, geoServerUrl):
32            self.id = id
33            self.name = name
34            self.geoServerUrl = geoServerUrl
35
36    def __call__(self, environ, start_response):
37        '''
38        Initialise the web service by overriding a method that always gets called upon controller construction.
39        More specifically, read in the config file.
40        '''
41
42        # The name of the config file is set in the server's "development.ini" file.
43        configFileName = config['app_conf']['csmlGrapher.configfile']
44       
45        # Load the configuration XML element (but only the part pertaining to the web service)
46        testConfig = ElementTree().parse(configFileName)
47        print testConfig
48        grapherConfig = (ElementTree().parse(configFileName))
49
50        # Initialise the datasets from the config file.
51        self.datasets = {}
52        datasetElements = grapherConfig.findall('Dataset')
53        for datasetElement in datasetElements:
54            id = datasetElement.get('id')
55            self.datasets[id] = self.Dataset(id, datasetElement.get('name'), datasetElement.find('GeoServerURL').text)
56
57        # Set other configurable variables
58        self.displayIntervalStart = dates.dateutil.parser.parse(grapherConfig.find('DisplayIntervalStart').text)
59        self.displayIntervalEnd = dates.dateutil.parser.parse(grapherConfig.find('DisplayIntervalEnd').text)
60        self.servedFromUrl = grapherConfig.find('ServedFromURL').text
61
62        print 'Config file parse finished.'
63
64        # Call the __call__ method of the parent class
65        return BaseController.__call__(self, environ, start_response)
66
67    def _datetimeToGeoServerDate(self, datetime):
68        monthCodes = ['JAN', 'FEB', 'MAR', 'APR', 'MAY', 'JUN', 'JUL', 'AUG', 'SEP', 'OCT', 'NOV', 'DEC']
69        return repr(datetime.day) + '-' + monthCodes[datetime.month-1] + '-' + repr(datetime.year)
70
71    def _retrieveCsmlPointSeriesFeature(self, dataset, station_name, feature_id):
72        '''
73        @return: A C{PointSeriesFeature} object representing the single CSML feature if the feature was found;
74        C{None} if a feature collection has been returned, but contains no CSML features (this happens when
75        there are no measured time points in the used time interval).
76        @throws: A C{LookupError} exception with a message if there is no response from the server.
77        '''
78     
79        # Prepare the request for the dataset's GeoServer
80        geoServerRequestUrl = urllib.quote(dataset.geoServerUrl + '?request=getfeature&service=wfs&version=1.1.0&typename=csml:PointSeriesFeature&filter=<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml"><ogc:And><ogc:PropertyIsEqualTo><ogc:PropertyName>gml:description</ogc:PropertyName><ogc:Literal>'+ station_name +'</ogc:Literal></ogc:PropertyIsEqualTo><ogc:PropertyIsEqualTo><ogc:PropertyName>csml:parameter/swe:Phenomenon/gml:name</ogc:PropertyName><ogc:Literal>'+ feature_id +'</ogc:Literal></ogc:PropertyIsEqualTo><ogc:PropertyIsBetween><ogc:PropertyName>csml:value/csml:PointSeriesCoverage/csml:pointSeriesDomain/csml:TimeSeries/csml:timePositionList</ogc:PropertyName><ogc:LowerBoundary><ogc:Literal>'+ self._datetimeToGeoServerDate(self.displayIntervalStart) +'</ogc:Literal></ogc:LowerBoundary><ogc:UpperBoundary><ogc:Literal>'+ self._datetimeToGeoServerDate(self.displayIntervalEnd) +'</ogc:Literal></ogc:UpperBoundary></ogc:PropertyIsBetween></ogc:And></ogc:Filter>', '/._:?&=')
81
82        # Acquire the GeoServer response and parse it into a CSMLFeatureCollection object
83        print geoServerRequestUrl
84        geoServerResponse = wget(geoServerRequestUrl)
85        if not geoServerResponse:
86            raise LookupError('Cannot acquire response from server (wrong URL or server down)')
87        csmlFeatureCollection = csml.parser.CSMLFeatureCollection()
88        csmlFeatureCollection.fromXML( XML(geoServerResponse) )
89
90        # If csmlFeatureCollection contains a single CSML feature, the isolate it. If it contains no feature,
91        # probably no data points are available for that interval.
92        try:
93            csmlFeature = csmlFeatureCollection.featureMembers
94        except AttributeError:
95            raise LookupError(
96                'No data points available for the given time interval. ' + 
97                'Perhaps the interval is shorter than the frequency of data points, ' +
98                'or collection of data for this feature was interrupted at the time.'
99                )
100
101        # Make sure that the feature is a PointSeriesFeature
102        if not isinstance(csmlFeature, csml.parser.PointSeriesFeature):
103            raise LookupError('CSML feature not a PointSeriesFeature')
104        csmlPointSeriesFeature = csmlFeature
105
106        return csmlPointSeriesFeature
107
108    def plot(self):
109        '''
110        Handler for plotting a specific PointSeries feature given by URL parameter "feature_id".
111        Retrieves the data from a web service, which should preferably run on the same machine.
112
113        Request parameters:
114                               dataset_id      Unique identifier of the dataset to use.
115                               feature_id      Unique identifier of the requested CSML PointSeries feature.
116                               station_name    Name of the requested station.
117                               
118        Response:              An image/png of the time series for the CSML PointSeries feature.
119        '''
120        def _plot_feature(feature, dataset):
121            '''
122            Plot a PointSeries feature using matplotlib, into a temporary file
123            '''
124           
125            # Make sure that time is considered to be UTC time. This is because all CSML data is supposed to be in UTC,
126            # but actually pylab is "time zone aware" and will interpret the time as local time zone time.
127            # Not quite sure if this would be a problem, but rather enforce it is UTC.
128            def _enforce_UTC_timezone(datestr):
129                if datestr[-1] == 'Z':
130                    return datestr
131                else:
132                    return datestr + 'Z'
133           
134            # Isolate time points
135            times=feature.value.pointSeriesDomain.timePositionList.CONTENT.split()
136            times = map(_enforce_UTC_timezone, times)
137
138            # Make a list of times as float numbers being days passed since start of epoch (here 01-01-0001)
139            elapsed_times = map(datestr2num, times)
140
141            # Get values of the measured quantity
142            ql = feature.value.rangeSet.quantityList
143            vals = map(float, ql.CONTENT.split())
144
145            # Make sure the times are sorted in ascending order (and rearrange the values accordingly),
146            # this is because we get them potentially unsorted from GeoServer. Also have to handle the sad fact,
147            # that sometimes we are getting duplicate time-value samples from the MIDAS dataset!
148            sortTimesExplicitly = True
149            if sortTimesExplicitly:
150                reorder = {}
151                for i in range(len(elapsed_times)):
152                    reorder[elapsed_times[i]] = i
153                elapsed_times = reorder.keys()
154                elapsed_times.sort()
155                times2 = []
156                vals2 = []
157                for et in elapsed_times:
158                    times2.append(times[reorder[et]])
159                    vals2.append(vals[reorder[et]])
160                times = times2
161                vals = vals2
162
163            print '---times (' + str(len(times)) + ' of them):\n' + str(times)
164            print '---vals (' + str(len(times)) + ' of them):\n' + str(vals)
165
166            # Note the start and end date (they are actually date+time), and time span between them
167
168            start_date = dates.num2date(elapsed_times[0])
169            end_date = dates.num2date(elapsed_times[-1])
170            span = elapsed_times[-1] - elapsed_times[0]
171
172            # Generate tick locator and formatter; these determine the time axis: locator determines tick positions
173            # and formatter format of time
174            print '---start_date:' + str(start_date)
175            print '---end_date:' + str(end_date)
176            print '---span: ' + str(span)
177            (tickLocator, tickFormatter) = dates.date_ticker_factory(span, numticks=8)
178
179            # Determine the units of measurement
180            uom=ql.uom.title()
181            if ql.uom.islower():
182                uom = uom.lower()
183            if ql.uom.isupper():
184                uom = uom.upper()
185
186            # Prepare the plot the figure (actual plotting actions are postponed until save)
187            fig = figure()
188            plot_date(elapsed_times, vals, 'b-', xdate=True, lw=1)
189            plot_date(elapsed_times, vals, 'go', markeredgecolor = 'g', xdate=True, lw=2)
190            ax = gca()
191            ax.xaxis.set_major_locator(tickLocator)
192            ax.xaxis.set_major_formatter(tickFormatter)
193            fig.autofmt_xdate()  # show times without overlaps
194            time_format = '%d-%b-%Y %H:%M:%S UTC'
195            xlabel('Times between %s and %s ' % (start_date.strftime(time_format), end_date.strftime(time_format)))
196            ylabel('Values [%s]' % uom)
197            title('"%s:%s" (%s station %s)' % (dataset.id, feature.id, dataset.name, feature.description.CONTENT))
198            grid(True)
199
200            # Save the figure to a temporary file
201            tempFile = NamedTemporaryFile(suffix='.png')
202            savefig(tempFile.name)
203           
204            return tempFile
205
206        def _set_response(tempFile):
207            '''
208            Set the WSGI response to an image, containing image read from a temporary location.
209            '''
210            img = Image.open(tempFile.name)
211            buf = StringIO()
212            img.save(buf, 'PNG')
213            response.content_type = 'image/png'
214            response.content = buf.getvalue()
215           
216        #----------------------
217
218        # Get parameters from the request object
219        try:
220            dataset_id = str(request.params['dataset_id'])     # convert back from Unicode
221            feature_id = str(request.params['feature_id'])     # convert back from Unicode
222            station_name = str(request.params['station_name']) # convert back from Unicode
223        except KeyError:
224            raise HTTPBadRequest('Parameters "dataset_id", "feature_id" and "station_name" must be supplied.')
225
226        # Try to retrieve the csmlPointSeriesFeature object from the appropriate dataset
227        try:
228            dataset = self.datasets[dataset_id]
229            csmlPointSeriesFeature = self._retrieveCsmlPointSeriesFeature(dataset, station_name, feature_id)
230        except LookupError, e:
231            errorMessage = e.args[0]
232            httpNotFound = HTTPNotFound()
233            httpNotFound.explanation = ''
234            httpNotFound.detail = errorMessage
235            raise httpNotFound
236
237        # Try to plot the feature into a temporary file, and put the contents of that file into the response
238        try:
239            tempFile = _plot_feature(csmlPointSeriesFeature, dataset)    # plot the feature into a temporary file
240            _set_response(tempFile)                                      # set the response as an image containing the plot
241        finally:
242            try:
243                tempFile.close()
244            except NameError:
245                pass  # tempFile undefined -- no need to close
246
247    def list(self):
248        '''
249        Handler for generating a list of CSML features per station, with dynamic links to the plot service.
250        Request parameters:
251                               dataset_id      Unique identifier of the dataset to use.
252                               station_name    Name of the requested station.
253                               
254        Response:              A text/html with the page containing links to the plot service on the same server.
255        '''
256
257        # Get parameters from the request object
258        try:
259            dataset_id = str(request.params['dataset_id'])      # convert back from Unicode
260            station_name = str(request.params['station_name'])  # convert back from Unicode
261        except KeyError:
262            raise HTTPBadRequest('Parameters "dataset_id" and "station_name" must be supplied.')
263
264        try:
265            # Identify the appropriate dataset.
266            try:
267                dataset = self.datasets[dataset_id]
268            except KeyError:
269                raise LookupError('Wrong dataset identifier.')
270           
271            # Prepare the request for the GeoServer -- to return a collection of np:Station, containing a single np:Station,
272            # which contains a list of CSML features.
273            geoServerRequestUrl = urllib.quote(dataset.geoServerUrl + '?request=getFeature&service=wfs&version=1.1.0&typename=np:Station&filter=<ogc:Filter xmlns:ogc="http://www.opengis.net/ogc" xmlns:gml="http://www.opengis.net/gml"><ogc:PropertyIsEqualTo><ogc:PropertyName>stationName</ogc:PropertyName><ogc:Literal>' + station_name + '</ogc:Literal></ogc:PropertyIsEqualTo></ogc:Filter>', '/._:?&=')
274
275            # Acquire the GeoServer response and parse it into a WFSStationCollection object
276            try:
277                geoServerResponse = wget(geoServerRequestUrl)
278            except URLError:
279                raise LookupError('Cannot access underlying WFS (server error, wrong network connection, or typo in the config file).')
280
281            wfsStationCollection = csml2kml.Station.WFSStationCollection()
282            wfsStationCollection.parseString(geoServerResponse)
283            if len(wfsStationCollection.stations) != 1:
284                raise LookupError('Requested station not found.')
285            wfsStation = wfsStationCollection.stations[0]
286        except LookupError, e:
287            errorMessage = e.args[0]
288            httpNotFound = HTTPNotFound()
289            httpNotFound.explanation = ''
290            httpNotFound.detail = errorMessage
291            raise httpNotFound
292
293        # Generate HTML with a list of CSML features contained in the station.
294        # Currently, this is done simply by directly generating HTML (i.e. no template was used).
295        htmlElement = Element('html')
296        SubElement(htmlElement, 'title').text = 'List of CSML features for station ' + station_name
297        bodyElement = SubElement(htmlElement, 'body')
298        SubElement(bodyElement, 'h2').text = 'List of CSML features for station ' + station_name
299        tableElement = SubElement(bodyElement, 'table')
300        tableElement.set('border', '1')
301        headingRowElement = SubElement(tableElement, 'tr')
302        SubElement(headingRowElement, 'th').text = 'Feature name'
303        SubElement(headingRowElement, 'th').text = 'Data collected since'
304        SubElement(headingRowElement, 'th').text = 'Data collected until '
305        for stationFeature in wfsStation.stationFeatures:
306            if self.displayIntervalStart >= stationFeature.collectBeginDate and self.displayIntervalEnd <= stationFeature.collectEndDate:
307                rowElement = SubElement(tableElement, 'tr')
308                featureNameElement = SubElement(rowElement, 'td')
309                anchorElement = SubElement(featureNameElement, 'a')
310                linkToGrapher = self.servedFromUrl + '/plot?dataset_id=' + dataset_id + '&station_name=' + station_name + '&feature_id=' + stationFeature.featureId
311                anchorElement.set('href', linkToGrapher)
312                anchorElement.text = stationFeature.featureId
313                SubElement(rowElement, 'td').text = self._datetimeToGeoServerDate(stationFeature.collectBeginDate)
314                SubElement(rowElement, 'td').text = self._datetimeToGeoServerDate(stationFeature.collectEndDate)
315
316        htmlStringIO = StringIO()
317        ElementTree(htmlElement).write(htmlStringIO)
318
319        response.content_type = 'text/html'
320        response.content = htmlStringIO.getvalue()
Note: See TracBrowser for help on using the repository browser.