source: mauRepo/HPFos/trunk/hpfos/HPFos/osImpl/myimpl.py @ 8580

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/HPFos/trunk/hpfos/HPFos/osImpl/myimpl.py@8580
Revision 8580, 21.9 KB checked in by mnagni, 7 years ago (diff)

Incomplete - # 22558: Formatting of start/end times to conform to OpenSearch? time extension
 http://team.ceda.ac.uk/trac/ceda/ticket/22558

a new string_to_datetime method has been added. It accepts the following format:

  • datatime with timezone 'YYYY-MM-DDTHH:mm:SS[+,-]hhmm'

o example: '1996-12-19T16:39:57-08:00'

  • datatime without timezone 'YYYY-MM-DDTHH:mm:SS'

o example: '1996-12-19T16:39:57' and is equivalent to '1996-12-19T16:39:57+00:00'

  • datatime without timezone 'YYYY-MM-DD'

o example: '1996-12-19' and is equivalent to '1996-12-19T00:00:00+00:00'

Following the OS Time extension specification, a new timeformat has been added to the string_to_datetime method

datatime with UTC timezone 'YYYY-MM-DDTHH:mm:SSZ' example: '1996-12-19T16:39:57Z' and is equivalent to '1996-12-19T16:39:57+00:00'

The previous format is equivalent to datatime without timezone

  • Property svn:mime-type set to text/plain
Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 5 May 2012
30
31@author: Maurizio Nagni
32'''
33from datetime import datetime, date
34from hpfos.libs.postgisutil import create_st_setSRID
35from ceda_markup.opensearch.osquery import OSQuery
36from ceda_markup.atom.atom import createID, createUpdated, \
37    createPublished, createEntry
38from ceda_markup.atom.info import createTitle, HTML_TYPE, createContent
39from ceda_markup.dc.dc import createDate
40from ceda_markup.gml.gml import createBeginPosition, createEndPosition, \
41    createTimePeriod, createValidTime, createEnvelope, createLowerCorner, \
42    createUpperCorner, createPosList, createLinearRing, createExterior, \
43    createPolygon, GML_NAMESPACE
44from ceda_markup.georss.georss import createWhere
45from ceda_markup.atom.link import REL_SEARCH, REL_ALTERNATE, createLink
46from ceda_markup.opensearch import filter_results, COUNT_DEFAULT, \
47    START_INDEX_DEFAULT, START_PAGE_DEFAULT, create_autodiscovery_link
48from ceda_markup.opensearch.template.osresponse import Result, Subresult
49from ceda_markup.opensearch.template.atom import OSAtomResponse
50from ceda_markup.opensearch.template.html import OSHTMLResponse
51from ea_model.iso_19108_2006_temporal_schema.temporal_objects.tm_instant \
52    import TM_Instant
53from ceda_markup.georss import create_where_from_postgis
54from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection \
55    import CEDA_ObservationCollection
56from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation \
57    import CEDA_Observation
58from ea_model.ceda_metadatamodel.ceda_result.ceda_result import CEDA_Result
59from ceda_markup.opensearch.os_param import OSParam
60from hpfos import __version__, __revision__
61from ceda_markup.atom.atom import ATOM_NAMESPACE
62from hpfos.HPFos.osImpl.commons import get_document, get_xml_document, \
63    from_pt_to_string, tm_InstantToDatetime, string_to_datetime
64from ceda_markup.opensearch.os_request import OS_NAMESPACE
65from xml.etree.ElementTree import _ElementInterface
66from ea_model.iso_19115_2006_metadata_corrigendum.\
67    extent_information.ex_geographicboundingbox import EX_GeographicBoundingBox
68
69GUID = 'guid'
70FILE_ID = 'guid'
71COLLECTION = 'collection'
72OBSERVATION = 'observation'
73RESULT = 'result'
74BBOX = 'bbox'
75DUMMY_GUID = 'dummy_guid'
76
77FATCAT_HOST = 'citest1.jc.rl.ac.uk'
78FATCAT_ROOT_PATH = 'fatcatOS'
79PROXY_URL = 'http://wwwcache.rl.ac.uk:8080'
80
81CEDA_TITLE = 'ceda_title'
82HPFOS_VERSION = __version__
83HPFOS_REVISION = __revision__
84
85HPFOS_ID = ''
86if HPFOS_REVISION != 'REVISION':
87    HPFOS_ID = '(v. %s rev. %s)' % (HPFOS_VERSION, HPFOS_REVISION)
88else:
89    HPFOS_ID = '(v. %s rev. %s)' % (HPFOS_VERSION, 'unknown')
90   
91HPFOS_TITLE = 'Discovery feed for Search Services %s' % (HPFOS_ID)
92
93
94
95class MyOSAtomResponse(OSAtomResponse):
96    '''
97    classdocs
98    '''
99
100    def __init__(self):
101        #query_params = {"q": "searchTerms", "pw":"startPage"}
102        super(MyOSAtomResponse, self).__init__()
103       
104        '''
105        Constructor
106        '''
107   
108    def _digest_fatcat_atom(self, context, results):
109        count, start_index, start_page = self._importCountAndPage(context)
110        entries = results.findall('{%s}entry' % (ATOM_NAMESPACE))
111        subresults = []
112        for entry in entries:
113            iid = entry.find('{%s}id' % (ATOM_NAMESPACE)).text.strip()
114            ititle = entry.find('{%s}title' % (ATOM_NAMESPACE)).text.strip()
115            kwargs = {}
116                         
117            element = entry.find('.//{%s}beginPosition' % (GML_NAMESPACE))
118            if element is not None:
119                kwargs['beginPosition'] = element.text.strip()
120                element = None
121           
122            element = entry.find('.//{%s}endPosition' % (GML_NAMESPACE))
123            if element is not None:
124                kwargs['endPosition'] = element.text.strip()
125                element = None           
126           
127            geometry = entry.find('.//{%s}posList' % (GML_NAMESPACE))
128            if geometry is not None:
129                kwargs['geometry'] = 'POLYGON((%s))' % geometry.text.strip()
130            else:
131                geometry = entry.find('.//{%s}Envelope' % (GML_NAMESPACE))
132                if geometry is not None:
133                    lc = geometry.find('./{%s}lowerCorner' % (GML_NAMESPACE)).text.strip()
134                    uc = geometry.find('./{%s}upperCorner' % (GML_NAMESPACE)).text.strip()
135                    kwargs['geometry'] = _create_box2d(lc, uc)
136
137
138            kwargs['enclosure'] = entry.findall("./{%s}link" % (ATOM_NAMESPACE))
139            #This can be activated on python >= 2.7
140            #kwargs['enclosure'] = entry.findall("./{%s}link[@rel='enclosure']" % (ATOM_NAMESPACE))                         
141            kwargs['description'] = 'no description'
142            subresults.append(Subresult(iid, ititle,
143                                        datetime.now().isoformat(), **kwargs))
144       
145        tot_results = int(results.find('{%s}totalResults'
146                                       % (OS_NAMESPACE)).text.replace('\n','').strip())
147        return Result(count, start_index, start_page, tot_results,
148                      subresult = subresults, title=HPFOS_TITLE)       
149       
150    def _check_text_filter(self, text, title, description):
151        words = text.split()
152        for word in words:
153            if word in title \
154                or word in description:
155                    return True
156        return False
157
158    def _check_start_filter(self, startTime, phenomenonTime):
159        pt = phenomenonTime
160        if isinstance(phenomenonTime, list):
161            if len(phenomenonTime) == 0:
162                return True
163            pt = phenomenonTime[0]
164         
165        if isinstance(pt, TM_Instant):
166            pt = tm_InstantToDatetime(pt)   
167        elif hasattr(pt, 'begin'):
168            pt = tm_InstantToDatetime(pt.begin)
169           
170        if isinstance(pt, datetime) and isinstance(startTime, date):
171            return pt.date() > startTime
172        if isinstance(pt, date) and isinstance(startTime, datetime):
173            return pt > startTime.date()   
174        return pt > startTime     
175       
176    def _check_end_filter(self, endTime, phenomenonTime):
177        pt = phenomenonTime
178        if isinstance(phenomenonTime, list):
179            if len(phenomenonTime) == 0:
180                return True
181            pt = phenomenonTime[0]
182
183        if isinstance(pt, TM_Instant):
184            pt = tm_InstantToDatetime(pt)   
185        elif hasattr(pt, 'end'):
186            pt = tm_InstantToDatetime(pt.end)
187       
188        if isinstance(pt, datetime) and isinstance(endTime, date):
189            return endTime < pt.date()
190        if isinstance(pt, date) and isinstance(endTime, datetime):
191            return endTime.date() < pt             
192        return endTime < pt
193
194    def apply_query_params(self, context, results):
195        subresults = []
196        if results is None:
197            return subresults
198        for result in results:         
199            result_guid = context['moles3EPB'].retrieveGUIDFromInstance(result)
200            if result_guid is None:
201                continue
202            ititle = self._extractTitle(result)
203
204            kwargs = {}               
205            #------------------------
206            phenomenonTime = result.phenomenonTime
207            if isinstance(result, list) and len(result.phenomenonTime) > 0:
208                phenomenonTime = result.phenomenonTime[0]                   
209            kwargs['beginPosition'], kwargs['endPosition'] = \
210                from_pt_to_string(phenomenonTime)
211            #------------------------                                                               
212            kwargs['geometry'] = self._new_extract_geographic_extent(result)
213            kwargs['description'] = result.description           
214
215            item = Subresult(result_guid.id, ititle, datetime.now().isoformat(),
216                             **kwargs)               
217            subresults.append(item)
218        return subresults
219
220    def digest_search_results(self, results, context):
221        if type(results) == CEDA_ObservationCollection:
222            results = results.member
223        elif type(results) == CEDA_Observation:
224            results = results.result
225        elif type(results) == CEDA_Result:
226            results = results.result
227        elif isinstance(results, _ElementInterface):
228            return self._digest_fatcat_atom(context, results)                   
229
230        count, start_index, start_page = self._importCountAndPage(context)
231        tot_results = 0
232        try:
233            tot_results = len(results)
234        except:
235            pass
236                   
237        # This check is a trick to speed up the response when no query parameter
238        # is required (all parameters are None). In this case the results are paged first
239        # then the subresults are generated.
240        # The opposite, filter first then the subresults are generated,
241        # if any query parameter is not None
242        if context['start'] is None \
243            or context['stop'] is None \
244            or context[BBOX] is None \
245            or context['q'] is None:
246            filtered = filter_results(results, count, start_index, start_page)
247            subresults = self.apply_query_params(context, filtered)
248        else:
249            filtered = self.apply_query_params(context, results)
250            subresults = filter_results(filtered, count, start_index, start_page)
251            tot_results = len(subresults)                                             
252               
253        return Result(count, start_index, start_page, tot_results, \
254                      subresult = subresults, title=HPFOS_TITLE)
255
256    def _new_extract_geographic_extent(self, result):       
257        if not hasattr(result, 'geographicExtent') \
258            or result.geographicExtent is None \
259            or len(result.geographicExtent) == 0:
260            return None
261       
262        ge = result.geographicExtent[0]
263        if not isinstance(ge, EX_GeographicBoundingBox):
264            return
265
266        return _create_box2d('%s %s' % (ge.southBoundLatitude, ge.westBoundLongitude),
267                             '%s %s' % (ge.northBoundLatitude, ge.eastBoundLongitude))
268                                   
269
270
271
272    def _extract_geographic_extent(self, result):       
273        if not hasattr(result, 'geographicExtent') \
274            or result.geographicExtent is None \
275            or len(result.geographicExtent) == 0:
276            return None, None
277       
278        ge = result.geographicExtent[0]
279        if not isinstance(ge, EX_GeographicBoundingBox):
280            return
281       
282        #returns lowerCorner, upperCorner
283        return [ge.southBoundLatitude, ge.westBoundLongitude], \
284                                    [ge.northBoundLatitude, ge.eastBoundLongitude]
285       
286       
287    def generateEntryLinks(self, entry, atomroot, path, linkid = None):
288        entry.append(create_autodiscovery_link(atomroot, path, self.extension, \
289                                               linkid, None, rel = REL_ALTERNATE))
290        entry.append(create_autodiscovery_link(atomroot, path, self.extension, \
291                                               linkid, None, rel = REL_SEARCH))               
292
293    def generate_entries(self, atomroot, subresults, path):
294        if isinstance(subresults, list) \
295                and len(subresults) > 0 \
296                and isinstance(subresults[0], _ElementInterface):
297            for entry in subresults:
298                atomroot.append(entry)
299            return
300       
301        entries = []
302       
303        for subresult in subresults:
304            #Here could loop over results
305            atomID = createID(path + subresult.id + '/' + self.extension, root = atomroot)
306            ititle = createTitle(root = atomroot,
307                                 body = subresult.title,
308                                 itype = HTML_TYPE)
309            atomContent = createContent(root = atomroot,
310                                        body = subresult.description,
311                                        itype = HTML_TYPE)
312            atomUpdated = createUpdated(subresult.updated, root = atomroot)
313            atomPublished = createPublished('TO_BE_DONE_2011-01-21T11:05:29.511Z',
314                                            root = atomroot)           
315            entry = createEntry(atomID, ititle, atomUpdated,
316                                published=atomPublished,
317                                content=atomContent, root = atomroot)
318            #xmlentry = entry.buildElement()
319           
320           
321            beginPosition = None
322            endPosition = None
323            if hasattr(subresult, 'beginPosition') \
324                    and subresult.beginPosition is not None:
325                beginPosition = subresult.beginPosition
326            if hasattr(subresult, 'endPosition') and subresult.endPosition is not None:               
327                endPosition = subresult.endPosition
328            self._append_valid_time(subresult, entry, atomroot, beginPosition, endPosition)           
329           
330            idate = createDate(root = atomroot,
331                               body = 'TO_BE_DONE_2002-10-18T08:07:37.387Z/2012-03-29T07:12:20.735Z')       
332            entry.append(idate)
333           
334            if hasattr(subresult, 'geometry') and subresult.geometry is not None:
335                where = create_where_from_postgis(subresult.geometry, atomroot)
336                entry.append(where)
337           
338            self.generateEntryLinks(entry, atomroot, path, subresult.id)
339            if hasattr(subresult, 'enclosure') and subresult.enclosure is not None:
340                for enclosure in subresult.enclosure:
341                    if enclosure.get('rel', None) == 'enclosure':
342                        entry.append(createLink(enclosure.get('href'),
343                                                rel = 'enclosure',
344                                                root = atomroot,
345                                                itype = enclosure.get('type'),
346                                                length = enclosure.get('length')))               
347                           
348            entries.append(entry)
349
350        for entry in entries:
351            atomroot.append(entry)
352
353    def _append_where(self, geometry, entry, atomroot):
354        if geometry.startswith('BOX2D('):
355            lc, uc = geometry[6:-1].split(',')
356            lowerCorner = createLowerCorner(atomroot, values = lc)
357            upperCorner = createUpperCorner(atomroot, values = uc)
358            where_body = createEnvelope(lowerCorner, upperCorner, atomroot)
359           
360        elif geometry.startswith('POLYGON(('):
361                posList = createPosList(root = atomroot,
362                                        values = [float(val) for val
363                                                  in geometry[9:-2].
364                                                        replace(',', ' ').split()],
365                                        srsDimension = '2')
366                linearRing = createLinearRing(root = atomroot, body = posList)
367                exterior = createExterior(root = atomroot, body = linearRing)
368                where_body = createPolygon(root = atomroot, body = exterior)       
369
370        where = createWhere(root = atomroot, body = where_body)
371        entry.append(where)
372
373    def _append_valid_time(self, subresult, entry, atomroot,
374                           beginPosition, endPosition):
375        #xmlentry = entry.buildElement()
376        if beginPosition is not None:
377            beginPosition = createBeginPosition(root = atomroot,
378                                                body = subresult.beginPosition)
379        if endPosition is not None:               
380            endPosition = createEndPosition(root = atomroot,
381                                            body = subresult.endPosition)               
382        timePeriod = createTimePeriod(root = atomroot,
383                                      begin = beginPosition, end = endPosition)       
384        validTime = createValidTime(root = atomroot, body = timePeriod)
385        if beginPosition is not None or endPosition is not None:
386            entry.append(validTime)
387
388    def _importCountAndPage(self, context):       
389        count = COUNT_DEFAULT
390        start_index = START_INDEX_DEFAULT
391        start_page = START_PAGE_DEFAULT
392       
393        try:
394            count = int(context['count'])
395        except:
396            pass       
397       
398        try:
399            start_index = int(context['startIndex'])
400        except:
401            pass
402       
403        try:
404            start_page = int(context['startPage'])
405        except:
406            pass
407       
408        return count, start_index, start_page
409
410    def _extractTitle(self, cedaObj):
411        if hasattr(cedaObj, 'identifier'):
412            for ident in cedaObj.identifier:
413                if ident.authority.title == CEDA_TITLE:
414                    return ident.code 
415
416class MyOSHTMLResponse(OSHTMLResponse):
417    '''
418    classdocs
419    '''
420
421    def __init__(self):
422        '''
423        Constructor
424        '''
425        super(MyOSHTMLResponse, self).__init__()
426       
427    def generateResponse(self, result, queries, ospath, **kwargs):
428        return result + " HTML!"
429       
430class MyOSQuery(OSQuery):
431    '''
432    classdocs
433    '''
434
435    def __init__(self):
436        '''
437            Constructor
438        '''       
439        param_1 = OSParam("count", "count")
440        param_2 = OSParam("startPage", "startPage")
441        param_3 = OSParam("startIndex", "startIndex")               
442        param_4 = OSParam("q", "searchTerms")           
443        param_5 = OSParam("uid", "uid",
444                          namespace = "http://a9.com/-/opensearch/extensions/geo/1.0/")       
445        param_6 = OSParam(BBOX, 'box',
446                          namespace = "http://a9.com/-/opensearch/extensions/geo/1.0/")       
447        param_7 = OSParam("start", "start",
448                          namespace = "http://a9.com/-/opensearch/extensions/time/1.0/")       
449        param_8 = OSParam("stop", "end",
450                          namespace = "http://a9.com/-/opensearch/extensions/time/1.0/")       
451        params = [param_1, param_2, param_3, param_4, param_5, param_6, param_7, param_8]
452        super(MyOSQuery, self).__init__(params)
453       
454    def do_search(self, context):
455        ibbox = None
456        if context.has_key(BBOX) and context[BBOX] is not None:
457            coords = context[BBOX].split(',')
458            try:
459                if len(coords) == 4:
460                    ibbox = create_st_setSRID(int(coords[0]),int(coords[1]),
461                                              int(coords[2]),int(coords[3]))
462            except:
463                pass
464       
465        if context.has_key(GUID) and 'FID' in context[GUID]:
466            return self._extractFatcatEntities(context[GUID])
467       
468        start = None
469        stop = None
470        if context['start'] is not None:
471            start = string_to_datetime(context['start'])
472        if context['stop'] is not None:
473            stop = string_to_datetime(context['stop'])           
474       
475        if not context.has_key(GUID) or context[GUID] is None:
476            return context['moles3EPB'].\
477                getObservationCollections(bbox = ibbox,
478                                          keywords = context['q'],
479                                          start = start,
480                                          stop = stop)       
481             
482        obj = context['moles3EPB'].getInstanceFromGUID(context[GUID])   
483        if obj is None:
484            return None
485        if type(obj) == CEDA_ObservationCollection:
486            return context['moles3EPB'].searchSelectiveLoadByInstance(obj, 'member')
487        elif type(obj) == CEDA_Observation:
488            for source in obj.result.source:
489                if source.function == 'search':
490                    return self._extractFatcatEntities(source.description)
491
492       
493    def _extractFatcatEntities(self, fc_resource_id):
494        path = '/%s/search/%s/atom/' % (FATCAT_ROOT_PATH, str(fc_resource_id))
495        return find_fatcat_atom_entity(host = FATCAT_HOST, path = path)
496   
497def find_fatcat_atom_entity(host = 'localhost', path = '', port = 80):   
498    source = get_document(host, path, port, proxy = PROXY_URL)
499    return get_xml_document(source)
500
501def _create_box2d(lc, uc):
502    '''
503        Creates a postgis-like BOX2D string.
504        **Parameters**
505            * lc: the box lower corner as a two floats-as-string space separated pair
506            * uc: the box upper corner as a two floats-as-string space separated pair
507        **Returns**
508            A string formatted as 'BOX2D(lc1 lc2, uc1 uc2)'
509    '''   
510    return 'BOX2D(%s, %s)' % (lc, uc)
Note: See TracBrowser for help on using the repository browser.