source: mauRepo/HPFos/trunk/hpfos/HPFos/osImpl/myimpl.py @ 8716

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/HPFos/trunk/hpfos/HPFos/osImpl/myimpl.py@8716
Revision 8716, 19.2 KB checked in by mnagni, 6 years ago (diff)

Incomplete - # 22576: Filtering on Files in a particular Result returns all Files rather than a subset.
 http://team.ceda.ac.uk/trac/ceda/ticket/22576

  • Property svn:mime-type set to text/plain
Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 5 May 2012
30
31@author: Maurizio Nagni
32'''
33from datetime import datetime
34from hpfos.libs.postgisutil import create_st_setSRID
35from ceda_markup.opensearch.osquery import OSQuery
36from ceda_markup.atom.atom import createID, createUpdated, \
37    createPublished, createEntry
38from ceda_markup.atom.info import createTitle, HTML_TYPE, createContent
39from ceda_markup.dc.dc import createDate
40from ceda_markup.gml.gml import createBeginPosition, createEndPosition, \
41    createTimePeriod, createValidTime, GML_NAMESPACE
42from ceda_markup.atom.link import REL_SEARCH, REL_ALTERNATE, createLink
43from ceda_markup.opensearch import filter_results, COUNT_DEFAULT, \
44    START_INDEX_DEFAULT, START_PAGE_DEFAULT, create_autodiscovery_link
45from ceda_markup.opensearch.template.osresponse import Result, Subresult
46from ceda_markup.opensearch.template.atom import OSAtomResponse
47from ceda_markup.opensearch.template.html import OSHTMLResponse
48from sqlalchemy.orm.collections import InstrumentedList
49from ceda_markup.georss import create_where_from_postgis
50from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection \
51    import CEDA_ObservationCollection
52from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation \
53    import CEDA_Observation
54from ceda_markup.opensearch.os_param import OSParam
55from hpfos import __version__, __revision__
56from ceda_markup.atom.atom import ATOM_NAMESPACE
57from hpfos.HPFos.osImpl.commons import get_document, get_xml_document, \
58    from_pt_to_string, string_to_datetime
59from ceda_markup.opensearch.os_request import OS_NAMESPACE
60from xml.etree.ElementTree import _ElementInterface
61from ea_model.iso_19115_2006_metadata_corrigendum.\
62    extent_information.ex_geographicboundingbox import EX_GeographicBoundingBox
63
64GUID = 'guid'
65FILE_ID = 'guid'
66COLLECTION = 'collection'
67OBSERVATION = 'observation'
68RESULT = 'result'
69BBOX = 'bbox'
70DUMMY_GUID = 'dummy_guid'
71
72FATCAT_HOST = 'citest1.jc.rl.ac.uk'
73FATCAT_ROOT_PATH = 'fatcatOS'
74PROXY_URL = 'http://wwwcache.rl.ac.uk:8080'
75
76CEDA_TITLE = 'ceda_title'
77HPFOS_VERSION = __version__
78HPFOS_REVISION = __revision__
79
80HPFOS_ID = ''
81if HPFOS_REVISION != 'REVISION':
82    HPFOS_ID = '(v. %s rev. %s)' % (HPFOS_VERSION, HPFOS_REVISION)
83else:
84    HPFOS_ID = '(v. %s rev. %s)' % (HPFOS_VERSION, 'unknown')
85   
86HPFOS_TITLE = 'Discovery feed for Search Services %s' % (HPFOS_ID)
87
88def append_valid_time(subresult, entry, atomroot,
89                       begin_position, end_position):
90    #xmlentry = entry.buildElement()
91    if begin_position is not None:
92        begin_position = createBeginPosition(root = atomroot,
93                                            body = subresult.beginPosition)
94    if end_position is not None:               
95        end_position = createEndPosition(root = atomroot,
96                                        body = subresult.endPosition)               
97    time_period = createTimePeriod(root = atomroot,
98                                  begin = begin_position, end = end_position)       
99    valid_time = createValidTime(root = atomroot, body = time_period)
100    if begin_position is not None or end_position is not None:
101        entry.append(valid_time)
102
103def extract_title(ceda_obj):
104    if hasattr(ceda_obj, 'identifier'):
105        for ident in ceda_obj.identifier:
106            if ident.authority.title == CEDA_TITLE:
107                return ident.code 
108
109def generate_url_id(url, iid = None):
110    if iid is None:
111        return "%s/search" % (url)
112   
113    return "%s/search/%s" % (url, iid)
114
115def new_extract_geographic_extent(result):       
116    if not hasattr(result, 'geographicExtent') \
117        or result.geographicExtent is None \
118        or len(result.geographicExtent) == 0:
119        return None
120   
121    geo_ex = result.geographicExtent[0]
122    if not isinstance(geo_ex, EX_GeographicBoundingBox):
123        return
124
125    return _create_box2d('%s %s' % (geo_ex.southBoundLatitude,
126                                    geo_ex.westBoundLongitude),
127                         '%s %s' % (geo_ex.northBoundLatitude,
128                                    geo_ex.eastBoundLongitude))
129
130def digest_fatcat_atom(context, results):
131    entries = results.findall('{%s}entry' % (ATOM_NAMESPACE))
132    subresults = []
133    for entry in entries:
134        iid = entry.find('{%s}id' % (ATOM_NAMESPACE)).text.strip()
135        ititle = entry.find('{%s}title' % (ATOM_NAMESPACE)).text.strip()
136        kwargs = {}
137                     
138        element = entry.find('.//{%s}beginPosition' % (GML_NAMESPACE))
139        if element is not None:
140            kwargs['beginPosition'] = element.text.strip()
141            element = None
142       
143        element = entry.find('.//{%s}endPosition' % (GML_NAMESPACE))
144        if element is not None:
145            kwargs['endPosition'] = element.text.strip()
146            element = None           
147       
148        geometry = entry.find('.//{%s}posList' % (GML_NAMESPACE))
149        if geometry is not None:
150            kwargs['geometry'] = 'POLYGON((%s))' % geometry.text.strip()
151        else:
152            geometry = entry.find('.//{%s}Envelope' % (GML_NAMESPACE))
153            if geometry is not None:
154                geo_lc = geometry.find('./{%s}lowerCorner'
155                                       % (GML_NAMESPACE)).text.strip()
156                geo_uc = geometry.find('./{%s}upperCorner'
157                                       % (GML_NAMESPACE)).text.strip()
158                kwargs['geometry'] = _create_box2d(geo_lc, geo_uc)
159
160
161        kwargs['enclosure'] = entry.findall("./{%s}link" % (ATOM_NAMESPACE))
162        #This can be activated on python >= 2.7
163        #kwargs['enclosure'] = entry.findall("./{%s}link[@rel='enclosure']"
164        #                            % (ATOM_NAMESPACE))                         
165        kwargs['description'] = 'no description'
166        subresults.append(Subresult(iid, ititle,
167                                    datetime.now().isoformat(), **kwargs))
168   
169    return subresults
170
171def apply_query_params(context, results):
172    # A cleaner implementation would require calls to
173    # db's merge_period_instant_views() but actually it does not collect
174    # infos about CEDA_Results
175   
176    subresults = []
177    if results is None:
178        return subresults
179    for result in results:         
180        result_guid = context['moles3EPB'].retrieveGUIDFromInstance(result)
181        if result_guid is None:
182            continue
183        ititle = extract_title(result)
184
185        kwargs = {}
186        #------------------------
187        phenomenon_time = result.phenomenonTime
188        if isinstance(result, list) and len(result.phenomenonTime) > 0:
189            phenomenon_time = result.phenomenonTime[0]                   
190        kwargs['beginPosition'], kwargs['endPosition'] = \
191            from_pt_to_string(phenomenon_time)
192        #------------------------
193       
194        #------------------------
195        kwargs['geometry'] = new_extract_geographic_extent(result)
196        #------------------------               
197
198        #------------------------                   
199        kwargs['description'] = result.description
200        #------------------------
201
202        item = Subresult(result_guid.id, ititle, datetime.now().isoformat(),
203                         **kwargs)               
204        subresults.append(item)
205    return subresults
206
207def import_count_and_page(context):
208    ret = []       
209   
210    try:
211        ret.append(int(context.get('count', COUNT_DEFAULT)))
212    except (ValueError, TypeError):
213        ret.append(COUNT_DEFAULT)     
214   
215    try:
216        ret.append(int(context.get('startIndex', START_INDEX_DEFAULT)))
217    except (ValueError, TypeError):
218        ret.append(START_INDEX_DEFAULT)
219   
220    try:
221        ret.append(int(context.get('startPage', START_PAGE_DEFAULT)))
222    except (ValueError, TypeError):
223        ret.append(START_PAGE_DEFAULT)
224   
225    return tuple(ret)
226
227class MyOSAtomResponse(OSAtomResponse):
228    '''
229    classdocs
230    '''
231
232    def __init__(self):
233        super(MyOSAtomResponse, self).__init__()       
234
235    def digest_search_results(self, results, context):
236        instances = None
237        tot_results = 0
238        if type(results) == CEDA_ObservationCollection:
239            instances = results.member
240        elif type(results) == tuple:
241            ids = [iid[0] for iid in results[1]]
242            instances = context['moles3EPB'].get_instance_by_ids(ids)
243            try:
244                tot_results = len(instances)
245            except TypeError:
246                pass
247        elif isinstance(results, _ElementInterface):
248            instances = digest_fatcat_atom(context, results)
249            tot_results = int(results.find('{%s}totalResults'
250                               % (OS_NAMESPACE)).text.replace('\n','').strip())                   
251
252        count, start_index, start_page = import_count_and_page(context)                       
253        subresults = filter_results(instances, count, start_index, start_page)
254       
255        if (type(instances) == list \
256                and type(instances[0]) == CEDA_ObservationCollection) \
257            or (type(instances) == InstrumentedList \
258                and type(instances[0]) == CEDA_Observation):
259            subresults = apply_query_params(context, subresults)
260                                                         
261        return Result(count, start_index, start_page, tot_results, \
262                      subresult = subresults, title=HPFOS_TITLE)
263       
264    def generateEntryLinks(self, entry, atomroot, path, params_model, context):
265        entry.append(create_autodiscovery_link(atomroot, path, \
266                                               params_model, context, \
267                                               self.extension, \
268                                               rel = REL_ALTERNATE))               
269       
270        entry.append(create_autodiscovery_link(atomroot, path, \
271                                           params_model, context, \
272                                           extension = self.extension, \
273                                           rel = REL_SEARCH))               
274
275    def generate_url(self, os_host_urlURL, context):
276        return generate_url_id(os_host_urlURL, context.get('guid')) 
277
278    def generate_entries(self, atomroot, subresults, path, params_model, context):
279        if subresults is None:
280            return
281       
282        if isinstance(subresults, list) \
283                and len(subresults) > 0 \
284                and isinstance(subresults[0], _ElementInterface):
285            for entry in subresults:
286                atomroot.append(entry)
287            return
288       
289        entries = []
290       
291        for subresult in subresults:
292            #Here could loop over results
293            entry_path = generate_url_id(path, subresult.id)
294            atom_id = createID(entry_path + '/' + self.extension, root = atomroot)
295            ititle = createTitle(root = atomroot,
296                                 body = subresult.title,
297                                 itype = HTML_TYPE)
298            atom_content = createContent(root = atomroot,
299                                        body = subresult.description,
300                                        itype = HTML_TYPE)
301            atom_updated = createUpdated(subresult.updated, root = atomroot)
302            atom_published = createPublished('TO_BE_DONE_2011-01-21T11:05:29.511Z',
303                                            root = atomroot)           
304            entry = createEntry(atom_id, ititle, atom_updated,
305                                published=atom_published,
306                                content=atom_content, root = atomroot)
307           
308            begin_position = None
309            end_position = None
310            if hasattr(subresult, 'beginPosition') \
311                    and subresult.beginPosition is not None:
312                begin_position = subresult.beginPosition
313            if hasattr(subresult, 'endPosition') \
314                    and subresult.endPosition is not None:               
315                end_position = subresult.endPosition
316            append_valid_time(subresult, entry, atomroot,
317                              begin_position, end_position)           
318           
319            idate = createDate(root = atomroot,
320                body = 'TO_BE_DONE_2002-10-18T08:07:37.387Z/2012-03-29T07:12:20.735Z')       
321            entry.append(idate)
322           
323            if hasattr(subresult, 'geometry') \
324                    and subresult.geometry is not None:
325                where = create_where_from_postgis(subresult.geometry, atomroot)
326                entry.append(where)
327           
328           
329           
330            self.generateEntryLinks(entry, atomroot, entry_path, \
331                                    params_model, context)
332            if hasattr(subresult, 'enclosure') \
333                    and subresult.enclosure is not None:
334                for enclosure in subresult.enclosure:
335                    if enclosure.get('rel', None) == 'enclosure':
336                        entry.append(createLink(enclosure.get('href'),
337                                            rel = 'enclosure',
338                                            root = atomroot,
339                                            itype = enclosure.get('type'),
340                                            length = enclosure.get('length')))               
341                           
342            entries.append(entry)
343
344        for entry in entries:
345            atomroot.append(entry)
346
347class MyOSHTMLResponse(OSHTMLResponse):
348    '''
349    classdocs
350    '''
351
352    def __init__(self):
353        '''
354        Constructor
355        '''
356        super(MyOSHTMLResponse, self).__init__()
357       
358    def generateResponse(self, result, queries, ospath, **kwargs):
359        return result + " HTML!"
360       
361class MyOSQuery(OSQuery):
362    '''
363    classdocs
364    '''
365
366    def __init__(self):
367        '''
368            Constructor
369        '''
370        params = []
371        params.append(OSParam("count", "count",
372                              namespace = OS_NAMESPACE))
373        params.append(OSParam("startPage", "startPage",
374                              namespace = OS_NAMESPACE))
375        params.append(OSParam("startIndex", "startIndex",
376                              namespace = OS_NAMESPACE))               
377        params.append(OSParam("q", "searchTerms",
378                              namespace = OS_NAMESPACE))         
379        params.append(OSParam("uid", "uid",
380                namespace = "http://a9.com/-/opensearch/extensions/geo/1.0/"))       
381        params.append(OSParam(BBOX, 'box',
382                namespace = "http://a9.com/-/opensearch/extensions/geo/1.0/"))       
383        params.append(OSParam("start", "start",
384                namespace = "http://a9.com/-/opensearch/extensions/time/1.0/"))       
385        params.append(OSParam("stop", "end",
386                namespace = "http://a9.com/-/opensearch/extensions/time/1.0/"))       
387        super(MyOSQuery, self).__init__(params)
388       
389    def do_search(self, context):
390        ibbox = None
391        if context.has_key(BBOX) and context[BBOX] is not None:
392            coords = context[BBOX].split(',')
393            try:
394                if len(coords) == 4:
395                    ibbox = create_st_setSRID(int(coords[0]),int(coords[1]),
396                                              int(coords[2]),int(coords[3]))
397            except:
398                pass
399       
400        if context.has_key(GUID) and 'FID' in context[GUID]:
401            return extract_fatcat_entities(context[GUID])
402       
403        start = None
404        stop = None
405        if context['start'] is not None:
406            start = string_to_datetime(context['start'])
407        if context['stop'] is not None:
408            stop = string_to_datetime(context['stop'])           
409       
410        if not context.has_key(GUID) or context[GUID] is None:
411            res = context['moles3EPB'].\
412                getObservationCollections_(bbox = ibbox,
413                                          keywords = context['q'],
414                                          start = start,
415                                          stop = stop)
416            return (CEDA_ObservationCollection, res)
417            #if res_ids is not None:
418            #    ids = [(id[0] for id in res_ids]     
419             
420        obj = context['moles3EPB'].getInstanceFromGUID(context[GUID])   
421        if obj is None:
422            return None
423        if type(obj) == CEDA_ObservationCollection:
424            return context['moles3EPB'].searchSelectiveLoadByInstance(obj, 'member')
425        elif type(obj) == CEDA_Observation:
426            '''
427                Sometimes returns empty because the Result associated with the
428                CedaObservation has not a pointer to the file catalog but a URL link.
429                The link may point either to a file or to a directory.
430            '''
431            for source in obj.result.source:
432                if source.function == 'search':
433                    return extract_fatcat_entities(source.description)
434            return None
435
436       
437def extract_fatcat_entities(fc_resource_id):
438    '''
439        Uses the Fatcat resource_id to build and submit an Opensearch query to Fatcat
440    '''
441    path = '/%s/search/%s/atom/' % (FATCAT_ROOT_PATH, str(fc_resource_id))
442    return find_fatcat_atom_entity(host = FATCAT_HOST, path = path)
443   
444def find_fatcat_atom_entity(host = 'localhost', path = '', port = 80):   
445    source = get_document(host, path, port, proxy = PROXY_URL)
446    return get_xml_document(source)
447
448def _create_box2d(lc, uc):
449    '''
450        Creates a postgis-like BOX2D string.
451        **Parameters**
452            * lc: the box lower corner as a two floats-as-string space separated pair
453            * uc: the box upper corner as a two floats-as-string space separated pair
454        **Returns**
455            A string formatted as 'BOX2D(lc1 lc2, uc1 uc2)'
456    '''   
457    return 'BOX2D(%s, %s)' % (lc, uc)
Note: See TracBrowser for help on using the repository browser.