Changeset 8505
- Timestamp:
- 10/08/12 14:00:34 (9 years ago)
- Location:
- mauRepo/HPFos/trunk
- Files:
-
- 8 edited
Legend:
- Unmodified
- Added
- Removed
-
mauRepo/HPFos/trunk/.pydevproject
r8500 r8505 11 11 <value>hpfos.HPFos.settings</value> 12 12 </pydev_variables_property> 13 13 14 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH"> 14 15 <path>/HPFos</path> -
mauRepo/HPFos/trunk/hpfos/HPFos/moles3epb.py
r8500 r8505 34 34 from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection 35 35 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 36 from ea_model.iso_19115_2006_metadata_corrigendum.reference_system_information.md_identifier import MD_Identifier37 from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_citation import CI_Citation38 36 from ea_model.moles3_4.observationcollection.mo_observationcollection import MO_ObservationCollection 39 37 from ea_model.moles3_4.observation.mo_observation import MO_Observation 40 from ea_model.ceda_metadatamodel.ceda_project.ceda_project import CEDA_Project41 38 from sqlalchemy import Table, Column, ForeignKey, Integer, String 42 39 from sqlalchemy.orm import mapper … … 69 66 self._dbManager.metadata.create_all() 70 67 71 def _initSearchIndexes(self):72 #To Be Done - CHECK IF THE COLUMN ALREADY EXISTS!73 # We don't want sqlalchemy to know about this column so we add it externally.74 try:75 self._dbManager.engine.execute("alter table md_identifier add column code_search_vector tsvector")76 77 # This indexes the tsvector column78 79 self._dbManager.engine.execute("create index md_identifier_code_search_index on md_identifier using gin(code_search_vector)")80 81 # This sets up the trigger that keeps the tsvector column up to date.82 self._dbManager.engine.execute("create trigger md_identifier_code_search_update before update or insert on md_identifier \83 for each row execute procedure tsvector_update_trigger('code_search_vector', 'pg_catalog.english', code)")84 except Exception as e:85 pass86 87 68 def _getSession(self): 88 69 if self._dbManager is not None: … … 102 83 103 84 def close(self): 104 return self._session.close() 105 106 def getObjectsByGE(self, class_type): 107 """ 108 Checks if a CEDA_Collection contains a given CEDA_Observation. 109 @param obs_coll_id: the CEDA_ObservationColleciton id 110 @param obs_id: the CEDA_Observation id 111 @return: True if the collection contains the given observation, False otherwise 112 """ 113 if isinstance(class_type, type(CEDA_ObservationCollection)) or isinstance(class_type, type(CEDA_Observation)): 114 return self._session.query(class_type).all() 115 return None 116 117 def getObservationCollections(self, bbox = None): 85 return self._session.close() 86 87 def getObservationCollections(self, bbox = None, keywords = '*'): 118 88 """ 119 89 Returns the stored CEDA_ObservationCollection eventually filtering them against a postgis goemetry … … 122 92 """ 123 93 124 collections = self._session.query(CEDA_ObservationCollection) 94 collections = self._session.query(CEDA_ObservationCollection) 95 res = [] 96 #collections = self._extract_observation_collection_by_title_keywords(keywords) 97 if collections is None: 98 return res 125 99 if bbox == None: 126 100 return collections.all() 127 101 128 res = [] 102 129 103 for collection in collections: 130 104 if len(collection.geographicExtent) > 0: … … 161 135 return self.search(CEDA_ObservationCollection, ceda_guid.ceda_observationcollection) 162 136 return None 163 164 165 def getObservationFromObservationCollection(self, obs_coll_id): 166 """ 167 Checks if a CEDA_Collection contains a given CEDA_Observation. 168 @param obs_coll_id: the CEDA_ObservationColleciton id 169 @return: the associated CEDA_Observation 170 """ 171 return self._session.query(CEDA_ObservationCollection, CEDA_Observation).filter(CEDA_ObservationCollection.id==obs_coll_id).all() 172 173 def extractObservationByTitleKeywords(self, keywords): 174 """ 175 Loooks for CEDA_Observation containing a specific title (observation.identifier.code) 176 @param keywords: a space separated terms string 177 @return: a tuple containing a CEDA_Observation satisfying the queryllection.idenfitier element having the title 137 138 def _extract_observation_collection_by_title_keywords(self, keywords): 139 """ 140 Loooks for CEDA_ObservationCollection containing a specific title (observationCollection.identifier.code) 141 @param keywords: a space separated terms string 178 142 """ 179 143 # search_vector is a ts_vector column. To search for terms, you use the … … 181 145 # used with @@. So this adds a where clause like "WHERE search_vector 182 146 # @@ plaint_tsquery(<search string>)" 183 q = self._session.query(CEDA_Observation ). \184 join(MO_Observation ).join(MO_ObservationCollection.identifier). \147 q = self._session.query(CEDA_ObservationCollection). \ 148 join(MO_ObservationCollection).join(MO_Observation.identifier). \ 185 149 filter('md_identifier.code_search_vector @@ to_tsquery(:terms)') 186 150 # This binds the :terms placeholder to the searchterms string. User input 187 151 # should always be put into queries this way to prevent SQL injection. 188 152 q = q.params(terms=keywords) 189 return q190 153 191 154 def search(self, clazz, inst_key): -
mauRepo/HPFos/trunk/hpfos/HPFos/osImpl/myimpl.py
r8500 r8505 31 31 @author: Maurizio Nagni 32 32 ''' 33 from xml.dom import minidom34 33 from datetime import datetime 35 34 from hpfos.libs.postgisutil import create_st_setSRID 36 35 from ceda_markup.opensearch.osquery import OSQuery 37 from ceda_markup.opensearch.osParam import OSParam38 36 from ceda_markup.atom.atom import createID, createUpdated,\ 39 createPublished, create AtomDocument, createEntry37 createPublished, createEntry 40 38 from ceda_markup.atom.info import createTitle, HTML_TYPE, createContent 41 39 from ceda_markup.dc.dc import createDate … … 43 41 createPolygon 44 42 from ceda_markup.georss.georss import createWhere 45 from ceda_markup.atom.link import REL_SEARCH, REL_SELF, REL_FIRST, REL_NEXT,\ 46 REL_LAST, REL_ALTERNATE 47 from ceda_markup.opensearch.os_response import createOpenSearchRespose 48 from ceda_markup.opensearch import filterResults, COUNT_DEFAULT,\ 49 START_INDEX_DEFAULT, START_PAGE_DEFAULT, createAutodiscoveryLink,\ 50 generateAutodiscoveryPath 51 from ceda_markup.opensearch.template.osresponse import Result 52 from xml.etree.ElementTree import tostring 43 from ceda_markup.atom.link import REL_SEARCH, REL_ALTERNATE 44 from ceda_markup.opensearch import filter_results, COUNT_DEFAULT,\ 45 START_INDEX_DEFAULT, START_PAGE_DEFAULT, create_autodiscovery_link 46 from ceda_markup.opensearch.template.osresponse import Result, Subresult 53 47 from ceda_markup.opensearch.template.atom import OSAtomResponse 54 48 from ceda_markup.opensearch.template.html import OSHTMLResponse … … 56 50 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 57 51 from ea_model.ceda_metadatamodel.ceda_result.ceda_result import CEDA_Result 52 from ceda_markup.opensearch.os_param import OSParam 58 53 59 54 GUID = 'guid' … … 90 85 return 1 91 86 92 def digest SearchResults(self, results, context):93 count, start Index, startPage = self._importCountAndPage(context)87 def digest_search_results(self, results, context): 88 count, start_index, start_page = self._importCountAndPage(context) 94 89 95 90 filtered = None … … 97 92 98 93 if type(results) == CEDA_ObservationCollection: 99 filtered = filter Results(results.member, count, startIndex, startPage)94 filtered = filter_results(results.member, count, start_index, start_page) 100 95 tot_results = self._get_tot_results(results.member) 101 96 elif type(results) == CEDA_Observation: 102 97 #to be done 103 filtered = filter Results(results.result, count, startIndex, startPage)98 filtered = filter_results(results.result, count, start_index, start_page) 104 99 tot_results = self._get_tot_results(results.result) 105 100 else: 106 filtered = filter Results(results, count, startIndex, startPage)101 filtered = filter_results(results, count, start_index, start_page) 107 102 tot_results = self._get_tot_results(results) 108 103 109 104 if filtered is None: 110 return Result(count, start Index, startPage, tot_results, subresult = [])105 return Result(count, start_index, start_page, tot_results, subresult = []) 111 106 112 107 subresults = [] … … 118 113 continue 119 114 ititle = self._extractTitle(result) 120 item = Subresult(result_guid.id, ititle, result.description, datetime.now().isoformat())115 item = Subresult(result_guid.id, ititle, datetime.now().isoformat(), description = result.description) 121 116 else: 122 item = Subresult(DUMMY_GUID, 'dummy_resultTitle', 'dummy_resultDescription', datetime.now().isoformat())117 item = Subresult(DUMMY_GUID, 'dummy_resultTitle', datetime.now().isoformat(), description = 'dummy_resultDescription') 123 118 subresults.append(item) 124 119 125 return Result(count, startIndex, startPage, tot_results, subresult = subresults) 126 127 def generateResponse(self, results, queries, osHostURL, context): 128 ospath = generateAutodiscoveryPath(osHostURL, None, self.extension, rel = None) 129 130 #Generates the ATOM document 131 atomdoc = createAtomDocument(ospath + "atom", results.title, results.updated) 132 133 #Generate feed's links 134 self.generateFeedLinks(atomdoc, ospath, None, results.totalResult, results.count) 135 136 #Inserts the OpenSearchResponse elements 137 createOpenSearchRespose(atomdoc, results.totalResult, results.startIndex, results.count, queries) 138 self.generateEntries(atomdoc, results.subresult, ospath) 139 140 reparsed = minidom.parseString(tostring(atomdoc)) 141 return reparsed.toprettyxml(indent=" ") 142 143 def generateFeedLinks(self, atomroot, path, linkid = None, total_results = None, step = 0): 144 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 145 linkid, startIndex = None, rel = REL_SEARCH)) 146 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 147 linkid, startIndex = 0, rel = REL_SELF)) 148 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 149 linkid, startIndex = 0, rel = REL_FIRST)) 150 151 if total_results > 2*step: 152 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 153 linkid, step, rel = REL_NEXT)) 154 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 155 linkid, total_results - (total_results % step), rel = REL_LAST)) 156 else: 157 if total_results > step: 158 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 159 linkid, step, rel = REL_NEXT)) 160 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 161 linkid, step, rel = REL_LAST)) 162 else: 163 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 164 linkid, 0, rel = REL_NEXT)) 165 atomroot.append(createAutodiscoveryLink(atomroot, path, self.extension, \ 166 linkid, step, rel = REL_LAST)) 120 return Result(count, start_index, start_page, tot_results, subresult = subresults) 167 121 168 122 def generateEntryLinks(self, entry, atomroot, path, linkid = None): 169 entry.append(create AutodiscoveryLink(atomroot, path, self.extension, linkid, None, rel = REL_ALTERNATE))170 entry.append(create AutodiscoveryLink(atomroot, path, self.extension, linkid, None, rel = REL_SEARCH))171 172 def generate Entries(self, atomroot, subresults, path):123 entry.append(create_autodiscovery_link(atomroot, path, self.extension, linkid, None, rel = REL_ALTERNATE)) 124 entry.append(create_autodiscovery_link(atomroot, path, self.extension, linkid, None, rel = REL_SEARCH)) 125 126 def generate_entries(self, atomroot, subresults, path): 173 127 entries = [] 174 128 … … 203 157 def _importCountAndPage(self, context): 204 158 count = COUNT_DEFAULT 205 start Index = START_INDEX_DEFAULT206 start Page = START_PAGE_DEFAULT159 start_index = START_INDEX_DEFAULT 160 start_page = START_PAGE_DEFAULT 207 161 208 162 try: … … 212 166 213 167 try: 214 start Index = int(context['startIndex'])168 start_index = int(context['startIndex']) 215 169 except: 216 170 pass 217 171 218 172 try: 219 start Page = int(context['startPage'])173 start_page = int(context['startPage']) 220 174 except: 221 175 pass 222 176 223 return count, start Index, startPage177 return count, start_index, start_page 224 178 225 179 def _extractTitle(self, cedaObj): … … 264 218 super(MyOSQuery, self).__init__(params) 265 219 266 def do Search(self, context):220 def do_search(self, context): 267 221 ibbox = None 268 222 if context is not None and context.has_key(BBOX) and context[BBOX] is not None: … … 284 238 elif type(obj) == CEDA_Observation: 285 239 return context['moles3EPB'].searchSelectiveLoadByInstance(obj, 'result') #need to add bbox & phenomTime 286 287 288 class Subresult(object):289 def __init__(self, iid, title, description, updated):290 '''291 Constructor292 @param id: a unique identifier, eventually an URI293 @param title: an atom.Entry instance294 @param updated: the last time the record was updated295 '''296 self.id = iid297 self.title = title298 self.description = description299 self.updated = updated -
mauRepo/HPFos/trunk/hpfos/HPFos/view/view.py
r8500 r8505 62 62 hostURL = _buildHostURL(request) 63 63 ospath = _buildDescriptionOsPath(hostURL, collection_guid, observation_guid, result_guid) 64 response = os_engine.get Description(ospath)64 response = os_engine.get_description(ospath) 65 65 context = {} 66 66 context['response'] = mark_safe(response) … … 69 69 def _doSearch(request, iformat, guid = None, result_guid = None): 70 70 hostURL = _buildHostURL(request) 71 context = os_engine.create QueryDictionary()71 context = os_engine.create_query_dictionary() 72 72 if request.GET is not None: 73 73 for param in request.GET.iteritems(): … … 80 80 context['moles3EPB'] = request.moles3EPB 81 81 82 response = os_engine.do Search(hostURL, iformat, context)82 response = os_engine.do_search(hostURL, iformat, context) 83 83 84 84 context = {} -
mauRepo/HPFos/trunk/hpfos/tests/__init__.py
r8354 r8505 1 from unittest import TestSuite 2 from hpfos.tests.testos import TestOS 3 4 def suite(): 5 suite = TestSuite() 6 suite.addTest(TestOS) 7 8 return suite -
mauRepo/HPFos/trunk/hpfos/tests/testos.py
r8500 r8505 33 33 from hpfos.HPFos.view.view import getDescription, doSearchL0 34 34 from hpfos.tests.testsetup import HPFosTest 35 from hpfos.HPFos.osImpl.myimpl import MyOSAtomResponse 36 from ceda_markup.opensearch.template.osresponse import Subresult, Result 37 from ceda_markup.opensearch.query import create_query 35 38 36 39 37 40 class TestOS(HPFosTest): 38 41 39 def testDescription(self):42 def description_test(self): 40 43 get_request = self.reqFactory.get('/description/') 41 44 res = getDescription(request = get_request) 42 45 print res 43 46 44 def testSearch(self):47 def search_test(self): 45 48 iformat = 'atom' 46 49 get_request = self.reqFactory.get('/atom/search', {'count': '3'}) … … 49 52 print res 50 53 51 def testOS(self): 52 self.testDescription() 53 self.testSearch() 54 def create_query_test(self): 55 context = self._generate_context() 56 query = create_query('atom', self.os_engine.os_query.params_model, context) 57 print query 58 return query 59 60 def generate_reponse_test(self): 61 ar = MyOSAtomResponse() 62 context = self._generate_context() 63 queries = self.create_query_test() 64 65 result = self._generate_results(count=7, start_index=3, start_page=1, tot_results=40) 66 response = ar.generate_response(result, queries, '', context) 67 self.assertTrue('<link href="/search/atom/?startIndex=3" rel="self" type="application/atom+xml"/>' in response, "Error") 68 self.assertTrue('<link href="/search/atom/?startIndex=1" rel="first" type="application/atom+xml"/>' in response, "Error") 69 self.assertTrue('<link href="/search/atom/?startIndex=10" rel="next" type="application/atom+xml"/>' in response, "Error") 70 self.assertTrue('<link href="/search/atom/?startIndex=38" rel="last" type="application/atom+xml"/>' in response, "Error") 71 72 result = self._generate_results(count=10, start_index=5, start_page=1, tot_results=20) 73 response = ar.generate_response(result, queries, '', context) 74 self.assertTrue('<link href="/search/atom/?startIndex=5" rel="self" type="application/atom+xml"/>' in response, "Error") 75 self.assertTrue('<link href="/search/atom/?startIndex=1" rel="first" type="application/atom+xml"/>' in response, "Error") 76 self.assertTrue('<link href="/search/atom/?startIndex=15" rel="next" type="application/atom+xml"/>' in response, "Error") 77 self.assertTrue('<link href="/search/atom/?startIndex=15" rel="last" type="application/atom+xml"/>' in response, "Error") 54 78 79 result = self._generate_results(count=2, start_index=2, start_page=1, tot_results=5) 80 response = ar.generate_response(result, queries, '', context) 81 self.assertTrue('<link href="/search/atom/?startIndex=2" rel="self" type="application/atom+xml"/>' in response, "Error") 82 self.assertTrue('<link href="/search/atom/?startIndex=1" rel="first" type="application/atom+xml"/>' in response, "Error") 83 self.assertTrue('<link href="/search/atom/?startIndex=4" rel="next" type="application/atom+xml"/>' in response, "Error") 84 self.assertTrue('<link href="/search/atom/?startIndex=4" rel="last" type="application/atom+xml"/>' in response, "Error") 85 86 result = self._generate_results(count=1, start_index=18, start_page=1, tot_results=20) 87 response = ar.generate_response(result, queries, '', context) 88 self.assertTrue('<link href="/search/atom/?startIndex=18" rel="self" type="application/atom+xml"/>' in response, "Error") 89 self.assertTrue('<link href="/search/atom/?startIndex=1" rel="first" type="application/atom+xml"/>' in response, "Error") 90 self.assertTrue('<link href="/search/atom/?startIndex=19" rel="next" type="application/atom+xml"/>' in response, "Error") 91 self.assertTrue('<link href="/search/atom/?startIndex=20" rel="last" type="application/atom+xml"/>' in response, "Error") 92 93 94 def _generate_results(self, count=1, start_index=0, start_page=1, tot_results=20): 95 subresults = [] 96 for index in range(tot_results): 97 subresults.append(Subresult('id_%s' % (str(index)), \ 98 'title_%s' % (str(index)) , \ 99 description = 'description_%s' % (str(index)))) 100 return Result(count, start_index, start_page, tot_results, subresult = subresults) 101 102 103 def _generate_context(self, count='1', start_index='0', start_page='1'): 104 context = {} 105 context['count'] = count 106 context['startIndex'] = start_index 107 context['startPage'] = start_page 108 return context -
mauRepo/HPFos/trunk/hpfos/tests/testsetup.py
r8500 r8505 25 25 26 26 self.epbRepo = HPFosTest.epbRepo 27 HPFosTest.osEngine = setUp()27 self.os_engine = setUp() 28 28 environ = { 29 29 'SERVER_NAME': 'fatcat.badc.rl.ac.uk', -
mauRepo/HPFos/trunk/resources/requirements.txt
r8466 r8505 7 7 # --extra-index-url http://ciprod1.cems.rl.ac.uk/pip 8 8 # to your pip install 9 ceda-markup==0.0. 69 ceda-markup==0.0.7 10 10 ceda-moles-model==0.1.5 11 11
Note: See TracChangeset
for help on using the changeset viewer.