source: mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py @ 8460

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py@8460
Revision 8460, 19.2 KB checked in by mnagni, 7 years ago (diff)

Complete - # 22489: CEDA Observation Collection - phenomenonTime
 http://team.ceda.ac.uk/trac/ceda/ticket/22489
Complete - # 22518: The description is broken
 http://team.ceda.ac.uk/trac/ceda/ticket/22518
Complete - # 22488: CEDA Observation Collection - Geographical Extent
 http://team.ceda.ac.uk/trac/ceda/ticket/22488

Now the Moles3EPB explicitly call the "synchronise" method in the SQLAlchemy mapped classes to assure the persistence of the data
Uses the CedaMolesModel? v 0.1.5 which correct a major problem in synchronise the instances with database

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 10 Jan 2012
30
31@author: mnagni
32'''
33from libs.epb import EPB
34from ea_model.moles3_4.observationcollection.mo_observationcollection import MO_ObservationCollection
35from ea_model.moles3_4.observation.mo_observation import MO_Observation
36from sqlalchemy import Table, Column, ForeignKey, Integer, String
37from sqlalchemy.orm import mapper
38from MolesManager.ceda_guid import CedaGUID
39from sqlalchemy.orm.collections import InstrumentedList
40from ea_model.iso_19115_2006_metadata_corrigendum.reference_system_information.md_identifier import MD_Identifier
41from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_citation import CI_Citation
42from ea_model.iso_19115_2006_metadata_corrigendum.extent_information.ex_geographicboundingbox import EX_GeographicBoundingBox
43from libs.postgisutil import unifyGeometries, create_st_setSRID, getBox2D,\
44    unifyGeometriesAsBBox
45from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection
46from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation
47from ea_model.ceda_metadatamodel.ceda_project.ceda_project import CEDA_Project
48from ea_model.moles3_4.utilities.mo_responsiblepartyinfo import MO_ResponsiblePartyInfo
49from ea_model.moles3_4.utilities.ceda_rolevalue import CEDA_RoleValue
50from MolesManager.codelist import MM_RoleValue, getCLValue
51from libs.migration.processor.commons import fromPhenomenonTimeToString
52from datetime import datetime
53from ascore.utils import synchAttributes
54
55class Moles3EPBFactory(EPB):
56   
57    def __init__(self, dbManager):
58        self._dbManager = dbManager
59        self._initCEDA_Customization()   
60       
61
62    def _initCEDA_Customization(self):
63        self._associateCEDA_GUID()
64        self._initSearchIndexes()       
65       
66    def _associateCEDA_GUID(self):
67        guid_table = Table('ceda_guid', self._dbManager.metadata, \
68                           Column('id', String, primary_key=True), \
69                           Column('ceda_observationcollection', Integer, ForeignKey('ceda_observationcollection.id')), 
70                           Column('ceda_observation', Integer, ForeignKey('ceda_observation.id')))
71        mapper(CedaGUID, guid_table)
72        self._dbManager.metadata.create_all()
73
74    def _initSearchIndexes(self):
75        #To Be Done - CHECK IF THE COLUMN ALREADY EXISTS!
76        # We don't want sqlalchemy to know about this column so we add it externally.
77        try:
78            self._dbManager.engine.execute("alter table md_identifier add column code_search_vector tsvector")                 
79
80            # This indexes the tsvector column
81
82            self._dbManager.engine.execute("create index md_identifier_code_search_index on md_identifier using gin(code_search_vector)")
83
84            # This sets up the trigger that keeps the tsvector column up to date.
85            self._dbManager.engine.execute("create trigger md_identifier_code_search_update before update or insert on md_identifier \
86                for each row execute procedure tsvector_update_trigger('code_search_vector', 'pg_catalog.english', code)")                       
87        except Exception as e:
88            pass
89
90    def _getSession(self):
91        if self._dbManager is not None:
92            return self._dbManager.createDbSession()               
93        return None
94       
95    def createEPB(self):
96        return Moles3EPB(self._getSession())
97
98class Moles3EPB(object):
99
100    def __init__(self, session):
101        self._session = session
102       
103    def close(self):
104        return self._session.close()       
105       
106    def searchEager(self, clazz, inst_id):
107        return EPB.searchEager(clazz, inst_id, self._session)     
108     
109    def _controlledCommit(self):
110        try:
111            self._session.commit()
112        except Exception as e:
113            print e
114     
115    def persistInstance(self, instance):
116        """
117            Adds a new migration object.
118            @param migrationObj: the migration object to add
119            @param session: an SQLAlchemy Session object. If not None the session IS NOT closed at the exit,
120            If None (default) a new Session is created from the underlying EPB and closed at the exit.
121            @return an updated, session independent, object instance reflecting the new persisted object
122        """
123        EPB.persistInstance(instance, self._session)       
124
125    def _mergeOrAddInSession(self, ceda_object):
126        try:
127            return self._session.merge(ceda_object)   
128        except:
129            self._session.add(ceda_object)
130            return ceda_object 
131     
132    def updateCedaObject(self, ceda_object, cols_to_update):
133        """
134            Update, eventually add to the session, and commit a CEDA Object in MOLES3 db.
135            @param ceda_object: the CEDA object to update
136            @param cols_to_update: a dictionary containing the columns to update for the given ceda_object and the desired value.
137            If the attribute is a list of objects the new instances are appended only if do not exist in the actual list
138            @return: the given instance with the updated attributes.
139        """
140        coll = self._mergeOrAddInSession(ceda_object)
141        if coll != None:       
142            for k,v in cols_to_update.items():
143                if hasattr(coll, k):                   
144                    coll_k = getattr(coll, k)                       
145                    if type(coll_k) == list or type(coll_k) == InstrumentedList:
146                        tmp_coll = []
147                        if type(v) == list or type(v) == InstrumentedList:
148                            tmp_coll.extend(v)
149                        else:
150                            tmp_coll.append(v)
151                        for item in tmp_coll:
152                            el = self._mergeOrAddInSession(item)
153                            if el not in coll_k:
154                                coll_k.append(el)
155                    else:
156                        el = self._mergeOrAddInSession(v)
157                        setattr(coll, k, el)
158        synchAttributes(coll)                                             
159        self._controlledCommit()
160        #return coll                                     
161
162    def getUnifyObservationCollectionGEAsBBox(self, collection):
163        """
164            Returns the union of the collections.member'a  GeographicExtension(s)
165            @param collection: an CEDA_ObservationColleciton instance 
166        """
167        bboxes = []
168        for member in collection.member:
169            for ge in member.geographicExtent:
170                bboxes.append(getGeograpicExtentGeometry(ge))
171       
172        return unifyGeometriesAsBBox(bboxes, self) 
173       
174    def getUnifyObservationCollectionPhenomenonTime(self, collection):
175        """
176            Returns the time period of the collections.member'a  phenomenonTime(s)
177            @param collection: an CEDA_ObservationColleciton instance
178            @return: a tuple (startDate, endDate)
179        """
180        ptStart = []
181        ptEnd = []     
182        for member in collection.member:
183            for pt in member.phenomenonTime:
184                ptString = fromPhenomenonTimeToString(pt)
185                if ptString[0] is not None:                   
186                    ptStart.append(datetime.strptime(ptString[0], '%Y-%m-%d'))
187                if ptString[1] is not None:                   
188                    ptEnd.append(datetime.strptime(ptString[1], '%Y-%m-%d'))
189        ptStart.sort()                   
190        ptEnd.sort()
191        start = None
192        end = None
193        #takes the earlier date
194        if len(ptStart) > 0:
195            start = ptStart[0]
196        #takes the latest date
197        if len(ptEnd) > 0:
198            end = ptEnd[len(ptEnd) - 1]
199        return start, end 
200       
201       
202       
203        #return unifyGeometriesAsBBox(bboxes, self)
204
205
206    def retrieveGUIDFromInstance(self, instance):
207        """
208            Returns the CedaGUID object associated with the given instance.
209            @param instance: an instance of CEDA_Observation os CEDA_ObservationCollection 
210        """
211        if instance is None or not hasattr(instance, 'id'):
212            return None
213        if type(instance) == CEDA_ObservationCollection:
214            return self._session.query(CedaGUID).filter(CedaGUID.ceda_observationcollection==instance.id).first()
215        elif type(instance) == CEDA_Observation:
216            return self._session.query(CedaGUID).filter(CedaGUID.ceda_observation==instance.id).first()       
217   
218    def observationCollectionHasObservation(self, obs_coll_id, obs_id):
219        """
220            Checks if a CEDA_Collection contains a given CEDA_Observation.
221            @param obs_coll_id: the CEDA_ObservationColleciton id
222            @param obs_id: the CEDA_Observation id
223            @return: True if the collection contains the given observation, False otherwise 
224        """
225        coll = self._session.query(CEDA_ObservationCollection).filter(CEDA_ObservationCollection.id==obs_coll_id).first()
226        obs = self._session.query(CEDA_Observation).filter(CEDA_Observation.id==obs_id).first()
227        return obs in coll.member
228
229    def observationAuthor(self, observation):
230        """
231            Lists the CEDA_Observation author.
232            @param observation: the CEDA_Observation inside which look for the author           
233            @return: True if the collection contains the given observation, False otherwise 
234        """
235       
236        # TO FIX!!!
237        for partyInfo in observation.relatedParty:
238            if partyInfo.role == getCLValue(MM_RoleValue.cl_author):
239                return partyInfo.party       
240
241       
242
243    def extractObservationByTitleKeywords(self, keywords):
244        """
245            Loooks for CEDA_Observation containing a specific title (observation.identifier.code)
246            @param keywords: a space separated terms string
247            @return: a tuple containing a CEDA_Observation satisfying the queryllection.idenfitier element having the title 
248        """               
249        # search_vector is a ts_vector column. To search for terms, you use the
250        # @@ operator. plainto_tsquery turns a string into a query that can be
251        # used with @@. So this adds a where clause like "WHERE search_vector
252        # @@ plaint_tsquery(<search string>)"
253        q = self._session.query(CEDA_Observation). \
254            join(MO_Observation).join(MO_Observation.identifier). \
255            filter('md_identifier.code_search_vector @@ to_tsquery(:terms)')
256        # This binds the :terms placeholder to the searchterms string. User input
257        # should always be put into queries this way to prevent SQL injection.
258        q = q.params(terms=keywords)
259        return q.all()
260
261
262    def extractCollectionIdentifierByTitle(self, i_title):
263        """
264            Searches for an MD_Identifier from a CEDA_ObservationCollection contains a specific title (observation.identifier.code)
265            @param i_title: the CEDA_ObservationCollection.identifier.title value to search for
266            @return: a tuple containing a CEDA_ObservationCollection and the CEDA_ObservationCollection.idenfitier element having the title 
267        """
268        return self._session.query(CEDA_ObservationCollection, MD_Identifier). \
269            join(MO_ObservationCollection).join(MO_ObservationCollection.identifier). \
270            join(MD_Identifier.authority).filter(CI_Citation.title.like('%' + i_title + '%'))
271
272    def extractObservationsForProject(self, project):
273        """
274            Searches for the CEDA_Observation associated with a CEDA_Project
275            @param project: a CEDA_Project instance
276            @return: a tuple containing the associated CEDA_Observation 
277        """
278        return self._session.query(CEDA_Observation). \
279            join(CEDA_Observation, MO_Observation.inSupportOf).filter(CEDA_Project.id == project.id)
280
281    def extractProjectObservationCollections(self, project):
282        """
283            Searches for the Observation_Collections associated with a CEDA_Project
284            @param project: a CEDA_Project instance
285            @return: a tuple containing the associated CEDA_ObservationCollection 
286        """
287        mo_obs = self._session.query(MO_Observation).join(CEDA_Project).filter(CEDA_Project.id == project.id).subquery()     
288        obsers = self._session.query(CEDA_Observation).join(mo_obs, CEDA_Observation.id == mo_obs.c.id).one()
289        #print "obsers: " + str(intSession.query(CEDA_Observation).join(mo_obs, CEDA_Observation.id == mo_obs.c.id).count())
290       
291        cos = self._session.query(CEDA_ObservationCollection).all()
292        co = self._session.query(MO_ObservationCollection).join(MO_ObservationCollection.member).filter(MO_ObservationCollection.member.contains(obsers))
293       
294        observations = self._session.query(MO_ObservationCollection).join(CEDA_Observation). \
295            filter(obsers.any(CEDA_Observation.id==obsers.c.id))
296        print "observation:" + str(observations.count())
297        return observations
298
299    def search(self, clazz, inst_id):
300        ret = EPB.search(clazz, inst_id, self._session)
301        return ret
302     
303    def searchSelectiveLoad(self, clazz, inst_id, attributes): 
304        """
305            Searches a required instance by id loading selectively \
306            the specified fields. The parameter "attributes" is a single string or a list of attributes
307            owned by the instance of "clazz". Furthermore such list may contain
308            also the children of the main attributes. For example "attrs" may look
309            like
310            ['resultAccumulation', 'identifier.authority', 'resultTime.position.dateTime8601.month', \
311                      'relatedParty.party', 'result.source.function', 'permission', \
312                      'geographicExtent', 'phenomenonTime', 'keywords', 'description', \
313                      'inSupportOf.abstract', 'dataLineage']
314            the first parameter refers to the main class so is equivalent to
315            clazz.resultAccumulation
316            the second parameter is equivalent to invoke
317            clazz.identifier.authority
318            As single string "attributes" could be as well just 'identifier.authority'
319            @param clazz: the class type to search for
320            @param inst_id: the instance id for which the search is done
321            @param attributes: a single string or a list of attributes to load
322            @param session: a session to use for the query. By default a new one is created automatically at start and closed at the end
323            @return the required instance             
324        """               
325        ret = EPB.searchSelectiveLoad(clazz, inst_id, attributes, self._session)
326        return ret   
327   
328    def loadAttributes(self, instance, attributes):
329        """
330            Returns the attribute of an instance. The parameter "attributes" is a single string or a list of attributes
331            owned by the instance of "clazz". Furthermore such list may contain
332            also the children of the main attributes. For example "attrs" may look
333            like
334            ['resultAccumulation', 'identifier.authority', 'resultTime.position.dateTime8601.month', \
335                      'relatedParty.party', 'result.source.function', 'permission', \
336                      'geographicExtent', 'phenomenonTime', 'keywords', 'description', \
337                      'inSupportOf.abstract', 'dataLineage']
338            the first parameter refers to the main class so is equivalent to
339            clazz.resultAccumulation
340            the second parameter is equivalent to invoke
341            clazz.identifier.authority
342            As single string "attributes" could be as well just 'identifier.authority'
343            @param instance: an instance containing the appropriate id
344            @param attributes: the attribute value required
345            @param session: the session to use for the operation
346            @return: the given instance filled with the required attributes.                     
347        """
348        instance = self._session.merge(instance)
349        EPB.loadAttributes(instance, attributes, self._session)                 
350        return instance
351
352    def executeNative(self, sqlNative):
353        return EPB.executeNative(sqlNative, self._session) 
354
355
356def getGeograpicExtentGeometry(ge):
357    '''
358        Creates the appropriate postgis geometry from a EX_GeographicExtent
359        @param ge: an EX_GeographicExtent instance
360        @return: a postgix text geometry
361    '''
362    if isinstance(ge, EX_GeographicBoundingBox):
363        return create_st_setSRID(ge.westBoundLongitude, ge.southBoundLatitude, \
364                       ge.eastBoundLongitude, ge.northBoundLatitude)
365    return None
366
367def getUnifyObservationCollectionPhenomenonTime(collection):
368    """
369        Returns the time period of the collections.member'a  phenomenonTime(s)
370        @param collection: an CEDA_ObservationColleciton instance
371        @return: a tuple (startDate, endDate) strings
372    """
373    dateFormat = '%Y-%m-%d'
374    ptStart = []
375    ptEnd = []     
376    for member in collection.member:
377        if member.phenomenonTime is None:
378            continue
379           
380        pt =  member.phenomenonTime
381        ptString = fromPhenomenonTimeToString(pt)
382        if ptString[0] is not None:                   
383            ptStart.append(datetime.strptime(ptString[0], dateFormat))
384        if ptString[1] is not None:                   
385            ptEnd.append(datetime.strptime(ptString[1], dateFormat))
386    ptStart.sort()                   
387    ptEnd.sort()
388    start = None
389    end = None
390    #takes the earlier date
391    if len(ptStart) > 0:
392        start = ptStart[0].strftime(dateFormat) 
393    #takes the latest date
394    if len(ptEnd) > 0:
395        end = ptEnd[len(ptEnd) - 1].strftime(dateFormat) 
396    return start, end
Note: See TracBrowser for help on using the repository browser.