source: mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py @ 8358

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py@8358
Revision 8358, 10.0 KB checked in by mnagni, 7 years ago (diff)

Major refactoring about migration (now handles better create/update, even if the single updates have to be quite fully implemented)
Added the connection pool from SQLAlchemy

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 15 Nov 2011
30
31@author: Maurizio Nagni
32'''
33from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import \
34    CEDA_ObservationCollection
35from libs.migration.exception.exceptions import MigrationObjectException, NoAssociatedAuthor,\
36    migrationObjectDescription, NoAssociatedDeployments
37from libs.migration.processor.commons import findDeploymentsInDE,\
38    createMD_Identifier, extractContent,\
39    hasMOSameHash, createCI_Citation, createCI_Date, findPublishedDate,\
40    isoDateTimeStringToTimeDate, findUpdatedDate, createDate,\
41    calculateHash, findDOIInMigrationDocument,\
42    hasMOBeenProcessed, getAtomDocumentHashByMO
43from libs.migration.processor.deployment import DeploymentProcessor
44from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue
45from logging import StreamHandler
46import logging
47from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_datetypecode import CI_DateTypeCode
48from MolesManager.ceda_guid import CedaGUID
49
50class DataEntityProcessor(object):
51    log = logging.getLogger('DataEntityProcessor')
52    log.addHandler(StreamHandler())
53    log.setLevel(logging.INFO) 
54    '''
55        Processes a DataEntityMigration item. Note that each DataEntity is associated to a "dataent_xxxx" file in Moles2
56    '''       
57    def __init__(self, dataEntityMigration, epbRepo):
58        '''
59            Initializes the class
60            @param _dataEntityMigration: the DataEntityMigration instance
61            @param epbRepo: an instance of EPBRepo
62        '''
63        #if dataEntityMigration is None:
64        #    raise MigrationObjectException("DataEntityProcessor cannot process an None item")
65        self._dataEntityMigration = dataEntityMigration       
66        self._dataEntityHasSameHash = hasMOSameHash(self._dataEntityMigration)       
67        self.epbRepo = epbRepo
68   
69    def _processCitation(self, ceda_observationCollection):
70        contentDict = extractContent(self._dataEntityMigration)
71        if not contentDict.has_key('citation'):
72            DataEntityProcessor.log.info("The migration object "+ migrationObjectDescription(self._dataEntityMigration) \
73                                         + " has not associated cedacat:citation")
74        else:
75            ci_dates = []
76            doc_date = findPublishedDate(self._dataEntityMigration)           
77            if doc_date:
78                i_date = createDate(isoDateTimeStringToTimeDate(doc_date))           
79                ci_dates.append(createCI_Date(CI_DateTypeCode.cl_publication, date = i_date))
80
81            doc_date = findUpdatedDate(self._dataEntityMigration)           
82            if doc_date:
83                i_date = createDate(isoDateTimeStringToTimeDate(doc_date))               
84                ci_dates.append(createCI_Date(CI_DateTypeCode.cl_revision, date = i_date))
85               
86            i_citation = createCI_Citation(title = 'ceda_moles2_citation', date=ci_dates)
87            newIdentifier = createMD_Identifier(code = contentDict['citation'], authority=i_citation)
88            ceda_observationCollection.identifier.append(newIdentifier)
89
90    def _execute(self, ceda_observationCollection): 
91        """
92            Creates a new CEDA_ObservationCollection instance in the Moles3DB using the self._dataEntityMigration object.
93            If successful adds the new instance ID to the related DataEntityMigration object
94            @return: the persisted CEDA_ObservationCollection element
95        """
96        if not self._dataEntityHasSameHash:
97            self._processCitation(ceda_observationCollection)
98            #self._processResult(ceda_observationCollection)
99       
100        #Is a first time process?
101        if not hasMOBeenProcessed(self._dataEntityMigration):
102            docHash = getAtomDocumentHashByMO(self._dataEntityMigration)
103            ceda_observationCollection.publicationState = MO_PublicationStateValue.cl_working
104            self.epbRepo.moles3EPB.persistInstance(ceda_observationCollection)
105            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \
106                {'ceda_observation_coll_id': ceda_observationCollection.id, \
107                 'doc_hash': docHash})
108
109        #Has to updated the hash?
110        if not self._dataEntityHasSameHash and hasMOBeenProcessed(self._dataEntityMigration):
111            docHash = getAtomDocumentHashByMO(self._dataEntityMigration)
112            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \
113                {'doc_hash': docHash})
114       
115        #Has a proper CEDAGUID?
116        if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observationCollection) is None:
117            #Adds the CedaGUID
118            ceda_guid = CedaGUID()
119            ceda_guid.id = calculateHash(self._dataEntityMigration.data_ent_id)
120            setattr(ceda_guid, 'ceda_observationcollection', ceda_observationCollection.id)
121            self.epbRepo.moles3EPB.persistInstance(ceda_guid)
122            DataEntityProcessor.log.info("GUID for this ObservationCollection: %s" % (ceda_guid.id))
123
124    def _processDOI(self, deploymentMigration, ceda_observation, deProcessor, single_deployment):       
125        doi = findDOIInMigrationDocument(deploymentMigration)                                                             
126        if single_deployment:
127            if doi is None:
128                doi = findDOIInMigrationDocument(self._dataEntityMigration)
129                    #collection_identifier = Moles3EPB.extractCollectionIdentifierByTitle(MD_CODE_MOLES2_CITATION, self.migrationSessions.molesSession)
130                    #if collection_identifier.count()==1:
131                    #    ceda_observation.identifier.append(collection_identifier.first())               
132        deProcessor.assignDOI(ceda_observation, doi)
133
134    def _processDeploymentMigration(self, deploymentMigration, single_deployment):                                     
135        deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration, self.epbRepo)
136        try:
137            DataEntityProcessor.log.info("Processing deployment: %s" % (migrationObjectDescription(deploymentMigration)))
138            ceda_observation = deProcessor.process()   
139            try:                           
140                self._processDOI(deploymentMigration, ceda_observation, deProcessor, single_deployment)
141            except Exception as ex:
142                pass                               
143        except NoAssociatedAuthor as ex:
144            raise ex                 
145        except Exception as ex:
146            #self.migrationSessions.molesSession.rollback()
147            #self.migrationSessions.migrationSession.rollback()               
148            raise MigrationObjectException(ex)                     
149       
150        return ceda_observation
151   
152    def process(self):
153        obsColl = None
154        exs = []
155        DataEntityProcessor.log.info("Processing dataEntity: %s" % (migrationObjectDescription(self._dataEntityMigration)))
156        try :
157            if self._dataEntityMigration.ceda_observation_coll_id:
158                obsColl = self.epbRepo.moles3EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id) 
159            else:
160                obsColl = CEDA_ObservationCollection()
161            self._execute(obsColl)                 
162        except Exception as ex:
163            exs.append(ex)
164            return exs       
165       
166        #retrieves the associated deployment links from the data_entity
167        deploymentsLinks = findDeploymentsInDE(self._dataEntityMigration)       
168        #retrieves the DataEntityMigration sorted by creation date
169        deploymentMigrations = self.epbRepo.migrationEPB.getAllDeploymentsMigrationByDataEntitySortedByDate( \
170                                                self._dataEntityMigration, deploymentsLinks)
171        howManydm = 0
172        if deploymentMigrations:
173            howManydm = len(deploymentMigrations)
174        if howManydm == 0:
175            exs.append(NoAssociatedDeployments(self._dataEntityMigration))
176        for deploymentMigration in deploymentMigrations:
177            try:
178                ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1)
179                #Is a first time process?
180                if not hasMOBeenProcessed(deploymentMigration):
181                    self.epbRepo.moles3EPB.updateCedaObject(obsColl, {'member': ceda_observation})
182                                     
183            except Exception as ex:
184                exs.append(ex) 
185        return exs
Note: See TracBrowser for help on using the repository browser.