source: mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py @ 8433

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py@8433
Revision 8433, 13.3 KB checked in by mnagni, 7 years ago (diff)

Two main new features:
1) the migration generate a report. The report is sent by mail to the subscribers defined in the settings.py
2) require to display a missing observation adds a message on the page and hides the some fields in the html form
3) a new javascript library (commons.js)

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 15 Nov 2011
30
31@author: Maurizio Nagni
32'''
33from libs.migration.exception.exceptions import MigrationObjectException, NoAssociatedAuthor,\
34    migrationObjectDescription, NoAssociatedDeployments,\
35    NoGeographicalExtensionException, NoCitationException
36from libs.migration.processor.commons import findDeploymentsInDE,\
37    createMD_Identifier, extractContent,\
38    hasMOSameHash, createCI_Citation, createCI_Date, findPublishedDate,\
39    isoDateTimeStringToTimeDate, findUpdatedDate, createDate,\
40    calculateHash, findDOIInMigrationDocument,\
41    hasMOBeenProcessed, getAtomDocumentHashByMO, extractTitle, extractSummary,\
42    createEX_GeographicBoundingBox
43from libs.migration.processor.deployment import DeploymentProcessor
44from logging import StreamHandler
45import logging
46from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_datetypecode import CI_DateTypeCode
47from MolesManager.ceda_guid import CedaGUID
48from MolesManager.codelist import MM_ObservationPublicationStateValue,\
49    getCLValue
50from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection
51CEDA_TITLE = 'ceda_title'
52
53class DataEntityProcessor(object):
54    log = logging.getLogger('DataEntityProcessor')
55    log.addHandler(StreamHandler())
56    log.setLevel(logging.INFO) 
57    '''
58        Processes a DataEntityMigration item. Note that each DataEntity is associated to a "dataent_xxxx" file in Moles2
59    '''       
60    def __init__(self, dataEntityMigration, epbRepo):
61        '''
62            Initializes the class
63            @param _dataEntityMigration: the DataEntityMigration instance
64            @param epbRepo: an instance of EPBRepo
65        '''
66        #if dataEntityMigration is None:
67        #    raise MigrationObjectException("DataEntityProcessor cannot process an None item")
68        self._dataEntityMigration = dataEntityMigration       
69        self._dataEntityHasSameHash = hasMOSameHash(self._dataEntityMigration)   
70        self._dataEntityHasBeenProcessed = hasMOBeenProcessed(self._dataEntityMigration)   
71        self.epbRepo = epbRepo
72        self._report = []
73
74    def _assignGeographicExtent(self, ceda_observationCollection):
75        bbox = self.epbRepo.moles3EPB.getUnifyObservationCollectionGEAsBBox(ceda_observationCollection)
76        if bbox is not None:
77            upperCornerData, lowerCornerData = bbox[4:len(bbox)-1].split(',')
78            east, north = upperCornerData.split()
79            west, south = lowerCornerData.split()
80            geographicExtent = createEX_GeographicBoundingBox(float(east), float(north), \
81                                                              float(west), float(south))
82            if self._dataEntityHasBeenProcessed:
83                DataEntityProcessor.log.warn('The _assignGeographicExtent update is skipped because not implemented')
84                return
85           
86            self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, {'geographicExtent': geographicExtent})                       
87        else:
88            self._report.append(NoGeographicalExtensionException(self._dataEntityMigration))         
89       
90
91    def _assignDescription(self, ceda_observationCollection):
92        description = extractSummary(self._dataEntityMigration)
93           
94        if description:
95            ceda_observationCollection.description = description
96   
97    def _processTitle(self, ceda_observationCollection):
98        ititle = extractTitle(self._dataEntityMigration)
99        if ceda_observationCollection.identifier:
100            for ident in ceda_observationCollection.identifier:
101                if ident.authority.title == CEDA_TITLE:
102                    if ident.code != ititle:
103                        ident.code = ititle
104                    else:
105                        return           
106 
107        #Else create new
108        i_citation = createCI_Citation(title = CEDA_TITLE)
109        newIdentifier = createMD_Identifier(code = ititle, authority=i_citation)
110       
111        if self._dataEntityHasBeenProcessed:
112            self._report.append('The _assignGeographicExtent update is skipped because not implemented')
113            DataEntityProcessor.log.warn('The _assignGeographicExtent update is skipped because not implemented')
114            return
115           
116        self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, {'identifier': newIdentifier})
117   
118    def _processCitation(self, ceda_observationCollection):
119        contentDict = extractContent(self._dataEntityMigration)
120        if not contentDict.has_key('citation'):
121            self._report(NoCitationException(self._dataEntityMigration))
122            DataEntityProcessor.log.info("The migration object "+ migrationObjectDescription(self._dataEntityMigration) \
123                                         + " has not associated cedacat:citation")
124            return
125
126        ci_dates = []
127        doc_date = findPublishedDate(self._dataEntityMigration)           
128        if doc_date:
129            i_date = createDate(isoDateTimeStringToTimeDate(doc_date))           
130            ci_dates.append(createCI_Date(getCLValue(CI_DateTypeCode.cl_publication), date = i_date))
131
132        doc_date = findUpdatedDate(self._dataEntityMigration)           
133        if doc_date:
134            i_date = createDate(isoDateTimeStringToTimeDate(doc_date))               
135            ci_dates.append(createCI_Date(getCLValue(CI_DateTypeCode.cl_revision), date = i_date))
136           
137        i_citation = createCI_Citation(title = 'ceda_moles2_citation', date=ci_dates)
138        newIdentifier = createMD_Identifier(code = contentDict['citation'], authority=i_citation)
139
140        if self._dataEntityHasBeenProcessed:
141            DataEntityProcessor.log.warn('The _assignGeographicExtent update is skipped because not implemented')
142            return
143       
144        ceda_observationCollection = self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, {'identifier': newIdentifier})
145       
146
147    def _execute(self, ceda_observationCollection): 
148        """
149            Creates a new CEDA_ObservationCollection instance in the Moles3DB using the self._dataEntityMigration object.
150            If successful adds the new instance ID to the related DataEntityMigration object
151            @return: the persisted CEDA_ObservationCollection element
152        """
153       
154        if not self._dataEntityHasSameHash:
155            self._processTitle(ceda_observationCollection)
156            self._assignDescription(ceda_observationCollection)
157            self._processCitation(ceda_observationCollection)
158       
159        #Is a first time process?
160        if not hasMOBeenProcessed(self._dataEntityMigration):
161            docHash = getAtomDocumentHashByMO(self._dataEntityMigration)
162            ceda_observationCollection.publicationState = getCLValue(MM_ObservationPublicationStateValue.cl_working)
163            self.epbRepo.moles3EPB.persistInstance(ceda_observationCollection)
164            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \
165                {'ceda_observation_coll_id': ceda_observationCollection.id, \
166                 'doc_hash': docHash})
167
168        #Has to updated the hash?
169        if not self._dataEntityHasSameHash and hasMOBeenProcessed(self._dataEntityMigration):
170            docHash = getAtomDocumentHashByMO(self._dataEntityMigration)
171            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \
172                {'doc_hash': docHash})
173       
174        #Has a proper CEDAGUID?
175        if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observationCollection) is None:
176            #Adds the CedaGUID
177            ceda_guid = CedaGUID()
178            ceda_guid.id = calculateHash(self._dataEntityMigration.data_ent_id)
179            setattr(ceda_guid, 'ceda_observationcollection', ceda_observationCollection.id)
180            self.epbRepo.moles3EPB.persistInstance(ceda_guid)
181            DataEntityProcessor.log.info("GUID for this ObservationCollection: %s" % (ceda_guid.id))
182
183    def _processDOI(self, deploymentMigration, ceda_observation, deProcessor, single_deployment):       
184        doi = findDOIInMigrationDocument(deploymentMigration)                                                             
185        if single_deployment:
186            if doi is None:
187                doi = findDOIInMigrationDocument(self._dataEntityMigration)
188                    #collection_identifier = Moles3EPB.extractCollectionIdentifierByTitle(MD_CODE_MOLES2_CITATION, self.migrationSessions.molesSession)
189                    #if collection_identifier.count()==1:
190                    #    ceda_observation.identifier.append(collection_identifier.first())               
191        deProcessor.assignDOI(ceda_observation, doi)
192
193    def _processDeploymentMigration(self, deploymentMigration, single_deployment):                                     
194        deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration, self.epbRepo)
195        try:
196            DataEntityProcessor.log.info("Processing deployment: %s" % (migrationObjectDescription(deploymentMigration)))
197            ceda_observation = deProcessor.process()   
198            try:                           
199                self._processDOI(deploymentMigration, ceda_observation, deProcessor, single_deployment)
200            except Exception as ex:
201                pass                               
202        except NoAssociatedAuthor as ex:
203            raise ex                 
204        except Exception as ex:               
205            raise MigrationObjectException(ex)                     
206       
207        return ceda_observation
208   
209    def process(self):
210        obsColl = None
211        exs = []
212        DataEntityProcessor.log.info("Processing dataEntity: %s" % (migrationObjectDescription(self._dataEntityMigration)))
213        try :
214            if self._dataEntityMigration.ceda_observation_coll_id:
215                #obsColl = self.epbRepo.moles3EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id)
216                obsColl = self.epbRepo.moles3EPB.searchSelectiveLoad(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id, \
217                                                           ['identifier'])
218            else:
219                obsColl = CEDA_ObservationCollection()
220            self._execute(obsColl)                 
221        except Exception as ex:
222            self._report.append(ex)
223            return self._report     
224       
225        #retrieves the associated deployment links from the data_entity
226        deploymentsLinks = findDeploymentsInDE(self._dataEntityMigration)       
227        #retrieves the DataEntityMigration sorted by creation date
228        deploymentMigrations = self.epbRepo.migrationEPB.getAllDeploymentsMigrationByDataEntitySortedByDate( \
229                                                self._dataEntityMigration, deploymentsLinks)
230   
231        if deploymentMigrations is None or len(deploymentMigrations) == 0:
232            self._report.append(NoAssociatedDeployments(self._dataEntityMigration))
233            return self._report
234       
235        howManydm = len(deploymentMigrations)           
236        for deploymentMigration in deploymentMigrations:
237            try:
238                ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1)
239                #Is a first time process?
240                if not self.epbRepo.moles3EPB.observationCollectionHasObservation(getattr(obsColl, 'id'), getattr(ceda_observation, 'id')):
241                    self.epbRepo.moles3EPB.updateCedaObject(obsColl, {'member': ceda_observation})
242                                     
243            except Exception as ex:
244                self._report.append(ex)               
245        self._assignGeographicExtent(obsColl)
246         
247        return self._report
Note: See TracBrowser for help on using the repository browser.