source: mauRepo/MolesManager/trunk/cedaMoles/libs/migration/processor/dataEntity.py @ 8517

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/cedaMoles/libs/migration/processor/dataEntity.py@8517
Revision 8517, 15.2 KB checked in by mnagni, 8 years ago (diff)

Incomplete - # 22534: Add versiojn number to the gui page
 http://team.ceda.ac.uk/trac/ceda/ticket/22534
New classes for the moles2gui library

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 15 Nov 2011
30
31@author: Maurizio Nagni
32'''
33from cedaMoles.libs.migration.exception.exceptions import MigrationObjectException, NoAssociatedAuthor,\
34    migrationObjectDescription, NoAssociatedDeployments,\
35    NoGeographicalExtensionException, NoCitationException
36from cedaMoles.libs.migration.processor.commons import find_deployments_in_de,\
37    create_md_identifier, extract_content,\
38    has_mo_same_hash, create_ci_citation, create_ci_date, find_published_date,\
39    isoDateTimeStringToTimeDate, find_updated_date, create_date,\
40    calculate_hash, find_doi_in_migration_document,\
41    has_mo_been_processed, get_atom_document_hash_by_mo, extract_title, extract_summary,\
42    create_ex_geographic_boundingbox, from_date_string_to_pt,\
43    compare_phenomenon_times, compareGeographicBoundingBoxes
44from cedaMoles.libs.migration.processor.deployment import DeploymentProcessor
45from logging import StreamHandler
46import logging
47from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_datetypecode import CI_DateTypeCode
48from cedaMoles.MolesManager.ceda_guid import CedaGUID
49from cedaMoles.MolesManager.codelist import MM_ObservationPublicationStateValue,\
50    getCLValue
51from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection
52from cedaMoles.MolesManager.moles3epb import unify_observation_collection_phenomenon_time
53CEDA_TITLE = 'ceda_title'
54
55class DataEntityProcessor(object):
56    log = logging.getLogger('DataEntityProcessor')
57    log.addHandler(StreamHandler())
58    log.setLevel(logging.INFO) 
59    '''
60        Processes a DataEntityMigration item. Note that each DataEntity is associated to a "dataent_xxxx" file in Moles2
61    '''       
62    def __init__(self, dataEntityMigration, epbRepo):
63        '''
64            Initializes the class
65            @param _dataEntityMigration: the DataEntityMigration instance
66            @param epbRepo: an instance of EPBRepo
67        '''
68        #if dataEntityMigration is None:
69        #    raise MigrationObjectException("DataEntityProcessor cannot process an None item")
70        self._dataEntityMigration = dataEntityMigration       
71        self._dataEntityHasSameHash = has_mo_same_hash(self._dataEntityMigration)   
72        self._dataEntityHasBeenProcessed = has_mo_been_processed(self._dataEntityMigration)   
73        self.epbRepo = epbRepo
74        self._report = []
75
76    def _assignGeographicExtent(self, ceda_observationCollection):
77        bbox = self.epbRepo.moles3EPB.getUnifyObservationCollectionGEAsBBox(ceda_observationCollection)
78        if bbox is not None:
79            upperCornerData, lowerCornerData = bbox[4:len(bbox)-1].split(',')
80            east, north = upperCornerData.split()
81            west, south = lowerCornerData.split()
82            geographicExtent = create_ex_geographic_boundingbox(float(east), float(north), \
83                                                              float(west), float(south))
84           
85            if len(ceda_observationCollection.geographicExtent) == 0 or \
86                (len(ceda_observationCollection.geographicExtent) > 0 and \
87                    not compareGeographicBoundingBoxes(geographicExtent, \
88                                                  ceda_observationCollection.geographicExtent[0])):
89                self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, \
90                                                        {'geographicExtent': geographicExtent})                       
91        else:
92            self._report.append(NoGeographicalExtensionException(self._dataEntityMigration))         
93       
94    def _assignPhenomenonTime(self, ceda_observationCollection):
95        start, end = unify_observation_collection_phenomenon_time(ceda_observationCollection)
96        dateString = start
97        if end is not None:
98            dateString = '%s/%s' % (start, end)
99        pt = from_date_string_to_pt(dateString)
100       
101        if pt is not None and ceda_observationCollection.phenomenonTime is not None \
102            and (len(ceda_observationCollection.phenomenonTime) == 0 \
103                 or (len(ceda_observationCollection.phenomenonTime) == 1 and \
104                     not compare_phenomenon_times(ceda_observationCollection.phenomenonTime[0], pt))):
105            self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, {'phenomenonTime': pt})       
106
107    def _assignDescription(self, ceda_observationCollection):
108        description = extract_summary(self._dataEntityMigration)
109           
110        if description:
111            ceda_observationCollection.description = description
112   
113    def _processTitle(self, ceda_observationCollection):
114        ititle = extract_title(self._dataEntityMigration)
115        if ceda_observationCollection.identifier:
116            for ident in ceda_observationCollection.identifier:
117                if ident.authority.title == CEDA_TITLE:
118                    if ident.code != ititle:
119                        ident.code = ititle
120                    else:
121                        return           
122 
123        #Else create new
124        i_citation = create_ci_citation(title = CEDA_TITLE)
125        newIdentifier = create_md_identifier(code = ititle, authority=i_citation)
126       
127        if self._dataEntityHasBeenProcessed:
128            self._report.append('The _assignGeographicExtent update is \
129                                skipped because not implemented')
130            DataEntityProcessor.log.warn('The _assignGeographicExtent \
131                                update is skipped because not implemented')
132            return
133           
134        self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, \
135                                                {'identifier': newIdentifier})
136   
137    def _processCitation(self, ceda_observationCollection):
138        contentDict = extract_content(self._dataEntityMigration)
139        if not contentDict.has_key('citation'):
140            self._report.append((NoCitationException(self._dataEntityMigration)))
141            DataEntityProcessor.log.info("The migration object "\
142                                + migrationObjectDescription(self._dataEntityMigration) \
143                                + " has not associated cedacat:citation")
144            return
145
146        ci_dates = []
147        doc_date = find_published_date(self._dataEntityMigration)           
148        if doc_date:
149            i_date = create_date(isoDateTimeStringToTimeDate(doc_date))           
150            ci_dates.append(create_ci_date(getCLValue(CI_DateTypeCode.cl_publication), date = i_date))
151
152        doc_date = find_updated_date(self._dataEntityMigration)           
153        if doc_date:
154            i_date = create_date(isoDateTimeStringToTimeDate(doc_date))               
155            ci_dates.append(create_ci_date(getCLValue(CI_DateTypeCode.cl_revision), date = i_date))
156           
157        i_citation = create_ci_citation(title = 'ceda_moles2_citation', date=ci_dates)
158        newIdentifier = create_md_identifier(code = contentDict['citation'], authority=i_citation)
159
160        if self._dataEntityHasBeenProcessed:
161            DataEntityProcessor.log.warn('The _assignGeographicExtent update is skipped because not implemented')
162            return
163       
164        ceda_observationCollection = self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, {'identifier': newIdentifier})
165       
166
167    def _execute(self, ceda_observationCollection): 
168        """
169            Creates a new CEDA_ObservationCollection instance in the Moles3DB using the self._dataEntityMigration object.
170            If successful adds the new instance ID to the related DataEntityMigration object
171            @return: the persisted CEDA_ObservationCollection element
172        """
173       
174        if not self._dataEntityHasSameHash:
175            self._processTitle(ceda_observationCollection)
176            self._assignDescription(ceda_observationCollection)
177            self._processCitation(ceda_observationCollection)
178       
179        #Is a first time process?
180        if not has_mo_been_processed(self._dataEntityMigration):
181            docHash = get_atom_document_hash_by_mo(self._dataEntityMigration)
182            ceda_observationCollection.publicationState = getCLValue(MM_ObservationPublicationStateValue.cl_working)
183            self.epbRepo.moles3EPB.persistInstance(ceda_observationCollection)
184            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \
185                {'ceda_observation_coll_id': ceda_observationCollection.id, \
186                 'doc_hash': docHash})
187
188        #Has to updated the hash?
189        if not self._dataEntityHasSameHash and has_mo_been_processed(self._dataEntityMigration):
190            docHash = get_atom_document_hash_by_mo(self._dataEntityMigration)
191            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \
192                {'doc_hash': docHash})
193       
194        #Has a proper CEDAGUID?
195        if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observationCollection) is None:
196            #Adds the CedaGUID
197            ceda_guid = CedaGUID()
198            ceda_guid.id = calculate_hash(self._dataEntityMigration.data_ent_id)
199            setattr(ceda_guid, 'ceda_observationcollection', ceda_observationCollection.id)
200            self.epbRepo.moles3EPB.persistInstance(ceda_guid)
201            DataEntityProcessor.log.info("GUID for this ObservationCollection: %s" % (ceda_guid.id))
202
203    def _processDOI(self, deploymentMigration, ceda_observation, deProcessor, single_deployment):       
204        doi = find_doi_in_migration_document(deploymentMigration)                                                             
205        if single_deployment:
206            if doi is None:
207                doi = find_doi_in_migration_document(self._dataEntityMigration)
208                    #collection_identifier = Moles3EPB.extractCollectionIdentifierByTitle(MD_CODE_MOLES2_CITATION, self.migrationSessions.molesSession)
209                    #if collection_identifier.count()==1:
210                    #    ceda_observation.identifier.append(collection_identifier.first())               
211        deProcessor.assignDOI(ceda_observation, doi)
212
213    def _assignObservationCollectionToProject(self, obsColl, ceda_observation):
214        if ceda_observation.inSupportOf is None:
215            return
216       
217        for proj_coll in ceda_observation.inSupportOf.observationCollection:
218            if proj_coll.id == obsColl.id:
219                return
220       
221        ceda_observation.inSupportOf.observationCollection.append(obsColl)
222
223    def _processDeploymentMigration(self, deploymentMigration, single_deployment):                                     
224        deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration, self.epbRepo)
225        try:
226            DataEntityProcessor.log.info("Processing deployment: %s" % (migrationObjectDescription(deploymentMigration)))
227            obs_ex_report, ceda_observation = deProcessor.process()
228            self._report.extend(obs_ex_report)
229            try:                           
230                self._processDOI(deploymentMigration, ceda_observation, deProcessor, single_deployment)
231            except Exception as ex:
232                pass                               
233        except NoAssociatedAuthor as ex:
234            raise ex                 
235        except Exception as ex:               
236            raise MigrationObjectException(ex)                     
237       
238        return ceda_observation
239   
240    def process(self):
241        obsColl = None
242        DataEntityProcessor.log.info("Processing dataEntity: %s" % (migrationObjectDescription(self._dataEntityMigration)))
243        try :
244            if self._dataEntityMigration.ceda_observation_coll_id:
245                #obsColl = self.epbRepo.moles3EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id)
246                obsColl = self.epbRepo.moles3EPB.searchSelectiveLoad(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id, \
247                                                           ['identifier'])
248            else:
249                obsColl = CEDA_ObservationCollection()
250            self._execute(obsColl)                 
251        except Exception as ex:
252            self._report.append(ex)
253            return self._report     
254       
255        #retrieves the associated deployment links from the data_entity
256        deploymentsLinks = find_deployments_in_de(self._dataEntityMigration)       
257        #retrieves the DataEntityMigration sorted by creation date
258        deploymentMigrations = self.epbRepo.migrationEPB.getAllDeploymentsMigrationByDataEntitySortedByDate( \
259                                                self._dataEntityMigration, deploymentsLinks)
260   
261        if deploymentMigrations is None or len(deploymentMigrations) == 0:
262            self._report.append(NoAssociatedDeployments(self._dataEntityMigration))
263            return self._report
264       
265        howManydm = len(deploymentMigrations)           
266        for deploymentMigration in deploymentMigrations:
267            try:
268                ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1)               
269                #Is a first time process?
270                if not self.epbRepo.moles3EPB.observationCollectionHasObservation(getattr(obsColl, 'id'), getattr(ceda_observation, 'id')):
271                    self.epbRepo.moles3EPB.updateCedaObject(obsColl, {'member': ceda_observation})
272               
273                self._assignObservationCollectionToProject(obsColl, ceda_observation)                   
274            except Exception as ex:
275                self._report.append(ex)               
276                       
277        self._assignGeographicExtent(obsColl)
278        self._assignPhenomenonTime(obsColl)       
279         
280        return self._report
Note: See TracBrowser for help on using the repository browser.