source: mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py @ 8460

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py@8460
Revision 8460, 34.9 KB checked in by mnagni, 7 years ago (diff)

Complete - # 22489: CEDA Observation Collection - phenomenonTime
 http://team.ceda.ac.uk/trac/ceda/ticket/22489
Complete - # 22518: The description is broken
 http://team.ceda.ac.uk/trac/ceda/ticket/22518
Complete - # 22488: CEDA Observation Collection - Geographical Extent
 http://team.ceda.ac.uk/trac/ceda/ticket/22488

Now the Moles3EPB explicitly call the "synchronise" method in the SQLAlchemy mapped classes to assure the persistence of the data
Uses the CedaMolesModel? v 0.1.5 which correct a major problem in synchronise the instances with database

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 15 Nov 2011
30
31@author: Maurizio Nagni
32'''
33from libs.migration.processor.commons import findMolesLineage,\
34    createMO_ResponsiblePartyInfo,\
35    DO_BADC, DO_NEODC, findAuthorsInResource, \
36    createMD_Identifier, extractSummary, extractQuality, \
37    hasMOSameHash, getAtomDocumentHashByMO, extractTitle,\
38        createCEDA_Result,\
39    createEX_GeographicBoundingBox, extractGeographicExtentInMigrationDocument, findDownloadLinksInMigrationDocument,\
40    extractContent, createCI_Citation, createCI_Date, createDate,\
41    createTM_Position, createTM_Instant, extractMolesCreationDate,\
42    createDateTime, isoDateTimeStringToTimeDate, extractMolesProviderID,\
43    DO_UKSSDC, createMO_Organization,\
44    createCI_Contact, createCI_Address, createCI_OnlineResource,\
45    createCI_Telephone, extractMolesTemporalRange, isoDateStringToTimeDate,\
46    createTM_Period, findAccessLinksInMigrationDocument,\
47    findLinksInDeployment, createMD_LegalConstraints,\
48    createDQ_Element, createDQ_ConformanceResult, findUpdatedDate,\
49    createMD_Metadata, createMO_OnlineResource, createCEDA_Review, calculateHash,\
50    createCI_ResponsibleParty, extractUpdateFrequency,\
51    findLinksInMigrationDocument, findSubTypeInDPT, extractMolesPublishedDate,\
52    createMD_Keywords, hasMOBeenProcessed, createMO_Individual,\
53    fromDateStringToPhenomenonTime, fromPhenomenonTimeToString,\
54    comparePhenomenonTimes, compareGeographicBoundingBoxes
55from libs.epb import EPB
56from libs.migration.processor.deployment_data import DeploymentDataProcessor
57from libs.migration.exception.exceptions import NoDataLineage,\
58    NoAssociatedAuthor, NoAssociatedDeployments,\
59    NoGeographicalExtensionException
60from libs.migration.authors import authors
61from logging import StreamHandler
62import logging
63from datetime import date
64from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_onlinefunctioncode import CI_OnLineFunctionCode
65from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_datetypecode import CI_DateTypeCode
66from ea_model.iso_19115_2006_metadata_corrigendum.constraint_information.md_restrictioncode import MD_RestrictionCode
67from copy import deepcopy
68import datetime
69from MolesManager.ceda_guid import CedaGUID
70from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_rolecode import CI_RoleCode
71from ea_model.iso_19115_2006_metadata_corrigendum.maintenance_information.md_maintenancefrequencycode import MD_MaintenanceFrequencyCode
72from ea_model.moles3_4.utilities.ceda_rolevalue import CEDA_RoleValue
73from ea_model.moles3_4.observation.mo_observationpublicationstatevalue import MO_ObservationPublicationStateValue
74from MolesManager.codelist import MM_RoleValue,\
75    MM_ObservationPublicationStateValue, getCLValue
76from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation
77from ea_model.ceda_metadatamodel.ceda_result.ceda_curationvalue import CEDA_CurationValue
78from ea_model.ceda_metadatamodel.ceda_utilities.ceda_reviewstatusvalue import CEDA_ReviewStatusValue
79from ea_model.ceda_metadatamodel.ceda_utilities.ceda_reviewfrequencyvalue import CEDA_ReviewFrequencyValue
80
81MET_GEO_FEATURE = 'Meteorological geographical features'
82ORTHOIMAGERY = 'Orthoimagery'
83           
84class DeploymentProcessor(object):
85    '''
86        Migrates a deployment element in a CEDA_Observation entity
87    '''
88    publisherName = 'CEDA'
89   
90    log = logging.getLogger('DeploymentProcessor')
91    log.addHandler(StreamHandler())
92    log.setLevel(logging.INFO)   
93    def __init__(self, dataEntityMigration, deploymentMigration, epbRepo):
94        '''
95            Initializes the class
96            @param dataEntityMigration: a DataEntityMigration instance
97            @param deploymentMigration: the DeploymentMigration instance
98            @param epbRepo: an instance of EPBRepo             
99        '''             
100        self._dataEntityMigration = dataEntityMigration
101        self._deploymentMigration = deploymentMigration
102        self.epbRepo = epbRepo
103        self._dataEntityHasSameHash = hasMOSameHash(self._dataEntityMigration)
104        self._deploymentHasSameHash = hasMOSameHash(self._deploymentMigration)
105        self._deploymentHasBeenProcessed = hasMOBeenProcessed(self._deploymentMigration)
106        self._report = []
107
108    def _addResponsiblePartyInfo(self, oldResponsiblePartyInfos, newResponsiblePartyInfo):
109        opi = None
110        for oldPartyInfo in oldResponsiblePartyInfos:
111            if oldPartyInfo.role == newResponsiblePartyInfo.role:
112                opi = oldPartyInfo
113                break
114           
115        if len(oldResponsiblePartyInfos) == 0 or opi is None:
116            oldResponsiblePartyInfos.append(newResponsiblePartyInfo)
117       
118        if opi is not None: 
119            for np in newResponsiblePartyInfo.party:
120                opi.party.append(np)
121       
122
123
124
125    def _extractIndividualsAndOrganizations(self, tmp_auth, whereAreAuthors):
126        if whereAreAuthors is None:
127            raise NoAssociatedDeployments("migrationObject is None")
128        try:
129            if tmp_auth['author'] == 'unknown':
130                doc_authors = findAuthorsInResource(self._dataEntityMigration)
131                tmp_auth = authors[doc_authors['authors']]
132           
133            ret = {'ind': [], 'org': [], 'co_ind': [], 'co_org': []}
134            if tmp_auth['type'] == 'ind':
135                ret['ind'].append(tmp_auth['author'])
136            elif tmp_auth['type'] == 'org':
137                ret['org'].append(tmp_auth['author'])           
138               
139            if tmp_auth['author'] == 'unknown':
140                DeploymentProcessor.log.debug("%s %s %s has unknown author" \
141                                             % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name))       
142
143            for item in tmp_auth['co_author_type']:           
144                if (tmp_auth['co_author'][tmp_auth['co_author_type'].index(item)] == 'unknown'):
145                    doc_authors = findAuthorsInResource(self._dataEntityMigration)
146                    tmp_auth = authors[doc_authors['contributors']]
147                    break
148                 
149            for index in range(len(tmp_auth['co_author_type'])): 
150                ptype = tmp_auth['co_author_type'][index]               
151                if ptype == 'ind':
152                    ret['co_ind'].append(tmp_auth['co_author'][index])
153                elif ptype == 'org':
154                    ret['co_org'].append(tmp_auth['co_author'][index])
155           
156                if (tmp_auth['co_author'][index] == 'unknown'):
157                    DeploymentProcessor.log.info("%s %s %s has unknown author" \
158                                                 % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name))
159        except Exception as e:
160            print e
161        return ret
162       
163       
164    def updateObservation(self):
165        return EPB.searchOrCreate(CEDA_Observation, self._deploymentMigration.ceda_observation_id)
166
167    def assignDOI(self, observation, doi):
168        if doi and doi.has_key('href'):
169            doi = doi['href'][22:]           
170           
171            #Check if a doi has been already assigned
172            observation = self.epbRepo.moles3EPB.loadAttributes(observation, 'identifier')
173            obs_identifier = observation.identifier
174            if obs_identifier:
175                for ident in obs_identifier:
176                    if ident.code == doi:
177                        return
178           
179            py_date = None
180            cited_responsible = createCI_ResponsibleParty(role=getCLValue(CI_RoleCode.cl_publisher), \
181                                                              organizationName='NERC - British Atmospheric Data Centre')
182            if doi.upper() == '10.5285/E8F43A51-0198-4323-A926-FE69225D57DD':
183                py_date = date(2011, 4, 1)
184            elif doi.upper() == '10.5285/78114093-E2BD-4601-8AE5-3551E62AEF2B':
185                py_date = date(2011, 11, 29)               
186            elif doi.upper() == '10.5285/DB8D8981-1A51-4D6E-81C0-CCED9B921390':
187                py_date = date(2012, 4, 16)
188            elif doi.upper() == '10.5285/639A3714-BC74-46A6-9026-64931F355E07':
189                py_date = date(2012, 4, 16)               
190               
191            if py_date:   
192                dt = createDate(py_date)
193                ci_date = createCI_Date(getCLValue(CI_DateTypeCode.cl_publication), date = dt)
194                i_authority = createCI_Citation("DOI", date = ci_date)
195                identifier = createMD_Identifier(code = doi, authority=i_authority)
196                self.epbRepo.moles3EPB.updateCedaObject(observation, {'identifier': identifier})
197                DeploymentProcessor.log.info("DOI: %s" % (doi))                                 
198
199    def _assignKeywords(self, ceda_observation):
200        if self._deploymentHasSameHash:
201            return
202       
203        provider_id = extractMolesProviderID(self._deploymentMigration)
204        i_keywords = []
205        if provider_id == DO_BADC:
206            i_keywords.append(MET_GEO_FEATURE)
207        if provider_id == DO_NEODC:
208            i_keywords.append(ORTHOIMAGERY)
209        if len(i_keywords) > 0:
210            #Is a first time process?
211            if not self._deploymentHasBeenProcessed: 
212                ceda_observation.keywords.append(createMD_Keywords(i_keywords))
213            else:
214                ceda_observation.keywords.keyword = i_keywords         
215
216    def _assignLineage(self, observation):
217        if self._deploymentHasSameHash:
218            return
219       
220        data_lineage = findMolesLineage(self._dataEntityMigration)
221        if data_lineage is None:
222            raise NoDataLineage(self._dataEntityMigration)
223       
224        if data_lineage != observation.dataLineage:
225            observation.dataLineage = data_lineage 
226
227    def _assignResult(self, observation):
228        if self._deploymentHasSameHash and self._dataEntityHasSameHash:
229            return
230
231        i_sources = []               
232        download = findDownloadLinksInMigrationDocument(self._deploymentMigration)
233        content = None
234        if len(download) == 0:
235            download = findDownloadLinksInMigrationDocument(self._dataEntityMigration)
236            content = extractContent(self._dataEntityMigration)
237        else:
238            content = extractContent(self._deploymentMigration)
239        for dwn in download:
240            int_description = None
241            int_applicationProfile = None
242            if content.has_key('formats'):
243                #int_applicationProfile = content['formats']
244                pass
245            if dwn['href'].startswith('http://badc.nerc.ac.uk/browse') or dwn['href'].startswith('http://neodc.nerc.ac.uk/browse'):
246                int_description = "download directly from archive"   
247            i_sources.append(createMO_OnlineResource(linkage = dwn['href'], name = dwn['title'], \
248                                                   function = getCLValue(CI_OnLineFunctionCode.cl_download), \
249                                                   description = int_description, applicationProfile = int_applicationProfile))
250           
251        dataentity_id = '%s__ATOM__%s' % (self._dataEntityMigration.doc_owner, self._dataEntityMigration.doc_name)
252        dataentity_id = dataentity_id.replace('.atom', '')           
253        infodb_de = self.epbRepo.infodbEPB.getCedaInfoApp_dataentityByDE_ID(dataentity_id)
254        i_logical_path = '/dummy'
255        if infodb_de is None:
256            i_logical_path = dwn['href'][dwn['href'].index('/browse/') + 7:]
257       
258        if infodb_de and infodb_de.has_key('logical_path'):
259            i_logical_path = infodb_de['logical_path']
260               
261        i_category = getCLValue(CEDA_CurationValue.cl_a)                                 
262        if infodb_de and infodb_de.has_key('category') and infodb_de['category']:
263            i_category = getCLValue(CEDA_CurationValue.from_string(infodb_de['category'].lower()))     
264
265        if not self._deploymentHasBeenProcessed:                     
266            observation.result = createCEDA_Result(i_category, i_logical_path, source = i_sources)
267            return 
268       
269        if observation.result.internalPath != i_logical_path:
270            observation.result.internalPath = i_logical_path
271           
272        if observation.result.curationCategory != i_category:
273            observation.result.curationCategory = i_category
274                       
275        #Still have to update observation.result.source
276
277    def _assignPublisherCurator(self, observation):
278        if self._deploymentHasSameHash:
279            return
280           
281        provider_id = extractMolesProviderID(self._deploymentMigration)
282        party = None
283        if provider_id == DO_BADC:
284            i_linkage = 'http://badc.rl.ac.uk'
285            i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'British Atmospheric Data Centre Website')
286            i_address = createCI_Address(deliveryPoint = ['British Atmospheric Data Centre, STFC Rutherford Appleton Laboratory'], \
287                                         electronicMailAddress=['badc@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford')                                   
288            i_phone = createCI_Telephone(voice=['+44(0)1235 446432'])                                 
289            contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources)                       
290            party = createMO_Organization(name = "NERC - British Atmospheric Data Centre", contactInfo = [contact])
291        elif provider_id == DO_NEODC:
292            i_linkage = 'http://www.neodc.rl.ac.uk'
293            i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'NERC Earth Observation Data Centre website')
294            i_address = createCI_Address(deliveryPoint = ['NERC - Earth Observation Data Centre, STFC Rutherford Appleton Laboratory'], \
295                                         electronicMailAddress=['neodc@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford')                                   
296            i_phone = createCI_Telephone(voice=['+44(0)1235 446432'])                                 
297            contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources)                       
298            party = createMO_Organization(name = 'NERC - Earth Observation Data Centre', contactInfo = [contact])
299        elif provider_id == DO_UKSSDC:
300            i_linkage = 'http://www.ukssdc.rl.ac.uk'
301            i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'UK Solar System Data Centre website')
302            i_address = createCI_Address(deliveryPoint = ['UK Solar System Data Centre, STFC Rutherford Appleton Laboratory'], \
303                                         electronicMailAddress=['support@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford')                                   
304            i_phone = createCI_Telephone(voice=['+44(0)1235 445173'])                                 
305            contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources)                       
306            party = createMO_Organization(name = 'NERC - UK Solar System Data Centre', contactInfo = [contact])
307       
308        if party and not self._deploymentHasBeenProcessed:
309            newrp = []
310            newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_publisher), [party]))
311            newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_custodian), deepcopy([party])))
312            newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_distributor), deepcopy([party])))
313            newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_pointofcontact), deepcopy([party])))
314            newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_metadataowner), deepcopy([party])))
315            newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_curator), deepcopy([party])))
316            self.epbRepo.moles3EPB.updateCedaObject(observation, {'relatedParty': newrp})           
317            return       
318        #Still have to update observation.result.source
319
320    def _assignQuality(self, observation):
321        if self._dataEntityHasSameHash:
322            return
323               
324        doc_quality = extractQuality(self._dataEntityMigration)
325        doc_date = findUpdatedDate(self._dataEntityMigration)
326        ci_dates = []           
327        if doc_date:
328            i_date = createDate(isoDateTimeStringToTimeDate(doc_date))               
329            ci_dates.append(createCI_Date(getCLValue(CI_DateTypeCode.cl_revision), date = i_date))           
330        else:
331            i_date = createDate(isoDateTimeStringToTimeDate(datetime.datetime.now()))
332            ci_dates.append(createCI_Date(getCLValue(CI_DateTypeCode.cl_creation), date = i_date))
333
334        i_specification = createCI_Citation(title = "CEDA Data Quality Statement", date=ci_dates)
335        i_dq_result = createDQ_ConformanceResult(doc_quality, True, i_specification)
336        i_quality_element = createDQ_Element(i_dq_result)
337       
338        if not self._deploymentHasBeenProcessed:     
339            observation.resultQuality.append(i_quality_element)
340            return               
341        #Still have to update observation.result.source
342
343    def _assignDescription(self, observation):
344        if self._dataEntityHasSameHash and self._deploymentHasSameHash:
345            return
346       
347        description = extractSummary(self._deploymentMigration)
348        if description is None:
349            description = extractSummary(self._dataEntityMigration)
350           
351        if description:
352            observation.description = description
353
354    def _assignTitle(self, observation): 
355        if self._dataEntityHasSameHash and self._deploymentHasSameHash:
356            return
357                     
358        doc_title = extractTitle(self._deploymentMigration)
359        if doc_title is None:
360            doc_title = extractTitle(self._dataEntityMigration)
361       
362        if doc_title.startswith('deployment_') or doc_title.startswith('Deployment_'):
363            links = findLinksInMigrationDocument(self._deploymentMigration)
364            dptList = links['DPT']
365            if links.has_key('DPT'):
366                doc_title = 'Data from ' + dptList[0]['title']
367                if len(dptList) > 2:
368                    for dpt in dptList[1:-2]:
369                        doc_title += ', ' + dpt['title']
370                    if len(dptList) > 1:
371                        doc_title += ' and ' + dptList[-1]
372                                                       
373            links = findLinksInDeployment(self._deploymentMigration)
374            if links.has_key('OBS'):
375                obsList = []
376                for obs in links['OBS']:
377                    observationStation = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, obs + '.atom')
378                    obsList.append((extractTitle(observationStation), findSubTypeInDPT(observationStation)))
379               
380                if obsList[0][1] in ['stationary platform' ,'moving platform', 'ship','aircraft','satellite','computer']:
381                    doc_title += ' on '
382                else : 
383                    doc_title += ' at '
384                    doc_title += obsList[0][0]
385                if len(obsList) > 2:
386                    for obs in obsList[1:-2]:
387                        doc_title += ', ' + obs[0]
388                    if len(obsList) > 1:
389                        doc_title += ' and ' + obsList[-1][0]
390           
391            if links.has_key('ACTIVITY'):             
392                for link in links['ACTIVITY']:
393                    activity = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, link + '.atom')                   
394     
395                    projSubType = findSubTypeInDPT(activity)
396                    doc_title += ' for the ' + extractTitle(activity)
397                    if projSubType[0:14] == 'dgActivityData':
398                        doc_title += ' ' + projSubType[14:]
399                    else :
400                        doc_title += ' ' + projSubType                                                   
401        else:             
402            if doc_title[0:10] != 'Data from' :
403                doc_title = "Data from " + doc_title           
404        auth = createCI_Citation(title = 'ceda_title')                 
405        identifier = createMD_Identifier(code = doc_title, authority = auth)
406       
407        if not self._deploymentHasBeenProcessed: 
408            observation.identifier.append(identifier)
409            return           
410        #Still have to update observation.identifier         
411
412    def _assignGeographicExtent(self, observation):
413        if self._dataEntityHasSameHash and self._deploymentHasSameHash:
414            return 
415       
416        ge = extractGeographicExtentInMigrationDocument(self._deploymentMigration)       
417        if ge is None:
418            ge = extractGeographicExtentInMigrationDocument(self._dataEntityMigration)
419       
420        if ge is None:
421            self._report.append(NoGeographicalExtensionException(self._deploymentMigration))
422            self._report.append(NoGeographicalExtensionException(self._dataEntityMigration))
423       
424        geographicExtent = createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south'])
425
426        if len(observation.geographicExtent) == 0 or \
427            (len(observation.geographicExtent) > 0 and \
428             not compareGeographicBoundingBoxes(geographicExtent, \
429                observation.geographicExtent[0])):
430            self.epbRepo.moles3EPB.updateCedaObject(observation, {'geographicExtent': geographicExtent})             
431           
432    def _assignCreationDate(self, observation):
433        if self._deploymentHasSameHash:
434            return 
435       
436        creation_date = extractMolesPublishedDate(self._deploymentMigration)
437        if creation_date is None:
438            creation_date = extractMolesCreationDate(self._deploymentMigration)
439        py_datetime = isoDateTimeStringToTimeDate(creation_date)
440        date_time = createDateTime(py_datetime)   
441        tm_position = createTM_Position(dateTime8601 = date_time)
442       
443        if not self._deploymentHasBeenProcessed: 
444            observation.resultTime = createTM_Instant(tm_position)
445            return       
446        #Still have to update observation.geographicExtent
447
448    def _assignPhenomenonTime(self, observation):
449        if self._deploymentHasSameHash and getattr(observation, 'phenomenonTime') != None:
450            return 
451             
452        doc_phenomenon_time = extractMolesTemporalRange(self._deploymentMigration) 
453        pt = fromDateStringToPhenomenonTime(doc_phenomenon_time)   
454       
455        if not self._deploymentHasBeenProcessed or not comparePhenomenonTimes(pt,observation.phenomenonTime):
456            self.epbRepo.moles3EPB.updateCedaObject(observation, {'phenomenonTime': pt})
457        #Now update phenomenonTime but has to remove the previous phenomenonTime               
458               
459    def _assignPermission(self, observation):
460        if self._deploymentHasSameHash and self._dataEntityHasSameHash:
461            return 
462       
463        access_link = findAccessLinksInMigrationDocument(self._deploymentMigration)
464        dwn_link = findDownloadLinksInMigrationDocument(self._deploymentMigration)
465        if len(access_link) == 0:
466            access_link = findAccessLinksInMigrationDocument(self._dataEntityMigration) 
467
468        i_accessConstraints = []
469        i_use_limitation = []
470       
471        permission = None
472        if len(access_link) == 0:
473            if len(dwn_link) == 0:
474                dwn_link = findDownloadLinksInMigrationDocument(self._dataEntityMigration)
475                if dwn_link and len(dwn_link) == 1:               
476                    i_use_limitation.append("These data are open access and available through %s." % (dwn_link[0]['href']) )
477                    #i_accessConstraints.append(MD_RestrictionCode.cl_)
478                    observation.permission = createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints)
479        else:
480            if access_link and len(access_link) == 1:
481                i_use_limitation.append("Access to these data is restricted. To obtain access please apply for access at: %s" % (access_link[0]['href']))
482                i_accessConstraints.append(getCLValue(MD_RestrictionCode.cl_restricted))
483                observation.permission = createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints)
484               
485        if not self._deploymentHasBeenProcessed:
486            observation.permission = permission       
487        #Still have to update observation.permission
488                                   
489        '''                               
490        contentDict = extractContent(self._deploymentMigration)
491        if not contentDict.has_key('access-restricted'):
492            contentDict = extractContent(self._dataEntityMigration)
493        '''           
494
495    def _assignMoles2Link(self, ceda_observation):
496        if self._deploymentHasSameHash:
497            return 
498               
499        i_code = 'http://badc.nerc.ac.uk/view/%s__ATOM__%s' % (self._deploymentMigration.doc_owner, self._deploymentMigration.doc_name)
500        i_code = i_code.replace('.atom', '')
501        #i_code = buildExistDocPath(self._deploymentMigration.doc_status, DT_DEPLOYMENTS, self._deploymentMigration.doc_owner, self._deploymentMigration.doc_name)
502        i_authority = createCI_Citation('moles2url')
503        identifier = createMD_Identifier(code = i_code, authority = i_authority)
504        if not self._deploymentHasBeenProcessed:             
505            ceda_observation.identifier.append(identifier)
506        #Still have to update observation.permission           
507
508    def _assignInternalReview(self, ceda_observation):
509        if self._deploymentHasBeenProcessed:             
510            return
511               
512        i_party = createMO_Individual(name = 'Graham Parton')
513        i_reviewer = createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_processor), [i_party])
514        ceda_observation.metadataManagement.append( \
515            createCEDA_Review(reviewer=i_reviewer, reviewFrequency=getCLValue(CEDA_ReviewFrequencyValue.cl_yearly), \
516                              reviewStatus=getCLValue(CEDA_ReviewStatusValue.cl_required))) 
517
518    def _assignLanguage(self, ceda_observation):
519        if self._deploymentHasBeenProcessed:             
520            return
521       
522        i_date_stamp = createDate(datetime.datetime.now())
523        #i_contact = createCI_Citation("", date = i_date_stamp)
524        i_contact = createCI_ResponsibleParty(getCLValue(CI_RoleCode.cl_user))
525        ceda_observation.metadata = createMD_Metadata(date_stamp=i_date_stamp, contact = [i_contact], language = "English")
526                       
527    def _processResultAccumulation(self, ceda_observation): 
528        if self._dataEntityHasSameHash:             
529            return                     
530
531            updateFrequency = extractUpdateFrequency(self._dataEntityMigration)
532            if updateFrequency:
533                resultAccumulation = MD_MaintenanceFrequencyCode.from_string(updateFrequency)
534                if not self._deploymentHasBeenProcessed:
535                    self.epbRepo.moles3EPB.updateCedaObject(ceda_observation, {'resultAccumulation': resultAccumulation})             
536                    return
537        #Still have to update observation.permission
538                   
539    def _assignName(self, observation):       
540        '''
541            @param relatedPartyInfos: a MO_ResponsiblePartyInfo list
542            @return True if the documents changed, False otherwise
543        '''
544        if self._deploymentHasSameHash and self._dataEntityHasSameHash:
545            return 
546       
547        whereAreAuthors = self._deploymentMigration       
548        doc_authors = findAuthorsInResource(self._deploymentMigration)       
549        if doc_authors['authors'] in [DO_BADC, DO_NEODC]:
550            doc_authors = findAuthorsInResource(self._dataEntityMigration)
551            whereAreAuthors = self._dataEntityMigration
552               
553        ind_names = []
554        org_names = []
555        if authors.has_key(doc_authors['authors']):
556            tmp_auth = authors[doc_authors['authors']]
557            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors)
558
559            if len(ret['ind']) > 0:
560                i_party = createMO_Individual(name = ret['ind'][0])
561                observation.relatedParty.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_author), [i_party]))                 
562            if len(ret['org']) > 0:
563                i_party = createMO_Organization(name = ret['org'][0])
564                observation.relatedParty.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_author), [i_party]))
565           
566            if len(ret['ind']) > 1:
567                ind_names.extend(ret['ind'][1:])                                                       
568            if len(ret['org']) > 1:
569                org_names.extend(ret['org'][1:])
570            if len(ret['co_ind']) > 0:                               
571                ind_names.extend(ret['co_ind'])
572            if len(ret['co_org']) > 0:                               
573                org_names.extend(ret['co_org'])                   
574                       
575        else:
576            raise NoAssociatedAuthor(doc_authors['authors'], migrationObject = whereAreAuthors)
577                     
578        if doc_authors['contributors'] and authors.has_key(doc_authors['contributors']):
579            tmp_auth = authors[doc_authors['contributors']]           
580            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors)
581            ind_names.extend(ret['ind'])
582            ind_names.extend(ret['co_ind'])
583            org_names.extend(ret['org'])
584            org_names.extend(ret['co_org'])
585           
586        i_party = []
587        for nm in ind_names:
588            i_party.append(createMO_Individual(name = nm))
589               
590        for nm in org_names:
591            i_party.append(createMO_Organization(name = nm))
592           
593        if i_party:
594            if not self._deploymentHasBeenProcessed:
595                rp = createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_coinvestigator), i_party)
596                self.epbRepo.moles3EPB.updateCedaObject(observation, {'relatedParty': rp})
597                return
598        #Still have to update observation.permission
599   
600    def _execute(self, ceda_observation):
601        self._assignKeywords(ceda_observation)
602        self._assignLineage(ceda_observation)
603        self._assignResult(ceda_observation)
604        self._assignPublisherCurator(ceda_observation)                       
605        self._assignQuality(ceda_observation)       
606        self._assignDescription(ceda_observation)
607        self._assignTitle(ceda_observation)   
608        self._assignGeographicExtent(ceda_observation)               
609        self._assignCreationDate(ceda_observation)       
610        self._assignPhenomenonTime(ceda_observation)       
611        self._assignPermission(ceda_observation)       
612        self._assignMoles2Link(ceda_observation)               
613        self._assignInternalReview(ceda_observation)       
614        self._assignLanguage(ceda_observation)
615        self._processResultAccumulation(ceda_observation)           
616        self._assignName(ceda_observation)
617        #self._assignDOI(ceda_observation)       
618       
619        #Is a first time process?
620        if not hasMOBeenProcessed(self._deploymentMigration):
621            ceda_observation.publicationState = getCLValue(MM_ObservationPublicationStateValue.cl_working)         
622            docHash = getAtomDocumentHashByMO(self._dataEntityMigration)
623            self.epbRepo.moles3EPB.persistInstance(ceda_observation)       
624            self.epbRepo.migrationEPB.updateMigrationObject(self._deploymentMigration, \
625                {'ceda_observation_id': ceda_observation.id,
626                 'doc_hash': docHash})               
627       
628        #Has a proper CEDAGUID?
629        if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observation) is None:       
630            ceda_guid = CedaGUID()
631            ceda_guid.id = calculateHash(self._deploymentMigration.depl_id)
632            ceda_guid.ceda_observation = ceda_observation.id
633            self.epbRepo.moles3EPB.persistInstance(ceda_guid)
634            DeploymentProcessor.log.info("GUID for this Observation: %s" % (ceda_guid.id))
635
636
637        if not self._deploymentHasBeenProcessed:               
638            deploymentDataProcessor = DeploymentDataProcessor(self._deploymentMigration, self.epbRepo)               
639            procedure = deploymentDataProcessor.createProcess()
640            project = deploymentDataProcessor.createProject() 
641            self.epbRepo.moles3EPB.updateCedaObject(ceda_observation, {'procedure': procedure, 'inSupportOf': project})
642
643        #Still have to update observation.procedure
644        #Still have to update observation.project
645       
646        return ceda_observation
647       
648    def process(self):
649        self._report = []
650        ceda_observation = None
651        #Moles3 object exists...
652        if self._deploymentMigration.ceda_observation_id:
653            ceda_observation = self.epbRepo.moles3EPB.search(CEDA_Observation, self._deploymentMigration.ceda_observation_id)
654        else:
655            #... does not exist so create it
656            ceda_observation = CEDA_Observation()
657   
658        self._execute(ceda_observation)   
659        return self._report, ceda_observation
Note: See TracBrowser for help on using the repository browser.