source: mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py @ 8199

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py@8199
Revision 8199, 23.5 KB checked in by mnagni, 8 years ago (diff)

Incomplete - # 22399: CEDA Observation - permission
 http://team.ceda.ac.uk/trac/ceda/ticket/22399

Line 
1'''
2Created on 15 Nov 2011
3
4@author: mnagni
5'''
6from libs.migration.processor.commons import findMolesLineage,\
7    createMO_ResponsiblePartyInfo,\
8    DO_BADC, DO_NEODC, findAuthorsInResource, CEDA,\
9    createMD_Identifier, extractSummary, extractQuality, \
10    hasAtomDocumentSameHash, getAtomDocumentHashByMO, extractTitle,\
11        createCEDA_Result,\
12    createEX_GeographicBoundingBox, extractGeographicExtentInMigrationDocument, findDownloadLinksInMigrationDocument,\
13    findDOIInMigrationDocument, extractContent, createCI_Citation, createCI_Date, createDate,\
14    createTM_Position, createTM_Instant, extractMolesCreationDate,\
15    createDateTime, isoDateTimeStringToTimeDate, extractMolesProviderID,\
16    DO_UKSSDC, createMO_ResponsiblePartyInfo_, createMO_Organization,\
17    createCI_Contact, createCI_Address, createCI_OnlineResource,\
18    createCI_Telephone, extractMolesTemporalRange, isoDateStringToTimeDate,\
19    createTM_Period, createCI_Individual, findAccessLinksInMigrationDocument,\
20    findLinksInDeployment, createMD_LegalConstraints
21from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue
22from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation
23from libs.migration.MigrationEPB import MigrationEPB
24from libs.epb import EPB
25from libs.migration.processor.deployment_data import DeploymentDataProcessor
26from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue
27from libs.migration.exception.exceptions import NoDataLineage,\
28    NoAssociatedAuthor
29from MolesManager.moles3epb import Moles3EPB
30from libs.migration.authors import authors
31from logging import StreamHandler
32import logging, datetime
33from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_onlinefunctioncode import CI_OnLineFunctionCode
34from libs.migration.processor.check.check import moles3Append
35from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_datetypecode import CI_DateTypeCode
36from ea_model.iso_19115_2006_metadata_corrigendum.constraint_information.md_restrictioncode import MD_RestrictionCode
37
38class DeploymentProcessor(object):
39    '''
40        Migrates a deployment element in a CEDA_Observation entity
41    '''
42    publisherName = 'CEDA'
43   
44    log = logging.getLogger('DeploymentProcessor')
45    log.addHandler(StreamHandler())
46    log.setLevel(logging.DEBUG)   
47    def __init__(self, dataEntityMigration, deploymentMigration, migrationSessions):
48        '''
49            Initializes the class
50            @param dataEntityMigration: a DataEntityMigration instance
51            @param deploymentMigration: the DeploymentMigration instance
52            @param migrationSessions: a MigrationSessions instance   
53        '''             
54        self._dataEntityMigration = dataEntityMigration
55        self._deploymentMigration = deploymentMigration
56        self._dataEntityHasSameHash = hasAtomDocumentSameHash(self._dataEntityMigration) and self._dataEntityMigration.doc_hash is not None
57        self._deploymentHasSameHash = hasAtomDocumentSameHash(self._deploymentMigration) and self._deploymentMigration.doc_hash is not None
58       
59        self._migrationSessions = migrationSessions
60
61    def _existsCEDAasPublisher(self):
62        for rp in self._ceda_observation.relatedParty:
63            if rp.role == MO_RoleValue.cl_publisher:
64                for party in rp.parties:
65                    if party.name == DeploymentProcessor.publisherName:
66                        return True
67        return False
68                   
69    def _extractResponsiblePartyInfo(self, authors = None, parsedParties = []):
70        '''
71            @param authors: the deployment's authors
72            @param parsedParties: a list to fill with the created MO_ResponsiblePartyInfos
73        '''
74        #First is assumend to be the author
75        parsedParties.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, authors[:1]))   
76        parsedParties.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_coinvestigator, authors[1:]))           
77
78    def _addResponsiblePartyInfo(self, oldResponsiblePartyInfos, newResponsiblePartyInfo):
79        opi = None
80        for oldPartyInfo in oldResponsiblePartyInfos:
81            if oldPartyInfo.role == newResponsiblePartyInfo.role:
82                opi = oldPartyInfo
83                break
84           
85        if len(oldResponsiblePartyInfos) == 0 or opi is None:
86            oldResponsiblePartyInfos.append(newResponsiblePartyInfo)
87       
88        if opi is not None: 
89            for np in newResponsiblePartyInfo.party:
90                opi.party.append(np)
91
92    def _appendResponsibleParty(self, observation, names, creation_function, role):
93        for nm in names:
94            i_party = creation_function(name = nm)
95            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(role, [i_party]))       
96
97    def _assignName(self, observation):       
98        '''
99            @param relatedPartyInfos: a MO_ResponsiblePartyInfo list
100            @return True if the documents changed, False otherwise
101        '''
102        whereAreAuthors = self._deploymentMigration       
103        doc_authors = findAuthorsInResource(self._deploymentMigration)       
104        if doc_authors['authors'] in [DO_BADC, DO_NEODC]:
105            doc_authors = findAuthorsInResource(self._dataEntityMigration)
106            whereAreAuthors = self._dataEntityMigration
107
108        i_party = createMO_Organization(name = CEDA)
109        observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_curator, [i_party]))                               
110       
111        if authors.has_key(doc_authors['authors']):
112            tmp_auth = authors[doc_authors['authors']]
113            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors)
114            if len(ret['ind']) > 0:
115                self._appendResponsibleParty(observation, ret['ind'], createCI_Individual, MO_RoleValue.cl_author)               
116            if len(ret['org']) > 0:
117                self._appendResponsibleParty(observation, ret['org'], createMO_Organization, MO_RoleValue.cl_author)                               
118            if len(ret['co_ind']) > 0:
119                self._appendResponsibleParty(observation, ret['co_ind'], createCI_Individual, MO_RoleValue.cl_coinvestigator)               
120            if len(ret['co_org']) > 0:
121                self._appendResponsibleParty(observation, ret['co_org'], createMO_Organization, MO_RoleValue.cl_coinvestigator)                                                                                         
122        else:
123            raise NoAssociatedAuthor(doc_authors['authors'], migrationObject = whereAreAuthors)
124       
125        if doc_authors['contributors'] is None:
126            return
127       
128        if authors.has_key(doc_authors['contributors']):
129            tmp_auth = authors[doc_authors['contributors']]           
130            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors)
131            if len(ret['ind']) > 0:
132                self._appendResponsibleParty(observation, ret['ind'], createCI_Individual, MO_RoleValue.cl_coinvestigator)
133            if len(ret['org']) > 0:
134                self._appendResponsibleParty(observation, ret['org'], createMO_Organization, MO_RoleValue.cl_coinvestigator)               
135            if len(ret['co_ind']) > 0:
136                self._appendResponsibleParty(observation, ret['co_ind'], createCI_Individual, MO_RoleValue.cl_coinvestigator)               
137            if len(ret['co_org']) > 0:
138                self._appendResponsibleParty(observation, ret['co_org'], createMO_Organization, MO_RoleValue.cl_coinvestigator)                                                       
139        else:
140            raise NoAssociatedAuthor(doc_authors['contributors'], migrationObject = whereAreAuthors)
141
142
143    def _extractIndividualsAndOrganizations(self, tmp_auth, whereAreAuthors):
144        if whereAreAuthors is None:
145            raise Exception("migrationObject is None")
146        try:
147            ret = {'ind': [], 'org': [], 'co_ind': [], 'co_org': []}
148            if tmp_auth['type'] == 'ind':
149                ret['ind'].append(tmp_auth['author'])
150            elif tmp_auth['type'] == 'org':
151                ret['org'].append(tmp_auth['author'])
152           
153            if tmp_auth['author'] == 'unknown':
154                DeploymentProcessor.log.info("%s %s %s has unknown author" \
155                                             % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name))       
156                 
157            for item in tmp_auth['co_author_type']:
158                if item == 'ind':
159                    ret['ind'].append(tmp_auth['co_author'][tmp_auth['co_author_type'].index(item)])
160                elif item == 'org':
161                    ret['org'].append(tmp_auth['co_author'][tmp_auth['co_author_type'].index(item)])
162           
163                if (tmp_auth['co_author'][tmp_auth['co_author_type'].index(item)] == 'unknown'):
164                    DeploymentProcessor.log.info("%s %s %s has unknown author" \
165                                                 % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name))
166        except Exception as e:
167            print e
168        return ret
169
170    def _assignLineage(self, observation):
171        data_lineage = findMolesLineage(self._dataEntityMigration)
172        if data_lineage is None:
173            raise NoDataLineage(self._dataEntityMigration)
174        Moles3EPB.addDataLineageToObservation(data_lineage, observation, self._migrationSessions.molesSession)       
175       
176    def _assignTitle(self, observation):       
177        doc_title = extractTitle(self._deploymentMigration)
178       
179        if doc_title.startswith('deployment_') or doc_title.startswith('Deployment_'):
180            doc_title = extractTitle(self._dataEntityMigration)           
181        auth = createCI_Citation(title = 'ceda_title')                 
182        Moles3EPB.addIdentifierToObservation(createMD_Identifier(code = doc_title, authority = auth), \
183                                             observation, self._migrationSessions.molesSession)
184
185    def _assignDescription(self, observation):
186        summary = extractSummary(self._deploymentMigration, self._dataEntityMigration)
187        Moles3EPB.addDescriptionToObservation(summary, observation, self._migrationSessions.molesSession)
188       
189    def _assignQuality(self, observation):
190        doc_quality = extractQuality(self._dataEntityMigration)
191        Moles3EPB.addDocumentQuality(doc_quality, observation, self._migrationSessions.molesSession)
192       
193    def updateObservation(self):
194        ceda_observation = EPB.searchOrCreate(CEDA_Observation, self._migrationSessions.molesSession, \
195                                              self._deploymentMigration.ceda_observation_id)
196        return ceda_observation
197
198
199    def _assignResult(self, observation):
200        # TDB - Check that if is an update or not!       
201        download = findDownloadLinksInMigrationDocument(self._deploymentMigration)
202        content = None
203        if len(download) == 0:
204            download = findDownloadLinksInMigrationDocument(self._dataEntityMigration)
205            content = extractContent(self._dataEntityMigration)
206        else:
207            content = extractContent(self._deploymentMigration) 
208        for dwn in download:
209            int_description = None
210            int_applicationProfile = None
211            if content.has_key('formats'):
212                int_applicationProfile = content['formats']
213            if dwn['href'].startswith('http://badc.nerc.ac.uk/browse') or dwn['href'].startswith('http://neodc.nerc.ac.uk/browse'):
214                int_description = "download directly from archive"               
215            observation.result = createCEDA_Result(linkage = dwn['href'], name = dwn['title'], \
216                                                   function = CI_OnLineFunctionCode.cl_download, \
217                                                   description = int_description, applicationProfile = int_applicationProfile)                               
218
219
220    def _assignGeographicExtent(self, observation):
221        # TDB - Check that if is an update or not!
222        ge = extractGeographicExtentInMigrationDocument(self._deploymentMigration)
223        if not ge:
224            ge = extractGeographicExtentInMigrationDocument(self._dataEntityMigration)
225        if ge:
226            moles3Append(observation, 'geographicExtent', createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south']))
227
228    def _assignCreationDate(self, observation):
229        # TDB - Check that if is an update or not!
230        creation_date = extractMolesCreationDate(self._deploymentMigration)
231        py_datetime = isoDateTimeStringToTimeDate(creation_date)
232        date_time = createDateTime(py_datetime)   
233        tm_position = createTM_Position(dateTime8601 = date_time)
234        observation.resultTime = createTM_Instant(tm_position)
235
236    def _assignDOI(self, observation):
237        # TDB - Check that if is an update or not!
238        doi = findDOIInMigrationDocument(self._deploymentMigration) 
239        if not doi:
240            doi = findDOIInMigrationDocument(self._dataEntityMigration)
241        if doi:
242            identifier = createMD_Identifier(code = doi['href'])
243            ci_date = None
244            if doi['href'] == '10.5285/E8F43A51-0198-4323-A926-FE69225D57DD':
245                #dt = createDateTime('21', '11', '04', '01', '07', '25', '35')
246                py_date = datetime.date(2011, 4, 1)
247                dt = createDate(py_date)
248                ci_date = createCI_Date(CI_DateTypeCode.cl_publication, date = dt)
249            elif doi['href'] == '10.5285/78114093-E2BD-4601-8AE5-3551E62AEF2B':
250                #dt = createDateTime('21', '11', '11', '29', '11', '37', '11')
251                py_date = datetime.date(2011, 11, 29)               
252                dt = createDate(py_date)               
253                ci_date = createCI_Date(CI_DateTypeCode.cl_publication, date = dt)
254            if ci_date:               
255                identifier.authority = createCI_Citation("", date = ci_date)
256            moles3Append(observation, 'identifier', identifier)                 
257            #Moles3EPB.addIdentifierToObservation(identifier, observation, self._migrationSessions.molesSession)                   
258
259    def _assignPublisherCurator(self, observation):       
260        provider_id = extractMolesProviderID(self._deploymentMigration)
261        party = None
262        if provider_id == DO_BADC:
263            i_linkage = 'http://badc.rl.ac.uk'
264            i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'British Atmospheric Data Centre Website')
265            i_address = createCI_Address(deliveryPoint = ['British Atmospheric Data Centre, STFC Rutherford Appleton Laboratory'], \
266                                         electronicMailAddress=['badc@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford')                                   
267            i_phone = createCI_Telephone(voice=['+44(0)1235 446432'])                                   
268            contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources)                       
269            party = createMO_Organization(name = "NERC - British Atmospheric Data Centre", contactInfo = [contact])
270        elif provider_id == DO_NEODC:
271            i_linkage = 'http://www.neodc.rl.ac.uk'
272            i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'NERC Earth Observation Data Centre website')
273            i_address = createCI_Address(deliveryPoint = ['NERC - Earth Observation Data Centre, STFC Rutherford Appleton Laboratory'], \
274                                         electronicMailAddress=['neodc@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford')                                   
275            i_phone = createCI_Telephone(voice=['+44(0)1235 446432'])                                 
276            contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources)                       
277            party = createMO_Organization(name = 'NERC - British Atmospheric Data Centre', contactInfo = [contact])
278        elif provider_id == DO_UKSSDC:
279            i_linkage = 'http://www.ukssdc.rl.ac.uk'
280            i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'UK Solar System Data Centre website')
281            i_address = createCI_Address(deliveryPoint = ['UK Solar System Data Centre, STFC Rutherford Appleton Laboratory'], \
282                                         electronicMailAddress=['support@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford')                                   
283            i_phone = createCI_Telephone(voice=['+44(0)1235 445173'])                                 
284            contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources)                       
285            party = createMO_Organization(name = 'NERC - UK Solar System Data Centre', contactInfo = [contact])
286       
287        if party:
288            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_publisher, [party]))
289            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_custodian, [party]))
290            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_distributor, [party]))
291            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_pointofcontact, [party]))
292            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_metadataowner, [party]))
293            observation.relatedParty.append(createMO_ResponsiblePartyInfo_(MO_RoleValue.cl_curator, [party]))                                                           
294
295    def _assignPhenomenonTime(self, observation):       
296        doc_phenomenon_time = extractMolesTemporalRange(self._deploymentMigration)
297       
298        if doc_phenomenon_time:
299            if '/' in doc_phenomenon_time:
300                period = doc_phenomenon_time.split('/')
301                begin_date = createDate(isoDateStringToTimeDate(period[0]))
302                begin_position = createTM_Position(date8601 = begin_date)
303                begin_tm_instant = createTM_Instant(begin_position)
304               
305                end_date = createDate(isoDateStringToTimeDate(period[1]))
306                end_position = createTM_Position(date8601 = end_date)
307                end_tm_instant = createTM_Instant(end_position)
308               
309                observation.phenomenonTime = createTM_Period(begin_tm_instant, end_tm_instant)
310            else:
311                observation.phenomenonTime = createTM_Position(date8601 = createDate(isoDateStringToTimeDate(doc_phenomenon_time)))         
312
313    def _assignPermission(self, observation):
314        access_link = findAccessLinksInMigrationDocument(self._deploymentMigration)
315        if len(access_link) == 0:
316            access_link = findAccessLinksInMigrationDocument(self._dataEntityMigration) 
317
318        i_accessConstraints = []
319        i_use_limitation = []
320       
321        if len(access_link) == 0:
322            dwn_link = findDownloadLinksInMigrationDocument(self._deploymentMigration)
323        if len(dwn_link) == 0:
324            dwn_link = findDownloadLinksInMigrationDocument(self._dataEntityMigration)
325            if dwn_link and len(dwn_link) == 1:               
326                i_use_limitation.append("These data are open access and available through %s." % (dwn_link['href']) )
327                #i_accessConstraints.append(MD_RestrictionCode.cl_)
328        else:
329            if access_link and len(access_link) == 1:
330                i_use_limitation.append("Access to these data is restricted. To obtain access please apply for access at: %s" % (access_link['href']))
331                i_accessConstraints.append(MD_RestrictionCode.cl_restricted)
332                           
333        createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints)
334       
335        '''                               
336        contentDict = extractContent(self._deploymentMigration)
337        if not contentDict.has_key('access-restricted'):
338            contentDict = extractContent(self._dataEntityMigration)
339        '''
340           
341    def _assignMoles2Link(self, ceda_observation):
342        i_code = 'http://badc.nerc.ac.uk/view/%s__ATOM__%s' % (self._deploymentMigration.doc_owner, self._deploymentMigration.doc_name)
343        i_code = i_code.replace('.atom', '')
344        #i_code = buildExistDocPath(self._deploymentMigration.doc_status, DT_DEPLOYMENTS, self._deploymentMigration.doc_owner, self._deploymentMigration.doc_name)
345        i_authority = createCI_Citation('moles2url')
346        identifier = createMD_Identifier(code = i_code, authority = i_authority)
347        moles3Append(ceda_observation, 'identifier', identifier)
348
349   
350    def createNewObservation(self):
351        ceda_observation = CEDA_Observation()
352        ceda_observation.publicationState = MO_PublicationStateValue.cl_working
353        self._assignQuality(ceda_observation)
354        self._assignLineage(ceda_observation)
355        self._assignDescription(ceda_observation)
356        self._assignName(ceda_observation)
357        self._assignTitle(ceda_observation)
358        #self._assignResult(ceda_observation)   
359        self._assignGeographicExtent(ceda_observation)   
360        self._assignDOI(ceda_observation)       
361        self._assignCreationDate(ceda_observation)
362        self._assignPublisherCurator(ceda_observation)       
363        self._assignPhenomenonTime(ceda_observation)       
364        self._assignPermission(ceda_observation)       
365        self._assignMoles2Link(ceda_observation)       
366        #self._migrationSessions.molesSession.commit()
367        #self._commitMoles3()
368        #self._commitMigration()
369        Moles3EPB.addCedaObject(ceda_observation, self._migrationSessions.molesSession, True)
370        MigrationEPB.associateObservationToDeployment(self._deploymentMigration, ceda_observation.id, self._migrationSessions.molesSession, True)
371        return ceda_observation
372
373    def _getObservation(self):
374        return EPB.search(CEDA_Observation, self._deploymentMigration.ceda_observation_id, self._migrationSessions.molesSession)
375           
376    def process(self):
377        ceda_observation = None
378        #Moles3 object exists...
379        if self._deploymentMigration.ceda_observation_id: 
380            # ...and the data entity document has not changed
381            if hasAtomDocumentSameHash(self._deploymentMigration):
382                ceda_observation = self._getObservation()
383            else:
384                ceda_observation = self.updateObservation()
385        else:
386            #... does not exist so create it
387            ceda_observation = self.createNewObservation()
388       
389        #process the CEDA_Observation.procedure
390        deploymentDataProcessor = DeploymentDataProcessor(self._deploymentMigration, self._migrationSessions)
391        links = findLinksInDeployment(self._deploymentMigration)
392        procedure = deploymentDataProcessor.createProcess(links)
393       
394        #Temporary commented because CEDA_Project.subProject is not correctly mapped to the DB
395        #project = deploymentDataProcessor.createProject(links)       
396        project = None
397        if procedure:
398            ceda_observation.procedure = procedure
399           
400        if project:
401            ceda_observation.inSupportOf = project         
402           
403        if procedure or project:                       
404            self._migrationSessions.molesSession.commit()
405           
406        if not self._deploymentHasSameHash:
407            self._deploymentMigration.doc_hash = getAtomDocumentHashByMO(self._deploymentMigration)
408            self._commitMigration()
409             
410       
411        return ceda_observation
Note: See TracBrowser for help on using the repository browser.