1 | ''' |
---|
2 | BSD Licence |
---|
3 | Copyright (c) 2012, Science & Technology Facilities Council (STFC) |
---|
4 | All rights reserved. |
---|
5 | |
---|
6 | Redistribution and use in source and binary forms, with or without modification, |
---|
7 | are permitted provided that the following conditions are met: |
---|
8 | |
---|
9 | * Redistributions of source code must retain the above copyright notice, |
---|
10 | this list of conditions and the following disclaimer. |
---|
11 | * Redistributions in binary form must reproduce the above copyright notice, |
---|
12 | this list of conditions and the following disclaimer in the documentation |
---|
13 | and/or other materials provided with the distribution. |
---|
14 | * Neither the name of the Science & Technology Facilities Council (STFC) |
---|
15 | nor the names of its contributors may be used to endorse or promote |
---|
16 | products derived from this software without specific prior written permission. |
---|
17 | |
---|
18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" |
---|
19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, |
---|
20 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
---|
21 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS |
---|
22 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, |
---|
23 | OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF |
---|
24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) |
---|
25 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
---|
26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
---|
27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
---|
28 | |
---|
29 | Created on 15 Nov 2011 |
---|
30 | |
---|
31 | @author: Maurizio Nagni |
---|
32 | ''' |
---|
33 | from cedaMoles.libs.migration.processor.commons import findMolesLineage, \ |
---|
34 | createMO_ResponsiblePartyInfo,\ |
---|
35 | DO_BADC, DO_NEODC, findAuthorsInResource, \ |
---|
36 | createMD_Identifier, extractSummary, extractQuality, \ |
---|
37 | hasMOSameHash, getAtomDocumentHashByMO, extractTitle,\ |
---|
38 | createCEDA_Result,\ |
---|
39 | createEX_GeographicBoundingBox, extractGeographicExtentInMigrationDocument, \ |
---|
40 | findDownloadLinksInMigrationDocument,\ |
---|
41 | extractContent, createCI_Citation, createCI_Date, createDate,\ |
---|
42 | createTM_Position, createTM_Instant, extractMolesCreationDate,\ |
---|
43 | createDateTime, isoDateTimeStringToTimeDate, extractMolesProviderID,\ |
---|
44 | DO_UKSSDC, createMO_Organization,\ |
---|
45 | createCI_Contact, createCI_Address, createCI_OnlineResource,\ |
---|
46 | createCI_Telephone, extractMolesTemporalRange,\ |
---|
47 | findAccessLinksInMigrationDocument,\ |
---|
48 | findLinksInDeployment, createMD_LegalConstraints,\ |
---|
49 | createDQ_Element, createDQ_ConformanceResult, findUpdatedDate,\ |
---|
50 | createMD_Metadata, createMO_OnlineResource, createCEDA_Review, calculateHash,\ |
---|
51 | createCI_ResponsibleParty, extractUpdateFrequency,\ |
---|
52 | findLinksInMigrationDocument, findSubTypeInDPT, extractMolesPublishedDate,\ |
---|
53 | createMD_Keywords, hasMOBeenProcessed, createMO_Individual,\ |
---|
54 | fromDateStringToPhenomenonTime, \ |
---|
55 | comparePhenomenonTimes, compareGeographicBoundingBoxes |
---|
56 | from cedaMoles.libs.epb import EPB |
---|
57 | from cedaMoles.libs.migration.processor.deployment_data import DeploymentDataProcessor |
---|
58 | from cedaMoles.libs.migration.exception.exceptions import NoDataLineage, \ |
---|
59 | NoAssociatedAuthor, NoAssociatedDeployments,\ |
---|
60 | NoGeographicalExtensionException |
---|
61 | from cedaMoles.libs.migration.authors import authors |
---|
62 | from logging import StreamHandler |
---|
63 | import logging |
---|
64 | from datetime import date |
---|
65 | from ea_model.iso_19115_2006_metadata_corrigendum.\ |
---|
66 | citation_and_responsible_party_information.ci_onlinefunctioncode \ |
---|
67 | import CI_OnLineFunctionCode |
---|
68 | from ea_model.iso_19115_2006_metadata_corrigendum.\ |
---|
69 | citation_and_responsible_party_information.ci_datetypecode \ |
---|
70 | import CI_DateTypeCode |
---|
71 | from ea_model.iso_19115_2006_metadata_corrigendum.\ |
---|
72 | constraint_information.md_restrictioncode import MD_RestrictionCode |
---|
73 | from copy import deepcopy |
---|
74 | import datetime |
---|
75 | from cedaMoles.MolesManager.ceda_guid import CedaGUID |
---|
76 | from ea_model.iso_19115_2006_metadata_corrigendum.\ |
---|
77 | citation_and_responsible_party_information.ci_rolecode import CI_RoleCode |
---|
78 | from ea_model.iso_19115_2006_metadata_corrigendum.\ |
---|
79 | maintenance_information.md_maintenancefrequencycode \ |
---|
80 | import MD_MaintenanceFrequencyCode |
---|
81 | from cedaMoles.MolesManager.codelist import MM_RoleValue, \ |
---|
82 | MM_ObservationPublicationStateValue, getCLValue |
---|
83 | from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation \ |
---|
84 | import CEDA_Observation |
---|
85 | from ea_model.ceda_metadatamodel.ceda_result.ceda_curationvalue \ |
---|
86 | import CEDA_CurationValue |
---|
87 | from ea_model.ceda_metadatamodel.ceda_utilities.ceda_reviewstatusvalue \ |
---|
88 | import CEDA_ReviewStatusValue |
---|
89 | from ea_model.ceda_metadatamodel.ceda_utilities.ceda_reviewfrequencyvalue \ |
---|
90 | import CEDA_ReviewFrequencyValue |
---|
91 | |
---|
92 | MET_GEO_FEATURE = 'Meteorological geographical features' |
---|
93 | ORTHOIMAGERY = 'Orthoimagery' |
---|
94 | |
---|
95 | class DeploymentProcessor(object): |
---|
96 | ''' |
---|
97 | Migrates a deployment element in a CEDA_Observation entity |
---|
98 | ''' |
---|
99 | publisherName = 'CEDA' |
---|
100 | |
---|
101 | log = logging.getLogger('DeploymentProcessor') |
---|
102 | log.addHandler(StreamHandler()) |
---|
103 | log.setLevel(logging.INFO) |
---|
104 | def __init__(self, data_entity_migration, deployment_migration, epbRepo): |
---|
105 | ''' |
---|
106 | Initializes the class |
---|
107 | @param dataEntityMigration: a DataEntityMigration instance |
---|
108 | @param deploymentMigration: the DeploymentMigration instance |
---|
109 | @param epbRepo: an instance of EPBRepo |
---|
110 | ''' |
---|
111 | self._data_entity_migration = data_entity_migration |
---|
112 | self._deployment_migration = deployment_migration |
---|
113 | self.epbRepo = epbRepo |
---|
114 | self._dataEntityHasSameHash = hasMOSameHash(self._data_entity_migration) |
---|
115 | self._deploymentHasSameHash = hasMOSameHash(self._deployment_migration) |
---|
116 | self._deploymentHasBeenProcessed = hasMOBeenProcessed(self._deployment_migration) |
---|
117 | self._report = [] |
---|
118 | |
---|
119 | def _addResponsiblePartyInfo(self, oldResponsiblePartyInfos, newResponsiblePartyInfo): |
---|
120 | opi = None |
---|
121 | for oldPartyInfo in oldResponsiblePartyInfos: |
---|
122 | if oldPartyInfo.role == newResponsiblePartyInfo.role: |
---|
123 | opi = oldPartyInfo |
---|
124 | break |
---|
125 | |
---|
126 | if len(oldResponsiblePartyInfos) == 0 or opi is None: |
---|
127 | oldResponsiblePartyInfos.append(newResponsiblePartyInfo) |
---|
128 | |
---|
129 | if opi is not None: |
---|
130 | for np in newResponsiblePartyInfo.party: |
---|
131 | opi.party.append(np) |
---|
132 | |
---|
133 | |
---|
134 | |
---|
135 | |
---|
136 | def _extractIndividualsAndOrganizations(self, tmp_auth, whereAreAuthors): |
---|
137 | if whereAreAuthors is None: |
---|
138 | raise NoAssociatedDeployments("migrationObject is None") |
---|
139 | try: |
---|
140 | if tmp_auth['author'] == 'unknown': |
---|
141 | doc_authors = findAuthorsInResource(self._data_entity_migration) |
---|
142 | tmp_auth = authors[doc_authors['authors']] |
---|
143 | |
---|
144 | ret = {'ind': [], 'org': [], 'co_ind': [], 'co_org': []} |
---|
145 | if tmp_auth['type'] == 'ind': |
---|
146 | ret['ind'].append(tmp_auth['author']) |
---|
147 | elif tmp_auth['type'] == 'org': |
---|
148 | ret['org'].append(tmp_auth['author']) |
---|
149 | |
---|
150 | if tmp_auth['author'] == 'unknown': |
---|
151 | DeploymentProcessor.log.debug("%s %s %s has unknown author" \ |
---|
152 | % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name)) |
---|
153 | |
---|
154 | for item in tmp_auth['co_author_type']: |
---|
155 | if (tmp_auth['co_author'][tmp_auth['co_author_type'].index(item)] == 'unknown'): |
---|
156 | doc_authors = findAuthorsInResource(self._data_entity_migration) |
---|
157 | tmp_auth = authors[doc_authors['contributors']] |
---|
158 | break |
---|
159 | |
---|
160 | for index in range(len(tmp_auth['co_author_type'])): |
---|
161 | ptype = tmp_auth['co_author_type'][index] |
---|
162 | if ptype == 'ind': |
---|
163 | ret['co_ind'].append(tmp_auth['co_author'][index]) |
---|
164 | elif ptype == 'org': |
---|
165 | ret['co_org'].append(tmp_auth['co_author'][index]) |
---|
166 | |
---|
167 | if (tmp_auth['co_author'][index] == 'unknown'): |
---|
168 | DeploymentProcessor.log.info("%s %s %s has unknown author" \ |
---|
169 | % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name)) |
---|
170 | except Exception as e: |
---|
171 | print e |
---|
172 | return ret |
---|
173 | |
---|
174 | |
---|
175 | def updateObservation(self): |
---|
176 | return EPB.searchOrCreate(CEDA_Observation, self._deployment_migration.ceda_observation_id) |
---|
177 | |
---|
178 | def assignDOI(self, observation, doi): |
---|
179 | if doi and doi.has_key('href'): |
---|
180 | doi = doi['href'][22:] |
---|
181 | |
---|
182 | #Check if a doi has been already assigned |
---|
183 | observation = self.epbRepo.moles3EPB.loadAttributes(observation, 'identifier') |
---|
184 | obs_identifier = observation.identifier |
---|
185 | if obs_identifier: |
---|
186 | for ident in obs_identifier: |
---|
187 | if ident.code == doi: |
---|
188 | return |
---|
189 | |
---|
190 | py_date = None |
---|
191 | cited_responsible = createCI_ResponsibleParty(role=getCLValue(CI_RoleCode.cl_publisher), \ |
---|
192 | organizationName='NERC - British Atmospheric Data Centre') |
---|
193 | if doi.upper() == '10.5285/E8F43A51-0198-4323-A926-FE69225D57DD': |
---|
194 | py_date = date(2011, 4, 1) |
---|
195 | elif doi.upper() == '10.5285/78114093-E2BD-4601-8AE5-3551E62AEF2B': |
---|
196 | py_date = date(2011, 11, 29) |
---|
197 | elif doi.upper() == '10.5285/DB8D8981-1A51-4D6E-81C0-CCED9B921390': |
---|
198 | py_date = date(2012, 4, 16) |
---|
199 | elif doi.upper() == '10.5285/639A3714-BC74-46A6-9026-64931F355E07': |
---|
200 | py_date = date(2012, 4, 16) |
---|
201 | |
---|
202 | if py_date: |
---|
203 | dt = createDate(py_date) |
---|
204 | ci_date = createCI_Date(getCLValue(CI_DateTypeCode.cl_publication), date = dt) |
---|
205 | i_authority = createCI_Citation("DOI", date = ci_date) |
---|
206 | identifier = createMD_Identifier(code = doi, authority=i_authority) |
---|
207 | self.epbRepo.moles3EPB.updateCedaObject(observation, {'identifier': identifier}) |
---|
208 | DeploymentProcessor.log.info("DOI: %s" % (doi)) |
---|
209 | |
---|
210 | def _assignKeywords(self, ceda_observation): |
---|
211 | if self._deploymentHasSameHash: |
---|
212 | return |
---|
213 | |
---|
214 | provider_id = extractMolesProviderID(self._deployment_migration) |
---|
215 | i_keywords = [] |
---|
216 | if provider_id == DO_BADC: |
---|
217 | i_keywords.append(MET_GEO_FEATURE) |
---|
218 | if provider_id == DO_NEODC: |
---|
219 | i_keywords.append(ORTHOIMAGERY) |
---|
220 | if len(i_keywords) > 0: |
---|
221 | #Is a first time process? |
---|
222 | if not self._deploymentHasBeenProcessed: |
---|
223 | ceda_observation.keywords.append(createMD_Keywords(i_keywords)) |
---|
224 | else: |
---|
225 | ceda_observation.keywords.keyword = i_keywords |
---|
226 | |
---|
227 | def _assignLineage(self, observation): |
---|
228 | if self._deploymentHasSameHash: |
---|
229 | return |
---|
230 | |
---|
231 | data_lineage = findMolesLineage(self._data_entity_migration) |
---|
232 | if data_lineage is None: |
---|
233 | raise NoDataLineage(self._data_entity_migration) |
---|
234 | |
---|
235 | if data_lineage != observation.dataLineage: |
---|
236 | observation.dataLineage = data_lineage |
---|
237 | |
---|
238 | def _assignResult(self, observation): |
---|
239 | if self._deploymentHasSameHash and self._dataEntityHasSameHash: |
---|
240 | return |
---|
241 | |
---|
242 | i_sources = [] |
---|
243 | download = findDownloadLinksInMigrationDocument(self._deployment_migration) |
---|
244 | content = None |
---|
245 | if len(download) == 0: |
---|
246 | download = findDownloadLinksInMigrationDocument(self._data_entity_migration) |
---|
247 | content = extractContent(self._data_entity_migration) |
---|
248 | else: |
---|
249 | content = extractContent(self._deployment_migration) |
---|
250 | for dwn in download: |
---|
251 | int_description = None |
---|
252 | int_applicationProfile = None |
---|
253 | if content.has_key('formats'): |
---|
254 | #int_applicationProfile = content['formats'] |
---|
255 | pass |
---|
256 | if dwn['href'].startswith('http://badc.nerc.ac.uk/browse') or dwn['href'].startswith('http://neodc.nerc.ac.uk/browse'): |
---|
257 | int_description = "download directly from archive" |
---|
258 | i_sources.append(createMO_OnlineResource(linkage = dwn['href'], name = dwn['title'], \ |
---|
259 | function = getCLValue(CI_OnLineFunctionCode.cl_download), \ |
---|
260 | description = int_description, applicationProfile = int_applicationProfile)) |
---|
261 | |
---|
262 | dataentity_id = '%s__ATOM__%s' % (self._data_entity_migration.doc_owner, self._data_entity_migration.doc_name) |
---|
263 | dataentity_id = dataentity_id.replace('.atom', '') |
---|
264 | infodb_de = self.epbRepo.infodbEPB.getCedaInfoApp_dataentityByDE_ID(dataentity_id) |
---|
265 | i_logical_path = '/dummy' |
---|
266 | if infodb_de is None: |
---|
267 | i_logical_path = dwn['href'][dwn['href'].index('/browse/') + 7:] |
---|
268 | |
---|
269 | if infodb_de and infodb_de.has_key('logical_path'): |
---|
270 | i_logical_path = infodb_de['logical_path'] |
---|
271 | |
---|
272 | i_category = getCLValue(CEDA_CurationValue.cl_a) |
---|
273 | if infodb_de and infodb_de.has_key('category') and infodb_de['category']: |
---|
274 | i_category = getCLValue(CEDA_CurationValue.from_string(infodb_de['category'].lower())) |
---|
275 | |
---|
276 | if not self._deploymentHasBeenProcessed: |
---|
277 | observation.result = createCEDA_Result(i_category, i_logical_path, source = i_sources) |
---|
278 | return |
---|
279 | |
---|
280 | if observation.result.internalPath != i_logical_path: |
---|
281 | observation.result.internalPath = i_logical_path |
---|
282 | |
---|
283 | if observation.result.curationCategory != i_category: |
---|
284 | observation.result.curationCategory = i_category |
---|
285 | |
---|
286 | #Still have to update observation.result.source |
---|
287 | |
---|
288 | def _assignPublisherCurator(self, observation): |
---|
289 | if self._deploymentHasSameHash: |
---|
290 | return |
---|
291 | |
---|
292 | provider_id = extractMolesProviderID(self._deployment_migration) |
---|
293 | party = None |
---|
294 | if provider_id == DO_BADC: |
---|
295 | i_linkage = 'http://badc.rl.ac.uk' |
---|
296 | i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'British Atmospheric Data Centre Website') |
---|
297 | i_address = createCI_Address(deliveryPoint = ['British Atmospheric Data Centre, STFC Rutherford Appleton Laboratory'], \ |
---|
298 | electronicMailAddress=['badc@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford') |
---|
299 | i_phone = createCI_Telephone(voice=['+44(0)1235 446432']) |
---|
300 | contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources) |
---|
301 | party = createMO_Organization(name = "NERC - British Atmospheric Data Centre", contactInfo = [contact]) |
---|
302 | elif provider_id == DO_NEODC: |
---|
303 | i_linkage = 'http://www.neodc.rl.ac.uk' |
---|
304 | i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'NERC Earth Observation Data Centre website') |
---|
305 | i_address = createCI_Address(deliveryPoint = ['NERC - Earth Observation Data Centre, STFC Rutherford Appleton Laboratory'], \ |
---|
306 | electronicMailAddress=['neodc@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford') |
---|
307 | i_phone = createCI_Telephone(voice=['+44(0)1235 446432']) |
---|
308 | contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources) |
---|
309 | party = createMO_Organization(name = 'NERC - Earth Observation Data Centre', contactInfo = [contact]) |
---|
310 | elif provider_id == DO_UKSSDC: |
---|
311 | i_linkage = 'http://www.ukssdc.rl.ac.uk' |
---|
312 | i_onlineResources = createCI_OnlineResource(linkage = i_linkage, name = 'UK Solar System Data Centre website') |
---|
313 | i_address = createCI_Address(deliveryPoint = ['UK Solar System Data Centre, STFC Rutherford Appleton Laboratory'], \ |
---|
314 | electronicMailAddress=['support@rl.ac.uk'], postalCode='OX11 0QX', country='UK', city='Harwell Oxford') |
---|
315 | i_phone = createCI_Telephone(voice=['+44(0)1235 445173']) |
---|
316 | contact = createCI_Contact(phone=i_phone, address=i_address, onlineResource=i_onlineResources) |
---|
317 | party = createMO_Organization(name = 'NERC - UK Solar System Data Centre', contactInfo = [contact]) |
---|
318 | |
---|
319 | if party and not self._deploymentHasBeenProcessed: |
---|
320 | newrp = [] |
---|
321 | newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_publisher), [party])) |
---|
322 | newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_custodian), deepcopy([party]))) |
---|
323 | newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_distributor), deepcopy([party]))) |
---|
324 | newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_pointofcontact), deepcopy([party]))) |
---|
325 | newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_metadataowner), deepcopy([party]))) |
---|
326 | newrp.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_curator), deepcopy([party]))) |
---|
327 | self.epbRepo.moles3EPB.updateCedaObject(observation, {'relatedParty': newrp}) |
---|
328 | return |
---|
329 | #Still have to update observation.result.source |
---|
330 | |
---|
331 | def _assignQuality(self, observation): |
---|
332 | if self._dataEntityHasSameHash: |
---|
333 | return |
---|
334 | |
---|
335 | doc_quality = extractQuality(self._data_entity_migration) |
---|
336 | doc_date = findUpdatedDate(self._data_entity_migration) |
---|
337 | ci_dates = [] |
---|
338 | if doc_date: |
---|
339 | i_date = createDate(isoDateTimeStringToTimeDate(doc_date)) |
---|
340 | ci_dates.append(createCI_Date(getCLValue(CI_DateTypeCode.cl_revision), date = i_date)) |
---|
341 | else: |
---|
342 | i_date = createDate(isoDateTimeStringToTimeDate(datetime.datetime.now())) |
---|
343 | ci_dates.append(createCI_Date(getCLValue(CI_DateTypeCode.cl_creation), date = i_date)) |
---|
344 | |
---|
345 | i_specification = createCI_Citation(title = "CEDA Data Quality Statement", date=ci_dates) |
---|
346 | i_dq_result = createDQ_ConformanceResult(doc_quality, True, i_specification) |
---|
347 | i_quality_element = createDQ_Element(i_dq_result) |
---|
348 | |
---|
349 | if not self._deploymentHasBeenProcessed: |
---|
350 | observation.resultQuality.append(i_quality_element) |
---|
351 | return |
---|
352 | #Still have to update observation.result.source |
---|
353 | |
---|
354 | def _assignDescription(self, observation): |
---|
355 | if self._dataEntityHasSameHash and self._deploymentHasSameHash: |
---|
356 | return |
---|
357 | |
---|
358 | description = extractSummary(self._deployment_migration) |
---|
359 | if description is None: |
---|
360 | description = extractSummary(self._data_entity_migration) |
---|
361 | |
---|
362 | if description: |
---|
363 | observation.description = description |
---|
364 | |
---|
365 | def _assignTitle(self, observation): |
---|
366 | if self._dataEntityHasSameHash and self._deploymentHasSameHash: |
---|
367 | return |
---|
368 | |
---|
369 | doc_title = extractTitle(self._deployment_migration) |
---|
370 | if doc_title is None: |
---|
371 | doc_title = extractTitle(self._data_entity_migration) |
---|
372 | |
---|
373 | if doc_title.startswith('deployment_') or doc_title.startswith('Deployment_'): |
---|
374 | links = findLinksInMigrationDocument(self._deployment_migration) |
---|
375 | dptList = links['DPT'] |
---|
376 | if links.has_key('DPT'): |
---|
377 | doc_title = 'Data from ' + dptList[0]['title'] |
---|
378 | if len(dptList) > 2: |
---|
379 | for dpt in dptList[1:-2]: |
---|
380 | doc_title += ', ' + dpt['title'] |
---|
381 | if len(dptList) > 1: |
---|
382 | doc_title += ' and ' + dptList[-1] |
---|
383 | |
---|
384 | links = findLinksInDeployment(self._deployment_migration) |
---|
385 | if links.has_key('OBS'): |
---|
386 | obsList = [] |
---|
387 | for obs in links['OBS']: |
---|
388 | observationStation = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deployment_migration, obs + '.atom') |
---|
389 | obsList.append((extractTitle(observationStation), findSubTypeInDPT(observationStation))) |
---|
390 | |
---|
391 | if obsList[0][1] in ['stationary platform' ,'moving platform', 'ship','aircraft','satellite','computer']: |
---|
392 | doc_title += ' on ' |
---|
393 | else : |
---|
394 | doc_title += ' at ' |
---|
395 | doc_title += obsList[0][0] |
---|
396 | if len(obsList) > 2: |
---|
397 | for obs in obsList[1:-2]: |
---|
398 | doc_title += ', ' + obs[0] |
---|
399 | if len(obsList) > 1: |
---|
400 | doc_title += ' and ' + obsList[-1][0] |
---|
401 | |
---|
402 | if links.has_key('ACTIVITY'): |
---|
403 | for link in links['ACTIVITY']: |
---|
404 | activity = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deployment_migration, link + '.atom') |
---|
405 | |
---|
406 | projSubType = findSubTypeInDPT(activity) |
---|
407 | doc_title += ' for the ' + extractTitle(activity) |
---|
408 | if projSubType[0:14] == 'dgActivityData': |
---|
409 | doc_title += ' ' + projSubType[14:] |
---|
410 | else : |
---|
411 | doc_title += ' ' + projSubType |
---|
412 | else: |
---|
413 | if doc_title[0:10] != 'Data from' : |
---|
414 | doc_title = "Data from " + doc_title |
---|
415 | auth = createCI_Citation(title = 'ceda_title') |
---|
416 | identifier = createMD_Identifier(code = doc_title, authority = auth) |
---|
417 | |
---|
418 | if not self._deploymentHasBeenProcessed: |
---|
419 | observation.identifier.append(identifier) |
---|
420 | return |
---|
421 | #Still have to update observation.identifier |
---|
422 | |
---|
423 | def _assignGeographicExtent(self, observation): |
---|
424 | if self._dataEntityHasSameHash and self._deploymentHasSameHash: |
---|
425 | return |
---|
426 | |
---|
427 | ge = extractGeographicExtentInMigrationDocument(self._deployment_migration) |
---|
428 | if ge is None: |
---|
429 | ge = extractGeographicExtentInMigrationDocument(self._data_entity_migration) |
---|
430 | |
---|
431 | if ge is None: |
---|
432 | self._report.append(NoGeographicalExtensionException(self._deployment_migration)) |
---|
433 | self._report.append(NoGeographicalExtensionException(self._data_entity_migration)) |
---|
434 | |
---|
435 | geographicExtent = createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south']) |
---|
436 | |
---|
437 | if len(observation.geographicExtent) == 0 or \ |
---|
438 | (len(observation.geographicExtent) > 0 and \ |
---|
439 | not compareGeographicBoundingBoxes(geographicExtent, \ |
---|
440 | observation.geographicExtent[0])): |
---|
441 | self.epbRepo.moles3EPB.updateCedaObject(observation, {'geographicExtent': geographicExtent}) |
---|
442 | |
---|
443 | def _assignCreationDate(self, observation): |
---|
444 | if self._deploymentHasSameHash: |
---|
445 | return |
---|
446 | |
---|
447 | creation_date = extractMolesPublishedDate(self._deployment_migration) |
---|
448 | if creation_date is None: |
---|
449 | creation_date = extractMolesCreationDate(self._deployment_migration) |
---|
450 | py_datetime = isoDateTimeStringToTimeDate(creation_date) |
---|
451 | date_time = createDateTime(py_datetime) |
---|
452 | tm_position = createTM_Position(dateTime8601 = date_time) |
---|
453 | |
---|
454 | if not self._deploymentHasBeenProcessed: |
---|
455 | observation.resultTime = createTM_Instant(tm_position) |
---|
456 | return |
---|
457 | #Still have to update observation.geographicExtent |
---|
458 | |
---|
459 | def _assignPhenomenonTime(self, observation): |
---|
460 | if self._deploymentHasSameHash and getattr(observation, 'phenomenonTime') != None: |
---|
461 | return |
---|
462 | |
---|
463 | doc_phenomenon_time = extractMolesTemporalRange(self._deployment_migration) |
---|
464 | pt = fromDateStringToPhenomenonTime(doc_phenomenon_time) |
---|
465 | |
---|
466 | if not self._deploymentHasBeenProcessed or not comparePhenomenonTimes(pt,observation.phenomenonTime): |
---|
467 | self.epbRepo.moles3EPB.updateCedaObject(observation, {'phenomenonTime': pt}) |
---|
468 | #Now update phenomenonTime but has to remove the previous phenomenonTime |
---|
469 | |
---|
470 | def _assignPermission(self, observation): |
---|
471 | if self._deploymentHasSameHash and self._dataEntityHasSameHash: |
---|
472 | return |
---|
473 | |
---|
474 | access_link = findAccessLinksInMigrationDocument(self._deployment_migration) |
---|
475 | dwn_link = findDownloadLinksInMigrationDocument(self._deployment_migration) |
---|
476 | if len(access_link) == 0: |
---|
477 | access_link = findAccessLinksInMigrationDocument(self._data_entity_migration) |
---|
478 | |
---|
479 | i_accessConstraints = [] |
---|
480 | i_use_limitation = [] |
---|
481 | |
---|
482 | permission = None |
---|
483 | if len(access_link) == 0: |
---|
484 | if len(dwn_link) == 0: |
---|
485 | dwn_link = findDownloadLinksInMigrationDocument(self._data_entity_migration) |
---|
486 | if dwn_link and len(dwn_link) == 1: |
---|
487 | i_use_limitation.append("These data are open access and available through %s." % (dwn_link[0]['href']) ) |
---|
488 | #i_accessConstraints.append(MD_RestrictionCode.cl_) |
---|
489 | observation.permission = createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints) |
---|
490 | else: |
---|
491 | if access_link and len(access_link) == 1: |
---|
492 | i_use_limitation.append("Access to these data is restricted. To obtain access please apply for access at: %s" % (access_link[0]['href'])) |
---|
493 | i_accessConstraints.append(getCLValue(MD_RestrictionCode.cl_restricted)) |
---|
494 | observation.permission = createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints) |
---|
495 | |
---|
496 | if not self._deploymentHasBeenProcessed: |
---|
497 | observation.permission = permission |
---|
498 | #Still have to update observation.permission |
---|
499 | |
---|
500 | ''' |
---|
501 | contentDict = extractContent(self._deployment_migration) |
---|
502 | if not contentDict.has_key('access-restricted'): |
---|
503 | contentDict = extractContent(self._data_entity_migration) |
---|
504 | ''' |
---|
505 | |
---|
506 | def _assignMoles2Link(self, ceda_observation): |
---|
507 | if self._deploymentHasSameHash: |
---|
508 | return |
---|
509 | |
---|
510 | i_code = 'http://badc.nerc.ac.uk/view/%s__ATOM__%s' % (self._deployment_migration.doc_owner, self._deployment_migration.doc_name) |
---|
511 | i_code = i_code.replace('.atom', '') |
---|
512 | #i_code = buildExistDocPath(self._deployment_migration.doc_status, DT_DEPLOYMENTS, self._deployment_migration.doc_owner, self._deployment_migration.doc_name) |
---|
513 | i_authority = createCI_Citation('moles2url') |
---|
514 | identifier = createMD_Identifier(code = i_code, authority = i_authority) |
---|
515 | if not self._deploymentHasBeenProcessed: |
---|
516 | ceda_observation.identifier.append(identifier) |
---|
517 | #Still have to update observation.permission |
---|
518 | |
---|
519 | def _assignInternalReview(self, ceda_observation): |
---|
520 | if self._deploymentHasBeenProcessed: |
---|
521 | return |
---|
522 | |
---|
523 | i_party = createMO_Individual(name = 'Graham Parton') |
---|
524 | i_reviewer = createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_processor), [i_party]) |
---|
525 | ceda_observation.metadataManagement.append( \ |
---|
526 | createCEDA_Review(reviewer=i_reviewer, reviewFrequency=getCLValue(CEDA_ReviewFrequencyValue.cl_yearly), \ |
---|
527 | reviewStatus=getCLValue(CEDA_ReviewStatusValue.cl_required))) |
---|
528 | |
---|
529 | def _assignLanguage(self, ceda_observation): |
---|
530 | if self._deploymentHasBeenProcessed: |
---|
531 | return |
---|
532 | |
---|
533 | i_date_stamp = createDate(datetime.datetime.now()) |
---|
534 | #i_contact = createCI_Citation("", date = i_date_stamp) |
---|
535 | i_contact = createCI_ResponsibleParty(getCLValue(CI_RoleCode.cl_user)) |
---|
536 | ceda_observation.metadata = createMD_Metadata(date_stamp=i_date_stamp, contact = [i_contact], language = "English") |
---|
537 | |
---|
538 | def _processResultAccumulation(self, ceda_observation): |
---|
539 | if self._dataEntityHasSameHash: |
---|
540 | return |
---|
541 | |
---|
542 | updateFrequency = extractUpdateFrequency(self._data_entity_migration) |
---|
543 | if updateFrequency: |
---|
544 | resultAccumulation = MD_MaintenanceFrequencyCode.from_string(updateFrequency) |
---|
545 | if not self._deploymentHasBeenProcessed: |
---|
546 | self.epbRepo.moles3EPB.updateCedaObject(ceda_observation, {'resultAccumulation': resultAccumulation}) |
---|
547 | return |
---|
548 | #Still have to update observation.permission |
---|
549 | |
---|
550 | def _assignName(self, observation): |
---|
551 | ''' |
---|
552 | @param relatedPartyInfos: a MO_ResponsiblePartyInfo list |
---|
553 | @return True if the documents changed, False otherwise |
---|
554 | ''' |
---|
555 | if self._deploymentHasSameHash and self._dataEntityHasSameHash: |
---|
556 | return |
---|
557 | |
---|
558 | whereAreAuthors = self._deployment_migration |
---|
559 | doc_authors = findAuthorsInResource(self._deployment_migration) |
---|
560 | if doc_authors['authors'] in [DO_BADC, DO_NEODC]: |
---|
561 | doc_authors = findAuthorsInResource(self._data_entity_migration) |
---|
562 | whereAreAuthors = self._data_entity_migration |
---|
563 | |
---|
564 | ind_names = [] |
---|
565 | org_names = [] |
---|
566 | if authors.has_key(doc_authors['authors']): |
---|
567 | tmp_auth = authors[doc_authors['authors']] |
---|
568 | ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) |
---|
569 | |
---|
570 | if len(ret['ind']) > 0: |
---|
571 | i_party = createMO_Individual(name = ret['ind'][0]) |
---|
572 | observation.relatedParty.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_author), [i_party])) |
---|
573 | if len(ret['org']) > 0: |
---|
574 | i_party = createMO_Organization(name = ret['org'][0]) |
---|
575 | observation.relatedParty.append(createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_author), [i_party])) |
---|
576 | |
---|
577 | if len(ret['ind']) > 1: |
---|
578 | ind_names.extend(ret['ind'][1:]) |
---|
579 | if len(ret['org']) > 1: |
---|
580 | org_names.extend(ret['org'][1:]) |
---|
581 | if len(ret['co_ind']) > 0: |
---|
582 | ind_names.extend(ret['co_ind']) |
---|
583 | if len(ret['co_org']) > 0: |
---|
584 | org_names.extend(ret['co_org']) |
---|
585 | |
---|
586 | else: |
---|
587 | raise NoAssociatedAuthor(doc_authors['authors'], migrationObject = whereAreAuthors) |
---|
588 | |
---|
589 | if doc_authors['contributors'] and authors.has_key(doc_authors['contributors']): |
---|
590 | tmp_auth = authors[doc_authors['contributors']] |
---|
591 | ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) |
---|
592 | ind_names.extend(ret['ind']) |
---|
593 | ind_names.extend(ret['co_ind']) |
---|
594 | org_names.extend(ret['org']) |
---|
595 | org_names.extend(ret['co_org']) |
---|
596 | |
---|
597 | i_party = [] |
---|
598 | for nm in ind_names: |
---|
599 | i_party.append(createMO_Individual(name = nm)) |
---|
600 | |
---|
601 | for nm in org_names: |
---|
602 | i_party.append(createMO_Organization(name = nm)) |
---|
603 | |
---|
604 | if i_party: |
---|
605 | if not self._deploymentHasBeenProcessed: |
---|
606 | rp = createMO_ResponsiblePartyInfo(getCLValue(MM_RoleValue.cl_coinvestigator), i_party) |
---|
607 | self.epbRepo.moles3EPB.updateCedaObject(observation, {'relatedParty': rp}) |
---|
608 | return |
---|
609 | #Still have to update observation.permission |
---|
610 | |
---|
611 | def _execute(self, ceda_observation): |
---|
612 | self._assignKeywords(ceda_observation) |
---|
613 | self._assignLineage(ceda_observation) |
---|
614 | self._assignResult(ceda_observation) |
---|
615 | self._assignPublisherCurator(ceda_observation) |
---|
616 | self._assignQuality(ceda_observation) |
---|
617 | self._assignDescription(ceda_observation) |
---|
618 | self._assignTitle(ceda_observation) |
---|
619 | self._assignGeographicExtent(ceda_observation) |
---|
620 | self._assignCreationDate(ceda_observation) |
---|
621 | self._assignPhenomenonTime(ceda_observation) |
---|
622 | self._assignPermission(ceda_observation) |
---|
623 | self._assignMoles2Link(ceda_observation) |
---|
624 | self._assignInternalReview(ceda_observation) |
---|
625 | self._assignLanguage(ceda_observation) |
---|
626 | self._processResultAccumulation(ceda_observation) |
---|
627 | self._assignName(ceda_observation) |
---|
628 | #self._assignDOI(ceda_observation) |
---|
629 | |
---|
630 | #Is a first time process? |
---|
631 | if not hasMOBeenProcessed(self._deployment_migration): |
---|
632 | ceda_observation.publicationState = getCLValue(MM_ObservationPublicationStateValue.cl_working) |
---|
633 | docHash = getAtomDocumentHashByMO(self._data_entity_migration) |
---|
634 | self.epbRepo.moles3EPB.persistInstance(ceda_observation) |
---|
635 | self.epbRepo.migrationEPB.updateMigrationObject(self._deployment_migration, \ |
---|
636 | {'ceda_observation_id': ceda_observation.id, |
---|
637 | 'doc_hash': docHash}) |
---|
638 | |
---|
639 | #Has a proper CEDAGUID? |
---|
640 | if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observation) is None: |
---|
641 | ceda_guid = CedaGUID() |
---|
642 | ceda_guid.id = calculateHash(self._deployment_migration.depl_id) |
---|
643 | ceda_guid.ceda_observation = ceda_observation.id |
---|
644 | self.epbRepo.moles3EPB.persistInstance(ceda_guid) |
---|
645 | DeploymentProcessor.log.info("GUID for this Observation: %s" % (ceda_guid.id)) |
---|
646 | |
---|
647 | if not self._deploymentHasBeenProcessed: |
---|
648 | deploymentDataProcessor = DeploymentDataProcessor(self._deployment_migration, self.epbRepo) |
---|
649 | procedure = deploymentDataProcessor.createProcess() |
---|
650 | project = deploymentDataProcessor.createProject() |
---|
651 | self.epbRepo.moles3EPB.updateCedaObject(ceda_observation, {'procedure': procedure, 'inSupportOf': project}) |
---|
652 | |
---|
653 | |
---|
654 | #Still have to update observation.procedure |
---|
655 | #Still have to update observation.project |
---|
656 | |
---|
657 | return ceda_observation |
---|
658 | |
---|
659 | def process(self): |
---|
660 | self._report = [] |
---|
661 | ceda_observation = None |
---|
662 | #Moles3 object exists... |
---|
663 | if self._deployment_migration.ceda_observation_id: |
---|
664 | ceda_observation = self.epbRepo.moles3EPB.search(CEDA_Observation, self._deployment_migration.ceda_observation_id) |
---|
665 | else: |
---|
666 | #... does not exist so create it |
---|
667 | ceda_observation = CEDA_Observation() |
---|
668 | |
---|
669 | self._execute(ceda_observation) |
---|
670 | return self._report, ceda_observation |
---|