source: mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py @ 8358

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py@8358
Revision 8358, 37.2 KB checked in by mnagni, 7 years ago (diff)

Major refactoring about migration (now handles better create/update, even if the single updates have to be quite fully implemented)
Added the connection pool from SQLAlchemy

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 15 Nov 2011
30
31@author: mnagni
32'''
33from ea_model.moles3_4.utilities.mo_responsiblepartyinfo import MO_ResponsiblePartyInfo
34from ea_model.iso_19115_2006_metadata_corrigendum.reference_system_information.md_identifier import MD_Identifier
35from httplib import HTTPConnection
36from xml.etree.ElementTree import XML, tostring
37import time, datetime
38from ea_model.upcomingiso.ci_organisation import CI_Organisation
39from ea_model.upcomingiso.ci_party import CI_Party
40from ea_model.upcomingiso.ci_individual import CI_Individual
41from libs.migration.exception.exceptions import NoDataLineage
42from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_citation import CI_Citation
43from ea_model.iso_19115_2006_metadata_corrigendum.data_quality_information.dq_element import DQ_Element
44from ea_model.iso_19115_2006_metadata_corrigendum.data_quality_information.dq_conformanceresult import DQ_ConformanceResult
45from hashlib import md5
46from xml.sax.saxutils import unescape, escape
47import html5lib
48from html5lib import treebuilders
49from ea_model.ceda_metadatamodel.ceda_result.ceda_result import CEDA_Result
50from ea_model.moles3_4.result.mo_onlineresource import MO_OnlineResource
51from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.url import URL
52from ea_model.iso_19115_2006_metadata_corrigendum.extent_information.ex_geographicboundingbox import EX_GeographicBoundingBox
53from ea_model.ceda_metadatamodel.ceda_computation.ceda_processing import CEDA_Processing
54from ea_model.ceda_metadatamodel.ceda_acquisition.ceda_instrument import CEDA_Instrument
55from ea_model.ceda_metadatamodel.ceda_observationprocess.ceda_compositeprocess import CEDA_CompositeProcess
56from ea_model.ceda_metadatamodel.ceda_acquisition.ceda_acquisition import CEDA_Acquisition
57from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_date import CI_Date
58from ea_model.iso_19103_2005_schema_language.basic_types.primitive.date_and_time.datetime import DateTime
59from ea_model.iso_19103_2005_schema_language.basic_types.primitive.date_and_time.date import Date
60from ea_model.iso_19108_2006_temporal_schema.temporal_reference_system.tm_position import TM_Position
61from ea_model.iso_19108_2006_temporal_schema.temporal_objects.tm_instant import TM_Instant
62from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_contact import CI_Contact
63from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_address import CI_Address
64from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_onlineresource import CI_OnlineResource
65from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_telephone import CI_Telephone
66from ea_model.iso_19108_2006_temporal_schema.temporal_objects.tm_period import TM_Period
67from ea_model.iso_19115_2006_metadata_corrigendum.constraint_information.md_constraints import MD_Constraints
68from ea_model.iso_19115_2006_metadata_corrigendum.constraint_information.md_legalconstraints import MD_LegalConstraints
69from ea_model.ceda_metadatamodel.ceda_project.ceda_project import CEDA_Project
70from ea_model.iso_19115_2006_metadata_corrigendum.metadata_entity_set_information.md_metadata import MD_Metadata
71from ea_model.ceda_metadatamodel.ceda_utilities.ceda_review import CEDA_Review
72from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_responsibleparty import CI_ResponsibleParty
73from ea_model.iso_19115_2006_metadata_corrigendum.maintenance_information.md_maintenancefrequencycode import MD_MaintenanceFrequencyCode
74from ea_model.iso_19115_2006_metadata_corrigendum.identification_information.md_keywords import MD_Keywords
75
76base = '/exist/rest/atoms'
77
78DS_pUBLISHED = 'published'
79DS_WORKING = 'working'
80DS_PUBLISHED = 'Published'
81docStatus = (DS_pUBLISHED, DS_WORKING, DS_PUBLISHED)
82
83DT_DEPLOYMENTS = 'deployments'
84DT_DATA_ENTITIES = 'data_entities'
85DT_DEPLOYMENT_DATA = 'deployment_data'
86DT_DATA_GRANULES = 'data_granules'
87docTypes = (DT_DEPLOYMENTS, DT_DATA_ENTITIES, DT_DEPLOYMENT_DATA, DT_DATA_GRANULES)
88
89DO_BADC = 'badc.nerc.ac.uk'
90DO_NEODC = 'neodc.nerc.ac.uk'
91DO_UKSSDC = 'ukssdc.ac.uk'
92
93CEDA = 'Centre for Environmental Data Archive'
94docOwners = (DO_BADC, DO_NEODC, DO_UKSSDC)
95
96atomNS = "{http://www.w3.org/2005/Atom}"
97existNS = "{http://exist.sourceforge.net/NS/exist}"
98molesNS = "{http://ndg.nerc.ac.uk/schema/moles2beta}"
99htmlNS = "{http://www.w3.org/1999/xhtml}"
100georssNS="{http://www.georss.org/georss/10}"
101gmlNS="{http://www.opengis.net/gml}"
102date_format = '%Y-%m-%d'
103datetime_format = date_format + 'T%H:%M:%SZ'
104ihost = 'bora.badc.rl.ac.uk'
105iport = '8080'
106
107linkMarkers = ['Deployment', 'DOWNLOAD', 'DOCUMENTATION', 'ACCESS', 'LOGO', 'ACTIVITY', 'DPT', 'OBS']
108
109#MD_Identifier codes
110MD_CODE_MOLES2_CITATION = 'ceda_moles2_citation'
111
112htmlParser = html5lib.HTMLParser(tree=treebuilders.getTreeBuilder("etree"), namespaceHTMLElements=False)
113
114def calculateHash(text):
115    """
116        Returns an md5 hexadecimal representation of the given text
117        @param text: the string to encode
118        @return: the hexadecimal md5 value of the given text
119    """
120    encoder = md5()
121    encoder.update(text)
122    return encoder.hexdigest()
123
124def buildExistDocPath(docStatus, docType, docOwner, docName):
125    '''
126        @param docStatus: one value from commons.docStatus
127        @param docType: one value from commons.docTypes
128        @param docOwner: one value from commons.docOwners
129        @param docName: one value from commons.docOwners       
130    '''       
131    return '%s/%s' % (buildExistOwnerPath(docStatus, docType, docOwner), docName)
132
133def buildExistOwnerPath(docStatus, docType, docOwner):
134    '''
135        @param docStatus: one value from commons.docStatus
136        @param docType: one value from commons.docCollections
137        @param docOwner: one value from commons.docOwners
138    '''       
139    return '%s/%s' % (buildExistTypePath(docStatus, docType), docOwner)
140
141def buildExistTypePath(docStatus, docType):
142    '''
143        @param docStatus: one value from commons.docStatus
144        @param docType: one value from commons.docCollections
145    '''       
146    return '%s/%s' % (buildExistStatusPath(docStatus), docType)
147
148def buildExistStatusPath(docStatus):
149    '''
150        @param docStatus: one value from commons.docStatus
151    '''       
152    return '/exist/rest/atoms/%s' % (docStatus)
153
154def getAtomDocumentByMO(migrationObject):
155    if migrationObject is None:
156        raise Exception("migrationObject is None")
157    mo_typeDict = {'DeploymentsMigration': DT_DEPLOYMENTS, 'DataEntityMigration': DT_DATA_ENTITIES, 'DeploymentDataMigration': DT_DEPLOYMENT_DATA}     
158    return getAtomDocumentAsElementtree(migrationObject.doc_status, mo_typeDict[type(migrationObject).__name__], migrationObject.doc_owner, migrationObject.doc_name)
159
160def getAtomDocumentHashByMO(migrationObject):
161    if migrationObject is None:
162        raise Exception("migrationObject is None")   
163    mo_typeDict = {'DeploymentsMigration': DT_DEPLOYMENTS, 'DataEntityMigration': DT_DATA_ENTITIES, 'DeploymentDataMigration': DT_DEPLOYMENT_DATA}     
164    text = _getAtomDocumentAsText(migrationObject.doc_status, mo_typeDict[type(migrationObject).__name__], migrationObject.doc_owner, migrationObject.doc_name)
165    return calculateHash(text)
166
167def getAtomDocumentHash(docStatus, docType, docOwner, docName):
168    source = buildExistDocPath(docStatus, docType, docOwner, docName)
169    text = _getDocument(source)
170    return calculateHash(text)
171
172def hasMOBeenProcessed(migrationObject):
173    '''
174        Checks if a migration object has been already processed.
175        @param migrationObject: an instance of DataEntityMigration or DeploymentsMigration
176        @return True if has been processed, otherwise False
177    '''
178    if migrationObject is None:
179        return False
180   
181    if not hasattr(migrationObject, 'doc_hash'):
182        return False
183    else:
184        if migrationObject.doc_hash is None:
185            return False
186       
187    return True
188
189def hasMOSameHash(migrationObject):
190    '''
191        Checks if a migration object has changed.
192        @param migrationObject: an instance of DataEntityMigration or DeploymentsMigration or DeploymentDataMigration
193        @return True if the hash of the actual document is the same of the migrationObject, otherwise False
194    '''
195    if not hasMOBeenProcessed(migrationObject):
196        return False
197    return getAtomDocumentHashByMO(migrationObject) == migrationObject.doc_hash
198
199def getAtomDocumentByType(migrationObject, docType):
200    if migrationObject is None:
201        raise Exception("migrationObject is None")   
202    return getAtomDocumentAsElementtree(migrationObject.doc_status, docType, migrationObject.doc_owner, migrationObject.doc_name)
203
204def _getAtomDocumentAsText(docStatus, docType, docOwner, docName):
205    source = buildExistDocPath(docStatus, docType, docOwner, docName)
206    return _getDocument(source)
207
208def getAtomDocumentAsElementtree(docStatus, docType, docOwner, docName):
209    source = buildExistDocPath(docStatus, docType, docOwner, docName)
210    return _getXMLDocument(source)
211
212def _getXMLDocument(source):
213    return XML(_getDocument(source))
214
215def stringToTimestamp(timestring):
216    '''
217        Return a timestamp such as is returned by time.time().
218        @param timestring: a time string formatted as '%Y-%m-%dT%H:%M:%SZ'
219    '''   
220    return datetime.datetime.fromtimestamp(time.mktime(time.strptime(timestring, datetime_format)))
221
222def isoDateStringToTimeDate(datestring):
223    '''
224        Return a datatime.datatime instance.
225        @param datestring: a date string formatted as '%Y-%m-%d'
226    '''
227    return datetime.datetime.strptime(datestring, date_format)
228
229def isoDateTimeStringToTimeDate(timestring):
230    '''
231        Return a datatime.datatime instance.
232        @param timestring: a time string formatted as '%Y-%m-%dT%H:%M:%SZ'
233    '''
234    return datetime.datetime.strptime(timestring, datetime_format)
235
236def _getDocument(source):
237    conn = HTTPConnection(host = ihost, port = iport)
238    conn.connect()
239    req = conn.request('GET', source)
240    res = conn.getresponse()
241    xmlDoc = res.read()
242    '''
243    print (xmlDoc)
244    '''
245    conn.close()
246    return xmlDoc
247
248def _returnNotNoneText(element):
249    if element is None:
250        return None
251    return element.text
252
253def findMolesCreationDate(resourceXML):
254    creationDate = resourceXML.find('%sentity/%smolesISO/%screated' % (molesNS, molesNS, molesNS))
255    return _returnNotNoneText(creationDate)
256
257def findMolesPublishedDate(resourceXML):
258    creationDate = resourceXML.find('%sentity/%smolesISO/%spublished' % (molesNS, molesNS, molesNS))
259    return _returnNotNoneText(creationDate)
260
261def findMolesLineage(dataEntityMigration):
262    resourceXML = getAtomDocumentByMO(dataEntityMigration)
263    lineage = resourceXML.find('%sentity/%smolesISO/%slineage' % (molesNS, molesNS, molesNS))
264    if lineage is None:
265        raise NoDataLineage(dataEntityMigration)
266    return lineage.text
267
268def extractMolesProviderID(migrationObject):
269    resourceXML = getAtomDocumentByMO(migrationObject)
270    provider_id = resourceXML.find('%sentity/%smolesISO/%sproviderID' % (molesNS, molesNS, molesNS))
271    return _returnNotNoneText(provider_id)
272
273def extractMolesQuality(migrationObject):
274    resourceXML = getAtomDocumentByMO(migrationObject)
275    quality = resourceXML.find('%sentity/%smolesISO/%squality' % (molesNS, molesNS, molesNS))
276    return _returnNotNoneText(quality)
277
278def extractMolesTemporalRange(migrationObject):
279    resourceXML = getAtomDocumentByMO(migrationObject)
280    temporal_range = resourceXML.find('%stemporalRange' % (molesNS))
281    return _returnNotNoneText(temporal_range)
282
283def extractMolesCreationDate(migrationObject):
284    resourceXML = getAtomDocumentByMO(migrationObject)
285    return findMolesCreationDate(resourceXML)
286
287def extractMolesPublishedDate(migrationObject):
288    resourceXML = getAtomDocumentByMO(migrationObject)
289    return findMolesPublishedDate(resourceXML)
290
291def extractQuality(dataEntityMigration):
292    resourceXML = getAtomDocumentByMO(dataEntityMigration)
293    quality = resourceXML.find('%sentity/%smolesISO/%squality' % (molesNS, molesNS, molesNS))
294    return _returnNotNoneText(quality)
295
296def extractUpdateFrequency(dataEntityMigration):
297    resourceXML = getAtomDocumentByMO(dataEntityMigration)
298    update_frequency = resourceXML.find('%sentity/%smolesISO/%supdateFrequency' % (molesNS, molesNS, molesNS))
299    return _returnNotNoneText(update_frequency)
300
301def extractContent(dataEntityMigration):
302    """
303        Returns a dictionary containing the div composing the
304        <content> element in a dataentity document.
305    """
306    resourceXML = getAtomDocumentByMO(dataEntityMigration)
307    content = resourceXML.find('%scontent' % (atomNS))
308    text = _returnNotNoneText(content)
309    contentDict = {}
310    if text:
311        doc = htmlParser.parse(unescape(text))
312        for el in doc.findall('body//div'):   
313            prop = el.get('property')
314            if prop:
315                if prop.startswith('cedacat'):
316                    contentDict[prop.split(':')[1]] = escape(tostring(el))
317    return contentDict
318
319def _extractAuthors(authorsCSV):
320    if authorsCSV is None:
321        return []
322    authors = authorsCSV.split(',')
323    for index in range(len(authors)):
324        authors[index] = authors[index].strip()
325        if len(authors[index]) == 0:
326            authors.remove(authors[index])
327    return authors
328
329def findAuthorsInResource(resourceMigration):
330    '''
331        Returns a dictionary with the following keys:
332        'authors': a list of string representing the authors
333        'contributors': a list of string representing the contributors
334    '''
335    ret = {}   
336    resourceXML = getAtomDocumentByMO(resourceMigration)
337    ret['authors'] = findAuthorInResource(resourceXML)
338    ret['contributors'] = findContributorInResource(resourceXML)
339    return ret
340
341def findAuthorInResource(resourceXML): 
342    author = resourceXML.find('%sauthor/%sname' % (atomNS, atomNS))
343    return _returnNotNoneText(author)
344
345def findContributorInResource(resourceXML): 
346    contributors = resourceXML.find('%scontributor/%sname' % (atomNS, atomNS))
347    return _returnNotNoneText(contributors)
348
349def findPublishedDate(resourceMigration):
350    resourceXML = getAtomDocumentByMO(resourceMigration)
351    publishedDate = resourceXML.find('%spublished' % (atomNS))
352    return _returnNotNoneText(publishedDate)
353
354def findUpdatedDate(resourceMigration):
355    resourceXML = getAtomDocumentByMO(resourceMigration)
356    publishedDate = resourceXML.find('%supdated' % (atomNS))
357    return _returnNotNoneText(publishedDate)
358
359def findSummary(resourceMigration):
360    """
361        Returns the <entry><summary> tag of an atom document
362        @param resourceMigration: a MigrationObject instance
363        @return: the <summary> value or None if empty
364    """
365    resourceXML = getAtomDocumentByMO(resourceMigration)
366    summary = resourceXML.find('%ssummary' % (atomNS))
367    return _returnNotNoneText(summary)
368
369def findID(dataEntXML):
370    ent_id = dataEntXML.find('%sid' % (atomNS))
371    return _returnNotNoneText(ent_id)
372
373def _updateLinksDict(linksDict, link, linkMarker):
374    if not linksDict.has_key(linkMarker):
375        linksDict[linkMarker] = []
376    rel = link.get('rel')
377    if rel and rel.endswith('/' + linkMarker):
378        linksDict[linkMarker].append({'href': link.get('href'), 'title': link.get('title')}) 
379
380def _extractLinks(dataEntXML, markers):
381    linksDict = {}
382    links = dataEntXML.findall('%slink' % (atomNS))
383    for link in links:
384        for marker in markers:
385            _updateLinksDict(linksDict, link, marker)       
386    return linksDict
387
388def findLinksInMigrationDocument(dataEntityMigration):
389    dataEntXML = getAtomDocumentByMO(dataEntityMigration)
390    linksDict = _extractLinks(dataEntXML, linkMarkers)
391    return linksDict
392
393
394
395def findDownloadLinksInMigrationDocument(migrationObject):
396    """
397        Return a list of dictionaries describing a <link rel="...DOWNLOAD..."> tag type
398        Each dictionary has two keys: 'href' and 'title'
399        @param migrationObject: the migration instance to retrieve and parse
400        @return: a list of dictionaries
401    """
402    linksDict = findLinksInMigrationDocument(migrationObject)
403    if linksDict.has_key('DOWNLOAD'):
404        return linksDict['DOWNLOAD']   
405    return {}       
406
407def findAccessLinksInMigrationDocument(migrationObject):
408    """
409        Return a list of dictionaries describing a <link rel="...ACCESS..."> tag type
410        Each dictionary has two keys: 'href' and 'title'
411        @param migrationObject: the migration instance to retrieve and parse
412        @return: a list of dictionaries
413    """
414    linksDict = findLinksInMigrationDocument(migrationObject)
415    if linksDict.has_key('ACCESS'):
416        return linksDict['ACCESS']     
417    return {}   
418
419def findDocumentationInMigrationDocument(migrationObject):
420    """
421        Return a list of dictionaries describing a <link rel="...DOCUMENTATION..."> tag type
422        Each dictionary has two keys: 'href' and 'title'
423        @param migrationObject: the migration instance to retrieve and parse
424        @return: a list of dictionaries
425    """
426    linksDict = findLinksInMigrationDocument(migrationObject)
427    if linksDict.has_key('DOCUMENTATION'):
428        return linksDict['DOCUMENTATION']     
429    return {}
430
431def findDOIInMigrationDocument(migrationObject):
432    """
433        Return a dictionary describing a <link rel="...DOCUMENTATION..."> tag type
434        The dictionary has two keys: 'href' and 'title'
435        @param migrationObject: the migration instance to retrieve and parse
436        @return: a dictionary relative to the DOI, None otherwise
437    """
438    for link in findDocumentationInMigrationDocument(migrationObject):
439        if link['href'].startswith('http://dx.doi.org/doi:'):
440            return link
441    return None
442
443def findDeploymentsInDE(dataEntityMigration):
444    linksDict = findLinksInMigrationDocument(dataEntityMigration)
445    links = _extractLinksByMarker(linksDict, 'Deployment')
446    return [depName + '.atom' for depName in links]
447
448def findSubTypeInDPT(resourceMigration):
449    resourceXML = getAtomDocumentByMO(resourceMigration)
450    categories = resourceXML.findall('%scategory' % (atomNS))
451    for category in categories:
452        if category.get("term") == "ATOM_SUBTYPE":
453            return category.get("label")   
454       
455def extractTitle(deploymentMigration):
456    resourceXML = getAtomDocumentByMO(deploymentMigration)
457    title = resourceXML.find('%stitle' % (atomNS))
458    return _returnNotNoneText(title)
459
460def extractSummary(deploymentMigration, dataEntityMigration):
461    resourceXML = getAtomDocumentByMO(deploymentMigration)
462    summary = resourceXML.find('%ssummary' % (atomNS))
463    ret = _returnNotNoneText(summary)
464    if ret:
465        return ret
466    resourceXML = getAtomDocumentByMO(dataEntityMigration)
467    summary = resourceXML.find('%ssummary' % (atomNS))
468    return _returnNotNoneText(summary)
469
470def extractGeographicExtentInMigrationDocument(migrationObject):
471    """
472        Extracts if existing the georss:where/gel:Enveloper/upper-lowerCorner elements.
473        @param migrationObject: a migration object to retrieve to parse for data
474        @return: None if no data are found, otherwise a dictionary with keys: 'east', 'north', 'west', 'south' where
475        the values are float
476    """
477    resourceXML = getAtomDocumentByMO(migrationObject)
478    upperCorner = resourceXML.find('%swhere/%sEnvelope/%supperCorner' % (georssNS, gmlNS, gmlNS))
479    lowerCorner = resourceXML.find('%swhere/%sEnvelope/%slowerCorner' % (georssNS, gmlNS, gmlNS))
480    ret = None
481    if upperCorner != None and lowerCorner != None:
482        upperCornerData = upperCorner.text.split()
483        lowerCornerData = lowerCorner.text.split()
484        ret = {'east': float(upperCornerData[0]), 'north': float(upperCornerData[1]), 'west': float(lowerCornerData[0]), 'south': float(lowerCornerData[1])}
485    return ret
486
487def findLinksInDeployment(migrationObject):
488    """
489        Returns a dictionary of links owned by the given dataEntity document
490        @param deploymentMigration: a MigrationObject instance
491        @return: a dictionary of links.
492    """
493    links = {}
494    linksDict = findLinksInMigrationDocument(migrationObject)
495    for marker in linkMarkers:   
496        links[marker] = _extractLinksByMarker(linksDict, marker)
497    return links
498
499def _extractLinksByMarker(linksDict, marker):
500    dpt = []
501    if linksDict.has_key(marker):
502        for link in linksDict[marker]:
503            try:
504                linkLongName = link['href'].split('/')[-1]
505                if '__ATOM__' in linkLongName:
506                    linkName = linkLongName.rsplit('__ATOM__')[1]
507                else:
508                    linkName = linkLongName
509                dpt.append(linkName)
510            except Exception as ex:
511                print "WARN - Cannot extractLinksByMarker %s" % (link)
512    return dpt
513
514
515def getResourceRefs(deploymentRefs):
516    '''
517        Returns a list of Elements representing the inner resource reference items
518        @param resourceRefs: the name of the eXist collection name below the 'deployments' one
519    ''' 
520    XMLDepl = _getXMLDocument(deploymentRefs)
521    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS))
522
523def getOwnerRefs(docStatus, docType, docOwner):
524    '''
525        Returns a list of Elements representing the inner resource reference items
526        @param resourceRefs: the name of the eXist collection name below the 'deployments' one
527    '''     
528    XMLDepl = _getXMLDocument(buildExistOwnerPath(docStatus, docType, docOwner))
529    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS))
530
531def getTypeRefs(docStatus, docType):
532    '''
533        Returns a list of Elements representing the inner resource reference items
534        @param resourceRefs: the name of the eXist collection name below the 'deployments' one
535    '''     
536    XMLDepl = _getXMLDocument(buildExistTypePath(docStatus, docType))
537    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS))
538
539
540
541def getCollectionRefs(publishedRefs):
542    '''
543        Returns a list of Elements representing the inner deployment reference items
544        @param basePublished: the name of the eXist collection name below the 'published' one
545    ''' 
546    XMLPubl = _getXMLDocument(publishedRefs)
547    return XMLPubl.findall('%scollection/%scollection' % (existNS, existNS))
548
549def getResource(source, resourceName):
550    resourceSource = '%s/%s' % (source, resourceName)
551    resourceDoc = _getDocument(resourceSource)
552    return XML(resourceDoc)
553
554def createMD_Keywords(keywords, k_type=None, thesaurusName=None):
555    '''
556        Creates a new MD_Keywords instance.
557        @param keywords: a String array       
558    '''   
559    md_keywords = MD_Keywords()
560    md_keywords.keyword.extend(keywords)
561    if k_type:
562        md_keywords.type = k_type
563    if thesaurusName:
564        md_keywords.thesaurusName = thesaurusName       
565    return md_keywords
566   
567
568def createDateTime(datetime):
569    '''
570        Creates a new DateTime instance.
571        @param datetime: a datetime.datetime instance       
572    '''
573    dateTime = DateTime()
574    dateTime.century = (datetime.year / 100) + 1
575    dateTime.year = datetime.year
576    dateTime.month = datetime.month       
577    dateTime.day = datetime.day
578    dateTime.hour = datetime.hour
579    dateTime.minute = datetime.minute
580    dateTime.second = datetime.second
581    dateTime.timeZone = datetime.tzinfo       
582    return dateTime       
583       
584def createDate(date):
585    '''
586        Creates a new Date instance.
587        @param date: a datetime.datetime instance
588    '''
589    idate = Date()
590    idate.century = (date.year / 100) + 1
591    idate.year = date.year
592    idate.month = date.month       
593    idate.day = date.day       
594    return idate       
595       
596def createTM_Position(anyOther = None, date8601 = None, dateTime8601 = None, time8601 = None):
597    '''
598        Creates a new TM_Position instance
599        @param anyOther: a TM_TemporalPosition instance
600        @param date8601: a Date instance
601        @param dateTime8601:a DateTime instance
602        @param time8601: a Time instance   
603    '''
604    tm_position = TM_Position()
605    if anyOther:
606        tm_position.anyOther = anyOther
607    if date8601:
608        tm_position.date8601 = date8601 
609    if dateTime8601:
610        tm_position.dateTime8601 = dateTime8601
611    if time8601:
612        tm_position.time8601 = time8601
613    return tm_position           
614
615def createTM_Instant(position):
616    '''
617        Creates a new TM_Position instance
618        @param position: a TM_Position instance
619    '''
620    tm_instant = TM_Instant()
621    tm_instant.position = position
622    return tm_instant
623
624def createTM_Period(begin, end):
625    '''
626        Creates a new TM_Position instance
627        @param position: a TM_Position instance
628    '''
629    tm_period = TM_Period()
630    tm_period.begin = begin
631    tm_period.end = end   
632    return tm_period
633
634def createCI_Address(deliveryPoint = None, electronicMailAddress = None, city = None, country = None, postalCode = None):
635    '''
636        Creates a new CI_Address instance
637        @param deliveryPoint: an array of Strings
638        @param electronicMailAddress: an array of Strings
639        @param city: a string
640        @param country: a string               
641        @param postalCode: a string       
642    '''
643    ci_address = CI_Address()
644    if deliveryPoint:
645        ci_address.deliveryPoint = deliveryPoint
646    if electronicMailAddress:
647        ci_address.electronicMailAddress = electronicMailAddress       
648    if postalCode:
649        ci_address.postalCode = postalCode       
650    if city:
651        ci_address.city = city       
652    if country:
653        ci_address.country = country       
654    if postalCode:
655        ci_address.postalCode = postalCode       
656    return ci_address
657
658def createCI_OnlineResource(linkage, name=None):
659    '''
660        Creates a new CI_OnlineResource instance
661        @param linkage: a string (the URL class is associated with a String)
662        @param name: a String
663    '''
664    ci_online_resource = CI_OnlineResource()
665    ci_online_resource.linkage = linkage
666    if name:
667        ci_online_resource.name = name
668    return ci_online_resource
669
670def createCI_Telephone(voice=None, facsimile=None):
671    '''
672        Creates a new CI_Telephone instance
673        @param voice: an array of strings       
674        @param facsimile: an array of strings
675    '''
676    ci_telephone = CI_Telephone()
677    if voice:
678        ci_telephone.voice = voice
679    if facsimile:
680        ci_telephone.facsimile = voice       
681    return ci_telephone
682
683def createCI_Contact(phone, address = None, onlineResource = None):
684    '''
685        Creates a new CI_Contact instance
686        @param phone: a CI_Telephone instance       
687        @param address: a CI_Address instance
688        @param onlineResource: a CI_OnlineResource instance
689    '''
690    ci_contact = CI_Contact()
691    ci_contact.phone = phone
692    if address:
693        ci_contact.address = address
694    if onlineResource:
695        ci_contact.onlineResource = onlineResource       
696    return ci_contact
697
698def createCI_Individual(name = None, contactInfo = None):
699    '''
700        Creates a new CI_Individual instance
701        @param name: a String
702        @param contactInfo: an array of CI_Contact       
703    '''   
704    ci_party = CI_Individual()
705    if name:
706        ci_party.name = name
707    if contactInfo:
708        ci_party.contactInfo = contactInfo       
709    return ci_party
710
711def createCI_Organization(name = None, contactInfo = None):
712    '''
713        Creates a new CI_Individual instance
714        @param name: a String
715        @param contactInfo: an array of CI_Contact       
716    '''   
717    ci_party = CI_Organisation()
718    if name:
719        ci_party.name = name
720    if contactInfo:
721        ci_party.contactInfo = contactInfo       
722    return ci_party
723
724def createMO_Organization(name = None, contactInfo = None):
725    '''
726        Creates a new MO_Organization instance.
727        TEMPORARILY RETURNS A CI_ORGANIZATION BECAUSE A DB SCHEMA PROBLEM!!!!
728        @param name: a String
729        @param contactInfo: an array of CI_Contact       
730    '''       
731    ci_party = CI_Organisation()
732    if name:
733        ci_party.name = name
734    if contactInfo:
735        ci_party.contactInfo.extend(contactInfo)         
736    return ci_party
737
738def createMO_ResponsiblePartyInfo(role, i_party):
739    """
740        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty
741        @param party: a list of MO_Organization/CI_Individual instances
742    """
743    mo_responsableInfo = MO_ResponsiblePartyInfo()
744    mo_responsableInfo.role = role
745    mo_responsableInfo.party.extend(i_party)
746    return mo_responsableInfo
747
748
749def createCI_Date(dateType, date = None):
750    """
751        Creates a new CI_Date
752        @param dateType: a CI_DateTypeCode value
753        @param date: a DateTime instance
754    """
755    ci_date = CI_Date()
756    ci_date.dateType = dateType
757    if date:
758        ci_date.date = date
759    return ci_date
760
761def createCI_Citation(title, date = None, citedResponsibleParty = None):
762    """
763        Creates a new CI_Citation
764        @param title: the CI_Citation title
765        @param date: an array of CI_Date instances
766        @param citedResponsibleParty: a list of CI_ResponsibleParty instances
767    """   
768    ci_citation = CI_Citation()
769    ci_citation.title = title
770    if date and type(date) == list:
771        ci_citation.date.extend(date)
772    if citedResponsibleParty:
773        ci_citation.extend(citedResponsibleParty)
774    return ci_citation
775
776def createMD_Constraints(useLimitation = None):
777    """
778        Creates a new MD_Constrains
779        @param useLimitation: a string array
780    """ 
781    md_constraints = MD_Constraints()
782    if useLimitation and type(useLimitation) == list:
783        md_constraints.useLimitation = useLimitation
784    return md_constraints
785
786def createMD_LegalConstraints(useLimitation = None, accessConstrains = None):
787    """
788        Creates a new MD_LegalConstrains
789        @param useLimitation: a string array
790        @param accessConstrains: an MD_RestrictionCode array
791    """ 
792    md_legalconstraints = MD_LegalConstraints()
793    if useLimitation and type(useLimitation) == list:
794        md_legalconstraints.useLimitation = useLimitation
795    if accessConstrains and type(accessConstrains) == list:
796        md_legalconstraints.accessConstrains = accessConstrains       
797    return md_legalconstraints
798
799def createMD_Identifier(code, authority = None):
800    """
801        Creates a new MD_Identifier
802        @param code: a String
803        @param authority: a CI_Citation instance
804    """
805    md_identifier = MD_Identifier()
806    md_identifier.code = code
807    if authority:
808        md_identifier.authority = authority
809    return md_identifier
810
811def createCI_ResponsibleParty(role, organizationName = None, individualName = None):
812    """
813        Creates a new CI_ResponsibeParty
814        @param role: a CI_RoleCode
815    """
816    ci_responsible_party = CI_ResponsibleParty()
817    ci_responsible_party.role = role
818    if organizationName:
819        ci_responsible_party.organisationName = organizationName
820    if individualName:
821        ci_responsible_party.individualName = individualName   
822    return ci_responsible_party
823
824def createMD_Metadata(date_stamp, contact, language = None):
825    """
826        Creates a new MD_Metadata
827        @param date_stamp: a Date instance
828        @param contacts: a CI_ResponsibleParty array instances
829        @param language: a string
830    """   
831    md_metadata = MD_Metadata()
832    md_metadata.dateStamp = date_stamp
833    for item in contact:
834        md_metadata.contact.append(item)
835    if language:
836        md_metadata.language = language   
837    return md_metadata
838
839def createMO_OnlineResource(linkage, instance = None, name = None, function = None, description = None, applicationProfile = None):
840    """
841        Creates a new CEDA_Result
842        @param linkage: the MO_OnlineResource.linkage.url.??? field
843        @param name: the MO_OnlineResource.linkage.name field
844        @param function: the MO_OnlineResource.function field
845        @param description: the MO_OnlineResource.description field
846        @param applicationProfile: the MO_OnlineResource.applicationProfile field
847    """
848    if instance is None:
849        on_line_resource = MO_OnlineResource()
850         
851    on_line_resource.linkage = linkage
852    if name and on_line_resource.name != name:         
853        on_line_resource.name = name
854    if function and on_line_resource.function != function:
855        on_line_resource.function = function
856    if description and on_line_resource.description != description:
857        on_line_resource.description = description
858    if applicationProfile and on_line_resource.applicationProfile != applicationProfile:
859        on_line_resource.applicationProfile = applicationProfile
860    return on_line_resource       
861
862def createCEDA_Result(curation_category, internal_path, source = None):
863    """
864        Creates a new CEDA_Result
865        @param curation_category: a CEDA_CurationValue instance
866        @param internal_path: a String
867        @param source: an array of MO_OnlineResource instances
868    """
869    ceda_result = CEDA_Result()
870    ceda_result.curationCategory = curation_category
871    ceda_result.internalPath = internal_path           
872    if source: 
873        ceda_result.source = source
874    return ceda_result
875
876
877
878def createDQ_ConformanceResult(explanation, pass_, specification):
879    """
880        Creates a DQ_ConformanceResult instance
881        @param explanation: a String
882        @param pass_: a boolean value
883        @param specification: a CI_Citation instance 
884    """
885    dq_conformanceResult = DQ_ConformanceResult()
886    dq_conformanceResult.explanation = explanation
887    dq_conformanceResult.pass_ = pass_
888    dq_conformanceResult.specification = specification
889    return dq_conformanceResult
890
891def createDQ_Element(result):
892    """
893        Creates a DQ_Element instance
894        @param result: a DQ_Result array (mix 1, max 2 items)
895    """
896    dq_element = DQ_Element()
897    if result and (type(result) == list) and (len(result) >= 1 and len(result) <= 2):
898        dq_element.result = result
899    return dq_element
900
901def createEX_GeographicBoundingBox(east, north, west, south):
902    """
903        Creates an EX_GeographicBoundingBox instance
904        @param east: the eastBoundLongitude attribute as float
905        @param north: the northBoundLongitude attribute as float
906        @param west: the westBoundLongitude attribute as float
907        @param south: the southBoundLongitude attribute as float               
908    """
909    ex_geographic_bb = EX_GeographicBoundingBox()
910    ex_geographic_bb.eastBoundLongitude = east
911    ex_geographic_bb.northBoundLatitude = north
912    ex_geographic_bb.westBoundLongitude = west
913    ex_geographic_bb.southBoundLatitude = south
914    return ex_geographic_bb
915   
916def createCEDA_Processing():
917    ceda_processing = CEDA_Processing()
918    return ceda_processing
919
920
921def createCEDA_Instrument():
922    ceda_instrument = CEDA_Instrument()
923    return ceda_instrument
924
925def createCEDA_CompositeProcess():
926    ceda_cp = CEDA_CompositeProcess()
927    return ceda_cp
928
929def createCEDA_Acquisition():
930    ceda_acquisition = CEDA_Acquisition()
931    return ceda_acquisition
932
933def createCEDA_Review(reviewer, reviewFrequency, reviewStatus):
934    """
935        Returns a new CEDA_Review
936        @param reviewer: an MO_ResponsibilityPartyInfo
937        @param reviewFrequency: a CEDA_ReviewFrequencyValue
938        @param reviewStatus: a CEDA_ReviewStatusValue
939    """
940    ceda_review = CEDA_Review()
941    ceda_review.reviewer = reviewer
942    ceda_review.reviewFrequency = reviewFrequency
943    ceda_review.reviewStatus = reviewStatus           
944    return ceda_review
945
946def createCEDA_Project(abstract = None, publication_state = None, documentation = None, project_resource=None):
947    ceda_project = CEDA_Project()
948    if abstract:
949        ceda_project.abstract = abstract
950    if publication_state:
951        ceda_project.publicationState = publication_state       
952    if documentation and type(documentation) == list:
953        ceda_project.documentation = documentation
954    if project_resource and type(project_resource) == list:
955        ceda_project.projectResource = project_resource   
956    return ceda_project   
Note: See TracBrowser for help on using the repository browser.