Changeset 8144 for mauRepo


Ignore:
Timestamp:
06/03/12 17:06:04 (8 years ago)
Author:
mnagni
Message:

Implementation of tickets #22386, #22395

Location:
mauRepo/MolesManager/trunk
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • mauRepo/MolesManager/trunk/resources/requirements.txt

    r8143 r8144  
    22SQLAlchemy==0.7.5 
    33psycopg2==2.4.2 
     4html5lib==0.95 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py

    r8091 r8144  
    77from ea_model.iso_19115_2006_metadata_corrigendum.reference_system_information.md_identifier import MD_Identifier 
    88from httplib import HTTPConnection 
    9 from xml.etree.ElementTree import XML 
     9from xml.etree.ElementTree import XML, ElementTree, fromstring, dump, tostring 
    1010import time, datetime 
    1111from ea_model.upcomingiso.ci_organisation import CI_Organisation 
     
    1717from ea_model.iso_19115_2006_metadata_corrigendum.data_quality_information.dq_conformanceresult import DQ_ConformanceResult 
    1818from hashlib import md5 
     19from xml.sax.saxutils import unescape, escape 
     20import html5lib 
     21from html5lib import sanitizer, treebuilders 
    1922 
    2023base = '/exist/rest/atoms' 
     
    4144existNS = "{http://exist.sourceforge.net/NS/exist}" 
    4245molesNS = "{http://ndg.nerc.ac.uk/schema/moles2beta}" 
     46htmlNS = "http://www.w3.org/1999/xhtml" 
    4347time_format = '%Y-%m-%dT%H:%M:%SZ' 
    4448ihost = 'bora.badc.rl.ac.uk' 
    4549iport = '8080' 
     50 
     51#MD_Identifier codes 
     52MD_CODE_MOLES2_CITATION = 'MOLES2_CITATION' 
     53 
     54htmlParser = html5lib.HTMLParser(tree=treebuilders.getTreeBuilder("etree"), namespaceHTMLElements=False) 
    4655 
    4756def calculateHash(text): 
     
    151160    quality = resourceXML.find('%sentity/%smolesISO/%squality' % (molesNS, molesNS, molesNS)) 
    152161    return _returnNotNoneText(quality) 
     162 
     163def extractContent(dataEntityMigration): 
     164    """ 
     165        Returns a dictionary containing the div composing the 
     166        <content> element in a dataentity document. 
     167    """ 
     168    resourceXML = getAtomDocumentByMO(dataEntityMigration) 
     169    content = resourceXML.find('%scontent' % (atomNS)) 
     170    text = _returnNotNoneText(content) 
     171    doc = htmlParser.parse(unescape(text)) 
     172    contentDict = {} 
     173    for el in doc.findall('body//div'):     
     174        prop = el.get('property') 
     175        if prop: 
     176            if prop.startswith('cedacat'): 
     177                contentDict[prop.split(':')[1]] = escape(tostring(el)) 
     178    return contentDict 
    153179 
    154180def _extractAuthors(authorsCSV): 
     
    343369    return ci_citation 
    344370 
    345 def createMD_Identifier(title = ""): 
     371def createMD_Identifier(title = "", code = ""): 
    346372    md_identifier = MD_Identifier() 
    347     md_identifier.code = "" 
     373    md_identifier.code = code 
    348374    md_identifier.authority = createCI_Citation(title) 
    349375    return md_identifier 
     
    360386    dq_element.result.append(dq_result) 
    361387    return dq_element 
     388 
     389 
     390 
     391 
     392 
     393 
     394 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py

    r8089 r8144  
    99from libs.epb import EPB 
    1010from libs.migration.MigrationEPB import MigrationEPB 
    11 from libs.migration.exception.exceptions import NoDataLineage 
     11from libs.migration.exception.exceptions import NoDataLineage,\ 
     12    MigrationObjectException 
    1213from libs.migration.processor.check.check import checkObservationCollection 
    13 from libs.migration.processor.commons import findDeploymentsInDE 
     14from libs.migration.processor.commons import findDeploymentsInDE,\ 
     15    createMD_Identifier, extractContent, MD_CODE_MOLES2_CITATION 
    1416from libs.migration.processor.deployment import DeploymentProcessor 
     17from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
    1518 
    1619class DataEntityProcessor(object): 
     
    2730        self.migrationSessions = migrationSessions 
    2831     
     32    def _processCitation(self, ceda_observationCollection): 
     33        contentDict = extractContent(self._dataEntityMigration)         
     34        ceda_observationCollection.identifier.append(createMD_Identifier(title = contentDict['citation'], code = MD_CODE_MOLES2_CITATION)) 
     35     
    2936    def _processObservationCollection(self): 
    3037        if self._dataEntityMigration is None: 
     
    3744            return ceda_observationCollection 
    3845 
     46        self._processCitation(ceda_observationCollection) 
     47        ceda_observationCollection.publicationState = MO_PublicationStateValue.cl_working 
    3948        #if not adds the new CEDA_ObservationCollection           
    4049        Moles3EPB.addCedaObject(ceda_observationCollection, self.migrationSessions.molesSession) 
     
    5968            self.migrationSessions.molesSession.rollback() 
    6069            self.migrationSessions.migrationSession.rollback()                     
    61             return ex                      
     70            raise MigrationObjectException(ex)                      
    6271         
    63         return None 
     72        return ceda_observation 
    6473     
    6574    def process(self): 
     
    7382        #retreives the DataEntityMigration sorted by creation date 
    7483        deploymentMigrations = MigrationEPB.getAllDeploymentsMigrationByDataEntitySortedByDate(self._dataEntityMigration, deploymentsLinks, self.migrationSessions.migrationSession) 
    75          
     84        howManydm = deploymentMigrations.count() 
    7685        for deploymentMigration in deploymentMigrations: 
    77             exs.append(self._processDeploymentMigration(deploymentMigration, cedaObservationCollection)) 
    78          
     86            try: 
     87                ceda_observation = self._processDeploymentMigration(deploymentMigration, cedaObservationCollection) 
     88                if howManydm > 1:                     
     89                    for identifier in cedaObservationCollection.identifier: 
     90                        if identifier.code == MD_CODE_MOLES2_CITATION: 
     91                            # TBD --- Here should compare if exists already 
     92                            ceda_observation.identifier.append(identifier) 
     93                            self.migrationSessions.molesSession.commit() 
     94            except Exception as ex: 
     95                exs.append(ex) 
    7996        return exs 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py

    r8143 r8144  
    1818from libs.migration.processor.deployment_data import DeploymentDataProcessor 
    1919from ea_model.iso_19115_2006_metadata_corrigendum.identification_information.md_progresscode import MD_ProgressCode 
     20from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
    2021 
    2122class DeploymentProcessor(object): 
     
    100101         
    101102        if not observationHasTitle(self._ceda_observation, doc_title):  
    102             self._ceda_observation.identifier.append(createMD_Identifier(title = doc_title)) 
     103            self._ceda_observation.identifier.append(createMD_Identifier(title = "Data from %s" % (doc_title))) 
    103104 
    104105    def _assignDescription(self): 
     
    143144    def _commitMoles3(self): 
    144145        if self._ceda_observation.id is None: 
    145             self._ceda_observation.status = MD_ProgressCode.cl_ongoing 
     146            self._ceda_observation.publicationState = MO_PublicationStateValue.cl_working 
    146147            Moles3EPB.addCedaObject(self._ceda_observation, self._migrationSessions.molesSession) 
    147148        self._migrationSessions.molesSession.commit() 
Note: See TracChangeset for help on using the changeset viewer.