Changeset 8147 for mauRepo


Ignore:
Timestamp:
12/03/12 15:43:46 (8 years ago)
Author:
mnagni
Message:

Implementation of tickets #22390

Location:
mauRepo/MolesManager/trunk/src
Files:
6 added
18 edited

Legend:

Unmodified
Added
Removed
  • mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py

    r8143 r8147  
    77from libs.commons_db import DbManager 
    88from libs.epb import EPB 
     9from libs.migration.exception.exceptions import NoDBManager 
     10from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection 
     11from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 
     12from libs.migration.processor.check.check import moles3Append 
     13from libs.migration.processor.commons import createDQ_Element 
    914 
    10 molesDB = DbManager(MOLES3_DB_CONNECTION, MOLES3_DB_SCRIPT) 
     15#molesDB = DbManager(MOLES3_DB_CONNECTION, MOLES3_DB_SCRIPT) 
    1116 
    1217class Moles3EPB(EPB): 
     18 
     19    _molesDB = None 
     20    #_migrationDB = DbManager(MIGRATION_DB_CONNECTION, MIGRATION_DB_SCRIPT) 
     21     
     22    @classmethod 
     23    def overrrideDBManager(self, dbManager): 
     24        """ 
     25            Sets the MigrationEPB libs.commons_db.DbManager 
     26        """ 
     27        Moles3EPB._molesDB = dbManager 
    1328 
    1429    @classmethod         
     
    3449        if commit: 
    3550            intSession.commit() 
    36         Moles3EPB._closeSession(session)  
     51        #Moles3EPB._closeSession(session)  
     52 
     53     
     54    @classmethod 
     55    def observationCollectionHasObservation(self, obs_coll_id, obs_id, session = None): 
     56        """ 
     57            Checks if a CEDA_Collection contains a given CEDA_Observation. 
     58            @param obs_coll_id: the CEDA_ObservationColleciton id 
     59            @param obs_id: the CEDA_Observation id 
     60            @return: True if the collection contains the given observation, False otherwise   
     61        """ 
     62        intSession = Moles3EPB._getSession(session) 
     63        return intSession.query(CEDA_ObservationCollection, CEDA_Observation).filter(CEDA_ObservationCollection.id==obs_coll_id).filter(CEDA_Observation.id==obs_id).count() > 0 
     64     
     65    @classmethod 
     66    def addObservationToObservationCollection(self, observationCollection, observation, session = None, commit = False): 
     67        """ 
     68            Adds an Observation instance to an ObservationCollection if still not part of the collection. 
     69            @param observationCollection: the collection to update 
     70            @param observation: the observation to add 
     71            @param session: the session to use for the operation 
     72            @param commit: if True commits at the end (defaul False)  
     73        """ 
     74        intSession = Moles3EPB._getSession(session) 
     75        if not self.observationCollectionHasObservation(observationCollection.id, observation.id, intSession): 
     76            moles3Append(observationCollection, 'member', observation) 
     77            if commit: 
     78                intSession.commit() 
     79    @classmethod 
     80    def addDocumentQuality(self, doc_quality, observation, session = None, commit = False): 
     81        """ 
     82            Adds an resultQuality element to an Observation if still not assigned or not equal. 
     83            @param doc_quality: the quality string to persist 
     84            @param observation: the observation to update 
     85            @param session: the session to use for the operation 
     86            @param commit: if True commits at the end (defaul False)  
     87        """         
     88        intSession = Moles3EPB._getSession(session) 
     89        for rq in observation.resultQuality: 
     90            for res in rq.result: 
     91                if hasattr(res, 'explaination') and res.explaination == doc_quality: 
     92                    return True 
     93                    moles3Append(observation, observation.resultQuality, createDQ_Element(explaination = doc_quality)) 
     94     
     95    @classmethod 
     96    def addDataLineageToObservation(self, data_lineage, observation, session = None, commit = False): 
     97        """ 
     98            Adds an data_lineage element to an Observation if still not assigned or not equal. 
     99            @param data_lineage: the quality string to persist 
     100            @param observation: the observation to update 
     101            @param session: the session to use for the operation 
     102            @param commit: if True commits at the end (defaul False)  
     103        """         
     104        intSession = Moles3EPB._getSession(session) 
     105        if observation.dataLineage != data_lineage: 
     106            observation.dataLineage = data_lineage 
     107        if commit: 
     108            intSession.commit()                 
     109 
     110    @classmethod 
     111    def addDescriptionToObservation(self, description, observation, session = None, commit = False): 
     112        """ 
     113            Adds an description element to an Observation if still not assigned or not equal. 
     114            @param description: the description string to persist 
     115            @param observation: the observation to update 
     116            @param session: the session to use for the operation 
     117            @param commit: if True commits at the end (defaul False)  
     118        """         
     119        intSession = Moles3EPB._getSession(session) 
     120        if observation.description != description: 
     121            observation.description = description 
     122        if commit: 
     123            intSession.commit() 
     124 
     125    @classmethod 
     126    def addRelatedPartyInfoToObservation(self, responsibleParty, observation, session = None, commit = False): 
     127        """ 
     128            Adds a relatedParty element to an Observation if still not assigned or not equal. 
     129            @param responsibleParty: the responsibleParty element to persist 
     130            @param observation: the observation to update 
     131            @param session: the session to use for the operation 
     132            @param commit: if True commits at the end (defaul False)  
     133        """           
     134        intSession = Moles3EPB._getSession(session) 
     135        if not responsibleParty in observation.relatedParty: 
     136            observation.relatedParty.append(responsibleParty)                            
     137            #moles3Append(observation, 'relatedParty', responsibleParty) 
     138        if commit: 
     139            intSession.commit() 
     140 
     141    @classmethod 
     142    def addIdentifierToObservation(self, identifier, observation, session = None, commit = False): 
     143        """ 
     144            Adds an identifier element to an Observation if still not assigned or not equal. 
     145            @param identifier: the responsibleParty element to persist 
     146            @param observation: the observation to update 
     147            @param session: the session to use for the operation 
     148            @param commit: if True commits at the end (defaul False)  
     149        """           
     150        intSession = Moles3EPB._getSession(session) 
     151        if not identifier in observation.identifier: 
     152            moles3Append(observation, 'identifier', identifier) 
     153        if commit: 
     154            intSession.commit() 
    37155 
    38156    @classmethod         
     
    41159 
    42160    @classmethod 
     161    def executeNative(self, sqlNative, session = None): 
     162        intSession = Moles3EPB._getSession(session) 
     163        return EPB.executeNative(sqlNative, intSession)   
     164 
     165    @classmethod 
    43166    def _getSession(self, session = None): 
    44         return EPB._getSession(molesDB, session) 
     167        if Moles3EPB._molesDB is None: 
     168            raise NoDBManager("Moles3EPB") 
     169        return EPB._getSession(Moles3EPB._molesDB, session) 
  • mauRepo/MolesManager/trunk/src/ascore/utils.py

    r8089 r8147  
    1111            return False 
    1212    return False 
     13 
     14def collectionProofHash(tupleToHash): 
     15    try: 
     16        return hash(tuple(_convertCollectionToList(tupleToHash))) 
     17    except Exception as e: 
     18        print e 
     19 
     20def _convertCollectionToList(tupleToHash, processedItems = []):  
     21    toHash = [] 
     22    for item in tupleToHash: 
     23        if item in processedItems: 
     24            continue 
     25        processedItems.append(item) 
     26        if type(item) == tuple or type(item) == list: 
     27            for innerItem in item: 
     28                toHash.extend(collectionProofHash(innerItem, processedItems)) 
     29        elif type(item) == dict: 
     30            for innerItem in item.keys(): 
     31                toHash.extend(collectionProofHash(innerItem, processedItems)) 
     32            for innerItem in item.values(): 
     33                toHash.extend(collectionProofHash(innerItem, processedItems)) 
     34        else: 
     35            toHash.append(item) 
     36    return toHash 
  • mauRepo/MolesManager/trunk/src/ea_model/ceda_metadatamodel/ceda_observation/ceda_observation.py

    r8091 r8147  
    2020 
    2121from ea_model.moles3_4.observation.mo_observation import  MO_Observation  
    22 from ascore.utils import has_value 
     22from ascore.utils import has_value, collectionProofHash 
    2323 
    2424''' 
     
    4646        self._geographicExtent_ex_geographicdescription = [] 
    4747        self._geographicExtent_ex_geographicboundingbox = [] 
    48         self.relatedParty = [] 
     48        #self.relatedParty = [] 
    4949        self._permission_md_securityconstraints = None 
    5050        self._permission_md_legalconstraints = None 
     
    289289        return id(self) == id(y) 
    290290 
    291     def __hash__(self): 
    292         return hash(self.__key()) 
     291    def __hash__(self):   
     292        return collectionProofHash(self.__key())       
  • mauRepo/MolesManager/trunk/src/ea_model/moles3_4/observation/mo_observation.py

    r8091 r8147  
    2020 
    2121from ea_model.iso_19156_2011_observations_and_measurements.observation_schema.observation.om_observation import  OM_Observation  
    22 from ascore.utils import has_value 
     22from ascore.utils import has_value, collectionProofHash 
    2323 
    2424''' 
     
    229229 
    230230    def __hash__(self): 
    231         return hash(self.__key()) 
     231        return collectionProofHash(self.__key()) 
  • mauRepo/MolesManager/trunk/src/ea_model/upcomingiso/ci_party.py

    r8091 r8147  
    1919 
    2020 
    21 from ascore.utils import has_value 
     21from ascore.utils import has_value, collectionProofHash 
    2222 
    2323''' 
     
    5656 
    5757    def __hash__(self): 
    58         return hash(self.__key()) 
     58        return collectionProofHash(self.__key()) 
  • mauRepo/MolesManager/trunk/src/ea_model/upcomingiso/ci_responsiblepartyinfo.py

    r8091 r8147  
    1919 
    2020 
    21 from ascore.utils import has_value 
     21from ascore.utils import has_value, collectionProofHash 
    2222 
    2323''' 
     
    5656 
    5757    def __hash__(self): 
    58         return hash(self.__key()) 
     58        return collectionProofHash(self.__key()) 
  • mauRepo/MolesManager/trunk/src/libs/commons_db.py

    r8089 r8147  
    66from sqlalchemy import MetaData, create_engine 
    77from sqlalchemy.orm import sessionmaker, scoped_session 
     8from sqlalchemy.sql.expression import text 
    89class DbManagerCollection(object): 
    910    """ 
     
    5657            dbSession.rollback() 
    5758            dbSession.close() 
    58             raise             
     59            raise 
     60         
     61    def executeNative(self, sqlNative): 
     62        t = text(sqlNative) 
     63        return self._connection.execute(t) 
    5964 
    6065def doInsertOrUpdate(items, session, update = True): 
  • mauRepo/MolesManager/trunk/src/libs/epb.py

    r8143 r8147  
    55''' 
    66from sqlalchemy.orm import subqueryload 
     7from sqlalchemy.sql.expression import text 
    78 
    89class EPB(object): 
     
    2627        return res 
    2728 
    28     @classmethod 
     29    @classmethod     
    2930    def searchOrCreate(self, clazz, session, clazz_id = None): 
    3031        if clazz_id is not None: 
     
    4243 
    4344    @classmethod 
     45    def executeNative(self, sqlNative, session): 
     46        return session.execute(text(sqlNative))      
     47 
     48    @classmethod 
    4449    def _getSession(self, dbManager, session = None): 
    4550        if session: 
  • mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py

    r8089 r8147  
    44@author: mnagni 
    55''' 
    6 from MolesManager.settings import MIGRATION_DB_SCRIPT, MIGRATION_DB_CONNECTION 
    76from libs.migration.db.classes import DeploymentDataMigration, DataEntityMigration,\ 
    87    DeploymentsMigration 
    98from sqlalchemy.sql.expression import asc 
    10 from libs.commons_db import DbManager 
    119from libs.epb import EPB 
     10from libs.migration.exception.exceptions import NoDBManager 
    1211 
    1312 
    1413class MigrationEPB(EPB): 
    1514     
    16     _migrationDB = DbManager(MIGRATION_DB_CONNECTION, MIGRATION_DB_SCRIPT) 
    17           
     15    _migrationDB = None 
     16    #_migrationDB = DbManager(MIGRATION_DB_CONNECTION, MIGRATION_DB_SCRIPT) 
     17     
     18    @classmethod 
     19    def overrrideDBManager(self, dbManager): 
     20        """ 
     21            Sets the MigrationEPB libs.commons_db.DbManager 
     22        """ 
     23        MigrationEPB._migrationDB = dbManager 
     24 
     25    @classmethod 
     26    def associateObservationCollectionToDataEntity(self, dataEntityMigration, obs_coll_id, session = None, commit = False): 
     27        intSession = MigrationEPB._getSession(session) 
     28        dataEntityMigration.ceda_observation_coll_id = obs_coll_id 
     29        if commit: 
     30            intSession.commit() 
     31 
     32    @classmethod 
     33    def associateObservationToDeployment(self, deploymentMigration, obs_id, session = None, commit = False): 
     34        intSession = MigrationEPB._getSession(session) 
     35        deploymentMigration.ceda_observation_id = obs_id 
     36        if commit: 
     37            intSession.commit()          
     38         
    1839    @classmethod 
    1940    def addMigration(self, migrationObj, session = None): 
     
    6384    @classmethod 
    6485    def _getSession(self, session = None): 
     86        if MigrationEPB._migrationDB is None: 
     87            raise NoDBManager("MigrationEPB") 
    6588        return EPB._getSession(MigrationEPB._migrationDB, session) 
  • mauRepo/MolesManager/trunk/src/libs/migration/client.py

    r8137 r8147  
    1010import time 
    1111from threading import Timer 
    12 from libs.migration.db.classes import MigrationObject 
    13 from libs.migration.processor.commons import DS_pUBLISHED, DO_BADC 
    14  
    15  
    1612 
    1713class MigrationThread(threading.Thread): 
     
    4541    def _migrate(self): 
    4642        lr = LoadResources() 
    47         lr.process() 
     43        ex = lr.process() 
     44        for e in ex: 
     45            print e 
    4846 
    4947        mp = MigrationProcess() 
    50         mp.process() 
    51  
    52 #record = Moles3EPB.search(CEDA_Observation, 12) 
    53 #print DJEncoder().encode(record) 
    54  
    55 lr = LoadResources() 
    56 mp = MigrationProcess() 
    57  
    58 dataEntity = MigrationObject() 
    59 dataEntity.doc_status = DS_pUBLISHED 
    60 dataEntity.doc_owner = DO_BADC 
    61 dataEntity.doc_name = 'dataent_csip.atom' 
    62  
    63 lr.process() 
    64 mp.process(dataEntity) 
    65  
    66  
    67  
    68  
    69  
     48        ex = mp.process() 
     49        for e in ex: 
     50            print e 
  • mauRepo/MolesManager/trunk/src/libs/migration/db/classes.py

    r8091 r8147  
    1515class DataEntityMigration(MigrationObject): 
    1616    def __init__(self):                    
    17         self.data_ent_id = None         
     17        self.data_ent_id = None     
    1818        self.ceda_observation_coll_id = None 
    1919        super(DataEntityMigration, self).__init__() 
  • mauRepo/MolesManager/trunk/src/libs/migration/exception/exceptions.py

    r8082 r8147  
    77def _migrationObjectDescription(migrationObject): 
    88    return "%s %s %s" % (migrationObject.doc_status, migrationObject.doc_owner, migrationObject.doc_name) 
     9 
     10class NoDBManager(Exception): 
     11    def __init__(self, value): 
     12        self.value = value 
     13    def __str__(self): 
     14        return "No DBManager has been set for %s" % (self.value)   
    915     
    1016class NoCreationDate(Exception): 
     
    1218        self.value = value 
    1319    def __str__(self): 
    14         return "The resource with ID %s has no creation date" % (self.value)   
     20        return "The resource %s has no creation date" % (self.value)   
     21     
     22class Moles3ObjectException(Exception): 
     23    def __init__(self, value): 
     24        self.value = value 
     25    def __str__(self): 
     26        return self.value     
     27     
     28class NoDocumentID(Exception): 
     29    def __init__(self, value): 
     30        self.value = value 
     31    def __str__(self): 
     32        return "The resource %s has no creation date" % (self.value)     
    1533 
    1634class MigrationObjectException(Exception): 
    17     def __init__(self, value): 
    18         self._migrationObject = value 
     35    """ 
     36        Raises a generic MigrationObject related exception.         
     37    """ 
     38    def __init__(self, migrationObject, comment = ''): 
     39        """ 
     40            Creates a new instance 
     41            @param migrationObject: the MigrationObject instance that raised the error 
     42            @param comment:additional comment  
     43        """         
     44        self.migrationObject = migrationObject 
     45        self.comment = comment 
    1946    def __str__(self): 
    20         return "Error on migration object %s " % (_migrationObjectDescription(self._migrationObject)) 
     47        return "Error on migration object %s " % (_migrationObjectDescription(self.migrationObject)) 
     48 
     49class NoAssociatedAuthor(MigrationObjectException): 
     50    """ 
     51        Raises a missing associatedAuthor exception.         
     52    """     
     53    def __init__(self, author, migrationObject = None, comment = ''): 
     54        self.author = author 
     55        super(NoAssociatedAuthor, self).__init__(migrationObject, comment = '') 
     56    def __str__(self): 
     57        return "The MigrationObject %s has no author mapping for %s" % (_migrationObjectDescription(self.migrationObject), self.author)         
    2158 
    2259class NoAssociatedDeployments(Exception): 
     
    2663        super(NoAssociatedDeployments, self).__init__() 
    2764    def __str__(self): 
    28         return "The DataEntity %s has no associated deployment named %s" % (_migrationObjectDescription(self._migrationObject), self._deploymentName) 
     65        return "The DataEntity %s has no associated deployment named %s" % (_migrationObjectDescription(self.migrationObject), self._deploymentName) 
    2966     
    3067class NoDataLineage(MigrationObjectException): 
    31     def __init__(self, value): 
    32         self._migrationObject = value         
    33         super(NoDataLineage, self).__init__(self._migrationObject) 
    34     def __str__(self): 
    35         return "The DataEntity %s has no moles:lineage value" % (_migrationObjectDescription(self._migrationObject)) 
     68    """ 
     69        Raises a missing dataLineage exception.         
     70    """     
     71    def __init__(self, migrationObject, comment = ''):         
     72        super(NoDataLineage, self).__init__(migrationObject, comment = '') 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/check/check.py

    r8091 r8147  
    44@author: mnagni 
    55''' 
     6from libs.migration.exception.exceptions import Moles3ObjectException 
    67def _updateParty(oldParty, newParty): 
    78    pass 
     
    5455                return 
    5556 
    56         #if here the relation has to be created 
    57         assign = [] 
    58         assign = ceda_observationCollection.member 
    59         assign.append(ceda_observation) 
    60         ceda_observationCollection.member = assign 
     57        moles3Append(ceda_observationCollection, 'member', ceda_observation) 
     58         
     59def moles3Append(parent, collectionAttributeName, itemToAppend): 
     60    """ 
     61        Appends a new item to a Moles3 object.  
     62        Because of the inheritance/composition structure of the Moles3 objects the associated ORM 
     63        has problem to simpy append an item to an existing ORM-controlled list. This method  
     64        copies all the existing items in a new list then append the new item and finally assign the  
     65        new list to the parent proper collection 
     66        @param parent: the instance containing the collection to update 
     67        @param collectionAttributeName: the name of the collection to update 
     68        @param itemToAppend: the item to append to the collectionAttribute list         
     69    """     
     70     
     71    if not hasattr(parent, collectionAttributeName): 
     72        raise Moles3ObjectException() 
     73     
     74    assign = [] 
     75    assign.extend(getattr(parent, collectionAttributeName)) 
     76    assign.append(itemToAppend) 
     77    try: 
     78        setattr(parent, collectionAttributeName, assign) 
     79    except Exception as e: 
     80        print e 
    6181                      
    6282                 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py

    r8144 r8147  
    1212from ea_model.upcomingiso.ci_party import CI_Party 
    1313from ea_model.upcomingiso.ci_individual import CI_Individual 
    14 from libs.migration.exception.exceptions import NoDataLineage 
     14from libs.migration.exception.exceptions import NoDataLineage,\ 
     15    NoAssociatedAuthor 
    1516from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_citation import CI_Citation 
    1617from ea_model.iso_19115_2006_metadata_corrigendum.data_quality_information.dq_element import DQ_Element 
     
    2021import html5lib 
    2122from html5lib import sanitizer, treebuilders 
     23from libs.migration.authors import authors 
    2224 
    2325base = '/exist/rest/atoms' 
     
    196198    ret = {}    
    197199    resourceXML = getAtomDocumentByMO(resourceMigration) 
    198     ret['authors'] = _extractAuthors(findAuthorInResource(resourceXML)) 
    199     ret['contributors'] = _extractAuthors(findContributorInResource(resourceXML)) 
     200    ret['authors'] = findAuthorInResource(resourceXML) 
     201    ret['contributors'] = findContributorInResource(resourceXML) 
    200202    return ret 
    201203 
     
    244246            return category.get("label")     
    245247         
    246 def extractAuthor(deploymentMigration): 
     248def extractTitle(deploymentMigration): 
    247249    resourceXML = getAtomDocumentByMO(deploymentMigration) 
    248250    title = resourceXML.find('%stitle' % (atomNS)) 
     
    325327    return XML(resourceDoc) 
    326328 
    327 def createMO_ResponsiblePartyInfo(role, name): 
    328     ''' 
    329         @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
    330         @param name: the CI_Party name 
    331     ''' 
    332     return createMO_ResponsiblePartyInfoAsCI_Individual(role, name) 
    333     #return _createMO_ResponsiblePartyInfo(role, name) 
    334  
    335329def createMO_ResponsiblePartyInfoAsCI_Organization(role, names): 
    336330    ''' 
     
    338332        @param names: the name assigned to each CI_Party 
    339333    ''' 
    340     return _createMO_ResponsiblePartyInfo(role, names, CI_Organisation) 
     334    return createMO_ResponsiblePartyInfo(role, names, CI_Organisation) 
    341335 
    342336def createMO_ResponsiblePartyInfoAsCI_Individual(role, names): 
     
    345339        @param names: the name assigned to each CI_Party 
    346340    ''' 
    347     return _createMO_ResponsiblePartyInfo(role, names, CI_Individual) 
    348  
    349 def _createMO_ResponsiblePartyInfo(role, names, partyType = CI_Party): 
     341    return createMO_ResponsiblePartyInfo(role, names, CI_Individual) 
     342 
     343def createMO_ResponsiblePartyInfo(role, names, partyType = CI_Party): 
    350344    """ 
    351345        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py

    r8144 r8147  
    1010from libs.migration.MigrationEPB import MigrationEPB 
    1111from libs.migration.exception.exceptions import NoDataLineage,\ 
    12     MigrationObjectException 
     12    MigrationObjectException, NoAssociatedAuthor 
    1313from libs.migration.processor.check.check import checkObservationCollection 
    1414from libs.migration.processor.commons import findDeploymentsInDE,\ 
    15     createMD_Identifier, extractContent, MD_CODE_MOLES2_CITATION 
     15    createMD_Identifier, extractContent, MD_CODE_MOLES2_CITATION,\ 
     16    hasAtomDocumentSameHash 
    1617from libs.migration.processor.deployment import DeploymentProcessor 
    1718from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
    1819 
    1920class DataEntityProcessor(object): 
     21     
    2022    ''' 
    2123        Processes a DataEntityMigration item. Note that each DataEntity is associated to a "dataent_xxxx" file in Moles2 
     
    2628            @param _dataEntityMigration: the DataEntityMigration instance 
    2729            @param migrationSessions: a MigrationSessions instance 
    28         '''           
     30        ''' 
     31        if dataEntityMigration is None: 
     32            raise MigrationObjectException("DataEntityProcessor cannot process an None item") 
    2933        self._dataEntityMigration = dataEntityMigration        
    3034        self.migrationSessions = migrationSessions 
    3135     
    3236    def _processCitation(self, ceda_observationCollection): 
    33         contentDict = extractContent(self._dataEntityMigration)         
     37        # TDB - Check that if is an update or not! 
     38        contentDict = extractContent(self._dataEntityMigration)        
    3439        ceda_observationCollection.identifier.append(createMD_Identifier(title = contentDict['citation'], code = MD_CODE_MOLES2_CITATION)) 
    3540     
    36     def _processObservationCollection(self): 
    37         if self._dataEntityMigration is None: 
    38             raise Exception 
    39                             
    40         #Does the migrationObject exists? 
    41         ceda_observationCollection = EPB.searchOrCreate(CEDA_ObservationCollection, self.migrationSessions.molesSession, self._dataEntityMigration.ceda_observation_coll_id) 
     41    def _getObservationCollection(self): 
     42        return EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id, self.migrationSessions.molesSession) 
     43     
     44    def _processObservationCollection(self):                                    
     45        #Moles3 object exists...  
     46        if self._dataEntityMigration.ceda_observation_coll_id:  
     47            # ...and the data entity document has not changed 
     48            if hasAtomDocumentSameHash(self._dataEntityMigration): 
     49                return self._getObservationCollection() 
     50            else: 
     51                self.updateObservationCollection() 
    4252 
    43         if ceda_observationCollection.id: 
    44             return ceda_observationCollection 
     53        #... does not exist so create it 
     54        return self.createNewObservationCollection() 
    4555 
     56    def updateObservationCollection(self): 
     57        """ 
     58            Updated the existing CEDA_ObservationCollection instance binded to the self._dataEntityMigration object. 
     59            @return: the persisted and updated CEDA_ObservationCollection element  
     60        """         
     61        ceda_observationCollection = self._getObservationCollection() 
     62        # TBD 
     63        #self._processCitation(ceda_observationCollection) 
     64        return ceda_observationCollection 
     65 
     66    def createNewObservationCollection(self):  
     67        """ 
     68            Creates a new CEDA_ObservationCollection instance in the Moles3DB using the self._dataEntityMigration object. 
     69            If successful adds the new instance ID to the related DataEntityMigration object 
     70            @return: the persisted CEDA_ObservationCollection element  
     71        """ 
     72        ceda_observationCollection = CEDA_ObservationCollection() 
    4673        self._processCitation(ceda_observationCollection) 
    4774        ceda_observationCollection.publicationState = MO_PublicationStateValue.cl_working 
    48         #if not adds the new CEDA_ObservationCollection           
    49         Moles3EPB.addCedaObject(ceda_observationCollection, self.migrationSessions.molesSession) 
    50         self.migrationSessions.molesSession.commit() 
    51          
    52         #associates the CEDA_ObservationCollection to the migrationObject 
    53         self._dataEntityMigration.ceda_observation_coll_id = ceda_observationCollection.id 
    54         self.migrationSessions.migrationSession.commit() 
     75                   
     76        Moles3EPB.addCedaObject(ceda_observationCollection, self.migrationSessions.molesSession, True) 
     77        MigrationEPB.associateObservationCollectionToDataEntity(self._dataEntityMigration, ceda_observationCollection.id, True) 
    5578        return ceda_observationCollection 
    5679 
     
    5982        try: 
    6083            ceda_observation = deProcessor.process() 
    61                          
    62             checkObservationCollection(cedaObservationCollection, ceda_observation) 
    63             self.migrationSessions.molesSession.commit() 
    64                          
    65             deploymentMigration.ceda_observation_coll_id = cedaObservationCollection.id 
    66             self.migrationSessions.migrationSession.commit()          
    67         except NoDataLineage as ex: 
     84            Moles3EPB.addObservationToObservationCollection(cedaObservationCollection, ceda_observation, self.migrationSessions.molesSession, True)             
     85            MigrationEPB.associateObservationCollectionToDataEntity(deploymentMigration, cedaObservationCollection.id, self.migrationSessions.migrationSession, True) 
     86        except NoAssociatedAuthor as ex: 
     87            raise ex                  
     88        except Exception as ex: 
    6889            self.migrationSessions.molesSession.rollback() 
    69             self.migrationSessions.migrationSession.rollback()                     
     90            self.migrationSessions.migrationSession.rollback()                
    7091            raise MigrationObjectException(ex)                      
    7192         
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py

    r8144 r8147  
    55''' 
    66from libs.migration.processor.commons import findMolesLineage,\ 
    7     createMO_ResponsiblePartyInfo, getAtomDocumentByMO,\ 
    8     DO_BADC, DO_NEODC, findAuthorsInResource, CEDA, extractAuthor,\ 
    9     createMD_Identifier, extractSummary, extractQuality, createDQ_Element,\ 
    10     hasAtomDocumentSameHash, getAtomDocumentHashByMO 
     7    createMO_ResponsiblePartyInfo,\ 
     8    DO_BADC, DO_NEODC, findAuthorsInResource, CEDA, extractTitle,\ 
     9    createMD_Identifier, extractSummary, extractQuality, \ 
     10    hasAtomDocumentSameHash, getAtomDocumentHashByMO, extractTitle,\ 
     11        createMO_ResponsiblePartyInfoAsCI_Organization,\ 
     12        createMO_ResponsiblePartyInfoAsCI_Individual 
    1113from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue 
    1214from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 
    13 from MolesManager.moles3epb import Moles3EPB 
    1415from libs.migration.MigrationEPB import MigrationEPB 
    1516from libs.epb import EPB 
    16 from libs.migration.processor.check.check import checkLineage, checkResponsiblePartyInfos,\ 
    17     observationHasTitle, observationHasQuality 
    1817from libs.migration.processor.deployment_data import DeploymentDataProcessor 
    19 from ea_model.iso_19115_2006_metadata_corrigendum.identification_information.md_progresscode import MD_ProgressCode 
    2018from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
     19from libs.migration.exception.exceptions import NoDataLineage,\ 
     20    NoAssociatedAuthor 
     21from MolesManager.moles3epb import Moles3EPB 
     22from libs.migration.authors import authors 
     23from logging import StreamHandler 
     24import logging 
    2125 
    2226class DeploymentProcessor(object): 
     
    2529    ''' 
    2630    publisherName = 'CEDA' 
    27     ''' 
    28         Processes a Deployment item.   
    29     '''     
     31     
     32    log = logging.getLogger('DeploymentProcessor') 
     33    log.addHandler(StreamHandler()) 
     34    log.setLevel(logging.DEBUG)     
    3035    def __init__(self, dataEntityMigration, deploymentMigration, migrationSessions): 
    3136        ''' 
     
    7378                opi.party.append(np) 
    7479 
    75     def _assignName(self):         
     80    def _assignName(self, observation):         
    7681        ''' 
    7782            @param relatedPartyInfos: a MO_ResponsiblePartyInfo list 
    7883            @return True if the documents changed, False otherwise 
    79         '''        
    80         newPartyInfos = [] 
     84        ''' 
     85        whereAreAuthors = self._deploymentMigration 
    8186         
    82         authors = findAuthorsInResource(self._deploymentMigration) 
     87        doc_authors = findAuthorsInResource(self._deploymentMigration) 
    8388         
    84         if (len(set(authors['authors']) & set([DO_BADC, DO_NEODC])) > 0): 
    85             authors = findAuthorsInResource(self._dataEntityMigration) 
    86              
    87         newPartyInfos.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_curator, [CEDA]))          
    88         newPartyInfos.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, authors['authors'])) 
    89         newPartyInfos.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_coinvestigator, authors['contributors'])) 
     89        if (len(set(doc_authors['authors']) & set([DO_BADC, DO_NEODC])) > 0): 
     90            doc_authors = findAuthorsInResource(self._dataEntityMigration) 
     91            whereAreAuthors = self._dataEntityMigration 
     92        Moles3EPB.addRelatedPartyInfoToObservation(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_curator, [CEDA]), observation, self._migrationSessions.molesSession) 
     93        tmp_auth = None 
     94        if authors.has_key(doc_authors['authors']): 
     95            tmp_auth = authors[doc_authors['authors']] 
     96            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) 
     97            if len(ret['ind']) > 0: 
     98                createMO_ResponsiblePartyInfoAsCI_Individual(MO_RoleValue.cl_author, [ret['ind']]) 
     99            if len(ret['org']) > 0: 
     100                createMO_ResponsiblePartyInfoAsCI_Organization(MO_RoleValue.cl_author, [ret['org']]) 
     101            if len(ret['co_ind']) > 0: 
     102                createMO_ResponsiblePartyInfoAsCI_Individual(MO_RoleValue.cl_author, [ret['co_ind']]) 
     103            if len(ret['co_org']) > 0: 
     104                createMO_ResponsiblePartyInfoAsCI_Organization(MO_RoleValue.cl_author, [ret['co_org']])                                                          
     105        else: 
     106            raise NoAssociatedAuthor(doc_authors['authors'], migrationObject = whereAreAuthors) 
    90107 
    91         checkResponsiblePartyInfos(self._ceda_observation.relatedParty, newPartyInfos) 
    92108 
    93     def _assignLineage(self): 
    94         checkLineage(self._ceda_observation, findMolesLineage(self._dataEntityMigration)) 
     109    def _extractIndividualsAndOrganizations(self, tmp_auth, whereAreAuthors):    
     110        ret = {'ind': [], 'org': [], 'co_ind': [], 'co_org': []} 
     111        if tmp_auth['type'] == 'ind': 
     112            ret['ind'].append(tmp_auth['author']) 
     113        elif tmp_auth['type'] == 'org': 
     114            ret['org'].append(tmp_auth['author']) 
    95115         
    96     def _assignTitle(self):        
    97         doc_title = extractAuthor(self._deploymentMigration) 
     116        if tmp_auth['author'] == 'unknown': 
     117            DeploymentProcessor.log.info("%s %s %s has unknown author" % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name))         
     118   
     119        for item in tmp_auth['co_author_type']: 
     120            if item == 'ind': 
     121                ret['ind'].append(tmp_auth['co_author'].index(item)) 
     122            elif item == 'org': 
     123                ret['org'].append(tmp_auth['co_author'].index(item)) 
     124         
     125            if (tmp_auth['co_author'].index(item) == 'unknown'): 
     126                DeploymentProcessor.log.info("%s %s %s has unknown author" % (whereAreAuthors.doc_status, whereAreAuthors.doc_owner, whereAreAuthors.doc_name)) 
     127        return ret 
     128 
     129    def _assignLineage(self, observation): 
     130        data_lineage = findMolesLineage(self._dataEntityMigration) 
     131        if data_lineage is None: 
     132            raise NoDataLineage(self._dataEntityMigration) 
     133        Moles3EPB.addDataLineageToObservation(data_lineage, observation, self._migrationSessions.molesSession)         
     134         
     135    def _assignTitle(self, observation):        
     136        doc_title = extractTitle(self._deploymentMigration) 
    98137         
    99138        if doc_title.startswith('deployment_'): 
    100             doc_title = extractAuthor(self._dataEntityMigration) 
     139            doc_title = extractTitle(self._dataEntityMigration)         
     140        Moles3EPB.addDataLineageToObservation(createMD_Identifier(title = "Data from %s" % (doc_title)), observation, self._migrationSessions.molesSession) 
     141 
     142    def _assignDescription(self, observation): 
     143        summary = extractSummary(self._deploymentMigration, self._dataEntityMigration) 
     144        Moles3EPB.addDescriptionToObservation(summary, observation, self._migrationSessions.molesSession) 
    101145         
    102         if not observationHasTitle(self._ceda_observation, doc_title):  
    103             self._ceda_observation.identifier.append(createMD_Identifier(title = "Data from %s" % (doc_title))) 
     146    def _assignQuality(self, observation): 
     147        doc_quality = extractQuality(self._dataEntityMigration) 
     148        Moles3EPB.addDocumentQuality(doc_quality, observation, self._migrationSessions.molesSession) 
     149         
     150    def updateObservation(self): 
     151        ceda_observation = EPB.searchOrCreate(CEDA_Observation, self._migrationSessions.molesSession, self._deploymentMigration.ceda_observation_id) 
     152        return ceda_observation 
    104153 
    105     def _assignDescription(self): 
    106         summary = extractSummary(self._deploymentMigration, self._dataEntityMigration)         
    107         self._ceda_observation.description = summary 
    108          
    109     def _assignQuality(self): 
    110         doc_quality = extractQuality(self._dataEntityMigration) 
    111          
    112         if not observationHasQuality(self._ceda_observation, doc_quality): 
    113             res = [] 
    114             for item in self._ceda_observation.resultQuality: 
    115                 res.append(item) 
    116             res.append(createDQ_Element(explaination = doc_quality)) 
    117             self._ceda_observation.resultQuality = res 
    118          
    119     def _processObj(self):          
    120         ''' 
    121             @param  cedaObservation: a cedaObservation instance  
    122         ''' 
    123         #These methods depend exclusively from  
    124         #self._dataEntityMigration and self._deploymentMigration 
    125         #consequently I calculate the hash for the referred documents.  
    126         #If both is the same as  
    127          
    128         if not self._dataEntityHasSameHash: 
    129             self._assignQuality() 
    130             self._assignLineage() 
    131          
    132         if not (self._dataEntityHasSameHash and self._deploymentHasSameHash): 
    133             self._assignDescription() 
    134             self._assignName() 
    135             self._assignTitle() 
    136          
    137         if not self._deploymentHasSameHash: 
    138             pass 
    139                           
    140         #If any of the previous methods has new values this function will return True 
    141         return not (self._dataEntityHasSameHash and self._deploymentHasSameHash)         
    142            
     154    def createNewObservation(self): 
     155        ceda_observation = CEDA_Observation() 
     156        ceda_observation.publicationState = MO_PublicationStateValue.cl_working 
     157        self._assignQuality(ceda_observation) 
     158        self._assignLineage(ceda_observation) 
     159        self._assignDescription(ceda_observation) 
     160        self._assignName(ceda_observation) 
     161        self._assignTitle(ceda_observation) 
     162        self._migrationSessions.molesSession.commit() 
     163        #self._commitMoles3() 
     164        #self._commitMigration() 
     165        Moles3EPB.addCedaObject(ceda_observation, self._migrationSessions.molesSession, True) 
     166        MigrationEPB.associateObservationToDeployment(self._deploymentMigration, ceda_observation.id, self._migrationSessions.molesSession, True) 
     167        return ceda_observation 
    143168 
    144     def _commitMoles3(self): 
    145         if self._ceda_observation.id is None: 
    146             self._ceda_observation.publicationState = MO_PublicationStateValue.cl_working 
    147             Moles3EPB.addCedaObject(self._ceda_observation, self._migrationSessions.molesSession) 
    148         self._migrationSessions.molesSession.commit() 
    149         self._deploymentMigration.ceda_observation_id = self._ceda_observation.id         
    150          
    151     def _commitMigration(self): 
    152         if self._deploymentMigration.depl_id is None: 
    153             MigrationEPB.addMigration(self._deploymentMigration, self._migrationSessions.migrationSession) 
    154                              
    155          
     169    def _getObservation(self): 
     170        return EPB.search(CEDA_Observation, self._deploymentMigration.ceda_observation_id, self._migrationSessions.molesSession) 
     171             
    156172    def process(self): 
    157         #self._deployment = getAtomDocumentByMO(self._deploymentMigration) 
    158         #retrieves the deployment'd document 
    159         self._ceda_observation = None 
    160  
    161         #Exists already an associated CEDA_Observation associated to this deployment? 
    162         if self._deploymentMigration is not None: 
    163             self._ceda_observation = EPB.searchOrCreate(CEDA_Observation, self._migrationSessions.molesSession, self._deploymentMigration.ceda_observation_id) 
    164  
    165         #fills/update the part of the CEDA_Observation                         
    166         if self._processObj(): 
    167             self._commitMoles3() 
    168             self._commitMigration() 
     173        ceda_observation = None 
     174        #Moles3 object exists...  
     175        if self._deploymentMigration.ceda_observation_id:  
     176            # ...and the data entity document has not changed 
     177            if hasAtomDocumentSameHash(self._deploymentMigration): 
     178                ceda_observation = self._getObservation() 
     179            else: 
     180                ceda_observation = self.updateObservation() 
     181        else: 
     182            #... does not exist so create it 
     183            ceda_observation = self.createNewObservation()  
    169184         
    170185        #process the CEDA_Observation.procedure 
     
    172187        procedure = deploymentDataProcessor.process() 
    173188        if procedure: 
    174             self._ceda_observation.procedure = procedure 
    175             self._commitMoles3() 
     189            ceda_observation.procedure = procedure 
     190            self._migrationSessions.molesSession.commit() 
    176191             
    177         if not self._deploymentHasSameHash:  
     192        if not self._deploymentHasSameHash: 
    178193            self._deploymentMigration.doc_hash = getAtomDocumentHashByMO(self._deploymentMigration) 
    179194            self._commitMigration() 
    180195              
    181196         
    182         return self._ceda_observation 
     197        return ceda_observation 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py

    r8091 r8147  
    88    buildExistDocPath, DT_DATA_ENTITIES, DT_DEPLOYMENTS, DT_DEPLOYMENT_DATA,\ 
    99    getOwnerRefs, getAtomDocumentAsElementtree, getAtomDocumentHash 
    10 from libs.migration.exception.exceptions import NoCreationDate 
     10from libs.migration.exception.exceptions import NoCreationDate, NoDocumentID 
    1111from libs.migration.db.classes import DeploymentDataMigration,\ 
    1212    DeploymentsMigration, DataEntityMigration 
     
    1919    def __init__(self): 
    2020        self._migrationSession = MigrationEPB.getNewMigrationSession() 
     21 
     22    def process(self): 
     23        """ 
     24            Loads all the folders below the atoms ('published', 'working', 'Published') folders 
     25        """  
     26        ex = []        
     27        for status in docStatus:        
     28            ex.extend(self._loadCollections(status, DT_DATA_ENTITIES, DataEntityMigration)) 
     29            ex.extend(self._loadCollections(status, DT_DEPLOYMENTS, DeploymentsMigration)) 
     30            ex.extend(self._loadCollections(status, DT_DEPLOYMENT_DATA, DeploymentDataMigration))                                                
    2131             
    22     def _loadDataEntities(self, docStatus): 
    23         self._loadCollections(docStatus, DT_DATA_ENTITIES, self._loadDataEntityDocs)         
     32        self._migrationSession.close() 
     33        return ex 
    2434 
    25     def _loadDeployments(self, docStatus): 
    26         self._loadCollections(docStatus, DT_DEPLOYMENTS, self._loadDeploymentsDocs)         
     35    def updateMigrationDoc(self, migrationClass, doc_id, docHash, docCreation): 
     36        """ 
     37            Updates an existing migration document 
     38            @param migrationClass: the migration class associated with the document 
     39            @param doc_id: the document id 
     40            @param docHash: the document hashCode calculated by libs.migration.processor.commons.calculateHash 
     41            @param docCreation:the creation date 
     42            @return: True if the document exists, False otherwise               
     43        """ 
     44        coll = MigrationEPB.search(migrationClass, doc_id, self._migrationSession) 
     45        if coll != None: 
     46            #... and has the same hash nothing has changed 
     47            if coll.doc_hash == docHash:                 
     48                return True 
     49            else: 
     50                coll.doc_creation = stringToTimestamp(docCreation) 
     51                coll.doc_hash = docHash 
     52                self._migrationSession.commit() 
     53                return True 
     54        return False 
    2755 
    28     def _loadDeploymentData(self, docStatus): 
    29         self._loadCollections(docStatus, DT_DEPLOYMENT_DATA, self._loadDeploymentDataDocs)             
    3056 
    31     def _loadCollections(self, docStatus, docType, processor): 
    32         refs = getCollectionRefs(buildExistTypePath(docStatus, docType))         
     57    def createMigrationDoc(self, migrationClass, doc_id, docName, docOwner, docStatus, docHash, docCreation): 
     58        coll = None 
     59        if migrationClass == DeploymentsMigration: 
     60            coll = DeploymentsMigration() 
     61            coll.depl_id = doc_id 
     62        elif  migrationClass == DataEntityMigration: 
     63            coll = DataEntityMigration() 
     64            coll.data_ent_id = doc_id 
     65        elif migrationClass == DeploymentDataMigration: 
     66            coll = DeploymentDataMigration() 
     67            coll.deployment_data_id = doc_id 
     68             
     69        self._processNotExistingDoc(coll, docName, docOwner, docStatus, docHash, docCreation) 
     70 
     71    def _loadCollections(self, docStatus, docType, migrationClass): 
     72        ex = [] 
     73        refs = getCollectionRefs(buildExistTypePath(docStatus, docType))             
    3374        for ref in refs: 
    3475            docOwner = ref.get('name')   
    3576            if docOwner.startswith('.'): 
    36                 continue                        
    37             processor(docStatus, docOwner)            
    38              
    39     def _loadDeploymentsDocs(self, docStatus, docOwner): 
    40         refs = getOwnerRefs(docStatus, DT_DEPLOYMENTS, docOwner)          
     77                continue                     
     78            ex.extend(self._loadMigrationDocs(docStatus, docOwner, docType, migrationClass)) 
     79 
     80        return ex 
     81 
     82    def _loadMigrationDocs(self, docStatus, docOwner, docType, migrationClass): 
     83        ex = [] 
     84        refs = getOwnerRefs(docStatus, docType, docOwner)                     
    4185        for ref in refs: 
    4286            docName = ref.get('name') 
    43             xmlDocument = getAtomDocumentAsElementtree(docStatus, DT_DEPLOYMENTS, docOwner, docName) 
    44             #docHash = getAtomDocumentHash(docStatus, DT_DEPLOYMENTS, docOwner, docName)             
     87            try: 
     88                self._processMigrationDoc(docName, docStatus, docOwner, docType, migrationClass) 
     89            except Exception as e: 
     90                if hasattr(e, 'value'): 
     91                    e.value = buildExistDocPath(docStatus, docType, docOwner, docName) 
     92                ex.append(e) 
     93        return ex 
     94 
     95    def _processMigrationDoc(self, docName, docStatus, docOwner, docType, migrationClass): 
     96        """ 
     97            Ingests documents like DataEntity, Deployment, Deployment_Data. The document is register in a DB table of the appropriate 
     98            type (DataEntityMigration, DeploymentsMigration, DeploymentDataMigration). 
     99            @param docName: the document name 
     100            @param docStatus: a common.docStatus item 
     101            @param docOwner: a common.docOwner item    
     102        """ 
     103        docHash = getAtomDocumentHash(docStatus, docType, docOwner, docName) 
     104        xmlDocument = getAtomDocumentAsElementtree(docStatus, docType, docOwner, docName) 
     105                                     
     106        docCreation = self._extractCreationDate(xmlDocument) 
     107        doc_id = findID(xmlDocument)  
     108         
     109        if self.updateMigrationDoc(migrationClass, doc_id, docHash, docCreation): 
     110            return 
     111 
     112        self.createMigrationDoc(migrationClass, doc_id, docName, docOwner, docStatus, docHash, docCreation) 
    45113             
    46             try: 
    47                 doc_creation = self._extractCreationDate(xmlDocument) 
    48             except NoCreationDate as ex: 
    49                 print ex 
    50                 continue 
    51              
    52             doc_id = findID(xmlDocument)  
    53             if doc_id is None: 
    54                 print (buildExistDocPath(docStatus, DT_DEPLOYMENTS, docOwner, docName)) 
     114    def _processNotExistingDoc(self, migrationObject, docName, docOwner, docStatus, docHash, docCreation): 
     115        migrationObject.doc_name = docName 
     116        migrationObject.doc_owner = docOwner 
     117        migrationObject.doc_status = docStatus 
     118        migrationObject.doc_hash = docHash                       
     119        migrationObject.doc_creation = stringToTimestamp(docCreation) 
     120        MigrationEPB.addMigration(migrationObject, self._migrationSession) 
     121        self._migrationSession.commit()     
    55122 
    56             add = False 
    57             coll = MigrationEPB.search(DeploymentsMigration, doc_id, self._migrationSession) 
    58             if coll is None:                 
    59                 coll = DeploymentsMigration() 
    60                 coll.depl_id = doc_id 
    61                 add = True 
    62                  
    63             #If exist and has the same hash nothing has changed 
    64             #if coll.doc_hash == docHash: 
    65             #    continue 
    66              
    67             #coll.doc_hash = docHash             
    68             coll.doc_name = docName 
    69             coll.doc_owner = docOwner 
    70             coll.doc_status = docStatus             
    71             coll.doc_creation = stringToTimestamp(doc_creation) 
    72              
    73             if add: 
    74                 MigrationEPB.addMigration(coll, self._migrationSession) 
    75                  
    76             self._migrationSession.commit() 
    77              
    78     def _loadDataEntityDocs(self, docStatus, docOwner): 
    79         refs = getOwnerRefs(docStatus, DT_DATA_ENTITIES, docOwner)  
    80         for ref in refs: 
    81             docName = ref.get('name') 
    82             xmlDocument = getAtomDocumentAsElementtree(docStatus, DT_DATA_ENTITIES, docOwner, docName) 
    83             #docHash = getAtomDocumentHash(docStatus, DT_DATA_ENTITIES, docOwner, docName)   
    84              
    85             try: 
    86                 doc_creation = self._extractCreationDate(xmlDocument)             
    87             except NoCreationDate as ex: 
    88                 print ex 
    89                 continue 
    90              
    91             doc_id = findID(xmlDocument) 
    92             if doc_id is None: 
    93                 print (buildExistDocPath(docStatus, DT_DEPLOYMENTS, docOwner, docName)) 
    94123 
    95                        
    96             add = False 
    97             coll = MigrationEPB.search(DataEntityMigration, doc_id, self._migrationSession) 
    98             if coll is None:                 
    99                 coll = DataEntityMigration() 
    100                 coll.data_ent_id = doc_id 
    101                 add = True 
    102              
    103             #If exist and has the same hash nothing has changed 
    104             #if coll.doc_hash == docHash: 
    105             #    continue 
    106              
    107             #coll.doc_hash = docHash 
    108             coll.doc_name = docName   
    109             coll.doc_owner = docOwner       
    110             coll.doc_status = docStatus 
    111             coll.doc_creation = stringToTimestamp(doc_creation)  
    112             if add: 
    113                 MigrationEPB.addMigration(coll, self._migrationSession) 
    114             self._migrationSession.commit() 
    115  
    116     def _loadDeploymentDataDocs(self, docStatus, docOwner): 
    117         refs = getOwnerRefs(docStatus, DT_DEPLOYMENT_DATA, docOwner) 
    118         for ref in refs: 
    119             docName = ref.get('name') 
    120             xmlDocument = getAtomDocumentAsElementtree(docStatus, DT_DEPLOYMENT_DATA, docOwner, docName) 
    121             #docHash = getAtomDocumentHash(docStatus, DT_DEPLOYMENT_DATA, docOwner, docName)                       
    122  
    123             try: 
    124                 doc_creation = self._extractCreationDate(xmlDocument)             
    125             except NoCreationDate as ex: 
    126                 print ex 
    127                 continue 
    128              
    129             doc_id = findID(xmlDocument) 
    130             if doc_id is None: 
    131                 print (buildExistDocPath(docStatus, DT_DEPLOYMENT_DATA, docOwner, docName)) 
    132  
    133             add = False 
    134             coll = MigrationEPB.search(DeploymentDataMigration, doc_id, self._migrationSession) 
    135             if coll is None:                 
    136                 coll = DeploymentDataMigration() 
    137                 coll.deployment_data_id = doc_id 
    138                 add = True 
    139              
    140             #If exist and has the same hash nothing has changed 
    141             #if coll.doc_hash == docHash: 
    142             #    continue 
    143              
    144             #coll.doc_hash = docHash 
    145             coll.doc_name = docName   
    146             coll.doc_owner = docOwner                       
    147             coll.doc_status = docStatus             
    148             coll.doc_creation = stringToTimestamp(doc_creation)  
    149             if add: 
    150                 MigrationEPB.addMigration(coll, self._migrationSession) 
    151             self._migrationSession.commit() 
     124    def _extractID(self, xmlDocument): 
     125        doc_id = findID(xmlDocument)  
     126        if doc_id is None: 
     127            raise NoDocumentID(doc_id) 
     128        return doc_id 
    152129 
    153130    def _extractCreationDate(self, xmlDocument): 
     
    157134        return creationDate 
    158135             
    159     def process(self): 
    160         """ 
    161             Loads all the folders below the atoms ('published', 'working', 'Published') folders 
    162         """ 
     136 
    163137         
    164         for status in docStatus:               
    165             self._loadDataEntities(status)             
    166             self._loadDeployments(status) 
    167             self._loadDeploymentData(status)             
    168         self._migrationSession.close() 
    169          
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/migrationProcess.py

    r8091 r8147  
    77from libs.migration.MigrationEPB import MigrationEPB 
    88from MolesManager.moles3epb import Moles3EPB 
    9 from libs.epb import EPB 
    109from libs.migration.migration_sessions import MigrationSessions 
    11 from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection 
     10import logging 
     11from logging import StreamHandler 
    1212 
    1313class MigrationProcess(): 
     14    log = logging.getLogger('MigrationProcess') 
     15    log.addHandler(StreamHandler()) 
     16    log.setLevel(logging.DEBUG) 
     17     
    1418    """ 
    1519        Manages the migration process from Moles2 to Moles3. 
     
    4347        for dataEntityMigration in [f for f in data_ents if f != None]:              
    4448            dep = DataEntityProcessor(dataEntityMigration, self.migrationSessions) 
    45             exs.append(dep.process()) 
    46         print exs  
     49            exs.extend(dep.process()) 
     50        return exs  
    4751         
    4852    def commitAll(self): 
Note: See TracChangeset for help on using the changeset viewer.