Changeset 8082


Ignore:
Timestamp:
31/01/12 17:10:32 (8 years ago)
Author:
mnagni
Message:

Refactored partially the migration code
Refactored substantially the Model2Form code
Now is visible the "cov" page is visible the relatedParty name (implementation as list possibly in the next commit)

Location:
mauRepo/MolesManager/trunk/src
Files:
1 added
3 deleted
18 edited

Legend:

Unmodified
Added
Removed
  • mauRepo/MolesManager/trunk/src/MolesManager/forms/date.py

    r8077 r8082  
    99from sqlalchemy.orm.properties import ColumnProperty, RelationshipProperty 
    1010import inspect 
     11from sqlalchemy.orm.util import class_mapper 
     12from sqlalchemy.orm.collections import InstrumentedList 
    1113     
    1214 
    13 def createForm(object = None): 
    14     form = Model2Form(object) 
     15def createForm(obj = None, mapper = None): 
     16    form = Model2Form(obj, mapper) 
    1517    return type('CoreForm', (CoreForm,), form._createFields())     
     18 
     19def createFields(obj = None, mapper = None): 
     20    form = Model2Form(obj, mapper) 
     21    return form._createFields()   
    1622 
    1723class CoreForm(Form): 
     
    2026 
    2127class Model2Form(object): 
    22     def __init__(self, object = None):                  
     28    def __init__(self, obj = None, mapper = None): 
     29        self._coreFields = {}                
    2330        self._instance = None 
    2431        self._clazz = None 
     32        self._mapper = mapper 
    2533         
    26         if inspect.isclass(object): 
     34        if inspect.isclass(obj): 
    2735            self._clazz = object 
    2836        else: 
    29             self._instance = object 
    30             self._clazz = type(object) 
     37            self._instance = obj 
     38            self._clazz = type(obj) 
     39 
    3140 
    3241    def _createFields(self): 
    33         coreFields = {} 
    34         for attrName in self._clazz.__dict__: 
    35             if not attrName.startswith('_'): 
    36                 if attrName == 'dataLineage': 
    37                     pass #print 'ciao' 
    38                 attrValue = getattr(self._clazz, attrName) 
    39                 if isinstance(attrValue, InstrumentedAttribute): 
    40                     try: 
    41                         coreFields[attrName] = self._manageInstrumentedAttribute(attrName, attrValue) 
    42                     except Exception as e: 
    43                         print e 
    44         return coreFields 
    45             
    46     def _manageInstrumentedAttribute(self, attrName, attrValue): 
    47         if hasattr(attrValue, 'property'): 
    48             if isinstance(attrValue.property, ColumnProperty):                 
    49                 return self._manageColumnProperty(attrName, attrValue.property) 
    50             elif isinstance(attrValue.property, RelationshipProperty):                     
    51                 return self._manageRelationshipProperty(attrName, attrValue.property) 
     42        instrList = [] 
     43        if self._mapper is None: 
     44            self._mapper = class_mapper(self._clazz) 
     45        if self._mapper is None: 
     46            raise Exception("Class not mapped %s" % (self._clazz)) 
     47        if not isinstance(self._instance, InstrumentedList): 
     48            self._loopOnProperties(self._instance) 
     49        else: 
     50            for item in self._instance: 
     51                instrList.append(self._loopOnProperties(item)) 
     52 
     53        return self._coreFields 
    5254     
    53     def _manageColumnProperty(self, attrName, attrValue): 
    54             if hasattr(attrValue, 'columns'): 
    55                 columns = getattr(attrValue, 'columns') 
    56                 if len(columns) > 0: 
    57                     return createFieldInstance(columns[0].type, **self._extractValue(attrName))                     
     55    def _loopOnProperties(self, instance): 
     56        for clazz_property in self._mapper.iterate_properties: 
     57            try: 
     58                if isinstance(clazz_property, ColumnProperty): 
     59                    self._manageColumnProperty(clazz_property, instance) 
     60                if isinstance(clazz_property, RelationshipProperty): 
     61                    self._manageRelationshipProperty(clazz_property, instance) 
     62            except Exception as e: 
     63                print e  
     64                 
     65    def _manageColumnProperty(self, clazz_property, instance): 
     66        if hasattr(clazz_property, 'columns'): 
     67            columns = getattr(clazz_property, 'columns') 
     68            if len(columns) > 0: 
     69                self._coreFields[clazz_property.key] = createFieldInstance(columns[0].type, **self._extractValue(clazz_property.key, instance))                     
    5870  
    59     def _manageRelationshipProperty(self, attrName, attrValue): 
    60         if (attrValue.direction is not None): 
    61             if (attrValue.direction.name == 'MANYTOONE'): 
     71    def _manageRelationshipProperty(self, relationship_property, instance): 
     72        if relationship_property.key == 'relatedParty': 
     73            print relationship_property.key 
     74        if (relationship_property.direction is not None): 
     75            if (relationship_property.direction.name == 'MANYTOONE'): 
    6276                pass 
    63             elif (attrValue.direction.name == 'ONETOMANY'):                                 
    64                 return createForm(getattr(self._instance, attrName)) 
     77            elif (relationship_property.direction.name == 'ONETOMANY'): 
     78                if relationship_property.key.startswith('_'): 
     79                    fields = createFields(getattr(instance, relationship_property.key), relationship_property.mapper) 
     80                    for key, value in fields.iteritems(): 
     81                        self._coreFields[key] = value 
     82                else:                            
     83                    self._coreFields[relationship_property.key] = createForm(getattr(instance, relationship_property.key), relationship_property.mapper) 
    6584     
    66     def _extractValue(self, attrName): 
     85    def _extractValue(self, property_name, instance): 
    6786        kwargs = {} 
    6887        if self._instance: 
    69             kwargs['initial'] = getattr(self._instance, attrName) 
     88            kwargs['initial'] = getattr(instance, property_name) 
    7089        return kwargs  
    7190 
  • mauRepo/MolesManager/trunk/src/MolesManager/forms/formsFactory.py

    r8076 r8082  
    1414       'Text': CharField, 
    1515       'TEXT': CharField, 
    16        'Integer': IntegerField, 
     16       'INTEGER': IntegerField,        
    1717       'NUMERIC': DecimalField, 
    1818     }     
  • mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py

    r8077 r8082  
    1313 
    1414    @classmethod         
    15     def addCEDA_Observation(self, ceda_observation, session = None): 
     15    def addCedaObject(self, ceda_observation, session = None): 
    1616        intSession = Moles3EPB._getSession(session) 
    1717        intSession.add(ceda_observation) 
  • mauRepo/MolesManager/trunk/src/MolesManager/molesSessionMiddleware.py

    r8077 r8082  
    66from MolesManager.moles3epb import Moles3EPB 
    77from libs.migration.client import MigrationThread 
    8 from MolesManager.settings import RUN_MIGRATION 
     8from MolesManager.settings import RUN_MIGRATION, MIGRATION_INTERVAL 
    99 
    1010class MolesSessionMiddleware(object): 
     
    1212    #This attribute should be not here. 
    1313    #unfortunately I cannot find any start/stop signals from Django 
    14     _migrationThread = MigrationThread(interval = 86400) 
     14    _migrationThread = MigrationThread(interval = MIGRATION_INTERVAL) 
    1515     
    1616    def _migration(self, runMigration = RUN_MIGRATION): 
  • mauRepo/MolesManager/trunk/src/MolesManager/settings.py

    r8078 r8082  
    163163MOLES3 = 'moles3' 
    164164#MOLES3_DB_CONNECTION = 'postgresql://user:pws@host:port/dbName' 
     165MOLES3_DB_CONNECTION = 'PG_MOLES3_DB_CONNECTION' 
    165166 
    166 MOLES3_DB_CONNECTION = 'PG_MOLES3_DB_CONNECTION' 
    167167MOLES3_DB_SCRIPT = SRC + '/sqlTables.py' 
    168168 
     
    173173 
    174174MIGRATION_DB_CONNECTION = 'PG_MIGRATION_DB_CONNECTION' 
     175 
    175176MIGRATION_DB_SCRIPT = SRC + '/libs/migration/db/migrationTables.py' 
    176177 
    177178RUN_MIGRATION = True 
     179#Default to one day 
     180MIGRATION_INTERVAL = 86400 
  • mauRepo/MolesManager/trunk/src/MolesManager/views/cedaObservationView.py

    r8077 r8082  
    2424    return sampleForm 
    2525    '''     
    26     #dp = getCEDA_ObservationById(objectId.cleaned_data['id']) 
    27     co = createForm(CEDA_Observation) 
    28     coObj = co(request.POST or None) 
     26    #co = createForm(CEDA_Observation) 
     27    #coObj = co(request.POST or None) 
    2928 
    3029    if objectId.is_valid(): 
     
    3635    c = {} 
    3736    objectId = ObjectById(request.POST or None) 
    38     form = __getSample(request, objectId)  
     37    form = __getSample(request, objectId)   
    3938    c['coObs'] = form 
    4039    c['objId'] = objectId 
  • mauRepo/MolesManager/trunk/src/dropAllTables.sql

    r8074 r8082  
    7474DROP TABLE IF EXISTS ci_series CASCADE; 
    7575DROP TABLE IF EXISTS ci_telephone CASCADE; 
     76DROP TABLE IF EXISTS class08_4 CASCADE; 
     77DROP TABLE IF EXISTS class09_1 CASCADE; 
     78DROP TABLE IF EXISTS class09_2 CASCADE; 
     79DROP TABLE IF EXISTS class09_3 CASCADE; 
     80DROP TABLE IF EXISTS class09_4 CASCADE; 
     81DROP TABLE IF EXISTS class10_1 CASCADE; 
     82DROP TABLE IF EXISTS class12_1 CASCADE; 
     83DROP TABLE IF EXISTS class12_2 CASCADE; 
     84DROP TABLE IF EXISTS class12_3 CASCADE; 
     85DROP TABLE IF EXISTS class12_4 CASCADE; 
    7686DROP TABLE IF EXISTS cl_classificationcollection CASCADE; 
    7787DROP TABLE IF EXISTS cl_classificationcollection_cv_surfacevaluepair CASCADE; 
     
    562572DROP TABLE IF EXISTS uomscale CASCADE; 
    563573DROP TABLE IF EXISTS url CASCADE; 
     574DROP TABLE IF EXISTS data_entity_migration CASCADE; 
     575DROP TABLE IF EXISTS deployment_data_migration CASCADE; 
     576DROP TABLE IF EXISTS deployments_migration CASCADE; 
     577 
    564578 
    565579 
  • mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py

    r8077 r8082  
    55''' 
    66from MolesManager.settings import MIGRATION_DB_SCRIPT, MIGRATION_DB_CONNECTION 
    7 from libs.migration.db.classes import ObservationMigration,\ 
    8     ObservationCollectionMigration 
     7from libs.migration.db.classes import DeploymentDataMigration, DataEntityMigration,\ 
     8    DeploymentsMigration 
    99from sqlalchemy.sql.expression import asc 
    1010from libs.commons_db import DbManager 
     
    1515     
    1616    _migrationDB = DbManager(MIGRATION_DB_CONNECTION, MIGRATION_DB_SCRIPT) 
    17  
     17          
    1818    @classmethod 
    19     def addObservationCollectionMigration(self, obsCollMigration, session = None): 
     19    def addMigration(self, migrationObj, session = None): 
    2020        intSession = MigrationEPB._getSession(session) 
    21         intSession.add(obsCollMigration) 
    22         EPB._closeSession(session) 
    23  
    24     @classmethod         
    25     def addObservationMigration(self, obsMigration, session = None): 
    26         intSession = MigrationEPB._getSession(session) 
    27         intSession.add(obsMigration) 
     21        intSession.add(migrationObj) 
    2822        EPB._closeSession(session)         
    2923 
    3024    @classmethod         
    31     def getObservationCollectionMigrationOrderByDate(self, session = None): 
     25    def getDataEntityMigrationOrderByDate(self, session = None): 
    3226        intSession = MigrationEPB._getSession(session)    
    33         data_ents = EPB.getAllObjects(ObservationCollectionMigration, intSession) 
    34         return data_ents.order_by(asc("data_ent_creation"))      
     27        data_ents = EPB.getAllObjects(DataEntityMigration, intSession) 
     28        return data_ents.order_by(asc("doc_creation"))      
    3529 
    3630    @classmethod 
    37     def getObservationMigrationByName(self, deplName, session = None): 
     31    def getDeploymentsMigrationByName(self, migrationObject, doc_name, session = None): 
    3832        intSession = MigrationEPB._getSession(session) 
    39         return intSession.query(ObservationMigration).filter(*[EPB.buildFilter('depl_name', deplName)]).first() 
     33        return intSession.query(DeploymentsMigration).filter(*[EPB.buildFilter('doc_status', migrationObject.doc_status)]).filter(*[EPB.buildFilter('doc_owner', migrationObject.doc_owner)]).filter(*[EPB.buildFilter('doc_name', doc_name)]).first() 
     34 
     35    @classmethod 
     36    def getDeploymentDataMigrationByName(self, dptName, session = None): 
     37        intSession = MigrationEPB._getSession(session) 
     38        return intSession.query(DeploymentDataMigration).filter(*[EPB.buildFilter('dpt_name', dptName)]).first() 
    4039 
    4140    @classmethod 
  • mauRepo/MolesManager/trunk/src/libs/migration/client.py

    r8077 r8082  
    2929    def run(self): 
    3030        while self._doRun: 
    31             #self._migrate() 
    32             #s = sched.scheduler(time.time, time.sleep) 
    3331            startTime = (int)(time.time()) 
    3432            timer = Timer(5, self._migrate, ()) 
     
    4341 
    4442    def _migrate(self): 
    45         #mEPB = MigrationEPB() 
    4643        lr = LoadResources() 
    4744        lr.process() 
     
    5754#t.stop() 
    5855 
     56#lr = LoadResources() 
     57#lr.process() 
    5958 
     59#mp = MigrationProcess() 
     60#mp.process() 
    6061 
    6162 
  • mauRepo/MolesManager/trunk/src/libs/migration/db/classes.py

    r8014 r8082  
    55''' 
    66 
    7 class ObservationCollectionMigration(object): 
     7class DataEntityMigration(object): 
    88    def __init__(self): 
    9         self.data_ent_name         
    10         self.data_ent_id 
    11         self.obs_coll_id 
    12         self.data_ent_creation 
    13         self.exist_path 
     9        self.doc_name = None 
     10        self.doc_owner = None 
     11        self.doc_status = None                     
     12        self.data_ent_id = None 
     13        self.ceda_observation_coll_id = None 
     14        self.doc_creation = None 
    1415 
    15 class ObservationMigration(object): 
     16class DeploymentsMigration(object): 
    1617    def __init__(self): 
    17         self.depl_name         
    18         self.depl_id 
    19         self.obs_id 
    20         self.depl_creation 
    21         self.exist_path 
     18        self.doc_name = None      
     19        self.doc_owner = None            
     20        self.doc_status = None         
     21        self.depl_id = None 
     22        self.ceda_observation_id = None 
     23        self.doc_creation = None 
     24         
     25class DeploymentDataMigration(object): 
     26    def __init__(self): 
     27        self.doc_name = None 
     28        self.doc_owner = None   
     29        self.doc_status = None                       
     30        self.deployment_data_id = None 
     31        self.doc_creation = None         
  • mauRepo/MolesManager/trunk/src/libs/migration/db/dbConnection.py

    r8054 r8082  
    3737 
    3838def getObservationMigrationByName(deplName, session): 
    39     return session.query(ObservationMigration).filter(*[_buildFilter('depl_name', deplName)]).first()             
     39    return session.query(ObservationMigration).filter(*[_buildFilter('doc_name', deplName)]).first()             
    4040 
    4141    
  • mauRepo/MolesManager/trunk/src/libs/migration/db/migrationTables.py

    r8015 r8082  
    77from sqlalchemy.types import TEXT, Integer, TIMESTAMP 
    88from sqlalchemy.orm import mapper 
    9 from libs.migration.db.classes import ObservationCollectionMigration,\ 
    10     ObservationMigration 
     9from libs.migration.db.classes import DeploymentDataMigration, DeploymentsMigration,\ 
     10    DataEntityMigration 
    1111 
    1212''' 
    1313Connects a dataEntity to its associated CEDA_Observation_Collection 
    1414'''         
    15 obs_coll_migration = Table('obs_coll_migration', metadata, 
     15data_entity_migration = Table('data_entity_migration', metadata, 
    1616            Column('data_ent_id', TEXT, primary_key=True, nullable=False), 
    17             Column('data_ent_name', TEXT),             
    18             Column('obs_coll_id', Integer), 
    19             Column('data_ent_creation', TIMESTAMP), 
    20             Column('exist_path', TEXT) 
     17            Column('doc_name', TEXT),             
     18            Column('doc_owner', TEXT),             
     19            Column('doc_status', TEXT),             
     20            Column('ceda_observation_coll_id', Integer), 
     21            Column('doc_creation', TIMESTAMP) 
    2122        ) 
    2223 
     
    2425Connects a deployment to its associated CEDA_Observation 
    2526''' 
    26 observation_migration = Table('observation_migration', metadata, 
     27deployments_migration = Table('deployments_migration', metadata, 
    2728            Column('depl_id', TEXT, primary_key=True, nullable=False), 
    28             Column('depl_name', TEXT),             
    29             Column('obs_id', Integer), 
    30             Column('depl_creation', TIMESTAMP), 
    31             Column('exist_path', TEXT)             
     29            Column('doc_name', TEXT),             
     30            Column('doc_owner', TEXT),             
     31            Column('doc_status', TEXT),                         
     32            Column('ceda_observation_id', Integer), 
     33            Column('doc_creation', TIMESTAMP)     
    3234        )    
    3335 
    34 mapper(ObservationCollectionMigration, obs_coll_migration)      
    35 mapper(ObservationMigration, observation_migration) 
     36deployment_data_migration = Table('deployment_data_migration', metadata, 
     37            Column('deployment_data_id', TEXT, primary_key=True, nullable=False), 
     38            Column('doc_name', TEXT),             
     39            Column('doc_owner', TEXT),             
     40            Column('doc_status', TEXT),            
     41            Column('doc_creation', TIMESTAMP)     
     42        )    
    3643 
    37 metadata.create_all(engine) 
     44mapper(DataEntityMigration, data_entity_migration)      
     45mapper(DeploymentsMigration, deployments_migration) 
     46mapper(DeploymentDataMigration, deployment_data_migration) 
  • mauRepo/MolesManager/trunk/src/libs/migration/exception/exceptions.py

    r8014 r8082  
    44@author: mnagni 
    55''' 
    6 class NoAssociatedDeployments(Exception): 
    7     def __init__(self, value): 
    8         self.value = value 
    9     def __str__(self): 
    10         return "The DataEntity with ID %s has no associated deployments" % (self.value) 
     6 
     7def _migrationObjectDescription(migrationObject): 
     8    return "%s %s %s" % (migrationObject.doc_status, migrationObject.doc_owner, migrationObject.doc_name) 
    119     
    1210class NoCreationDate(Exception): 
     
    1513    def __str__(self): 
    1614        return "The resource with ID %s has no creation date" % (self.value)   
     15 
     16class MigrationObjectException(Exception): 
     17    def __init__(self, value): 
     18        self._migrationObject = value 
     19    def __str__(self): 
     20        return "Error on migration object %s " % (_migrationObjectDescription(self._migrationObject)) 
     21 
     22class NoAssociatedDeployments(Exception): 
     23    def __init__(self, data_entityMigration, deploymentName): 
     24        self._migrationObject = data_entityMigration 
     25        self._deploymentName = deploymentName         
     26        super(NoAssociatedDeployments, self).__init__() 
     27    def __str__(self): 
     28        return "The DataEntity %s has no associated deployment named %s" % (_migrationObjectDescription(self._migrationObject), self._deploymentName) 
    1729     
    18 class NoDataLineage(Exception): 
     30class NoDataLineage(MigrationObjectException): 
    1931    def __init__(self, value): 
    20         ''' 
    21         @param value: the full eXist path of the associated resource  
    22         '''        
    23         self.value = value 
     32        self._migrationObject = value         
     33        super(NoDataLineage, self).__init__(self._migrationObject) 
    2434    def __str__(self): 
    25         return "The resource with path %s has no moles:lineage value" % (self.value)         
     35        return "The DataEntity %s has no moles:lineage value" % (_migrationObjectDescription(self._migrationObject)) 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py

    r8074 r8082  
    55''' 
    66from ea_model.moles3_4.utilities.mo_responsiblepartyinfo import MO_ResponsiblePartyInfo 
    7 from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue 
    8 from ea_model.moles3_4.utilities.mo_organization import MO_Organization 
    97from ea_model.iso_19115_2006_metadata_corrigendum.reference_system_information.md_identifier import MD_Identifier 
    108from httplib import HTTPConnection 
    119from xml.etree.ElementTree import XML 
    12 import time, datetime, re 
     10import time, datetime 
    1311from ea_model.upcomingiso.ci_organisation import CI_Organisation 
    1412from ea_model.upcomingiso.ci_party import CI_Party 
    1513from ea_model.upcomingiso.ci_individual import CI_Individual 
     14from libs.migration.exception.exceptions import NoDataLineage 
    1615 
    1716base = '/exist/rest/atoms' 
    18 docStatus = ('published', 'working', 'Published')  
    19 docCollections = ('deployments', 'data_entities', 'deployment_data', 'data_granules') 
     17 
     18DS_pUBLISHED = 'published' 
     19DS_WORKING = 'working' 
     20DS_PUBLISHED = 'Published' 
     21docStatus = (DS_pUBLISHED, DS_WORKING, DS_PUBLISHED) 
     22 
     23DT_DEPLOYMENTS = 'deployments' 
     24DT_DATA_ENTITIES = 'data_entities' 
     25DT_DEPLOYMENT_DATA = 'deployment_data' 
     26DT_DATA_GRANULES = 'data_granules' 
     27docTypes = (DT_DEPLOYMENTS, DT_DATA_ENTITIES, DT_DEPLOYMENT_DATA, DT_DATA_GRANULES) 
     28 
     29DO_BADC = 'badc.nerc.ac.uk' 
     30DO_NEODC = 'neodc.nerc.ac.uk' 
     31DO_UKSSDC = 'ukssdc.ac.uk' 
     32docOwners = (DO_BADC, DO_NEODC, DO_UKSSDC) 
    2033 
    2134atomNS = "{http://www.w3.org/2005/Atom}" 
     
    2639iport = '8080' 
    2740 
    28 def getDocument(source): 
     41def buildExistDocPath(docStatus, docType, docOwner, docName): 
     42    ''' 
     43        @param docStatus: one value from commons.docStatus 
     44        @param docType: one value from commons.docCollections  
     45        @param docOwner: one value from commons.docOwners 
     46        @param docName: one value from commons.docOwners         
     47    '''         
     48    return '%s/%s' % (buildExistOwnerPath(docStatus, docType, docOwner), docName) 
     49 
     50def buildExistOwnerPath(docStatus, docType, docOwner): 
     51    ''' 
     52        @param docStatus: one value from commons.docStatus 
     53        @param docType: one value from commons.docCollections  
     54        @param docOwner: one value from commons.docOwners 
     55    '''         
     56    return '%s/%s' % (buildExistTypePath(docStatus, docType), docOwner) 
     57 
     58def buildExistTypePath(docStatus, docType): 
     59    ''' 
     60        @param docStatus: one value from commons.docStatus 
     61        @param docType: one value from commons.docCollections  
     62    '''         
     63    return '%s/%s' % (buildExistStatusPath(docStatus), docType) 
     64 
     65def buildExistStatusPath(docStatus): 
     66    ''' 
     67        @param docStatus: one value from commons.docStatus  
     68    '''         
     69    return '/exist/rest/atoms/%s' % (docStatus) 
     70 
     71def getAtomDocumentByMO(migrationObject): 
     72    mo_typeDict = {'DeploymentsMigration': DT_DEPLOYMENTS, 'DataEntityMigration': DT_DATA_ENTITIES, 'DeploymentDataMigration': DT_DEPLOYMENT_DATA}      
     73    return getAtomDocument(migrationObject.doc_status, mo_typeDict[type(migrationObject).__name__], migrationObject.doc_owner, migrationObject.doc_name) 
     74 
     75def getAtomDocumentByType(migrationObject, docType): 
     76    return getAtomDocument(migrationObject.doc_status, docType, migrationObject.doc_owner, migrationObject.doc_name) 
     77 
     78def getAtomDocument(docStatus, docType, docOwner, docName): 
     79    source = buildExistDocPath(docStatus, docType, docOwner, docName) 
     80    return getXMLDocument(source) 
     81 
     82def getXMLDocument(source): 
     83    return XML(_getDocument(source)) 
     84 
     85def stringToTimestamp(timestring): 
     86    return datetime.datetime.fromtimestamp(time.mktime(time.strptime(timestring, time_format))) 
     87 
     88def _getDocument(source): 
    2989    conn = HTTPConnection(host = ihost, port = iport) 
    3090    conn.connect() 
     
    3898    return xmlDoc 
    3999 
    40 def getXMLDocument(source): 
    41     return XML(getDocument(source)) 
    42  
    43 def stringToTimestamp(timestring): 
    44     return datetime.datetime.fromtimestamp(time.mktime(time.strptime(timestring, time_format))) 
    45  
    46100def _returnNotNoneText(element): 
    47101    if element is None: 
     
    53107    return _returnNotNoneText(creationDate) 
    54108 
    55 def findMolesLineage(resourceXML): 
     109def findMolesLineage(dataEntityMigration): 
     110    resourceXML = getAtomDocumentByMO(dataEntityMigration) 
    56111    lineage = resourceXML.find('%sentity/%smolesISO/%slineage' % (molesNS, molesNS, molesNS)) 
    57     return _returnNotNoneText(lineage) 
    58  
    59 def findAuthorInResource(resourceXML): 
     112    if lineage is None: 
     113        raise NoDataLineage(dataEntityMigration) 
     114    return lineage.text 
     115 
     116def findAuthorInResource(dataEntityMigration):     
     117    resourceXML = getAtomDocumentByMO(dataEntityMigration) 
    60118    author = resourceXML.find('%sauthor/%sname' % (atomNS, atomNS)) 
    61119    return _returnNotNoneText(author) 
    62120 
    63121def findID(dataEntXML): 
    64     id = dataEntXML.find('%sid' % (atomNS)) 
    65     return _returnNotNoneText(id) 
     122    ent_id = dataEntXML.find('%sid' % (atomNS)) 
     123    return _returnNotNoneText(ent_id) 
    66124 
    67125def extractLinks(dataEntXML, markers): 
     
    73131    return linksDict 
    74132 
    75 def extractDataEntityLinks(dataEntXML): 
     133def _extractDataEntityLinks(dataEntXML): 
    76134    markers = ['Deployment', 'DOWNLOAD', 'DOCUMENTATION', 'ACCESS', 'LOGO'] 
    77     return extractLinks(dataEntXML, markers) 
    78  
    79 def extractDeploymentLinks(dataEntXML): 
    80     markers = ['ACTIVITY', 'DPT', 'OBS'] 
    81135    return extractLinks(dataEntXML, markers) 
    82136 
     
    88142        linksDict[linkMarker].append(link.get('href')) 
    89143 
    90 def findDeploymentsInDE(dataEntXML): 
    91     linksDict = extractDataEntityLinks(dataEntXML) 
    92     depls = [] 
    93     if linksDict.has_key('Deployment'): 
    94         for link in linksDict['Deployment']: 
     144def findDeploymentsInDE(dataEntityMigration): 
     145    dataEntXML = getAtomDocumentByMO(dataEntityMigration) 
     146    linksDict = _extractDataEntityLinks(dataEntXML) 
     147    return _extractLinksByMarker(linksDict, 'Deployment') 
     148 
     149def findLinksInDeployment(dataEntXML): 
     150    """ 
     151        Returns a dictionary of links owned by the given dataEntity document 
     152        @param dataEntXML: a DataEntity atom document 
     153        @return: a dictionary of links. The possible keys are ['ACTIVITY', 'DPT', 'OBS'] 
     154    """ 
     155    links = {} 
     156    markers = ['ACTIVITY', 'DPT', 'OBS']  
     157    linksDict = extractLinks(dataEntXML, markers) 
     158    for marker in markers:     
     159        links[marker] = _extractLinksByMarker(linksDict, marker) 
     160    return links 
     161 
     162def _extractLinksByMarker(linksDict, marker): 
     163    dpt = [] 
     164    if linksDict.has_key(marker): 
     165        for link in linksDict[marker]: 
    95166            try: 
    96167                linkLongName = link.split('/')[-1] 
    97168                linkName = linkLongName.rsplit('__ATOM__')[1] 
    98                 depls.append(linkName) 
     169                dpt.append(linkName) 
    99170            except Exception as ex: 
    100171                print ex 
    101     return depls 
     172    return dpt 
     173 
    102174 
    103175def getResourceRefs(deploymentRefs): 
     
    109181    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS)) 
    110182 
     183def getOwnerRefs(docStatus, docType, docOwner): 
     184    ''' 
     185        Returns a list of Elements representing the inner resource reference items 
     186        @param resourceRefs: the name of the eXist collection name below the 'deployments' one 
     187    '''       
     188    XMLDepl = getXMLDocument(buildExistOwnerPath(docStatus, docType, docOwner)) 
     189    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS)) 
     190 
     191def getTypeRefs(docStatus, docType): 
     192    ''' 
     193        Returns a list of Elements representing the inner resource reference items 
     194        @param resourceRefs: the name of the eXist collection name below the 'deployments' one 
     195    '''       
     196    XMLDepl = getXMLDocument(buildExistTypePath(docStatus, docType)) 
     197    return XMLDepl.findall('%scollection/%sresource' % (existNS, existNS)) 
     198 
     199 
     200 
    111201def getCollectionRefs(publishedRefs): 
    112202    ''' 
     
    119209def getResource(source, resourceName): 
    120210    resourceSource = '%s/%s' % (source, resourceName) 
    121     resourceDoc = getDocument(resourceSource) 
     211    resourceDoc = _getDocument(resourceSource) 
    122212    return XML(resourceDoc) 
    123213 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py

    r8077 r8082  
    55''' 
    66from libs.migration.processor.deployment import DeploymentProcessor 
    7 from libs.migration.exception.exceptions import NoAssociatedDeployments,\ 
    8     NoDataLineage 
    9 from libs.migration.processor.commons import getResource, findDeploymentsInDE,\ 
    10     getResourceRefs, getXMLDocument 
     7from libs.migration.exception.exceptions import NoDataLineage,\ 
     8    NoAssociatedDeployments 
     9from libs.migration.processor.commons import findDeploymentsInDE 
    1110from libs.migration.MigrationEPB import MigrationEPB 
     11from libs.epb import EPB 
     12from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection 
     13from MolesManager.moles3epb import Moles3EPB 
    1214 
    1315class DataEntityProcessor(object): 
    1416    ''' 
    1517        Processes a DataEntity item. 
    16         @var _de_path: the full eXist path 
    17         @var session: an sqlAlchemy Session 
     18        @var _dataEntityMigration: a DataEntityMigration instance 
    1819        @var resource: an elementTree Element representing XML document processed by this instance 
    1920        @var _depl: a list of Deployment items associated with this DataEntity     
    2021    '''     
    2122     
    22     def __init__(self, de_path, migrationProcess): 
     23    def __init__(self, dataEntityMigration, migrationProcess): 
    2324        ''' 
    2425            Initializes the class 
    25             @param de_path: the full eXist path 
    26             @param session: an sqlAlchemy Session 
     26            @param _dataEntityMigration: ta DataEntityMigration instance 
     27            @param migrationProcess: a MigrationProcess instance 
    2728        '''           
    28         self._de_path = de_path 
     29        self._dataEntityMigration = dataEntityMigration 
    2930        self._migrationProcess = migrationProcess 
    3031        self._migrationSession = migrationProcess._migrationSession 
    3132        self._molesSession = migrationProcess._molesSession 
    3233        self.resource = None 
    33         self._depls = 0         
     34        self._depls = 0 
    3435 
    35     def _createCEDA_Observation(self, resource): 
    36         ''' 
    37         author = findAuthorInResource(resource) 
    38         id = findDEorDepl_ID(resource) 
    39         mo_obs_coll = CEDA_ObservationCollection() 
    40         mo_obs_coll.relatedParty = [] 
    41         mo_obs_coll.identifier = []         
    42         mo_obs_coll.relatedParty.append(createMO_ResponsiblePartyInfo(author)) 
    43         mo_obs_coll.identifier.append(createMD_Identifier(id)) 
    44         return mo_obs_coll 
    45         ''' 
    46  
    47     def _getIfExists(self, resource): 
    48         ''' 
    49         id = findDEorDepl_ID(resource) 
    50         ''' 
    51  
    52     def _ingestDataEntity(self, base, owner): 
    53         resource = getResource(base, owner) 
    54          
    55         self._getIfExists(resource) 
    56          
    57         deplNames = findDeploymentsInDE(resource) 
    58         if len(deplNames) == 0: 
    59             raise NoAssociatedDeployments('%s/%s' % (base, owner)) 
    60         ceda_observations = [] 
    61         for deplName in deplNames: 
    62             deplProcessor = DeploymentProcessor(self._baseStatus, self._owner, deplName) 
    63             ceda_observations.append(deplProcessor.process()) 
    64         mo_obs_coll = self._createCEDA_Observation(resource) 
    65         mo_obs_coll.member = ceda_observations 
    66         ''' 
    67         Do Insert or doUpdate??? 
    68         doInsert(mo_obs_coll, self._session) 
    69         ''' 
    70          
    71         print mo_obs_coll 
    72          
    73  
    74     def _processDataEntity(self):  
    75         baseOwner = '%s/%s' % (self._baseDE, self._owner)         
    76         refs = getResourceRefs(baseOwner) 
    77         for ref in refs: 
    78             refName = ref.get('name') 
     36    def _processObj(self, ceda_observationCollection): 
     37        exs = [] 
     38        #loops over the deployments  
     39        for deplName in findDeploymentsInDE(self._dataEntityMigration): 
     40            deploymentMigration = MigrationEPB.getDeploymentsMigrationByName(self._dataEntityMigration, deplName + '.atom', self._migrationSession) 
     41            #retrieve the deployment's path 
     42            #deplObj = MigrationEPB.getDeploymentsMigrationByName(self._dataEntityMigration.doc_status, self._dataEntityMigration.doc_owner, deplName + '.atom', session = self._migrationSession)             
     43            #if deplObj is None: 
     44            #    print "cannot find Deployment for %s %s %s" % (self._dataEntityMigration.doc_status, self._dataEntityMigration.doc_owner, deplName + '.atom') 
     45            #    continue 
     46            if deploymentMigration is None: 
     47                exs.append(NoAssociatedDeployments(self._dataEntityMigration, deplName + '.atom')) 
     48                continue 
     49                                         
     50            deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration, self._migrationProcess) 
    7951            try: 
    80                 self._ingestDataEntity(baseOwner, refName) 
    81             except NoAssociatedDeployments as dee: 
    82                 print dee 
    83              
     52                new_ceda_obs = deProcessor.process() 
     53                 
     54                for old_ceda_obs in ceda_observationCollection.member: 
     55                    if new_ceda_obs.ceda_observation_id == old_ceda_obs.ceda_observation_id: 
     56                        ceda_observationCollection.member.remove(old_ceda_obs)                     
     57                        new_ceda_obs.append(new_ceda_obs) 
     58                    self._migrationProcess.commitAll()                 
     59            except NoDataLineage as ex: 
     60                    self._migrationProcess.rollbackAll() 
     61                    exs.append(ex) 
     62                    print ex 
     63        return exs             
     64                     
    8465     
    8566    def process(self): 
    86         exs = [] 
    87         #Retrieve the DataEntity 
    88         self.dataEntity = getXMLDocument(self._de_path) 
    89         #extracts the dataentity deployments  
    90         self._depls = findDeploymentsInDE(self.dataEntity) 
    91         #loops over the deployments  
    92         for deplName in self._depls: 
    93             #retrieve the deployment's path 
    94             deplObj = MigrationEPB.getObservationMigrationByName(deplName + '.atom', self._migrationSession)             
    95             if deplObj is None: 
    96                 print "cannot find deployment %s for resource %s" % (deplName, self._de_path) 
    97                 continue                              
    98             depl = DeploymentProcessor(deplObj, self) 
    99             try: 
    100                 #Processes the deployment 
    101                 depl.process() 
    102                 self._migrationProcess.commitAll()                 
    103             except NoDataLineage as ex: 
    104                 self._migrationProcess.rollbackAll() 
    105                 exs.append(ex) 
    106                 print ex 
    107         return exs;         
     67        ceda_observationCollection = None 
    10868         
     69        add = False 
     70        #Exists already an associated CEDA_Observation associated to this deployment? 
     71        if self._dataEntityMigration.ceda_observation_coll_id is not None: 
     72            ceda_observationCollection = EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id, self._migrationProcess._migrationSession) 
     73 
     74        #If does not exist creates a new one             
     75        if ceda_observationCollection is None: 
     76            add = True 
     77            ceda_observationCollection = CEDA_ObservationCollection()           
     78 
     79        #fills/update the CEDA_ObservationCollection 
     80        exs = self._processObj(ceda_observationCollection) 
     81 
     82        if add: 
     83            Moles3EPB.addCedaObject(ceda_observationCollection, self._migrationProcess._molesSession) 
     84        self._migrationProcess._molesSession.commit() 
     85        self._dataEntityMigration.ceda_observation_coll_id = ceda_observationCollection.id         
     86         
     87        if add: 
     88            MigrationEPB.addMigration(self._dataEntityMigration, self._migrationProcess._migrationSession) 
     89        return exs 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py

    r8077 r8082  
    44@author: mnagni 
    55''' 
    6 from libs.migration.processor.commons import getXMLDocument,\ 
    7     findMolesLineage,\ 
    8     findAuthorInResource, createMO_ResponsiblePartyInfo 
     6from libs.migration.processor.commons import findMolesLineage,\ 
     7    findAuthorInResource, createMO_ResponsiblePartyInfo, getAtomDocumentByMO,\ 
     8    DO_BADC, DO_NEODC 
    99from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue 
    1010from libs.migration.exception.exceptions import NoDataLineage 
     
    2121    publisherName = 'CEDA' 
    2222    ''' 
    23         Processes a Deployment item. 
    24         @var _deplObj: a ObservationMigration instance 
    25         @var _dataEntity: the associated DataEntityProcessor 
    26         @var _resource: an elementTree Element representing XML document processed by this instance  
    27         @var _ceda_observation: the CEDA_Observation associated with this instance   
     23        Processes a Deployment item.   
    2824    '''     
    29      
    30     def __init__(self, deplObj, deProcessor): 
     25    def __init__(self, dataEntityMigration, deploymentMigration, migrationProcess): 
    3126        ''' 
    3227            Initializes the class 
    33             @param deplObj: a ObservationMigration entity 
    34             @param deProcessor: the associated DataEntityProcessor 
     28            @param dataEntityMigration: a DataEntityMigration instance 
     29            @param deploymentMigration: the DeploymentMigration instance 
     30            @param migrationProcess: a MigrationProcess instance             
    3531        '''               
    36         self._deplObj = deplObj 
    37         self._deProcessor = deProcessor 
    38  
    39         self._migrationSession = deProcessor._migrationSession 
    40         self._molesSession = deProcessor._molesSession 
     32        self._dataEntityMigration = dataEntityMigration 
     33        self._deploymentMigration = deploymentMigration 
     34        self._migrationProcess = migrationProcess 
    4135 
    4236    def _existsCEDAasPublisher(self): 
     
    5650         
    5751        parsedParties = [] 
    58         if (len(set(authors) & set(['badc.nerc.ac.uk', 'neodc.nerc.ac.uk'])) > 0): 
     52        if (len(set(authors) & set([DO_BADC, DO_NEODC])) > 0): 
    5953            parsedParties.append(self._createResponsibleParty('Centre for Environmental Data Archive', MO_RoleValue.cl_curator)) 
    60             authorsCSV = findAuthorInResource(self._deProcessor.dataEntity) 
     54            authorsCSV = findAuthorInResource(self._dataEntityMigration) 
    6155            authors = self._extractAuthors(authorsCSV) 
    6256         
     
    140134            @param relatedPartyInfos: a MO_ResponsiblePartyInfo list 
    141135        '''         
    142         authorsCSV = findAuthorInResource(self._deployment) 
     136        authorsCSV = findAuthorInResource(self._deploymentMigration) 
    143137        if authorsCSV is None: 
    144138            return 
     
    159153 
    160154    def _assignLineage(self): 
    161         self._ceda_observation.dataLineage = findMolesLineage(self._deProcessor.dataEntity) 
    162         if self._ceda_observation.dataLineage is None: 
    163             raise NoDataLineage(self._de_path) 
     155        self._ceda_observation.dataLineage = findMolesLineage(self._dataEntityMigration) 
    164156         
    165157    def _assignTitle(self): 
    166158        pass  
     159 
     160    def _assignProcess(self): 
     161        pass 
     162    ''' 
     163        links = findLinksInDeployment(self._deployment) 
     164        for dpt in links['DPT']: 
     165            dptObj = MigrationEPB.getDPTMigrationByName(dpt + '.atom', self._migrationSession) 
     166        print links 
     167    ''' 
     168  
    167169         
    168170    def _processObj(self, cedaObservation): 
    169         ''' 
    170             @param  cedaObservation: a relatedPartiesInfo instance  
     171        # 
     172        ''' 
     173            @param  cedaObservation: a cedaObservation instance  
    171174        ''' 
    172175        self._assignName(cedaObservation.relatedParty) 
    173176        self._assignLineage() 
    174177        self._assignTitle() 
     178        self._assignProcess()         
    175179         
    176180         
    177181    def process(self): 
    178         self._de_path = '%s/%s' % (self._deplObj.exist_path, self._deplObj.depl_name) 
    179         print    self._de_path 
    180         #retrieves the deployment'd document          
    181         self._deployment = getXMLDocument(self._de_path) 
     182        self._deployment = getAtomDocumentByMO(self._deploymentMigration) 
     183        #retrieves the deployment'd document 
    182184        self._ceda_observation = None 
    183185         
    184186        add = False 
    185187        #Exists already an associated CEDA_Observation associated to this deployment? 
    186         if self._deplObj is not None and self._deplObj.obs_id is not None: 
    187             self._ceda_observation = EPB.search(CEDA_Observation, self._deplObj.obs_id, self._migrationSession) 
     188        if self._deploymentMigration is not None and self._deploymentMigration.ceda_observation_id is not None: 
     189            self._ceda_observation = EPB.search(CEDA_Observation, self._deploymentMigration.ceda_observation_id, self._migrationProcess._molesSession) 
    188190 
    189191        #If does not exist creates a new one             
    190192        if self._ceda_observation is None: 
    191193            add = True 
    192             self._ceda_observation = CEDA_Observation() 
    193             self._ceda_observation.relatedParty = [] 
     194            self._ceda_observation = CEDA_Observation()             
    194195 
    195196        #fills/update the CEDA_Observation                         
     
    197198         
    198199        if add: 
    199             Moles3EPB.addCEDA_Observation(self._ceda_observation, self._molesSession) 
    200         self._molesSession.commit() 
    201         self._deplObj.obs_id = self._ceda_observation.id 
     200            Moles3EPB.addCedaObject(self._ceda_observation, self._migrationProcess._molesSession) 
     201        self._migrationProcess._molesSession.commit() 
     202        self._deploymentMigration.ceda_observation_id = self._ceda_observation.id 
    202203         
    203204        if add: 
    204             MigrationEPB.addObservationMigration(self._deplObj, self._migrationSession) 
     205            MigrationEPB.addMigration(self._deploymentMigration, self._migrationProcess._migrationSession) 
    205206        return self._ceda_observation 
    206207 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py

    r8077 r8082  
    44@author: mnagni 
    55''' 
    6 from libs.migration.processor.commons import docStatus, base, getCollectionRefs,\ 
    7     findMolesCreationDate, findID, stringToTimestamp,\ 
    8     getResourceRefs, getXMLDocument 
     6from libs.migration.processor.commons import docStatus, getCollectionRefs,\ 
     7    findMolesCreationDate, findID, stringToTimestamp, buildExistTypePath,\ 
     8    buildExistDocPath, DT_DATA_ENTITIES, DT_DEPLOYMENTS, DT_DEPLOYMENT_DATA,\ 
     9    getOwnerRefs, getAtomDocument 
    910from libs.migration.exception.exceptions import NoCreationDate 
    10 from libs.migration.db.classes import ObservationMigration,\ 
    11     ObservationCollectionMigration     
     11from libs.migration.db.classes import DeploymentDataMigration,\ 
     12    DeploymentsMigration, DataEntityMigration 
    1213from libs.migration.MigrationEPB import MigrationEPB 
    1314 
    1415class LoadResources(object): 
    15      
     16    """ 
     17     Loads informations from the eXist database to populate reference table for the migrations 
     18    """ 
    1619    def __init__(self): 
    1720        self._migrationSession = MigrationEPB.getNewMigrationSession() 
    1821             
    19     def _loadDataEntities(self, base): 
    20         baseDE = '%s/%s' % (base, 'data_entities') 
    21         refs = getCollectionRefs(baseDE) 
     22    def _loadDataEntities(self, docStatus): 
     23        self._loadCollections(docStatus, DT_DATA_ENTITIES, self._loadDataEntityDocs)         
     24 
     25    def _loadDeployments(self, docStatus): 
     26        self._loadCollections(docStatus, DT_DEPLOYMENTS, self._loadDeploymentsDocs)         
     27 
     28    def _loadDeploymentData(self, docStatus): 
     29        self._loadCollections(docStatus, DT_DEPLOYMENT_DATA, self._loadDeploymentDataDocs)             
     30 
     31    def _loadCollections(self, docStatus, docType, processor): 
     32        refs = getCollectionRefs(buildExistTypePath(docStatus, docType))         
    2233        for ref in refs: 
    23             owner = ref.get('name') 
    24             if owner.startswith('.'): 
    25                 continue          
    26             self._loadDataEntityOwner(baseDE, owner) 
    27  
    28     def _loadDeployments(self, base): 
    29         baseDE = '%s/%s' % (base, 'deployments') 
    30         refs = getCollectionRefs(baseDE) 
     34            docOwner = ref.get('name')   
     35            if docOwner.startswith('.'): 
     36                continue                        
     37            processor(docStatus, docOwner)            
     38             
     39    def _loadDeploymentsDocs(self, docStatus, docOwner): 
     40        refs = getOwnerRefs(docStatus, DT_DEPLOYMENTS, docOwner)          
    3141        for ref in refs: 
    32             owner = ref.get('name')   
    33             if owner.startswith('.'): 
    34                 continue                        
    35             self._loadDeploymentsOwner(baseDE, owner) 
    36              
    37     def _loadDeploymentsOwner(self, base, owner): 
    38         baseOwner = '%s/%s' % (base, owner) 
    39         refs = getResourceRefs(baseOwner)          
    40         for ref in refs: 
    41             deName = ref.get('name') 
    42             resourceSource = '%s/%s' % (baseOwner, deName) 
    43             res = getXMLDocument(resourceSource) 
    44             
     42            docName = ref.get('name') 
     43            xmlDocument = getAtomDocument(docStatus, DT_DEPLOYMENTS, docOwner, docName) 
    4544             
    4645            try: 
    47                 deCreationDate = findMolesCreationDate(res)             
    48                 if deCreationDate is None: 
    49                     raise NoCreationDate(resourceSource) 
     46                doc_creation = self._extractCreationDate(xmlDocument) 
    5047            except NoCreationDate as ex: 
    5148                print ex 
    5249                continue 
    5350             
    54             deID = findID(res)  
    55             if deID is None: 
    56                 print (resourceSource) 
     51            doc_id = findID(xmlDocument)  
     52            if doc_id is None: 
     53                print (buildExistDocPath(docStatus, DT_DEPLOYMENTS, docOwner, docName)) 
    5754 
    5855            add = False 
    59             coll = MigrationEPB.search(ObservationMigration, deID, self._migrationSession) 
     56            coll = MigrationEPB.search(DeploymentsMigration, doc_id, self._migrationSession) 
    6057            if coll is None:                 
    61                 coll = ObservationMigration() 
    62                 coll.depl_id = deID 
     58                coll = DeploymentsMigration() 
     59                coll.depl_id = doc_id 
    6360                add = True 
    6461                 
    6562             
    66             coll.depl_name = deName 
    67             coll.depl_creation = stringToTimestamp(deCreationDate)  
    68             coll.exist_path = baseOwner 
     63            coll.doc_name = docName 
     64            coll.doc_owner = docOwner 
     65            coll.doc_status = docStatus             
     66            coll.doc_creation = stringToTimestamp(doc_creation) 
    6967             
    7068            if add: 
    71                 MigrationEPB.addObservationMigration(coll, self._migrationSession) 
     69                MigrationEPB.addMigration(coll, self._migrationSession) 
    7270                 
    7371            self._migrationSession.commit() 
    7472             
    75     def _loadDataEntityOwner(self, base, owner): 
    76         baseOwner = '%s/%s' % (base, owner) 
    77         refs = getResourceRefs(baseOwner)  
     73    def _loadDataEntityDocs(self, docStatus, docOwner): 
     74        refs = getOwnerRefs(docStatus, DT_DATA_ENTITIES, docOwner)  
    7875        for ref in refs: 
    79             deName = ref.get('name') 
    80             resourceSource = '%s/%s' % (baseOwner, deName) 
    81             res = getXMLDocument(resourceSource)   
     76            docName = ref.get('name') 
     77            xmlDocument = getAtomDocument(docStatus, DT_DATA_ENTITIES, docOwner, docName)   
    8278             
    8379            try: 
    84                 deCreationDate = findMolesCreationDate(res)             
    85                 if deCreationDate is None: 
    86                     raise NoCreationDate(resourceSource) 
     80                doc_creation = self._extractCreationDate(xmlDocument)             
    8781            except NoCreationDate as ex: 
    8882                print ex 
    8983                continue 
    9084             
    91             deID = findID(res) 
    92             if deID is None: 
    93                 print (resourceSource) 
     85            doc_id = findID(xmlDocument) 
     86            if doc_id is None: 
     87                print (buildExistDocPath(docStatus, DT_DEPLOYMENTS, docOwner, docName)) 
    9488 
    9589                       
    9690            add = False 
    97             coll = MigrationEPB.search(ObservationCollectionMigration, deID, self._migrationSession) 
     91            coll = MigrationEPB.search(DataEntityMigration, doc_id, self._migrationSession) 
    9892            if coll is None:                 
    99                 coll = ObservationCollectionMigration() 
    100                 coll.data_ent_id = deID 
     93                coll = DataEntityMigration() 
     94                coll.data_ent_id = doc_id 
    10195                add = True 
    10296             
    103             coll.data_ent_name = deName             
    104             coll.data_ent_creation = stringToTimestamp(deCreationDate)  
    105             coll.exist_path = baseOwner 
     97            coll.doc_name = docName   
     98            coll.doc_owner = docOwner       
     99            coll.doc_status = docStatus 
     100            coll.doc_creation = stringToTimestamp(doc_creation)  
    106101            if add: 
    107                 MigrationEPB.addObservationCollectionMigration(coll, self._migrationSession) 
     102                MigrationEPB.addMigration(coll, self._migrationSession) 
    108103            self._migrationSession.commit() 
     104 
     105    def _loadDeploymentDataDocs(self, docStatus, docOwner): 
     106        refs = getOwnerRefs(docStatus, DT_DEPLOYMENT_DATA, docOwner) 
     107        for ref in refs: 
     108            docName = ref.get('name') 
     109            xmlDocument = getAtomDocument(docStatus, DT_DEPLOYMENT_DATA, docOwner, docName)             
     110 
     111            try: 
     112                doc_creation = self._extractCreationDate(xmlDocument)             
     113            except NoCreationDate as ex: 
     114                print ex 
     115                continue 
     116             
     117            doc_id = findID(xmlDocument) 
     118            if doc_id is None: 
     119                print (buildExistDocPath(docStatus, DT_DEPLOYMENT_DATA, docOwner, docName)) 
     120 
     121            add = False 
     122            coll = MigrationEPB.search(DeploymentDataMigration, doc_id, self._migrationSession) 
     123            if coll is None:                 
     124                coll = DeploymentDataMigration() 
     125                coll.deployment_data_id = doc_id 
     126                add = True 
     127             
     128            coll.doc_name = docName   
     129            coll.doc_owner = docOwner                       
     130            coll.doc_status = docStatus             
     131            coll.doc_creation = stringToTimestamp(doc_creation)  
     132            if add: 
     133                MigrationEPB.addMigration(coll, self._migrationSession) 
     134            self._migrationSession.commit() 
     135 
     136    def _extractCreationDate(self, xmlDocument): 
     137        creationDate = findMolesCreationDate(xmlDocument)             
     138        if creationDate is None: 
     139            raise NoCreationDate(creationDate) 
     140        return creationDate 
    109141             
    110142    def process(self): 
     143        """ 
     144            Loads all the folders below the atoms ('published', 'working', 'Published') folders 
     145        """ 
    111146        ''' 
    112147            docStatus = ('published', 'working', 'Published')  
    113148        ''' 
    114         for status in docStatus:            
    115             baseStatus = '%s/%s' % (base, status) 
    116             self._loadDataEntities(baseStatus)             
    117             self._loadDeployments(baseStatus) 
    118         self._migrationSession.close()                              
     149        for status in docStatus:               
     150            self._loadDataEntities(status)             
     151            self._loadDeployments(status) 
     152            self._loadDeploymentData(status)             
     153        self._migrationSession.close() 
     154         
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/migrationProcess.py

    r8077 r8082  
    1414        self._molesSession = Moles3EPB.getNewMolesSession() 
    1515     
    16     def process(self):     
    17         sorted_data_ents = MigrationEPB.getObservationCollectionMigrationOrderByDate(session = self._migrationSession) 
     16    def process(self): 
     17        sorted_data_ents = MigrationEPB.getDataEntityMigrationOrderByDate(session = self._migrationSession) 
    1818        exs = [] 
    1919        #loops over the sorted data entities 
    20         for de in sorted_data_ents:     
    21             de_path = '%s/%s' % (de.exist_path, de.data_ent_name) 
    22             dep = DataEntityProcessor(de_path, self) 
     20        for dataEntityMigration in sorted_data_ents:                 
     21            dep = DataEntityProcessor(dataEntityMigration, self) 
    2322            exs.append(dep.process()) 
    2423            #self.commitAll() 
Note: See TracChangeset for help on using the changeset viewer.