Changeset 8077
- Timestamp:
- 27/01/12 17:33:18 (8 years ago)
- Location:
- mauRepo/MolesManager/trunk/src
- Files:
-
- 2 added
- 12 edited
- 1 moved
Legend:
- Unmodified
- Added
- Removed
-
mauRepo/MolesManager/trunk/src/MolesManager/dataaccess.py
r8054 r8077 5 5 ''' 6 6 from ea_model.iso_19103_2005_schema_language.basic_types.primitive.date_and_time.date import Date 7 from ea_model.ceda_moles.ceda_observation.ceda_observation import CEDA_Observation 8 from sqlalchemy.orm.util import class_mapper 7 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 9 8 10 9 def _buildFilter(key, keyValue): -
mauRepo/MolesManager/trunk/src/MolesManager/forms/date.py
r7976 r8077 34 34 for attrName in self._clazz.__dict__: 35 35 if not attrName.startswith('_'): 36 if attrName == 'dataLineage': 37 pass #print 'ciao' 36 38 attrValue = getattr(self._clazz, attrName) 37 39 if isinstance(attrValue, InstrumentedAttribute): -
mauRepo/MolesManager/trunk/src/MolesManager/molesSessionMiddleware.py
r8054 r8077 4 4 @author: mnagni 5 5 ''' 6 from libs.migration.pjb import PJB 6 from MolesManager.moles3epb import Moles3EPB 7 from libs.migration.client import MigrationThread 8 from MolesManager.settings import RUN_MIGRATION 7 9 8 10 class MolesSessionMiddleware(object): 11 12 #This attribute should be not here. 13 #unfortunately I cannot find any start/stop signals from Django 14 _migrationThread = MigrationThread(interval = 86400) 15 16 def _migration(self, runMigration = RUN_MIGRATION): 17 if runMigration and not MolesSessionMiddleware._migrationThread.isAlive(): 18 #t.setDaemon(False) 19 MolesSessionMiddleware._migrationThread.start() 20 9 21 """ 10 Represents the access to the Moles database 22 Represents the access to the Moles database. 23 Creates from the existing db connections pools a new session 24 to be used by all the DB operations involved in the actual HTTPRequest 11 25 """ 12 26 13 def process_request(self, request): 14 request.moles_session = PJB.getNewMolesSession() 27 def process_request(self, request): 28 self._migration() #see the note on MolesSessionMiddleware._migration 29 30 request.moles_session = Moles3EPB.getNewMolesSession() 15 31 ''' 16 32 request.db_sessions = {} … … 40 56 return 41 57 session.rollback() 42 session.close() 43 44 @classmethod 45 def search(self, request, clazz, inst_id): 46 return PJB.search(clazz, inst_id, request.moles_session) 47 58 session.close() -
mauRepo/MolesManager/trunk/src/MolesManager/settings.py
r8074 r8077 162 162 #moles3 database connection 163 163 MOLES3 = 'moles3' 164 #MOLES3_DB_CONNECTION = 'postgresql://user:pws@host:port/dbName' 165 MOLES3_DB_CONNECTION = 'PG_MOLES3_DB_CONNECTION' 164 MOLES3_DB_CONNECTION = 'postgresql://user:pws@host:port/dbName' 165 166 #MOLES3_DB_CONNECTION = 'PG_MOLES3_DB_CONNECTION' 166 167 MOLES3_DB_SCRIPT = SRC + '/sqlTables.py' 167 168 … … 169 170 #moles2 to moles3 migration database connection 170 171 MIGRATION = 'migration' 171 #MIGRATION_DB_CONNECTION = 'postgresql://user:pws@host:port/dbName' 172 MIGRATION_DB_CONNECTION = 'PG_MIGRATION_DB_CONNECTION' 172 MIGRATION_DB_CONNECTION = 'postgresql://user:pws@host:port/dbName' 173 174 #MIGRATION_DB_CONNECTION = 'PG_MIGRATION_DB_CONNECTION' 173 175 MIGRATION_DB_SCRIPT = SRC + '/libs/migration/db/migrationTables.py' 174 176 177 RUN_MIGRATION = True -
mauRepo/MolesManager/trunk/src/MolesManager/views/cedaObservationView.py
r8054 r8077 9 9 from MolesManager.forms.commons import ObjectById 10 10 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 11 from MolesManager.moles SessionMiddleware import MolesSessionMiddleware11 from MolesManager.moles3epb import Moles3EPB 12 12 13 13 … … 30 30 if objectId.is_valid(): 31 31 id = objectId.cleaned_data['obj_id'] 32 dp = Moles SessionMiddleware.search(request, CEDA_Observation, objectId.cleaned_data['obj_id'])32 dp = Moles3EPB.search(CEDA_Observation, objectId.cleaned_data['obj_id'], request.moles_session) 33 33 return createForm(dp) 34 34 return None -
mauRepo/MolesManager/trunk/src/libs/commons_db.py
r8054 r8077 19 19 20 20 class DbManager(object): 21 22 _metadata = MetaData() 21 23 """ 22 24 Represents a database instance managed by an SQLAlchemy engine. … … 30 32 self._script = script 31 33 self._engine = create_engine(self._connection, echo=True) 32 if (self._script): 33 self._metadata = MetaData()34 execfile(self._script, {'metadata': self._metadata, 'engine': self._engine})34 if (self._script): 35 execfile(self._script, {'metadata': DbManager._metadata, 'engine': self._engine}) 36 DbManager._metadata.create_all(bind=self._engine) 35 37 self._session = scoped_session(sessionmaker(bind=self._engine)) 36 38 -
mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py
r8054 r8077 4 4 @author: mnagni 5 5 ''' 6 from MolesManager.settings import MOLES3_DB_CONNECTION, MOLES3_DB_SCRIPT,\ 7 MIGRATION_DB_SCRIPT, MIGRATION_DB_CONNECTION, MIGRATION, MOLES3 6 from MolesManager.settings import MIGRATION_DB_SCRIPT, MIGRATION_DB_CONNECTION 8 7 from libs.migration.db.classes import ObservationMigration,\ 9 8 ObservationCollectionMigration 10 9 from sqlalchemy.sql.expression import asc 11 from libs.commons_db import DbManager, DbManagerCollection 10 from libs.commons_db import DbManager 11 from libs.epb import EPB 12 12 13 molesDB = DbManager(MOLES3_DB_CONNECTION, MOLES3_DB_SCRIPT)14 migrationDB = DbManager(MIGRATION_DB_CONNECTION, MIGRATION_DB_SCRIPT)15 dbManagers = {MOLES3: molesDB, MIGRATION: migrationDB}16 dbManagerCollection = DbManagerCollection(dbManagers)17 13 18 class PJB(object): 19 20 @classmethod 21 def buildFilter(self, key, keyValue): 22 return '%s = \'%s\'' % (key, keyValue) 23 24 @classmethod 25 def search(self, clazz, inst_id, session): 26 res = session.query(clazz).get(inst_id) 27 if res is None: 28 return None 29 return res 14 class MigrationEPB(EPB): 15 16 _migrationDB = DbManager(MIGRATION_DB_CONNECTION, MIGRATION_DB_SCRIPT) 30 17 31 18 @classmethod 32 19 def addObservationCollectionMigration(self, obsCollMigration, session = None): 33 intSession = PJB._getSession(session, MIGRATION)20 intSession = MigrationEPB._getSession(session) 34 21 intSession.add(obsCollMigration) 35 PJB._closeSession(session)22 EPB._closeSession(session) 36 23 37 24 @classmethod 38 25 def addObservationMigration(self, obsMigration, session = None): 39 intSession = PJB._getSession(session, MIGRATION)26 intSession = MigrationEPB._getSession(session) 40 27 intSession.add(obsMigration) 41 PJB._closeSession(session)28 EPB._closeSession(session) 42 29 43 30 @classmethod 44 31 def getObservationCollectionMigrationOrderByDate(self, session = None): 45 intSession = PJB._getSession(session, MIGRATION)46 data_ents = PJB.getAllObjects(ObservationCollectionMigration, intSession)32 intSession = MigrationEPB._getSession(session) 33 data_ents = EPB.getAllObjects(ObservationCollectionMigration, intSession) 47 34 return data_ents.order_by(asc("data_ent_creation")) 48 35 49 36 @classmethod 50 37 def getObservationMigrationByName(self, deplName, session = None): 51 intSession = PJB._getSession(session, MIGRATION) 52 return intSession.query(ObservationMigration).filter(*[PJB.buildFilter('depl_name', deplName)]).first() 53 54 @classmethod 55 def addCEDA_Observation(self, ceda_observation, session = None): 56 intSession = PJB._getSession(session, MOLES3) 57 intSession.add(ceda_observation) 58 PJB._closeSession(session) 38 intSession = MigrationEPB._getSession(session) 39 return intSession.query(ObservationMigration).filter(*[EPB.buildFilter('depl_name', deplName)]).first() 59 40 60 41 @classmethod 61 42 def getNewMigrationSession(self): 62 return dbManagerCollection.createSession(MIGRATION) 63 64 @classmethod 65 def getNewMolesSession(self): 66 return dbManagerCollection.createSession(MOLES3) 43 return MigrationEPB._getSession() 67 44 68 45 @classmethod 69 def getAllObjects(self, clazz, session): 70 res = session.query(clazz) 71 if res is None: 72 return None 73 return res 74 75 @classmethod 76 def _getSession(self, session = None, sessionName = ""): 77 if session: 78 return session 79 return dbManagerCollection.createSession(sessionName) 80 81 @classmethod 82 def _closeSession(self, session): 83 if session is not None: 84 return 85 session.commit() 86 session.close() 46 def _getSession(self, session = None): 47 return EPB._getSession(MigrationEPB._migrationDB, session) -
mauRepo/MolesManager/trunk/src/libs/migration/client.py
r8074 r8077 5 5 ''' 6 6 from libs.migration.processor.loadResources import LoadResources 7 from libs.migration.pjb import PJB8 7 from libs.migration.processor.migrationProcess import MigrationProcess 9 8 10 ''' 11 exist/rest/atoms/published/data_entities/neodc.nerc.ac.uk 12 ''' 9 import threading 10 import time 11 from threading import Timer 13 12 14 13 15 16 14 class MigrationThread(threading.Thread): 15 """ 16 Constructs a scheduler for the Moles2 to Moles3 migration. 17 @param interval: define the minimal time, in seconds, between an execution and the successive. 18 If the current migration run takes more than the defined interval the next will starts 19 as soon the previous ends. 20 """ 21 def __init__(self, interval=600): 22 threading.Thread.__init__(self) 23 self._doRun = True 24 self.interval = interval 25 26 def stop(self): 27 self._doRun = False 28 29 def run(self): 30 while self._doRun: 31 #self._migrate() 32 #s = sched.scheduler(time.time, time.sleep) 33 startTime = (int)(time.time()) 34 timer = Timer(5, self._migrate, ()) 35 timer.start() 36 timer.join() 37 diffTime = startTime + self.interval - (int)(time.time()) 38 if diffTime > 0: 39 time.sleep(diffTime) 40 41 def _printTime(self): 42 print "From print_time", time.time() 43 44 def _migrate(self): 45 #mEPB = MigrationEPB() 46 lr = LoadResources() 47 lr.process() 48 49 mp = MigrationProcess() 50 mp.process() 51 52 #t = MigrationThread(interval = 1000) 53 #t.setDaemon(False) 54 #t.start() 55 #print "ciao" 56 #time.sleep(600) 57 #t.stop() 58 59 17 60 18 61 19 62 20 63 ''' 21 So it works: 22 Loads the eXist resources 23 Loops over a docStatus 24 Loops over the data_entities 25 For each data_entity loop over its deployments 64 26 65 ''' 27 66 28 29 30 '''31 ceda_observation = CEDA_Observation()32 ceda_observation.dataLineage = 'mauData'33 ceda_observation.relatedParty = []34 rp = MO_ResponsiblePartyInfo()35 rp.role = MO_RoleValue.cl_author36 rp.party = []37 party = CI_Party()38 party.name = 'Mau'39 rp.party.append(party)40 ceda_observation.relatedParty.append(rp)41 doInsertOrUpdate([ceda_observation], session, update = False)42 session.commit()43 44 45 res = session.query(CEDA_Observation).filter(*[_buildFilter('ceda_observation_id', 2)])46 rp = res[0].relatedParty[0]47 print rp48 '''49 50 pjb = PJB()51 52 lr = LoadResources(pjb)53 lr.process()54 55 56 mp = MigrationProcess(pjb)57 mp.process()58 59 -
mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py
r8063 r8077 9 9 from libs.migration.processor.commons import getResource, findDeploymentsInDE,\ 10 10 getResourceRefs, getXMLDocument 11 from libs.migration. pjb import PJB11 from libs.migration.MigrationEPB import MigrationEPB 12 12 13 13 class DataEntityProcessor(object): … … 92 92 for deplName in self._depls: 93 93 #retrieve the deployment's path 94 deplObj = PJB.getObservationMigrationByName(deplName + '.atom', self._migrationSession)94 deplObj = MigrationEPB.getObservationMigrationByName(deplName + '.atom', self._migrationSession) 95 95 if deplObj is None: 96 96 print "cannot find deployment %s for resource %s" % (deplName, self._de_path) -
mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py
r8074 r8077 10 10 from libs.migration.exception.exceptions import NoDataLineage 11 11 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 12 from libs.migration.pjb import PJB13 12 import re 13 from MolesManager.moles3epb import Moles3EPB 14 from libs.migration.MigrationEPB import MigrationEPB 15 from libs.epb import EPB 14 16 15 17 class DeploymentProcessor(object): … … 183 185 #Exists already an associated CEDA_Observation associated to this deployment? 184 186 if self._deplObj is not None and self._deplObj.obs_id is not None: 185 self._ceda_observation = PJB.search(CEDA_Observation, self._deplObj.obs_id, self._migrationSession)187 self._ceda_observation = EPB.search(CEDA_Observation, self._deplObj.obs_id, self._migrationSession) 186 188 187 189 #If does not exist creates a new one … … 195 197 196 198 if add: 197 PJB.addCEDA_Observation(self._ceda_observation, self._molesSession)199 Moles3EPB.addCEDA_Observation(self._ceda_observation, self._molesSession) 198 200 self._molesSession.commit() 199 201 self._deplObj.obs_id = self._ceda_observation.id 200 202 201 203 if add: 202 PJB.addObservationMigration(self._deplObj, self._migrationSession)204 MigrationEPB.addObservationMigration(self._deplObj, self._migrationSession) 203 205 return self._ceda_observation 204 206 -
mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py
r8054 r8077 9 9 from libs.migration.exception.exceptions import NoCreationDate 10 10 from libs.migration.db.classes import ObservationMigration,\ 11 ObservationCollectionMigration 12 from libs.migration. pjb import PJB11 ObservationCollectionMigration 12 from libs.migration.MigrationEPB import MigrationEPB 13 13 14 14 class LoadResources(object): 15 15 16 def __init__(self, pjb): 17 self._migrationSession = PJB.getNewMigrationSession() 18 self._molesSession = PJB.getNewMolesSession() 16 def __init__(self): 17 self._migrationSession = MigrationEPB.getNewMigrationSession() 19 18 20 19 def _loadDataEntities(self, base): … … 58 57 59 58 add = False 60 coll = PJB.search(ObservationMigration, deID, self._migrationSession)59 coll = MigrationEPB.search(ObservationMigration, deID, self._migrationSession) 61 60 if coll is None: 62 61 coll = ObservationMigration() … … 70 69 71 70 if add: 72 PJB.addObservationMigration(coll, self._migrationSession)71 MigrationEPB.addObservationMigration(coll, self._migrationSession) 73 72 74 73 self._migrationSession.commit() … … 96 95 97 96 add = False 98 coll = PJB.search(ObservationCollectionMigration, deID, self._migrationSession)97 coll = MigrationEPB.search(ObservationCollectionMigration, deID, self._migrationSession) 99 98 if coll is None: 100 99 coll = ObservationCollectionMigration() … … 106 105 coll.exist_path = baseOwner 107 106 if add: 108 PJB.addObservationCollectionMigration(coll, self._migrationSession)107 MigrationEPB.addObservationCollectionMigration(coll, self._migrationSession) 109 108 self._migrationSession.commit() 110 109 … … 117 116 self._loadDataEntities(baseStatus) 118 117 self._loadDeployments(baseStatus) 119 self._migrationSession.close() 120 self._molesSession.close() 118 self._migrationSession.close() -
mauRepo/MolesManager/trunk/src/libs/migration/processor/migrationProcess.py
r8054 r8077 5 5 ''' 6 6 from libs.migration.processor.dataEntity import DataEntityProcessor 7 from libs.migration.pjb import PJB 7 from libs.migration.MigrationEPB import MigrationEPB 8 from MolesManager.moles3epb import Moles3EPB 8 9 9 10 class MigrationProcess(): 10 11 11 def __init__(self, pjb): 12 self._pjb = pjb 13 self._migrationSession = PJB.getNewMigrationSession() 14 self._molesSession = PJB.getNewMolesSession() 12 def __init__(self): 13 self._migrationSession = MigrationEPB.getNewMigrationSession() 14 self._molesSession = Moles3EPB.getNewMolesSession() 15 15 16 16 def process(self): 17 sorted_data_ents = PJB.getObservationCollectionMigrationOrderByDate(session = self._migrationSession)17 sorted_data_ents = MigrationEPB.getObservationCollectionMigrationOrderByDate(session = self._migrationSession) 18 18 exs = [] 19 19 #loops over the sorted data entities … … 22 22 dep = DataEntityProcessor(de_path, self) 23 23 exs.append(dep.process()) 24 self.commitAll()24 #self.commitAll() 25 25 print exs 26 26 -
mauRepo/MolesManager/trunk/src/sqlTables.py
r8074 r8077 417 417 from ea_model.iso_19109_2005_application_schema.general_feature_model.gf_associationrole import GF_AssociationRole 418 418 419 clear_mappers()420 metadata = MetaData()421 419 def next_id(connection, seq_name): 422 420 seq = Sequence(seq_name) … … 9596 9594 class_mapper(TP_DirectedSolid).add_properties({'topo': relationship(TP_Solid, uselist=False, backref='tp_directedsolid_topo', primaryjoin=tp_solid_table.c.tp_directedsolid_topo_id==tp_directedsolid_table.c.id),'hub': relationship(TP_Face, secondary=tp_face_tp_directedsolid_table),'_tp_directedtopo': relationship(TP_DirectedTopo, uselist=False, backref='tp_directedsolid__tp_directedtopo', primaryjoin=tp_directedtopo_table.c.tp_directedsolid__tp_directedtopo_id==tp_directedsolid_table.c.id)}) 9597 9595 9598 metadata.create_all(engine) 9599 9600 9596
Note: See TracChangeset
for help on using the changeset viewer.