Changeset 8054
- Timestamp:
- 10/01/12 17:15:33 (9 years ago)
- Location:
- mauRepo/MolesManager/trunk/src
- Files:
-
- 3 added
- 1 deleted
- 12 edited
Legend:
- Unmodified
- Added
- Removed
-
mauRepo/MolesManager/trunk/src/MolesManager/dataaccess.py
r8015 r8054 4 4 @author: mnagni 5 5 ''' 6 from MolesManager.settings import MOLES3DB7 6 from ea_model.iso_19103_2005_schema_language.basic_types.primitive.date_and_time.date import Date 8 7 from ea_model.ceda_moles.ceda_observation.ceda_observation import CEDA_Observation 9 8 from sqlalchemy.orm.util import class_mapper 10 9 11 10 def _buildFilter(key, keyValue): 12 11 return '%s = \'%s\'' % (key, keyValue) 13 12 14 def _checkSession(session): 15 if session is None: 16 return MOLES3DB.getSession() 17 else: 18 return session 19 20 def doInsert(toStoreInstance, session = None): 21 session = _checkSession(session) 13 def doInsert(toStoreInstance, session): 22 14 session.add(toStoreInstance) 23 15 session.commit() 24 16 25 def getCEDA_ObservationById(id, session = None): 26 session = _checkSession(session) 27 res = session.query(CEDA_Observation).filter(*[_buildFilter('ceda_observation_id', id)]) 17 def getCEDA_ObservationById(id, session): 18 res = session.query(CEDA_Observation) 28 19 print(res) 29 20 return res.all() 30 21 31 def getDateById(id, session = None): 32 session = _checkSession(session) 22 def getDateById(id, session): 33 23 res = session.query(Date).filter(*[_buildFilter('date_id', id)]) 34 24 print(res) -
mauRepo/MolesManager/trunk/src/MolesManager/settings.py
r8042 r8054 1 1 # Django settings for MolesManager project. 2 2 import os 3 from db_manager.dbConnection import DBConnection, DBScript 3 4 4 5 5 DEBUG = True … … 102 102 'django.contrib.auth.middleware.AuthenticationMiddleware', 103 103 'django.contrib.messages.middleware.MessageMiddleware', 104 'molesSessionMiddleware.MolesSessionMiddleware', 104 105 ) 105 106 106 ROOT_URLCONF = 'MolesManager.urls' 107 107 … … 162 162 } 163 163 } 164 165 #moles3 database connection 164 166 MOLES3 = 'moles3' 165 MOLES2 = 'moles2' 167 MOLES3_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3' 168 MOLES3_DB_SCRIPT = '/misc/frigid3/humid1/mnagni/workspaces/moles3/MolesManager/src/sqlTables.py' 166 169 167 MOLES3DB_SCRIPT = '/misc/frigid3/humid1/mnagni/workspaces/moles3/MolesManager/src/sqlTables.py' 168 MOLES3DB = DBConnection('postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3') 169 MOLES3DB.initTables(DBScript(MOLES3DB_SCRIPT)) 170 171 #moles2 to moles3 migration database connection 172 MIGRATION = 'migration' 173 MIGRATION_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3' 174 MIGRATION_DB_SCRIPT = '/misc/frigid3/humid1/mnagni/workspaces/moles3/MolesManager/src/libs/migration/db/migrationTables.py' 175 -
mauRepo/MolesManager/trunk/src/MolesManager/static/templates/cedaObservation.html
r8042 r8054 14 14 15 15 <div class="fieldWrapper"> 16 <label for="id_dataLineage">dataLineage</label> {{ coObs.dataLineage 17 }} <label for="id_relatedParty">relatedParty</label> {{ 18 coObs.relatedParty }} 16 <label for="id_dataLineage">dataLineage</label> {{ coObs.dataLineage }} 17 <label for="id_relatedParty">relatedParty</label> {{ coObs.relatedParty }} 19 18 </div> 20 19 </form> -
mauRepo/MolesManager/trunk/src/MolesManager/views/cedaObservationView.py
r8016 r8054 4 4 @author: mnagni 5 5 ''' 6 from MolesManager.dataaccess import getCEDA_ObservationById7 6 from MolesManager.forms.date import createForm 8 7 from django.shortcuts import render_to_response 9 8 from django.core.context_processors import csrf 10 9 from MolesManager.forms.commons import ObjectById 11 from ea_model.ceda_moles.ceda_observation.ceda_observation import CEDA_Observation 10 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 11 from MolesManager.molesSessionMiddleware import MolesSessionMiddleware 12 12 13 13 … … 28 28 coObj = co(request.POST or None) 29 29 30 if objectId.is_valid() and coObj.is_valid():31 return coObj32 33 30 if objectId.is_valid(): 34 31 id = objectId.cleaned_data['obj_id'] 35 dp = getCEDA_ObservationById(objectId.cleaned_data['obj_id'])36 return createForm(dp [0])32 dp = MolesSessionMiddleware.search(request, CEDA_Observation, objectId.cleaned_data['obj_id']) 33 return createForm(dp) 37 34 return None 38 35 def coView(request): -
mauRepo/MolesManager/trunk/src/db_manager/dbConnection.py
r8014 r8054 14 14 class DBConnection(object): 15 15 16 def __init__(self, dbConnectionString ):16 def __init__(self, dbConnectionString, dbInitializer = []): 17 17 self._dbConnectionString = dbConnectionString 18 self._engine = None 19 self._metadata = None 20 self._Session = None 21 self._dbInitializer = [] 22 23 def initTables(self, dbInitializer): 24 if(dbInitializer): 25 dbInitializer.initTables(self._getMetadata(), self._getEngine()) 26 self._dbInitializer.append(dbInitializer) 18 for initalizer in dbInitializer: 19 initalizer.initTables(self._metadata, self._engine) 27 20 28 21 def getSession(self): 29 return (self._getSession())() 22 return (self._session)() 23 24 @property 25 def _metadata(self): 26 if not self.__dict__.has_key('_metadata'): 27 self._metadata = MetaData() 28 return self.__dict__['_metadata'] 29 30 @_metadata.setter 31 def _metadata(self, value): 32 self.__dict__['_metadata'] = value 30 33 31 def _getEngine(self): 32 if not self._engine: 34 @property 35 def _engine(self): 36 if not self.__dict__.has_key('_engine'): 33 37 self._engine = create_engine(self._dbConnectionString, echo=True) 34 return self._engine 35 36 def _getMetadata(self): 37 if not self._metadata: 38 self._metadata = MetaData() 39 return self._metadata 38 return self.__dict__['_engine'] 39 40 @_engine.setter 41 def _engine(self, value): 42 self.__dict__['_engine'] = value 40 43 41 def _getSession(self): 42 if not self._Session: 43 self._Session = sessionmaker(bind=self._getEngine()) 44 return self._Session 44 @property 45 def _session(self): 46 if not self.__dict__.has_key('_session'): 47 self._session = sessionmaker(bind=self._engine) 48 return self.__dict__['_session'] 49 50 @_session.setter 51 def _session(self, value): 52 self.__dict__['_session'] = value 45 53 46 54 -
mauRepo/MolesManager/trunk/src/libs/commons_db.py
r8014 r8054 4 4 @author: mnagni 5 5 ''' 6 from sqlalchemy import MetaData, create_engine 7 from sqlalchemy.orm import sessionmaker, scoped_session 8 class DbManagerCollection(object): 9 """ 10 Represents a collection of DbManagers. The key of the dictionary passed as argument to the constructor 11 will be used to retrieve a specific db instance. 12 """ 13 def __init__(self, dbManagers = {}): 14 self._dbManagers = dbManagers 15 16 def createSession(self, name): 17 if self._dbManagers.has_key(name): 18 return self._dbManagers[name].createDbSession() 19 20 class DbManager(object): 21 """ 22 Represents a database instance managed by an SQLAlchemy engine. 23 """ 24 def __init__(self, connection, script): 25 """ 26 :param connection: the URL used to open a connection to the database 27 :param script: a python module defining the classes mapping 28 """ 29 self._connection = connection 30 self._script = script 31 self._engine = create_engine(self._connection, echo=True) 32 if (self._script): 33 self._metadata = MetaData() 34 execfile(self._script, {'metadata': self._metadata, 'engine': self._engine}) 35 self._session = scoped_session(sessionmaker(bind=self._engine)) 36 37 def createDbSession(self): 38 """ 39 Returns a brand new Session 40 """ 41 return self._session() 42 43 def closeDbSession(self, dbSession): 44 """ 45 Commits and closes the given session instance 46 :param dbSession: a Session instance 47 """ 48 try: 49 dbSession.commit() 50 dbSession.close() 51 except: 52 dbSession.rollback() 53 dbSession.close() 54 raise 6 55 7 56 def doInsertOrUpdate(items, session, update = True): -
mauRepo/MolesManager/trunk/src/libs/migration/client.py
r8044 r8054 4 4 @author: mnagni 5 5 ''' 6 from libs.migration.db.dbConnection import MOLES3DB, \7 getObservationCollectionMigrationOrderByDate, _buildFilter8 from libs.migration.processor.dataEntity import DataEntityProcessor9 6 from libs.migration.processor.loadResources import LoadResources 10 from ea_model.moles3_4.utilities.mo_responsiblepartyinfo import MO_ResponsiblePartyInfo 11 from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue 12 from ea_model.upcomingiso.ci_party import CI_Party 13 from libs.commons_db import doInsertOrUpdate 7 from libs.migration.pjb import PJB 8 from libs.migration.processor.migrationProcess import MigrationProcess 14 9 15 10 ''' … … 18 13 19 14 20 session = MOLES3DB.getSession() 15 16 17 18 21 19 22 20 ''' … … 50 48 ''' 51 49 52 53 lr = LoadResources( session)50 pjb = PJB() 51 lr = LoadResources(pjb) 54 52 lr.process() 55 53 56 sorted_data_ents = getObservationCollectionMigrationOrderByDate(session) 57 exs = [] 58 #loops over the sorted data entities 59 for de in sorted_data_ents: 60 de_path = '%s/%s' % (de.exist_path, de.data_ent_name) 61 dep = DataEntityProcessor(de_path, session) 62 exs.append(dep.process()) 63 #session.commit() 64 print exs 54 mp = MigrationProcess(pjb) 55 mp.process() 65 56 66 57 -
mauRepo/MolesManager/trunk/src/libs/migration/db/dbConnection.py
r8042 r8054 13 13 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 14 14 15 ''' 15 16 SITE_ROOT = '/misc/frigid3/humid1/mnagni/workspaces/moles3/MolesManager/src' 16 17 … … 18 19 MigrationDB_SCRIPT = SITE_ROOT + '/libs/migration/db/migrationTables.py' 19 20 20 _MOLES3DB = DBConnection('postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3') 21 _MOLES3DB.initTables(DBScript(MOLES3DB_SCRIPT)) 22 _MOLES3DB.initTables(DBScript(MigrationDB_SCRIPT)) 21 initializer = [] 22 initializer.append(DBScript(MOLES3DB_SCRIPT)) 23 initializer.append(DBScript(MigrationDB_SCRIPT)) 24 _MOLES3DB = DBConnection('postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3', initializer) 23 25 MOLES3DB = _MOLES3DB 26 ''' 24 27 25 28 … … 36 39 return session.query(ObservationMigration).filter(*[_buildFilter('depl_name', deplName)]).first() 37 40 38 def getObservationCollectionMigrationOrderByDate(session): 39 data_ents = getAllObjects(ObservationCollectionMigration, session) 40 return data_ents.order_by(asc("data_ent_creation")) 41 41 42 42 43 -
mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py
r8016 r8054 11 11 from xml.etree.ElementTree import XML 12 12 import time, datetime 13 from libs.migration.db.classes import ObservationMigration14 13 from ea_model.upcomingiso.ci_organisation import CI_Organisation 15 14 -
mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py
r8044 r8054 9 9 from libs.migration.processor.commons import getResource, findDeploymentsInDE,\ 10 10 getResourceRefs, getDocument, getXMLDocument 11 from libs.migration. db.dbConnection import getObservationMigrationByName11 from libs.migration.pjb import PJB 12 12 13 13 class DataEntityProcessor(object): … … 20 20 ''' 21 21 22 def __init__(self, de_path, session):22 def __init__(self, de_path, migrationProcess): 23 23 ''' 24 24 Initializes the class … … 27 27 ''' 28 28 self._de_path = de_path 29 self.session = session 29 self._migrationProcess = migrationProcess 30 self._migrationSession = migrationProcess._migrationSession 31 self._molesSession = migrationProcess._molesSession 30 32 self.resource = None 31 33 self._depls = 0 … … 90 92 for deplName in self._depls: 91 93 #retrieve the deployment's path 92 deplObj = getObservationMigrationByName(deplName + '.atom', self.session)94 deplObj = PJB.getObservationMigrationByName(deplName + '.atom', self._migrationSession) 93 95 if deplObj is None: 94 96 print "cannot find deployment %s for resource %s" % (deplName, self._de_path) … … 98 100 #Processes the deployment 99 101 depl.process() 100 self. session.commit()102 self._migrationProcess.commitAll() 101 103 except NoDataLineage as ex: 102 self. session.rollback()104 self._migrationProcess.rollbackAll() 103 105 exs.append(ex) 104 106 print ex -
mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py
r8044 r8054 12 12 from libs.migration.exception.exceptions import NoDataLineage 13 13 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 14 from libs.migration.pjb import PJB 14 15 15 16 class DeploymentProcessor(object): … … 34 35 self._deplObj = deplObj 35 36 self._deProcessor = deProcessor 37 38 self._migrationSession = deProcessor._migrationSession 39 self._molesSession = deProcessor._molesSession 36 40 37 41 def _existsCEDAasPublisher(self): … … 154 158 self._deployment = getXMLDocument(self._de_path) 155 159 self._ceda_observation = None 156 isUpdate = True 160 161 add = False 157 162 #Exists already an associated CEDA_Observation associated to this deployment? 158 163 if self._deplObj is not None and self._deplObj.obs_id is not None: 159 self._ceda_observation = findCEDAObservationByDeplID(self._deplObj.obs_id, self._deProcessor.session)164 self._ceda_observation = PJB.search(CEDA_Observation, self._deplObj.obs_id, self._migrationSession) 160 165 161 166 #If does not exist creates a new one 162 167 if self._ceda_observation is None: 163 isUpdate = False168 add = True 164 169 self._ceda_observation = CEDA_Observation() 165 170 self._ceda_observation.relatedParty = [] … … 167 172 #fills/update the CEDA_Observation 168 173 self._processObj(self._ceda_observation) 169 #insert/update the CEDA_Observation 170 doInsertOrUpdate([self._ceda_observation], self._deProcessor.session, isUpdate) 171 self._deProcessor.session.commit() 174 175 if add: 176 PJB.addCEDA_Observation(self._ceda_observation, self._molesSession) 177 self._molesSession.commit() 172 178 self._deplObj.obs_id = self._ceda_observation.id 173 doInsertOrUpdate([self._deplObj], self._deProcessor.session, True) 174 self._deProcessor.session.commit() 179 180 if add: 181 PJB.addObservationMigration(self._deplObj, self._migrationSession) 175 182 return self._ceda_observation 176 183 -
mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py
r8016 r8054 5 5 ''' 6 6 from libs.migration.processor.commons import docStatus, base, getCollectionRefs,\ 7 getResource,findMolesCreationDate, findID, stringToTimestamp,\7 findMolesCreationDate, findID, stringToTimestamp,\ 8 8 getResourceRefs, getXMLDocument 9 from libs.migration.db.classes import ObservationCollectionMigration,\10 ObservationMigration11 9 from libs.migration.exception.exceptions import NoCreationDate 12 from libs.commons_db import getObjectId, doInsertOrUpdate 10 from libs.migration.db.classes import ObservationMigration,\ 11 ObservationCollectionMigration 12 from libs.migration.pjb import PJB 13 13 14 14 class LoadResources(object): 15 15 16 def __init__(self, session): 17 self._session = session 16 def __init__(self, pjb): 17 self._migrationSession = PJB.getNewMigrationSession() 18 self._molesSession = PJB.getNewMolesSession() 18 19 19 20 def _loadDataEntities(self, base): … … 56 57 print (resourceSource) 57 58 58 59 isUpdate = True 60 coll = getObjectId(ObservationMigration, deID, self._session) 59 add = False 60 coll = PJB.search(ObservationMigration, deID, self._migrationSession) 61 61 if coll is None: 62 62 coll = ObservationMigration() 63 63 coll.depl_id = deID 64 isUpdate = False 64 add = True 65 65 66 66 67 coll.depl_name = deName 67 68 coll.depl_creation = stringToTimestamp(deCreationDate) 68 coll.exist_path = baseOwner 69 doInsertOrUpdate([coll], self._session, isUpdate) 70 self._session.commit() 69 coll.exist_path = baseOwner 70 71 if add: 72 PJB.addObservationMigration(coll, self._migrationSession) 73 74 self._migrationSession.commit() 71 75 72 76 def _loadDataEntityOwner(self, base, owner): … … 91 95 92 96 93 isUpdate = True94 coll = getObjectId(ObservationCollectionMigration, deID, self._session)97 add = False 98 coll = PJB.search(ObservationCollectionMigration, deID, self._migrationSession) 95 99 if coll is None: 96 100 coll = ObservationCollectionMigration() 97 101 coll.data_ent_id = deID 98 isUpdate = False102 add = True 99 103 100 104 coll.data_ent_name = deName 101 105 coll.data_ent_creation = stringToTimestamp(deCreationDate) 102 coll.exist_path = baseOwner 103 doInsertOrUpdate([coll], self._session, isUpdate) 104 self._session.commit() 106 coll.exist_path = baseOwner 107 if add: 108 PJB.addObservationCollectionMigration(coll, self._migrationSession) 109 self._migrationSession.commit() 105 110 106 111 def process(self): … … 111 116 baseStatus = '%s/%s' % (base, status) 112 117 self._loadDataEntities(baseStatus) 113 self._loadDeployments(baseStatus) 118 self._loadDeployments(baseStatus) 119 self._migrationSession.close() 120 self._molesSession.close()
Note: See TracChangeset
for help on using the changeset viewer.