Changeset 8545
- Timestamp:
- 30/08/12 08:40:42 (9 years ago)
- Location:
- mauRepo/MolesManager/trunk/cedaMoles
- Files:
-
- 3 added
- 9 edited
Legend:
- Unmodified
- Added
- Removed
-
mauRepo/MolesManager/trunk/cedaMoles/MolesManager/db/after_flush.py
r8514 r8545 31 31 @author: mnagni 32 32 ''' 33 from cedaMoles.MolesManager.db.partyIndexes import addPartyIndexes, deletePartyIndexes,\33 from cedaMoles.MolesManager.db.partyIndexes import addPartyIndexes, \ 34 34 updatePartyIndexes 35 35 from ea_model.moles3_4.utilities.mo_responsiblepartyinfo import MO_ResponsiblePartyInfo 36 def after Flush(session, flush_context):36 def after_flush(session, flush_context): 37 37 # created a new one? 38 38 for instance in session.new: … … 43 43 for instance in session.dirty: 44 44 if isinstance(instance, MO_ResponsiblePartyInfo): 45 updatePartyIndexes(instance, session) 45 updatePartyIndexes(instance, session) 46 46 47 47 # deleted an existing one -
mauRepo/MolesManager/trunk/cedaMoles/MolesManager/db/partyIndexes.py
r8514 r8545 54 54 self.party_order = None 55 55 56 def associate MOParty_indexes(metadata):56 def associate_moparty_indexes(metadata): 57 57 ''' 58 58 Creates the table used to store the index associated per ResponsiblePartyIngo -
mauRepo/MolesManager/trunk/cedaMoles/MolesManager/moles3epb.py
r8538 r8545 32 32 ''' 33 33 from cedaMoles.libs.epb import EPB 34 from cedaMoles.MolesManager.db.tm_instant import customize_tm_instant 35 from cedaMoles.MolesManager.db.search_indexes import init_search_indexes 36 from cedaMoles.MolesManager.db.ceda_guid import init_ceda_guid 37 from cedaMoles.MolesManager.db.after_flush import after_flush 34 38 from ea_model.moles3_4.observationcollection.mo_observationcollection \ 35 39 import MO_ObservationCollection 36 40 from ea_model.moles3_4.observation.mo_observation import MO_Observation 37 from sqlalchemy import Table, Column, ForeignKey, Integer, String, event 38 from sqlalchemy.orm import mapper 41 from sqlalchemy import event 39 42 from cedaMoles.MolesManager.ceda_guid import CedaGUID 40 43 from sqlalchemy.orm.collections import InstrumentedList … … 52 55 from ea_model.ceda_metadatamodel.ceda_project.ceda_project import CEDA_Project 53 56 from cedaMoles.MolesManager.codelist import MM_RoleValue, getCLValue 54 from cedaMoles.libs.migration.processor.commons import from_pt_to_string55 from datetime import datetime56 57 from ascore.utils import synchAttributes 57 from cedaMoles.MolesManager.db.partyIndexes import associateMOParty_indexes 58 from cedaMoles.MolesManager.db.after_flush import afterFlush 58 from cedaMoles.MolesManager.db.partyIndexes import associate_moparty_indexes 59 59 60 60 class Moles3EPBFactory(EPB): … … 64 64 65 65 def _init_ceda_customization(self): 66 self._associate_ceda_guid() 67 associateMOParty_indexes(self._db_manager.metadata) 68 self._initSearchIndexes() 69 70 def _associate_ceda_guid(self): 71 guid_table = Table('ceda_guid', self._db_manager.metadata, 72 Column('id', String, primary_key=True), 73 Column('ceda_observationcollection', 74 Integer, ForeignKey('ceda_observationcollection.id')), 75 Column('ceda_observation', 76 Integer, ForeignKey('ceda_observation.id')), 77 Column('ceda_project', 78 Integer, ForeignKey('ceda_project.id'))) 79 mapper(CedaGUID, guid_table) 80 self._db_manager.metadata.create_all() 81 82 def _initSearchIndexes(self): 83 #To Be Done - CHECK IF THE COLUMN ALREADY EXISTS! 84 # We don't want sqlalchemy to know about this column so we add it externally. 85 try: 86 self._db_manager.engine.execute("alter table md_identifier \ 87 add column code_search_vector tsvector") 88 89 # This indexes the tsvector column 90 91 self._db_manager.engine.execute("create index \ 92 md_identifier_code_search_index on md_identifier using gin(code_search_vector)") 93 94 # This sets up the trigger that keeps the tsvector column up to date. 95 self._db_manager.engine.execute("create trigger \ 96 md_identifier_code_search_update \ 97 before update or insert on md_identifier \ 98 for each row execute procedure \ 99 tsvector_update_trigger('code_search_vector', \ 100 'pg_catalog.english', code)") 101 except Exception: 102 pass 66 init_ceda_guid(self._db_manager.metadata) 67 associate_moparty_indexes(self._db_manager.metadata) 68 init_search_indexes(self._db_manager.engine) 69 customize_tm_instant(self._db_manager.engine, self._db_manager.metadata) 103 70 104 71 def createEPB(self): 105 72 session = self._get_session() 106 event.listen(session, 'after_flush', after Flush)73 event.listen(session, 'after_flush', after_flush) 107 74 return Moles3EPB(session) 108 75 … … 137 104 138 105 **Returns** 139 An updated, session independent, object instance reflecting the new persisted object 106 An updated, session independent, object instance reflecting 107 the new persisted object 140 108 """ 141 109 EPB.persistInstance(instance, self._session) … … 225 193 @param collection: an CEDA_ObservationColleciton instance 226 194 """ 195 if not hasattr(collection, 'member'): 196 return None 197 227 198 bboxes = [] 228 if not hasattr(collection, 'member'):229 return bboxes230 199 for member in collection.member: 231 200 for ge in member.geographicExtent: … … 428 397 """ 429 398 return "%s-%s-%s" % (dt.year, dt.month, dt.day) 430 431 def unify_observation_collection_phenomenon_time(collection):432 """433 Returns the time period of the collections.member'a phenomenonTime(s)434 @param collection: an CEDA_ObservationColleciton instance435 @return: a tuple (startDate, endDate) strings436 """437 dateFormat = '%Y-%m-%d'438 ptStart = []439 ptEnd = []440 for member in collection.member:441 if member.phenomenonTime is None:442 continue443 444 pt = member.phenomenonTime445 ptString = from_pt_to_string(pt)446 if ptString[0] is not None:447 ptStart.append(datetime.strptime(ptString[0], dateFormat))448 if ptString[1] is not None:449 ptEnd.append(datetime.strptime(ptString[1], dateFormat))450 ptStart.sort()451 ptEnd.sort()452 start = None453 end = None454 #takes the earlier date455 if len(ptStart) > 0:456 start = _tmpstrftime(ptStart[0])457 #takes the latest date458 if len(ptEnd) > 0:459 end = _tmpstrftime(ptEnd[len(ptEnd) - 1])460 return start, end -
mauRepo/MolesManager/trunk/cedaMoles/MolesManager/views/moles2gui/__init__.py
r8537 r8545 11 11 from cedaMoles.libs.epb import isPrimitive 12 12 13 """ 13 ''' 14 14 From now on we define: 15 15 - cedaObject any instances or class defined in from the CedaMoles3 model 16 - jsonObject any json representation generated by the CedaMolesGuiAdapter.encode method 17 """ 18 19 _encodeMapper = {'ea_model.iso_19115_2006_metadata_corrigendum.\ 16 - jsonObject any json representation generated 17 by the CedaMolesGuiAdapter.encode method 18 ''' 19 20 _ENCODE_MAPPER = {'ea_model.iso_19115_2006_metadata_corrigendum.\ 20 21 extent_information.ex_geographicboundingbox.EX_GeographicBoundingBox': 21 22 ('cedaMoles.MolesManager.views.moles2gui.ex.geographic_extent', … … 333 334 # Is a mapped class? 334 335 key = "%s.%s" % (cedaObj.__module__, cedaObj.__class__.__name__) 335 if _ encodeMapper.has_key(key):336 module, clazz = _ encodeMapper[key]336 if _ENCODE_MAPPER.has_key(key): 337 module, clazz = _ENCODE_MAPPER[key] 337 338 encoder = dynamicallyLoadClass(module, clazz) 338 339 return encoder.cedaObjToAdapter(cedaObj) -
mauRepo/MolesManager/trunk/cedaMoles/libs/migration/MigrationEPB.py
r8538 r8545 30 30 """ 31 31 Adds a new migration object. 32 @param migrationObj: the migration object to add 33 @param session: an SQLAlchemy Session object. If None (default) the method creates 34 @return an updated, session independent, object instance reflecting the new persisted object 35 """ 32 33 **Parameters** 34 * instance: the instance to persist 35 36 **Raise** 37 * Exception both `moles3EPB` and `cedaMoleObj` are `None` 38 """ 36 39 EPB.persistInstance(instance, self._session) 37 40 38 def getAllDeploymentsMigrationByDataEntitySortedByDate(self, dataEntity, deploymentNames): 39 return EPB.getAllObjects(DeploymentsMigration, self._session).filter(DeploymentsMigration.doc_name.in_(deploymentNames)).order_by(asc("doc_creation")).all() 41 def getAllDeploymentsMigrationByDataEntitySortedByDate(self, dataEntity, 42 deploymentNames): 43 return EPB.getAllObjects(DeploymentsMigration, self._session).\ 44 filter(DeploymentsMigration.doc_name.in_(deploymentNames)).\ 45 order_by(asc("doc_creation")).all() 40 46 41 47 def _getMigrationObject(self, mo_type, doc_status, doc_owner, doc_name): 42 return self._session.query(mo_type).filter(*[EPB.buildFilter('doc_name', doc_name)]).first() 48 return self._session.query(mo_type).\ 49 filter(*[EPB.buildFilter('doc_name', doc_name)]).first() 43 50 44 51 def _getMigrationObjectByName(self, mo_type, migration_object, doc_name): 45 return self._getMigrationObject(mo_type, migration_object.doc_status, migration_object.doc_owner, doc_name) 52 return self._getMigrationObject(mo_type, migration_object.doc_status, 53 migration_object.doc_owner, doc_name) 46 54 47 55 def get_deployment_data_migration_by_name(self, migration_object, doc_name): 48 56 if migration_object is None: 49 57 raise Exception("migrationObject is None") 50 return self._getMigrationObjectByName(DeploymentDataMigration, migration_object, doc_name) 58 return self._getMigrationObjectByName(DeploymentDataMigration, 59 migration_object, doc_name) 51 60 52 61 def getDataEntityMigrationbyPath(self, migration_object): … … 57 66 ret = [] 58 67 if migration_object: 59 ret.append(self._session.query(DataEntityMigration).filter(*[EPB.buildFilter('doc_name', migration_object.doc_name)]).first()) 68 ret.append(self._session.query(DataEntityMigration).\ 69 filter(*[EPB.buildFilter('doc_name', migration_object.doc_name)]).\ 70 first()) 60 71 else: #then process all the DataEntities 61 72 res = self._session.query(DataEntityMigration).all() -
mauRepo/MolesManager/trunk/cedaMoles/libs/migration/processor/commons.py
r8522 r8545 625 625 tm_instant = TM_Instant() 626 626 tm_instant.position = position 627 if hasattr(tm_instant, 'ceda_timestamp'): 628 if isinstance(position.date8601, Date): 629 idate = position.date8601 630 tm_instant.ceda_timestamp = datetime.datetime(int(idate.year), 631 int(idate.month), 632 int(idate.day)) 633 elif isinstance(position.dateTime8601, DateTime): 634 idate = position.dateTime8601 635 tm_instant.ceda_timestamp = datetime.datetime(int(idate.year), 636 int(idate.month), 637 int(idate.day), 638 int(idate.hour), 639 int(idate.minute), 640 int(idate.second)) 627 641 return tm_instant 628 642 … … 955 969 return ceda_project 956 970 957 def from_date_string_to_pt( doc_phenomenon_time):971 def from_date_string_to_pt(start, end): 958 972 """ 959 973 Transforms a date string like '2002-07-22' (startDate) in a TM_Instant instance or … … 961 975 @param doc_phenomenon_time: a date string in the expected format 962 976 """ 963 if doc_phenomenon_time: 964 pt = None 965 if '/' in doc_phenomenon_time: 966 period = doc_phenomenon_time.split('/') 967 begin_date = create_date(isoDateStringToTimeDate(period[0])) 968 begin_position = create_tm_position(date8601 = begin_date) 969 begin_tm_instant = create_tm_instant(begin_position) 970 971 end_date = create_date(isoDateStringToTimeDate(period[1])) 972 end_position = create_tm_position(date8601 = end_date) 973 end_tm_instant = create_tm_instant(end_position) 974 975 pt = create_tm_period(begin_tm_instant, end_tm_instant) 976 else: 977 tm_position = create_tm_position(date8601 = \ 978 create_date(isoDateStringToTimeDate(doc_phenomenon_time))) 979 pt = create_tm_instant(tm_position) 980 return pt 981 977 pt = None 978 if end is not None and start is not None: 979 begin_date = start 980 if isinstance(start, str): 981 begin_date = isoDateStringToTimeDate(start) 982 begin_position = create_tm_position(date8601 = create_date(begin_date)) 983 begin_tm_instant = create_tm_instant(begin_position) 984 985 end_date = end 986 if isinstance(end, str): 987 end_date = isoDateStringToTimeDate(end) 988 end_position = create_tm_position(date8601 = create_date(end_date)) 989 end_tm_instant = create_tm_instant(end_position) 990 pt = create_tm_period(begin_tm_instant, end_tm_instant) 991 elif start is not None: 992 begin_date = start 993 if isinstance(start, str): 994 begin_date = isoDateStringToTimeDate(start) 995 tm_position = create_tm_position(date8601 = create_date(begin_date)) 996 pt = create_tm_instant(tm_position) 997 return pt 998 999 def unify_observation_collection_phenomenon_time(collection): 1000 """ 1001 Returns the time period of the collections.member'a phenomenonTime(s) 1002 @param collection: an CEDA_ObservationColleciton instance 1003 @return: a tuple (startDate, endDate) strings 1004 """ 1005 ptStart = [] 1006 ptEnd = [] 1007 for member in collection.member: 1008 if member.phenomenonTime is None: 1009 continue 1010 pt = member.phenomenonTime 1011 if hasattr(pt, 'start'): 1012 ptStart.append(pt.start.ceda_timestamp) 1013 elif hasattr(pt, 'end'): 1014 ptEnd.append(pt.end.ceda_timestamp) 1015 elif hasattr(pt, 'ceda_timestamp'): 1016 ptStart.append(pt.ceda_timestamp) 1017 start = None 1018 end = None 1019 if len(ptStart) > 0: 1020 start = min(ptStart) 1021 #takes the latest date 1022 if len(ptEnd) > 0: 1023 end = max(ptEnd) 1024 return start, end 1025 1026 ''' 982 1027 def from_pt_to_string(phenomenonTime): 983 1028 """ … … 998 1043 endDate = _tm_InstantToString(phenomenonTime.end) 999 1044 return startDate, endDate 1000 1045 ''' 1001 1046 def compare_phenomenon_times(p1, p2): 1002 s1 = from_pt_to_string(p1) 1003 s2 = from_pt_to_string(p2) 1004 return s1[0] == s2[0] and s1[1] == s2[1] 1005 1047 if hasattr(p1, 'start') and hasattr(p1, 'end') \ 1048 and hasattr(p2, 'start') and hasattr(p2, 'end'): 1049 return (p1.start.ceda_timestamp == p2.start.ceda_timestamp) \ 1050 and (p1.end.ceda_timestamp == p2.end.ceda_timestamp) 1051 1052 if hasattr(p1, 'ceda_timestamp') and hasattr(p2, 'ceda_timestamp'): 1053 return p1.ceda_timestamp == p2.ceda_timestamp 1054 1055 return False 1056 1057 ''' 1006 1058 def _tm_InstantToString(tm_instant): 1007 1059 idate = tm_instant.position.date8601 1008 1060 return str(datetime.date(int(idate.year), int(idate.month), int(idate.day))) 1061 ''' 1009 1062 1010 1063 def _from_geographic_bb_to_string(gbb): -
mauRepo/MolesManager/trunk/cedaMoles/libs/migration/processor/dataEntity.py
r8537 r8545 41 41 has_mo_been_processed, get_atom_document_hash_by_mo, extract_title, extract_summary, \ 42 42 create_ex_geographic_boundingbox, from_date_string_to_pt, \ 43 compare_phenomenon_times, compareGeographicBoundingBoxes 43 compare_phenomenon_times, compareGeographicBoundingBoxes,\ 44 unify_observation_collection_phenomenon_time 44 45 from cedaMoles.libs.migration.processor.deployment import DeploymentProcessor 45 46 from logging import StreamHandler … … 52 53 from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection \ 53 54 import CEDA_ObservationCollection 54 from cedaMoles.MolesManager.moles3epb import unify_observation_collection_phenomenon_time55 55 CEDA_TITLE = 'ceda_title' 56 56 … … 78 78 79 79 def _assignGeographicExtent(self, ceda_observationCollection): 80 bbox = self.epbRepo.moles3EPB.getUnifyObservationCollectionGEAsBBox(ceda_observationCollection) 81 if bbox is not None: 82 upperCornerData, lowerCornerData = bbox[4:len(bbox)-1].split(',') 83 east, north = upperCornerData.split() 84 west, south = lowerCornerData.split() 85 geographicExtent = create_ex_geographic_boundingbox(float(east), \ 86 float(north), \ 87 float(west), \ 88 float(south)) 89 90 if len(ceda_observationCollection.geographicExtent) == 0 or \ 91 (len(ceda_observationCollection.geographicExtent) > 0 and \ 92 not compareGeographicBoundingBoxes(geographicExtent, \ 93 ceda_observationCollection.geographicExtent[0])): 94 self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, \ 95 {'geographicExtent': geographicExtent}) 96 else: 97 self._report.append(NoGeographicalExtensionException(self._dataEntityMigration)) 80 bbox = self.epbRepo.moles3EPB.\ 81 getUnifyObservationCollectionGEAsBBox(ceda_observationCollection) 82 83 if bbox is None: 84 self._report.\ 85 append(NoGeographicalExtensionException(self._dataEntityMigration)) 86 return 87 88 upperCornerData, lowerCornerData = bbox[4:len(bbox)-1].split(',') 89 east, north = upperCornerData.split() 90 west, south = lowerCornerData.split() 91 geographicExtent = create_ex_geographic_boundingbox(float(east), 92 float(north), 93 float(west), 94 float(south)) 95 96 if len(ceda_observationCollection.geographicExtent) == 0 or \ 97 (len(ceda_observationCollection.geographicExtent) > 0 and \ 98 not compareGeographicBoundingBoxes(geographicExtent, \ 99 ceda_observationCollection.geographicExtent[0])): 100 self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, 101 {'geographicExtent': geographicExtent}) 102 98 103 99 104 def _assignPhenomenonTime(self, ceda_observationCollection): … … 102 107 if end is not None: 103 108 dateString = '%s/%s' % (start, end) 104 pt = from_date_string_to_pt( dateString)109 pt = from_date_string_to_pt(start, end) 105 110 106 111 if pt is not None \ … … 301 306 self._assignObservationCollectionToProject(obs_coll, ceda_observation) 302 307 except Exception as ex: 303 self._report.append(ex) 308 self._report.append(ex) 304 309 305 310 self._assignGeographicExtent(obs_coll) -
mauRepo/MolesManager/trunk/cedaMoles/libs/migration/processor/deployment.py
r8538 r8545 540 540 return 541 541 542 doc_phenomenon_time = extract_moles_temporal_range(self._deployment_migration) 543 pt = from_date_string_to_pt(doc_phenomenon_time) 544 542 doc_phenomenon_time = extract_moles_temporal_range(self._deployment_migration) 543 544 start, end = doc_phenomenon_time, None 545 if '/' in doc_phenomenon_time: 546 start, end = doc_phenomenon_time.split('/') 547 pt = from_date_string_to_pt(start, end) 545 548 if self._deploymentHasBeenProcessed \ 546 549 or observation.phenomenonTime is not None \ 547 550 and not compare_phenomenon_times(pt,observation.phenomenonTime): 548 551 self.epbRepo.moles3EPB.updateCedaObject(observation, {'phenomenonTime': pt}) 549 #Now update phenomenonTime but has to remove the previous phenomenonTime 550 552 #Now update phenomenonTime but has to remove the previous phenomenonTime 551 553 elif not self._deploymentHasBeenProcessed: 552 554 observation.phenomenonTime = pt 553 554 555 def _assignPermission(self, observation): 555 556 if self._deploymentHasSameHash and self._dataEntityHasSameHash: -
mauRepo/MolesManager/trunk/cedaMoles/tests/migration/commontest.py
r8496 r8545 6 6 from unittest import TestCase 7 7 from cedaMoles.libs.migration.processor.commons import isoDateTimeStringToTimeDate,\ 8 from_date_string_to_pt, from_pt_to_string,\8 from_date_string_to_pt,\ 9 9 compare_phenomenon_times 10 10 … … 31 31 temporalRange = '2002-07-22/2011-08-06' 32 32 pt = from_date_string_to_pt(temporalRange) 33 ptString = from_pt_to_string(pt)34 33 startDate, endDate = temporalRange.split('/') 35 self.assertTrue(startDate == pt String[0], "Error")36 self.assertTrue(endDate == pt String[1], "Error")34 self.assertTrue(startDate == pt.start.ceda_timestamp, "Error") 35 self.assertTrue(endDate == pt.end.ceda_timestamp, "Error") 37 36 38 37 temporalRange = '2002-07-22' 39 38 pt = from_date_string_to_pt(temporalRange) 40 ptString = from_pt_to_string(pt)41 39 startDate, endDate = temporalRange, None 42 self.assertTrue(startDate == pt String[0], "Error")43 self.assertTrue(endDate == pt String[1], "Error")40 self.assertTrue(startDate == pt.start.ceda_timestamp, "Error") 41 self.assertTrue(endDate == pt.end.ceda_timestamp, "Error") 44 42 45 43 def testComparePhenomenonTimes(self):
Note: See TracChangeset
for help on using the changeset viewer.