- Timestamp:
- 10/07/12 15:15:27 (9 years ago)
- Location:
- mauRepo/MolesManager/trunk
- Files:
-
- 8 edited
Legend:
- Unmodified
- Added
- Removed
-
mauRepo/MolesManager/trunk/resources/requirements.txt
r8445 r8460 11 11 # --extra-index-url http://ciprod1.cems.rl.ac.uk/pip 12 12 # to your pip install 13 ceda-moles-model==0.1. 413 ceda-moles-model==0.1.5 -
mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py
r8445 r8460 121 121 @return an updated, session independent, object instance reflecting the new persisted object 122 122 """ 123 self._session.add(instance) 124 self._session.commit() 125 123 EPB.persistInstance(instance, self._session) 124 125 def _mergeOrAddInSession(self, ceda_object): 126 try: 127 return self._session.merge(ceda_object) 128 except: 129 self._session.add(ceda_object) 130 return ceda_object 126 131 127 132 def updateCedaObject(self, ceda_object, cols_to_update): … … 133 138 @return: the given instance with the updated attributes. 134 139 """ 135 coll = None 136 try: 137 if ceda_object in self._session: 138 coll = self._session.merge(ceda_object) 139 else: 140 self._session.add(ceda_object) 141 coll = ceda_object 142 except Exception as e: 143 print e 140 coll = self._mergeOrAddInSession(ceda_object) 144 141 if coll != None: 145 142 for k,v in cols_to_update.items(): 146 if hasattr(coll, k): 147 val = None 148 try: 149 val = self._session.merge(v) 150 except Exception: 151 val = v 143 if hasattr(coll, k): 152 144 coll_k = getattr(coll, k) 153 145 if type(coll_k) == list or type(coll_k) == InstrumentedList: 154 146 tmp_coll = [] 155 if type(v al) == list or type(val) == InstrumentedList:156 tmp_coll.extend(v al)147 if type(v) == list or type(v) == InstrumentedList: 148 tmp_coll.extend(v) 157 149 else: 158 tmp_coll.append(v al)150 tmp_coll.append(v) 159 151 for item in tmp_coll: 160 if item not in coll_k: 161 coll_k.append(item) 152 el = self._mergeOrAddInSession(item) 153 if el not in coll_k: 154 coll_k.append(el) 162 155 else: 163 setattr(coll, k, val) 156 el = self._mergeOrAddInSession(v) 157 setattr(coll, k, el) 164 158 synchAttributes(coll) 165 159 self._controlledCommit() 166 return coll160 #return coll 167 161 168 162 def getUnifyObservationCollectionGEAsBBox(self, collection): -
mauRepo/MolesManager/trunk/src/libs/epb.py
r8445 r8460 34 34 from sqlalchemy.sql.expression import text 35 35 from MolesManager.forms.date import methodsWithDecorator 36 from sqlalchemy.orm.util import identity_key 36 37 37 38 class EPB(object): … … 158 159 159 160 @classmethod 161 def persistInstance(cls, instance, session): 162 """ 163 Adds a new migration object. 164 @param migrationObj: the migration object to add 165 @param session: an sqlalchemy Session object. If None (default) the method creates 166 @return an updated, session independant, object instance reflecting the new persisted object 167 """ 168 session.add(instance) 169 session.commit() 170 id_key = identity_key(instance=instance) 171 instance = EPB.search(id_key[0], id_key[1], session) 172 #instance = ret 173 174 @classmethod 160 175 def executeNative(self, sqlNative, session): 161 176 return session.execute(text(sqlNative)) -
mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py
r8433 r8460 32 32 def close(self): 33 33 return self._session.close() 34 35 def persistInstance(self, instance): 36 """ 37 Adds a new migration object. 38 @param migrationObj: the migration object to add 39 @param session: an SQLAlchemy Session object. If None (default) the method creates 40 @return an updated, session independent, object instance reflecting the new persisted object 41 """ 42 EPB.persistInstance(instance, self._session) 43 #return ret 34 44 35 45 def getAllDeploymentsMigrationByDataEntitySortedByDate(self, dataEntity, deploymentNames): -
mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py
r8445 r8460 959 959 tm_position = createTM_Position(date8601 = createDate(isoDateStringToTimeDate(doc_phenomenon_time))) 960 960 pt = createTM_Instant(tm_position) 961 return pt961 return pt 962 962 963 963 def fromPhenomenonTimeToString(phenomenonTime): … … 988 988 return str(datetime.date(int(idate.year), int(idate.month), int(idate.day))) 989 989 990 def fromGeographicBoundingBoxToString(g eographicBoundingBox):991 if g eographicBoundingBoxis None:990 def fromGeographicBoundingBoxToString(gbb): 991 if gbb is None: 992 992 return None 993 return ('%f %f,%f %f') % (geographicBoundingBox.eastBoundLongitude, geographicBoundingBox.northBoundLatitude, \ 994 geographicBoundingBox.westBoundLongitude, geographicBoundingBox.southBoundLatitude) 993 return '{0} {1},{2} {3}'.format(gbb.eastBoundLongitude, gbb.northBoundLatitude, gbb.westBoundLongitude, gbb.southBoundLatitude) 995 994 996 995 def compareGeographicBoundingBoxes(gb1, gb2): -
mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py
r8445 r8460 99 99 pt = fromDateStringToPhenomenonTime(dateString) 100 100 101 if ceda_observationCollection.phenomenonTime is not None \101 if pt is not None and ceda_observationCollection.phenomenonTime is not None \ 102 102 and (len(ceda_observationCollection.phenomenonTime) == 0 \ 103 103 or (len(ceda_observationCollection.phenomenonTime) == 1 and \ … … 211 211 try: 212 212 DataEntityProcessor.log.info("Processing deployment: %s" % (migrationObjectDescription(deploymentMigration))) 213 ceda_observation = deProcessor.process() 213 obs_ex_report, ceda_observation = deProcessor.process() 214 self._report.extend(obs_ex_report) 214 215 try: 215 216 self._processDOI(deploymentMigration, ceda_observation, deProcessor, single_deployment) … … 225 226 def process(self): 226 227 obsColl = None 227 exs = []228 228 DataEntityProcessor.log.info("Processing dataEntity: %s" % (migrationObjectDescription(self._dataEntityMigration))) 229 229 try : … … 252 252 for deploymentMigration in deploymentMigrations: 253 253 try: 254 ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1) 254 ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1) 255 255 #Is a first time process? 256 256 if not self.epbRepo.moles3EPB.observationCollectionHasObservation(getattr(obsColl, 'id'), getattr(ceda_observation, 'id')): -
mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py
r8445 r8460 52 52 createMD_Keywords, hasMOBeenProcessed, createMO_Individual,\ 53 53 fromDateStringToPhenomenonTime, fromPhenomenonTimeToString,\ 54 comparePhenomenonTimes 54 comparePhenomenonTimes, compareGeographicBoundingBoxes 55 55 from libs.epb import EPB 56 56 from libs.migration.processor.deployment_data import DeploymentDataProcessor 57 57 from libs.migration.exception.exceptions import NoDataLineage,\ 58 NoAssociatedAuthor, NoAssociatedDeployments 58 NoAssociatedAuthor, NoAssociatedDeployments,\ 59 NoGeographicalExtensionException 59 60 from libs.migration.authors import authors 60 61 from logging import StreamHandler … … 103 104 self._deploymentHasSameHash = hasMOSameHash(self._deploymentMigration) 104 105 self._deploymentHasBeenProcessed = hasMOBeenProcessed(self._deploymentMigration) 106 self._report = [] 105 107 106 108 def _addResponsiblePartyInfo(self, oldResponsiblePartyInfos, newResponsiblePartyInfo): … … 412 414 return 413 415 414 ge = extractGeographicExtentInMigrationDocument(self._deploymentMigration) 415 if not ge:416 ge = extractGeographicExtentInMigrationDocument(self._deploymentMigration) 417 if ge is None: 416 418 ge = extractGeographicExtentInMigrationDocument(self._dataEntityMigration) 417 if ge: 418 geographicExtent = createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south']) 419 if self._deploymentHasBeenProcessed: 420 DeploymentProcessor.log.warn('The _assignGeographicExtent update is skipped because not implemented') 421 observation.geographicExtent.append(geographicExtent) 422 else: 423 print "No Geographic Extent" 424 return 425 #Still have to update observation.geographicExtent 419 420 if ge is None: 421 self._report.append(NoGeographicalExtensionException(self._deploymentMigration)) 422 self._report.append(NoGeographicalExtensionException(self._dataEntityMigration)) 423 424 geographicExtent = createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south']) 425 426 if len(observation.geographicExtent) == 0 or \ 427 (len(observation.geographicExtent) > 0 and \ 428 not compareGeographicBoundingBoxes(geographicExtent, \ 429 observation.geographicExtent[0])): 430 self.epbRepo.moles3EPB.updateCedaObject(observation, {'geographicExtent': geographicExtent}) 426 431 427 432 def _assignCreationDate(self, observation): … … 642 647 643 648 def process(self): 649 self._report = [] 644 650 ceda_observation = None 645 651 #Moles3 object exists... … … 648 654 else: 649 655 #... does not exist so create it 650 ceda_observation = ceda_observation =CEDA_Observation()656 ceda_observation = CEDA_Observation() 651 657 652 658 self._execute(ceda_observation) 653 return ceda_observation659 return self._report, ceda_observation -
mauRepo/MolesManager/trunk/src/libs/migration/tests/migrationprocess.py
r8433 r8460 45 45 from MolesManager.settings import EVENTS_DB 46 46 from libs.migration.client import EPBRepo, generateMigrationReport 47 from django.core.mail import send_mail48 47 49 48 class LoadResourceTest(TestCase): … … 88 87 89 88 #Has 3 DOI 90 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_chobs.atom')89 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_chobs.atom') 91 90 92 91 #IASI 93 self.processDataEntity(DO_NEODC, DS_pUBLISHED, 'dataent_12417810458627666.atom')94 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_VIRTEM.atom')95 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12162569915925921.atom')92 #self.processDataEntity(DO_NEODC, DS_pUBLISHED, 'dataent_12417810458627666.atom') 93 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_VIRTEM.atom') 94 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12162569915925921.atom') 96 95 97 96 #ECMWF 98 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-ERA.atom')99 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-OP.atom')100 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12458543158227759.atom')101 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-E40.atom')102 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-TRJ.atom')97 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-ERA.atom') 98 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-OP.atom') 99 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12458543158227759.atom') 100 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-E40.atom') 101 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-TRJ.atom') 103 102 104 103 … … 106 105 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-TRJ.atom') 107 106 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_UGAMPO3.atom') 108 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12282368470627062.atom')107 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12282368470627062.atom') 109 108 110 109 #self.processDataEntity()
Note: See TracChangeset
for help on using the changeset viewer.