Changeset 8433 for mauRepo/MolesManager
- Timestamp:
- 04/07/12 12:28:21 (9 years ago)
- Location:
- mauRepo/MolesManager/trunk/src
- Files:
-
- 1 added
- 14 edited
Legend:
- Unmodified
- Added
- Removed
-
mauRepo/MolesManager/trunk/src/MolesManager/djencoder.py
r8426 r8433 51 51 else: 52 52 self.__markers[id(value)] = value 53 '''54 53 if isinstance(value, str) or isinstance(value, unicode): 55 54 self.__markers[id(value)] = escapeForJSON(value) 56 55 else: 57 56 self.__markers[id(value)] = value 58 '''59 57 return d 60 58 61 59 def encodeToJSON(toEncode): 62 return escapeForJSON(DJEncoder().encode(toEncode))60 return DJEncoder().encode(toEncode) 63 61 64 62 def escapeForJSON(toEscape): -
mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py
r8430 r8433 104 104 return EPB.searchEager(clazz, inst_id, self._session) 105 105 106 def _controlledCommit(self): 107 try: 108 self._session.commit() 109 except Exception as e: 110 print e 111 106 112 def persistInstance(self, instance): 107 113 """ … … 111 117 If None (default) a new Session is created from the underlying EPB and closed at the exit. 112 118 @return an updated, session independent, object instance reflecting the new persisted object 113 """ 114 EPB.persistInstance(instance, self._session) 119 """ 120 self._session.add(instance) 121 self._session.commit() 115 122 116 123 … … 152 159 else: 153 160 setattr(coll, k, val) 154 self._ session.commit()161 self._controlledCommit() 155 162 return coll 156 163 -
mauRepo/MolesManager/trunk/src/MolesManager/settings.py
r8408 r8433 232 232 SERVE_STATIC_CONTENT = False 233 233 234 reportMigrationTo = ['maurizio.nagni@stfc.ac.uk', 'm.nagni@gmail.com'] 235 234 236 # Local modifications to settings are imported from settings_local.py 235 237 from settings_local import * -
mauRepo/MolesManager/trunk/src/MolesManager/static/js/cedaObservation.js
r8426 r8433 1 define(['dojo', ' ceda/cedaol', 'dojox/lang/functional'], function(dojo, cedaol, functional){1 define(['dojo', 'dojox/lang/functional', 'ceda/cedaol', 'ceda/common'], function(dojo, functional, cedaol, common){ 2 2 //console.log("loads cedaObservation.js") 3 3 // function <nameOfFunction>(parameter, parameter2..etc coming in){ … … 235 235 } 236 236 237 238 237 require(['dojo/ready', 'dojo/parser', 'dijit/registry', 'dojo/_base/json', 'dojo/store/Memory', 239 238 'dijit/Tree', 'dijit/form/Button', … … 257 256 258 257 var json = dojo.byId('coObs_id').value; 258 if (json == ''){ 259 var title_id = new dojo.html.set(dojo.byId("title_id"),'<h3>The required item does not exist.</h3>'); 260 common.fadeOutNode("lower_section") 261 common.fadeOutNode("upper_section") 262 return 263 } 264 259 265 var guid = dojo.byId('guid_id').value; 260 266 coObs = dojo.fromJson(json); … … 345 351 }); 346 352 }); 347 -
mauRepo/MolesManager/trunk/src/MolesManager/urls.py
r8300 r8433 21 21 urlpatterns += patterns('MolesManager.views.cedaObservationView', 22 22 23 (r'cov/(\ d+)$', 'coView'),23 (r'cov/(\w+)$', 'coView'), 24 24 ) 25 25 -
mauRepo/MolesManager/trunk/src/MolesManager/views/cedaObservationView.py
r8426 r8433 34 34 from django.core.context_processors import csrf 35 35 from MolesManager.djencoder import encodeToJSON 36 from django.utils.safestring import mark_safe36 #from django.utils.safestring import mark_safe 37 37 from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 38 38 … … 40 40 41 41 def coView(request, obs_id_str): 42 obs_id = int(obs_id_str) 43 return splash(request, obs_id) 42 record = None 43 try: 44 obs_id = int(obs_id_str) 45 record = _getCedaObservation(request, obs_id) 46 except ValueError: 47 pass 44 48 45 def splash(request, obs_id):46 """47 Processes and display a given CEDA_Observation id48 @param request: an HTTP request49 @param obs_id: the CEDA_Observation id50 @return: an HTTPResponse51 """52 record = None53 if obs_id:54 record = _getCedaObservation(request, obs_id)55 49 return _finalizeResponse(request, record) 56 50 … … 58 52 c = {} 59 53 if record: 60 c['coObs'] = mark_safe(encodeToJSON(record))54 c['coObs'] = encodeToJSON(record) 61 55 guid = request.moles3EPB.retrieveGUIDFromInstance(record) 62 56 if guid: -
mauRepo/MolesManager/trunk/src/libs/epb.py
r8425 r8433 33 33 from sqlalchemy.orm import subqueryload 34 34 from sqlalchemy.sql.expression import text 35 from sqlalchemy.orm.util import identity_key36 35 37 36 class EPB(object): … … 157 156 158 157 @classmethod 159 def persistInstance(cls, instance, session):160 """161 Adds a new migration object.162 @param migrationObj: the migration object to add163 @param session: an sqlalchemy Session object. If None (default) the method creates164 @return an updated, session independant, object instance reflecting the new persisted object165 """166 session.add(instance)167 session.commit()168 id_key = identity_key(instance=instance)169 instance = EPB.search(id_key[0], id_key[1], session)170 #instance = ret171 172 @classmethod173 158 def executeNative(self, sqlNative, session): 174 159 return session.execute(text(sqlNative)) -
mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py
r8424 r8433 32 32 def close(self): 33 33 return self._session.close() 34 35 def persistInstance(self, instance):36 """37 Adds a new migration object.38 @param migrationObj: the migration object to add39 @param session: an SQLAlchemy Session object. If None (default) the method creates40 @return an updated, session independent, object instance reflecting the new persisted object41 """42 EPB.persistInstance(instance, self._session)43 #return ret44 34 45 35 def getAllDeploymentsMigrationByDataEntitySortedByDate(self, dataEntity, deploymentNames): … … 108 98 else: 109 99 setattr(coll, k, val) 110 EPB.persistInstance(coll, self._session) 100 self._session.add(coll) 101 self._session.commit() 111 102 112 103 def search(self, clazz, inst_id): -
mauRepo/MolesManager/trunk/src/libs/migration/client.py
r8358 r8433 38 38 from threading import Timer 39 39 from libs.migration.processor.EPBRepo import EPBRepo 40 from django.core.mail import send_mail 41 from MolesManager.settings import reportMigrationTo 42 from datetime import datetime 40 43 41 44 class MigrationThread(threading.Thread): … … 67 70 if diffTime > 0: 68 71 time.sleep(diffTime) 69 70 def _printTime(self):71 print "From print_time", time.time()72 72 73 73 def _migrate(self): 74 74 lr = LoadResources(self.epbRepo) 75 75 ex = lr.process() 76 for e in ex:77 print e78 76 79 77 mp = MigrationProcess(self.epbRepo) 80 ex = mp.process() 81 for e in ex: 82 print e 78 ex.extend(mp.process()) 79 generateMigrationReport(ex) 80 81 def generateMigrationReport(messages): 82 report = "Migration Report - " + str(datetime.today()) 83 for e in messages: 84 report = "%s\n\n%s" % (report, e) 85 86 send_mail('Migration Report', report, 'moles2.migration@ceda.rl.ac.uk', 87 reportMigrationTo, fail_silently=False) -
mauRepo/MolesManager/trunk/src/libs/migration/exception/exceptions.py
r8358 r8433 9 9 raise Exception("Migration object is None") 10 10 try: 11 return " %s %s%s" % (migrationObject.doc_status, migrationObject.doc_owner, migrationObject.doc_name)11 return "STATUS:%s OWNER:%s NAME:%s" % (migrationObject.doc_status, migrationObject.doc_owner, migrationObject.doc_name) 12 12 except Exception as ex: 13 13 print ex … … 46 46 return "Error on migration object %s " % (migrationObjectDescription(self.migrationObject)) 47 47 48 class NoGeographicalExtensionException(MigrationObjectException): 49 """ 50 Raises a generic MigrationObject related exception. 51 """ 52 def __init__(self, migrationObject, comment = ''): 53 """ 54 Creates a new instance 55 @param migrationObject: the MigrationObject instance that raised the error 56 @param comment:additional comment 57 """ 58 super(NoGeographicalExtensionException, self).__init__(migrationObject, comment = '') 59 def __str__(self): 60 return "No Geographical Extension associated with migration object %s " % (migrationObjectDescription(self.migrationObject)) 61 62 class NoCitationException(MigrationObjectException): 63 """ 64 Raises a generic MigrationObject related exception. 65 """ 66 def __init__(self, migrationObject, comment = ''): 67 """ 68 Creates a new instance 69 @param migrationObject: the MigrationObject instance that raised the error 70 @param comment:additional comment 71 """ 72 super(NoCitationException, self).__init__(migrationObject, comment = '') 73 def __str__(self): 74 return "No cedacat:citation associated with migration object %s " % (migrationObjectDescription(self.migrationObject)) 75 48 76 class NoAssociatedAuthor(MigrationObjectException): 49 77 """ … … 57 85 58 86 class NoAssociatedDeployments(Exception): 59 def __init__(self, data_entityMigration, deploymentName = None):87 def __init__(self, migrationObject, deploymentName = None): 60 88 """ 61 @param data_entityMigration: the data_entity migration associated89 @param migrationObject: the associated migrationObject 62 90 @param deploymentName: the associated deployment document name 63 91 """ 64 self._migrationObject = data_entityMigration92 self._migrationObject = migrationObject 65 93 self._deploymentName = deploymentName 66 94 super(NoAssociatedDeployments, self).__init__() 67 95 def __str__(self): 68 96 if self._deploymentName: 69 return "The DataEntity %s has no associated deployment named %s" % (migrationObjectDescription(self. migrationObject), self._deploymentName)97 return "The DataEntity %s has no associated deployment named %s" % (migrationObjectDescription(self._migrationObject), self._deploymentName) 70 98 else: 71 return "The DataEntity %s has no associated deployment" % (migrationObjectDescription(self. migrationObject))99 return "The DataEntity %s has no associated deployment" % (migrationObjectDescription(self._migrationObject)) 72 100 73 101 class NoDataLineage(MigrationObjectException): … … 76 104 """ 77 105 def __init__(self, migrationObject, comment = ''): 78 super(NoDataLineage, self).__init__(migrationObject, comment = '') 106 super(NoDataLineage, self).__init__(migrationObject, comment = '') -
mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py
r8424 r8433 54 54 from ea_model.moles3_4.utilities.mo_individual import MO_Individual 55 55 from ea_model.moles3_4.utilities.mo_organisation import MO_Organisation 56 from ea_model.moles3_4.utilities import mo_organisation57 56 from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_date import CI_Date 58 57 from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_citation import CI_Citation … … 780 779 Creates a new MD_Identifier 781 780 @param code: a String 782 @param authority: a CI_Citation instance 781 @param authority: a CI_Citation instance 783 782 """ 784 783 md_identifier = MD_Identifier() 784 if code == None: 785 raise NoNullableElement() 785 786 md_identifier.code = code 786 787 if authority: … … 933 934 if project_resource and type(project_resource) == list: 934 935 ceda_project.projectResource = project_resource 935 return ceda_project 936 return ceda_project 937 -
mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py
r8424 r8433 32 32 ''' 33 33 from libs.migration.exception.exceptions import MigrationObjectException, NoAssociatedAuthor,\ 34 migrationObjectDescription, NoAssociatedDeployments 34 migrationObjectDescription, NoAssociatedDeployments,\ 35 NoGeographicalExtensionException, NoCitationException 35 36 from libs.migration.processor.commons import findDeploymentsInDE,\ 36 37 createMD_Identifier, extractContent,\ … … 69 70 self._dataEntityHasBeenProcessed = hasMOBeenProcessed(self._dataEntityMigration) 70 71 self.epbRepo = epbRepo 72 self._report = [] 71 73 72 74 def _assignGeographicExtent(self, ceda_observationCollection): … … 84 86 self.epbRepo.moles3EPB.updateCedaObject(ceda_observationCollection, {'geographicExtent': geographicExtent}) 85 87 else: 86 print "No Geographic Extent" 87 return 88 self._report.append(NoGeographicalExtensionException(self._dataEntityMigration)) 88 89 89 90 … … 109 110 110 111 if self._dataEntityHasBeenProcessed: 112 self._report.append('The _assignGeographicExtent update is skipped because not implemented') 111 113 DataEntityProcessor.log.warn('The _assignGeographicExtent update is skipped because not implemented') 112 114 return … … 117 119 contentDict = extractContent(self._dataEntityMigration) 118 120 if not contentDict.has_key('citation'): 121 self._report(NoCitationException(self._dataEntityMigration)) 119 122 DataEntityProcessor.log.info("The migration object "+ migrationObjectDescription(self._dataEntityMigration) \ 120 123 + " has not associated cedacat:citation") … … 199 202 except NoAssociatedAuthor as ex: 200 203 raise ex 201 except Exception as ex: 202 #self.migrationSessions.molesSession.rollback() 203 #self.migrationSessions.migrationSession.rollback() 204 except Exception as ex: 204 205 raise MigrationObjectException(ex) 205 206 … … 219 220 self._execute(obsColl) 220 221 except Exception as ex: 221 exs.append(ex)222 return exs222 self._report.append(ex) 223 return self._report 223 224 224 225 #retrieves the associated deployment links from the data_entity … … 229 230 230 231 if deploymentMigrations is None or len(deploymentMigrations) == 0: 231 return exs 232 233 howManydm = 0 234 if deploymentMigrations: 235 howManydm = len(deploymentMigrations) 236 if howManydm == 0: 237 exs.append(NoAssociatedDeployments(self._dataEntityMigration)) 232 self._report.append(NoAssociatedDeployments(self._dataEntityMigration)) 233 return self._report 234 235 howManydm = len(deploymentMigrations) 238 236 for deploymentMigration in deploymentMigrations: 239 237 try: … … 244 242 245 243 except Exception as ex: 246 exs.append(ex)244 self._report.append(ex) 247 245 self._assignGeographicExtent(obsColl) 248 246 249 return exs247 return self._report -
mauRepo/MolesManager/trunk/src/libs/migration/processor/migrationProcess.py
r8361 r8433 67 67 dep = DataEntityProcessor(dataEntityMigration, self.epbRepo) 68 68 exs.extend(dep.process()) 69 MigrationProcess.log.info("Done")70 69 return exs -
mauRepo/MolesManager/trunk/src/libs/migration/tests/migrationprocess.py
r8391 r8433 44 44 from libs.migration.InfodbEPB import InfodbEPBFactory 45 45 from MolesManager.settings import EVENTS_DB 46 from libs.migration.client import EPBRepo 46 from libs.migration.client import EPBRepo, generateMigrationReport 47 from django.core.mail import send_mail 47 48 48 49 class LoadResourceTest(TestCase): … … 90 91 91 92 #IASI 92 #self.processDataEntity(DO_NEODC, DS_pUBLISHED, 'dataent_12417810458627666.atom')93 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_VIRTEM.atom')94 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12162569915925921.atom')93 self.processDataEntity(DO_NEODC, DS_pUBLISHED, 'dataent_12417810458627666.atom') 94 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_VIRTEM.atom') 95 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12162569915925921.atom') 95 96 96 97 #ECMWF 97 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-ERA.atom') 98 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-OP.atom') 99 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12458543158227759.atom') 100 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-E40.atom') 101 #self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-TRJ.atom') 98 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-ERA.atom') 99 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-OP.atom') 100 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12458543158227759.atom') 101 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-E40.atom') 102 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-TRJ.atom') 103 104 105 self.processDataEntity(DO_NEODC, DS_pUBLISHED, 'dataent_11750994700119761.atom') 106 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_ECMWF-TRJ.atom') 107 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_UGAMPO3.atom') 108 self.processDataEntity(DO_BADC, DS_pUBLISHED, 'dataent_12282368470627062.atom') 109 110 #self.processDataEntity() 102 111 103 112 104 def processDataEntity(self, doc_owner, doc_status, doc_name): 105 dataEntity = MigrationObject() 106 dataEntity.doc_owner = doc_owner 107 dataEntity.doc_status = doc_status 108 dataEntity.doc_name = doc_name 113 def processDataEntity(self, doc_owner = None, doc_status = None, doc_name = None): 114 dataEntity = None 115 if doc_owner is not None and doc_status is not None and doc_name is not None: 116 dataEntity = MigrationObject() 117 dataEntity.doc_owner = doc_owner 118 dataEntity.doc_status = doc_status 119 dataEntity.doc_name = doc_name 109 120 ex = self.mp.process(dataEntity) 110 for e in ex: 111 print e 112 121 generateMigrationReport(ex) 113 122
Note: See TracChangeset
for help on using the changeset viewer.