Changeset 8254 for mauRepo


Ignore:
Timestamp:
18/04/12 22:27:24 (8 years ago)
Author:
mnagni
Message:

Complete - # 22386: CEDA Observation - title
 http://team.ceda.ac.uk/trac/ceda/ticket/22386

Location:
mauRepo/MolesManager/trunk/src/libs/migration
Files:
10 edited

Legend:

Unmodified
Added
Removed
  • mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py

    r8245 r8254  
    5454    def getAllDeploymentsMigrationByDataEntitySortedByDate(self, dataEntity, deploymentNames, session = None): 
    5555        intSession = MigrationEPB._getSession(session) 
    56         return EPB.getAllObjects(DeploymentsMigration, intSession).filter(*[EPB.buildFilter('doc_status', dataEntity.doc_status)]).filter(*[EPB.buildFilter('doc_owner', dataEntity.doc_owner)]).filter(DeploymentsMigration.doc_name.in_(deploymentNames)).order_by(asc("doc_creation")) 
     56        return EPB.getAllObjects(DeploymentsMigration, intSession).filter(*[EPB.buildFilter('doc_owner', dataEntity.doc_owner)]).filter(DeploymentsMigration.doc_name.in_(deploymentNames)).order_by(asc("doc_creation"))         
     57        #return EPB.getAllObjects(DeploymentsMigration, intSession).filter(*[EPB.buildFilter('doc_status', dataEntity.doc_status)]).filter(*[EPB.buildFilter('doc_owner', dataEntity.doc_owner)]).filter(DeploymentsMigration.doc_name.in_(deploymentNames)).order_by(asc("doc_creation")) 
    5758 
    5859    @classmethod 
  • mauRepo/MolesManager/trunk/src/libs/migration/authors.py

    r8147 r8254  
    112112'STFC RAL Space Remote Sensing Group':{'type':'org','author':'STFC RAL Space Remote Sensing Group','co_author_type':[],'co_author':[]}, 
    113113'STFC Rutherford Appleton Laboratory':{'type':'org','author':'STFC Rutherford Appleton Laboratory','co_author_type':[],'co_author':[]}, 
     114'Science and Technology Facilities Council (STFC), Chilbolton Facility for Atmospheric and Radio Research, [S. A. Callaghan, J. Waight, C. J. Walden, J. Agnew and S. Ventouras]':{'type':'org','author':'Science and Technology Facilities Council (STFC)','co_author_type':['org','ind','ind','ind','ind','ind'],'co_author':['Chilbolton Facility for Atmospheric and Radio Research (CFARR)', 'S.A. Callaghan', 'J. Waight','C.J. Walden','J Agnew','S. Ventouras']},  
    114115'The International Service of Geomagnetic Indices':{'type':'org','author':'The International Service of Geomagnetic Indices','co_author_type':[],'co_author':[]}, 
    115116'The Max-Planck-Institut fur Biogeochemie, Jena':{'type':'org','author':'Max-Planck-Institut fur Biogeochemie, Jena','co_author_type':[],'co_author':[]}, 
  • mauRepo/MolesManager/trunk/src/libs/migration/exception/exceptions.py

    r8180 r8254  
    6161 
    6262class NoAssociatedDeployments(Exception): 
    63     def __init__(self, data_entityMigration, deploymentName): 
     63    def __init__(self, data_entityMigration, deploymentName = None): 
     64        """ 
     65            @param data_entityMigration: the data_entity migration associated 
     66            @param deploymentName: the associated deployment document name  
     67        """ 
    6468        self._migrationObject = data_entityMigration 
    6569        self._deploymentName = deploymentName         
    6670        super(NoAssociatedDeployments, self).__init__() 
    6771    def __str__(self): 
    68         return "The DataEntity %s has no associated deployment named %s" % (_migrationObjectDescription(self.migrationObject), self._deploymentName) 
     72        if self._deploymentName: 
     73            return "The DataEntity %s has no associated deployment named %s" % (_migrationObjectDescription(self.migrationObject), self._deploymentName) 
     74        else: 
     75            return "The DataEntity %s has no associated deployment" % (_migrationObjectDescription(self.migrationObject)) 
    6976     
    7077class NoDataLineage(MigrationObjectException): 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py

    r8247 r8254  
    359359def findDocumentationInMigrationDocument(migrationObject): 
    360360    """ 
    361         Return a list of dictionaries describing a <link rel="...DOCUMENT..."> tag type 
     361        Return a list of dictionaries describing a <link rel="...DOCUMENTATION..."> tag type 
    362362        Each dictionary has two keys: 'href' and 'title' 
    363363        @param migrationObject: the migration instance to retrieve and parse 
     
    365365    """ 
    366366    linksDict = findLinksInMigrationDocument(migrationObject) 
    367     if linksDict.has_key('DOCUMENT'): 
    368         return linksDict['DOCUMENT']      
     367    if linksDict.has_key('DOCUMENTATION'): 
     368        return linksDict['DOCUMENTATION']      
    369369    return {} 
    370370 
     
    682682    return ci_date 
    683683 
    684 def createCI_Citation(title, date = None): 
     684def createCI_Citation(title, date = None, citedResponsibleParty = None): 
    685685    """ 
    686686        Creates a new CI_Citation 
    687         @param titile: the CI_Citation title 
     687        @param title: the CI_Citation title 
    688688        @param date: an array of CI_Date instances 
     689        @param citedResponsibleParty: a list of CI_ResponsibleParty instances 
    689690    """     
    690691    ci_citation = CI_Citation() 
     
    692693    if date and type(date) == list: 
    693694        ci_citation.date = date 
     695    if citedResponsibleParty: 
     696        ci_citation.extend(citedResponsibleParty) 
    694697    return ci_citation 
    695698 
     
    720723    """ 
    721724        Creates a new MD_Identifier 
    722         @param title: the MD_Identifier.authority.title field 
    723         @param code: the MD_Identifier.code field 
    724         @param date: the MD_Identifier.authority.date field 
     725        @param code: a String 
     726        @param authority: a CI_Citation instance 
    725727    """ 
    726728    md_identifier = MD_Identifier() 
     
    730732    return md_identifier 
    731733 
    732 def createCI_ResponsibleParty(role): 
     734def createCI_ResponsibleParty(role, organizationName = None, individualName = None): 
    733735    """ 
    734736        Creates a new CI_ResponsibeParty 
     
    737739    ci_responsible_party = CI_ResponsibleParty() 
    738740    ci_responsible_party.role = role 
     741    if organizationName: 
     742        ci_responsible_party.organisationName = organizationName 
     743    if individualName: 
     744        ci_responsible_party.individualName = individualName     
    739745    return ci_responsible_party 
    740746 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py

    r8247 r8254  
    1010from libs.migration.MigrationEPB import MigrationEPB 
    1111from libs.migration.exception.exceptions import MigrationObjectException, NoAssociatedAuthor,\ 
    12     _migrationObjectDescription 
     12    _migrationObjectDescription, NoAssociatedDeployments 
    1313from libs.migration.processor.commons import findDeploymentsInDE,\ 
    1414    createMD_Identifier, extractContent, MD_CODE_MOLES2_CITATION,\ 
    1515    hasAtomDocumentSameHash, createCI_Citation, createCI_Date, findPublishedDate,\ 
    1616    isoDateTimeStringToTimeDate, findUpdatedDate, createDate,\ 
    17     calculateHash, extractUpdateFrequency 
     17    calculateHash, extractUpdateFrequency, findDOIInMigrationDocument 
    1818from libs.migration.processor.deployment import DeploymentProcessor 
    1919from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
     
    113113        return ceda_observationCollection 
    114114 
    115     def _processDeploymentMigration(self, deploymentMigration, cedaObservationCollection):                                       
     115    def _processDeploymentMigration(self, deploymentMigration, single_deployment):                                       
    116116        deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration, self.migrationSessions) 
    117117        try: 
     
    121121                    if updateFrequency: 
    122122                        ceda_observation.resultAccumulation = MD_MaintenanceFrequencyCode.from_string(updateFrequency) 
    123             Moles3EPB.addObservationToObservationCollection(cedaObservationCollection, ceda_observation, self.migrationSessions.molesSession, True)             
    124             MigrationEPB.associateObservationCollectionToDataEntity(deploymentMigration, \ 
    125                          cedaObservationCollection.id, self.migrationSessions.migrationSession, True) 
     123             
     124            doi = findDOIInMigrationDocument(deploymentMigration)                                                               
     125            if single_deployment: 
     126                if doi is None: 
     127                    doi = findDOIInMigrationDocument(self._dataEntityMigration) 
     128                #collection_identifier = Moles3EPB.extractCollectionIdentifierByTitle(MD_CODE_MOLES2_CITATION, self.migrationSessions.molesSession) 
     129                #if collection_identifier.count()==1: 
     130                #    ceda_observation.identifier.append(collection_identifier.first()) 
     131             
     132            deProcessor.assignDOI(ceda_observation, doi) 
     133            self.migrationSessions.molesSession.commit() 
    126134        except NoAssociatedAuthor as ex: 
    127135            raise ex                  
     
    160168                                                self._dataEntityMigration, deploymentsLinks, self.migrationSessions.migrationSession) 
    161169        howManydm = deploymentMigrations.count() 
     170        if howManydm == 0: 
     171            exs.append(NoAssociatedDeployments(self._dataEntityMigration)) 
    162172        for deploymentMigration in deploymentMigrations: 
    163173            try: 
    164                 ceda_observation = self._processDeploymentMigration(deploymentMigration, cedaObservationCollection)                 
    165                 if howManydm == 1: 
    166                     collection_identifier = Moles3EPB.extractCollectionIdentifierByTitle(MD_CODE_MOLES2_CITATION, self.migrationSessions.molesSession) 
    167                     if collection_identifier.count()==1: 
    168                         ceda_observation.identifier.append(collection_identifier.firt()) 
    169                         self.migrationSessions.molesSession.commit()                 
     174                ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1) 
     175                cedaObservationCollection.member.append(ceda_observation)                                        
    170176            except Exception as ex: 
    171177                exs.append(ex) 
    172178            except RuntimeError as er: 
    173179                print er                 
     180        self.migrationSessions.molesSession.commit()                         
     181                 
    174182        return exs 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py

    r8252 r8254  
    238238                    projSubType = findSubTypeInDPT(activity) 
    239239                    doc_title += ' for the ' + extractTitle(activity) 
    240                     if projSubType[0:8] == 'Activity': 
    241                         doc_title += ' ' + projSubType[8:] 
     240                    if projSubType[0:8] == 'dgActivityData': 
     241                        doc_title += ' ' + projSubType[:13] 
    242242                    else : 
    243243                        doc_title += ' ' + projSubType 
     
    340340        observation.resultTime = createTM_Instant(tm_position) 
    341341 
    342     def _assignDOI(self, observation): 
    343         # TDB - Check that if is an update or not! 
    344         doi = findDOIInMigrationDocument(self._deploymentMigration)  
    345         if not doi: 
    346             doi = findDOIInMigrationDocument(self._dataEntityMigration) 
     342    def assignDOI(self, observation, doi): 
    347343        if doi: 
    348             identifier = createMD_Identifier(code = doi['href']) 
    349             ci_date = None 
    350             if doi['href'] == '10.5285/E8F43A51-0198-4323-A926-FE69225D57DD': 
    351                 #dt = createDateTime('21', '11', '04', '01', '07', '25', '35') 
     344            doi = doi['href'][4:]             
     345            py_date = None 
     346            cited_responsible = createCI_ResponsibleParty(role=CI_RoleCode.cl_publisher, \ 
     347                                                              organizationName='NERC - British Atmospheric Data Centre') 
     348            if doi == '10.5285/E8F43A51-0198-4323-A926-FE69225D57DD': 
    352349                py_date = date(2011, 4, 1) 
     350            elif doi == '10.5285/78114093-E2BD-4601-8AE5-3551E62AEF2B': 
     351                py_date = date(2011, 11, 29)                 
     352            elif doi == '10.5285/DB8D8981-1A51-4D6E-81C0-CCED9B921390': 
     353                py_date = date(2012, 4, 16) 
     354            elif doi == '10.5285/639A3714-BC74-46A6-9026-64931F355E07': 
     355                py_date = date(2012, 4, 16)                 
     356                 
     357            if py_date:    
    353358                dt = createDate(py_date) 
    354359                ci_date = createCI_Date(CI_DateTypeCode.cl_publication, date = dt) 
    355             elif doi['href'] == '10.5285/78114093-E2BD-4601-8AE5-3551E62AEF2B': 
    356                 #dt = createDateTime('21', '11', '11', '29', '11', '37', '11') 
    357                 py_date = date(2011, 11, 29)                 
    358                 dt = createDate(py_date)                 
    359                 ci_date = createCI_Date(CI_DateTypeCode.cl_publication, date = dt) 
    360             if ci_date:                 
    361                 identifier.authority = createCI_Citation("", date = ci_date) 
    362             observation.identifier.append(identifier)                                      
     360                i_authority = createCI_Citation("DOI", date = ci_date) 
     361                identifier = createMD_Identifier(code = doi, authority=i_authority)              
     362                observation.identifier.append(identifier)                                      
    363363 
    364364    def _assignPublisherCurator(self, observation):         
     
    479479        self._assignResult(ceda_observation)     
    480480        self._assignGeographicExtent(ceda_observation)     
    481         self._assignDOI(ceda_observation)         
     481        #self._assignDOI(ceda_observation)         
    482482        self._assignCreationDate(ceda_observation) 
    483483        self._assignPublisherCurator(ceda_observation)         
  • mauRepo/MolesManager/trunk/src/libs/migration/tests/dropAllTables.sql

    r8147 r8254  
    556556DROP TABLE IF EXISTS deployments_migration CASCADE; 
    557557DROP TABLE IF EXISTS cv_solidvaluepair_cv_discretecoverage CASCADE; 
    558  
    559  
     558DROP TABLE IF EXISTS ceda_gui CASCADE; 
    560559 
    561560DROP SEQUENCE IF EXISTS angle_id_seq CASCADE; 
  • mauRepo/MolesManager/trunk/src/libs/migration/tests/migrationprocess.py

    r8245 r8254  
    99from libs.migration.processor.loadResources import LoadResources 
    1010from libs.migration.db.classes import MigrationObject 
    11 from libs.migration.processor.commons import DO_BADC, DS_pUBLISHED 
     11from libs.migration.processor.commons import DO_BADC, DS_pUBLISHED, DS_PUBLISHED 
    1212from MolesManager.moles3epb import Moles3EPB 
    1313from libs.migration.processor.migrationProcess import MigrationProcess 
     
    3636        lr = LoadResources() 
    3737        ex = [] 
    38         ex = lr.process() 
     38        #ex = lr.process() 
    3939        for e in ex: 
    4040            print e 
     
    4747        dataEntity.doc_status = DS_pUBLISHED 
    4848        dataEntity.doc_owner = DO_BADC 
    49         dataEntity.doc_name = 'dataent_csip.atom' 
    50         #dataEntity.doc_name = 'DE_dcaa78b2-4008-11e0-88c9-00e081470265.atom' 
     49        #dataEntity.doc_name = 'dataent_csip.atom' 
     50         
     51        dataEntity.doc_status = DS_PUBLISHED 
     52        dataEntity.doc_name = 'DE_095e8da2-cf02-11e0-8b7a-00e081470265.atom' 
    5153         
    5254        ex = mp.process(dataEntity) 
    53         ex = mp.process() 
     55        #ex = mp.process() 
    5456        for e in ex: 
    5557            print e 
  • mauRepo/MolesManager/trunk/src/libs/migration/tests/moles3epbmothods.py

    r8217 r8254  
    1212from logging import StreamHandler 
    1313from test_utils import createObservationCollection, createObservation, createProject 
     14from testconfig import TESTMOLES3_DB_CONNECTION 
     15from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection 
    1416from ea_model.moles3_4.observationcollection.mo_observationcollection import MO_ObservationCollection 
    1517 
     
    1921    def setUp(self):     
    2022        Moles3EPBTest.molesDB = DbManager(MOLES3_DB_CONNECTION, MOLES3_DB_SCRIPT) 
     23        #Moles3EPBTest.molesDB = DbManager(TESTMOLES3_DB_CONNECTION, MOLES3_DB_SCRIPT) 
    2124        Moles3EPB.overrrideDBManager(Moles3EPBTest.molesDB) 
    2225        self._initSearchIndexes(Moles3EPBTest.molesDB)         
     
    3033 
    3134    def runTest(self): 
    32         #self.checkExtractCollectionIdentifierByTitle()    
    33         #self.checkExtractObservationByTitleKeywords()  
     35        self.checkExtractCollectionIdentifierByTitle()    
     36        self.checkExtractObservationByTitleKeywords()  
    3437        self.checkExtractProjectObservationCollections() 
     38        #self.checkExtractObservationCollectionsForObservation() 
     39 
     40    def checkExtractObservationCollectionsForObservation(self): 
     41        session = Moles3EPB.getNewMolesSession(); 
     42        #coll = session.query(CEDA_ObservationCollection). \ 
     43        #    join(MO_ObservationCollection).join(MO_ObservationCollection.member) 
     44        coll = session.query(CEDA_ObservationCollection) 
     45        colls = coll.all() 
     46        print colls 
    3547 
    3648    def checkExtractCollectionIdentifierByTitle(self): 
  • mauRepo/MolesManager/trunk/src/libs/migration/tests/testconfig.py

    r8222 r8254  
    88INFO_DB_CONNECTION = 'postgresql://cedainfo:ler239b@bora.badc.rl.ac.uk:5432/cedainfo' 
    99 
     10TESTMIGRATION_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/ceda_moles3_test' 
     11TESTMOLES3_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/ceda_moles3_test' 
     12 
    1013from libs.migration.db.migrationTables import doTables as doMigration 
    1114MIGRATION_DB_SCRIPT = doMigration 
Note: See TracChangeset for help on using the changeset viewer.