Changeset 8358


Ignore:
Timestamp:
17/06/12 15:40:15 (8 years ago)
Author:
mnagni
Message:

Major refactoring about migration (now handles better create/update, even if the single updates have to be quite fully implemented)
Added the connection pool from SQLAlchemy

Location:
mauRepo/MolesManager/trunk/src
Files:
16 edited

Legend:

Unmodified
Added
Removed

  • mauRepo/MolesManager/trunk/src/MolesManager/moles3epb.py

    r8350 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 10 Jan 2012 
    330 
    … …  
    532''' 
    633from libs.epb import EPB 
    7 from libs.migration.exception.exceptions import NoDBManager 
    834from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import CEDA_ObservationCollection 
    935from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 
    … …  
    127153            @param instance: an instance of CEDA_Observation os CEDA_ObservationCollection   
    128154        """ 
    129         if instance is None: 
     155        if instance is None or not hasattr(instance, 'id'): 
    130156            return None 
    131157        if type(instance) == CEDA_ObservationCollection: 

  • mauRepo/MolesManager/trunk/src/MolesManager/molesSessionMiddleware.py

    r8351 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 9 Jan 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from MolesManager.moles3epb import Moles3EPBFactory 
    … …  
    1239from libs.migration.InfodbEPB import InfodbEPBFactory 
    1340from libs.migration.MigrationEPB import MigrationEPBFactory 
    14 from sqlalchemy.exc import SQLAlchemyError 
     41from sqlalchemy import exc 
    1542from django.core.context_processors import csrf 
    1643from django.shortcuts import render_to_response 
    … …  
    4976        self._migration() #see the note on  MolesSessionMiddleware._migration       
    5077     
    51     """ 
    52     def _getMolesSession(self): 
    53         ''' 
    54             @deprecated: db session is going to be removed from all GUI-related instances 
    55         ''' 
    56         if not MolesSessionMiddleware._epbInitialized: 
    57             self._doInitialization() 
    58  
    59         return self._getNewMolesSession() 
    60     """ 
    61      
    6278    def _migration(self, runMigration = RUN_MIGRATION): 
    6379        if runMigration and not MolesSessionMiddleware._migrationThread.isAlive(): 
    6480            #t.setDaemon(False) 
    6581            MolesSessionMiddleware._migrationThread.start() 
    66              
    67     """ 
    68         Represents the access to the Moles database. 
    69         Creates from the existing db connections pools a new session 
    70         to be used by all the DB operations involved in the actual HTTPRequest 
    71     """   
    72      
     82    
    7383    def process_request(self, request):   
    7484        try: 
    … …  
    7787                   
    7888            request.moles3EPB = MolesSessionMiddleware._moles3EPBFactory.createEPB()                    
    79         except SQLAlchemyError as ex: 
    80             MolesSessionMiddleware._epbInitialized = False 
    81             return self.process_exception(request, ex) 
     89        except exc.DBAPIError, e: 
     90            # an exception is raised, Connection is invalidated. 
     91            #if e.connection_invalidated: 
     92            #    print "Connection was invalidated!" 
     93            return self.process_exception(request, e) 
    8294 
    8395    def process_response(self, request, response): 
    … …  
    88100 
    89101    def process_exception(self, request, exception): 
    90         try: 
    91             c = {} 
    92             c['exception'] = exception 
    93             c.update(csrf(request)) 
    94             return render_to_response('errorTemplate.html', c) 
    95         except SQLAlchemyError as ex: 
    96             MolesSessionMiddleware._epbInitialized = False             
    97          
     102        c = {} 
     103        c['exception'] = exception 
     104        c.update(csrf(request)) 
     105        return render_to_response('errorTemplate.html', c)         

  • mauRepo/MolesManager/trunk/src/libs/commons_db.py

    r8344 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 17 Nov 2011 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from sqlalchemy import MetaData, create_engine 
    7 from sqlalchemy.orm import sessionmaker, scoped_session 
     34from sqlalchemy.orm import sessionmaker 
    835from sqlalchemy.sql.expression import text 
    936class DbManagerCollection(object): 
    … …  
    2552        Represents a database instance managed by an SQLAlchemy engine.         
    2653    """ 
    27     def __init__(self, connection, script = None, sql_echo=False, session_event_manager = None): 
     54    def __init__(self, connection, script = None, sql_echo=False, session_event_manager = None, ipool_size = 10): 
    2855        """ 
    2956          @param connection: the URL used to open a connection to the database 
    … …  
    3158          @param sql_echo: sets the SQLalchemy create_engine.echo param. Default is False  
    3259          @param session_event_manager: a python method defining session events \ 
    33           (see http://docs.sqlalchemy.org/en/latest/orm/events.html?highlight=session%20events#sqlalchemy.orm.events.SessionEvents)      
     60          (see http://docs.sqlalchemy.org/en/latest/orm/events.html?highlight=session%20events#sqlalchemy.orm.events.SessionEvents) 
     61          @param ipool_size: sets the connections pool size (note that this is the maximum size not the number of the pre-allocated ones)       
    3462        """ 
    3563        self._connection = connection         
    3664        self._script = script 
    37         self.engine = None 
    38         try: 
    39             self.engine = create_engine(self._connection, echo=sql_echo) 
    40         except Exception as e: 
    41             print e 
     65        self.engine = create_engine(self._connection, pool_size = ipool_size, pool_recycle=3600, echo=sql_echo) 
    4266        self.metadata = MetaData(bind=self.engine)         
    4367        self._session = None 
    … …  
    4670            self._script(self.metadata) 
    4771            self.metadata.create_all(bind=self.engine) 
    48             #self._session = scoped_session(sessionmaker()) 
    4972        if self.engine: 
    5073            self.engine.connect() 
    … …  
    5578           @return: a sqlalchemy.orm.session.Session instance if the ORM is fully configured, otherwise a sqlalchemy.engine.base.Connection instance  
    5679        """ 
    57         #session = scoped_session(sessionmaker(bind=self.engine))() 
    5880        session = sessionmaker(bind=self.engine)()         
    5981        if self._session_event_manager: 
    6082            self._session_event_manager(session) 
    6183        return session         
    62         #if self._session: 
    63         #    ret = self._session() 
    64         #    if self._session_event_manager: 
    65         #        self._session_event_manager(ret) 
    66         #    return ret 
    67         #if self.engine: 
    68         #    return self.engine.connect() 
    6984 
    7085    def closeDbSession(self, dbSession): 
    … …  
    8499        t = text(sqlNative) 
    85100        return self._connection.execute(t) 
     101 
     102class NoDBManager(Exception): 
     103    def __init__(self, value): 
     104        self.value = value 
     105    def __str__(self): 
     106        return "No DBManager has been set for %s" % (self.value)  
    86107 
    87108def doInsertOrUpdate(items, session, update = True): 

  • mauRepo/MolesManager/trunk/src/libs/epb.py

    r8344 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 10 Jan 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from sqlalchemy.orm import subqueryload 

  • mauRepo/MolesManager/trunk/src/libs/migration/client.py

    r8334 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 9 Nov 2011 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from libs.migration.processor.loadResources import LoadResources 
    … …  
    2249     
    2350    def __init__(self, moles3EPB, migrationEPB, infodbEPB, interval=600): 
    24         EPBRepo.moles3EPB = moles3EPB 
    25         EPBRepo.migrationEPB = migrationEPB 
    26         EPBRepo.infodbEPB = infodbEPB 
     51        self.epbRepo = EPBRepo(moles3EPB, migrationEPB, infodbEPB) 
    2752         
    2853        threading.Thread.__init__(self) 
    … …  
    4772 
    4873    def _migrate(self): 
    49         lr = LoadResources() 
     74        lr = LoadResources(self.epbRepo) 
    5075        ex = lr.process() 
    5176        for e in ex: 
    5277            print e 
    5378 
    54         mp = MigrationProcess() 
     79        mp = MigrationProcess(self.epbRepo) 
    5580        ex = mp.process() 
    5681        for e in ex: 

  • mauRepo/MolesManager/trunk/src/libs/migration/exception/exceptions.py

    r8269 r8358  
    1212    except Exception as ex: 
    1313        print ex 
    14  
    15 class NoDBManager(Exception): 
    16     def __init__(self, value): 
    17         self.value = value 
    18     def __str__(self): 
    19         return "No DBManager has been set for %s" % (self.value)   
    2014     
    2115class NoCreationDate(Exception): 

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/EPBRepo.py

    r8326 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 1 Jun 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633class EPBRepo(object): 
    7     moles3EPB = None 
    8     migrationEPB = None 
    9     infodbEPB = None 
     34     
     35    def __init__(self, moles3EPB, migrationEPB, infodbEPB):   
     36        self.moles3EPB = moles3EPB 
     37        self.migrationEPB = migrationEPB 
     38        self.infodbEPB = infodbEPB 
     39         
     40     
     41     

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py

    r8323 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 15 Nov 2011 
    330 
    … …  
    143170    return calculateHash(text) 
    144171 
    145 def hasAtomDocumentSameHash(migrationObject): 
     172def hasMOBeenProcessed(migrationObject): 
     173    ''' 
     174        Checks if a migration object has been already processed. 
     175        @param migrationObject: an instance of DataEntityMigration or DeploymentsMigration  
     176        @return True if has been processed, otherwise False 
     177    ''' 
     178    if migrationObject is None: 
     179        return False 
     180     
     181    if not hasattr(migrationObject, 'doc_hash'): 
     182        return False 
     183    else: 
     184        if migrationObject.doc_hash is None: 
     185            return False 
     186         
     187    return True 
     188 
     189def hasMOSameHash(migrationObject): 
     190    ''' 
     191        Checks if a migration object has changed. 
     192        @param migrationObject: an instance of DataEntityMigration or DeploymentsMigration or DeploymentDataMigration 
     193        @return True if the hash of the actual document is the same of the migrationObject, otherwise False 
     194    ''' 
     195    if not hasMOBeenProcessed(migrationObject): 
     196        return False 
    146197    return getAtomDocumentHashByMO(migrationObject) == migrationObject.doc_hash 
    147198 
    … …  
    786837    return md_metadata 
    787838 
    788 def createMO_OnlineResource(linkage, name = None, function = None, description = None, applicationProfile = None): 
     839def createMO_OnlineResource(linkage, instance = None, name = None, function = None, description = None, applicationProfile = None): 
    789840    """ 
    790841        Creates a new CEDA_Result 
    … …  
    795846        @param applicationProfile: the MO_OnlineResource.applicationProfile field 
    796847    """ 
    797     on_line_resource = MO_OnlineResource()     
     848    if instance is None: 
     849        on_line_resource = MO_OnlineResource() 
     850          
    798851    on_line_resource.linkage = linkage 
    799     if name:  
     852    if name and on_line_resource.name != name:         
    800853        on_line_resource.name = name 
    801     if function: 
     854    if function and on_line_resource.function != function: 
    802855        on_line_resource.function = function 
    803     if description: 
     856    if description and on_line_resource.description != description: 
    804857        on_line_resource.description = description 
    805     if applicationProfile: 
     858    if applicationProfile and on_line_resource.applicationProfile != applicationProfile: 
    806859        on_line_resource.applicationProfile = applicationProfile 
    807860    return on_line_resource         

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py

    r8351 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 15 Nov 2011 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    6 from MolesManager.moles3epb import Moles3EPB 
    733from ea_model.ceda_metadatamodel.ceda_observationcollection.ceda_observationcollection import \ 
    834    CEDA_ObservationCollection 
    9 from libs.migration.MigrationEPB import MigrationEPB 
    1035from libs.migration.exception.exceptions import MigrationObjectException, NoAssociatedAuthor,\ 
    1136    migrationObjectDescription, NoAssociatedDeployments 
    1237from libs.migration.processor.commons import findDeploymentsInDE,\ 
    1338    createMD_Identifier, extractContent,\ 
    14     hasAtomDocumentSameHash, createCI_Citation, createCI_Date, findPublishedDate,\ 
     39    hasMOSameHash, createCI_Citation, createCI_Date, findPublishedDate,\ 
    1540    isoDateTimeStringToTimeDate, findUpdatedDate, createDate,\ 
    16     calculateHash, extractUpdateFrequency, findDOIInMigrationDocument,\ 
    17     getAtomDocumentHash 
     41    calculateHash, findDOIInMigrationDocument,\ 
     42    hasMOBeenProcessed, getAtomDocumentHashByMO 
    1843from libs.migration.processor.deployment import DeploymentProcessor 
    1944from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
    … …  
    2247from ea_model.iso_19115_2006_metadata_corrigendum.citation_and_responsible_party_information.ci_datetypecode import CI_DateTypeCode 
    2348from MolesManager.ceda_guid import CedaGUID 
    24 from ea_model.iso_19115_2006_metadata_corrigendum.maintenance_information.md_maintenancefrequencycode import MD_MaintenanceFrequencyCode 
    25 from libs.migration.processor.EPBRepo import EPBRepo 
    2649 
    2750class DataEntityProcessor(object): 
    … …  
    3255        Processes a DataEntityMigration item. Note that each DataEntity is associated to a "dataent_xxxx" file in Moles2 
    3356    '''         
    34     def __init__(self, dataEntityMigration): 
     57    def __init__(self, dataEntityMigration, epbRepo): 
    3558        ''' 
    3659            Initializes the class 
    3760            @param _dataEntityMigration: the DataEntityMigration instance 
     61            @param epbRepo: an instance of EPBRepo 
    3862        ''' 
    39         if dataEntityMigration is None: 
    40             raise MigrationObjectException("DataEntityProcessor cannot process an None item") 
    41         self._dataEntityMigration = dataEntityMigration        
     63        #if dataEntityMigration is None: 
     64        #    raise MigrationObjectException("DataEntityProcessor cannot process an None item") 
     65        self._dataEntityMigration = dataEntityMigration         
     66        self._dataEntityHasSameHash = hasMOSameHash(self._dataEntityMigration)        
     67        self.epbRepo = epbRepo 
    4268     
    4369    def _processCitation(self, ceda_observationCollection): 
    44         # TDB - Check that if is an update or not! 
    4570        contentDict = extractContent(self._dataEntityMigration) 
    4671        if not contentDict.has_key('citation'): 
    … …  
    6287            newIdentifier = createMD_Identifier(code = contentDict['citation'], authority=i_citation) 
    6388            ceda_observationCollection.identifier.append(newIdentifier) 
    64      
    65     def _getObservationCollection(self): 
    66         return EPBRepo.moles3EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id) 
    67      
    68     def _processObservationCollection(self):                                    
    69         #Moles3 object exists...  
    70         if self._dataEntityMigration.ceda_observation_coll_id:  
    71             # ...and the data entity document has not changed 
    72             if hasAtomDocumentSameHash(self._dataEntityMigration): 
    73                 return self._getObservationCollection() 
    74             else: 
    75                 self.updateObservationCollection() 
    7689 
    77         #... does not exist so create it 
    78         return self.createNewObservationCollection() 
    79  
    80     def updateObservationCollection(self): 
    81         """ 
    82             Updated the existing CEDA_ObservationCollection instance binded to the self._dataEntityMigration object. 
    83             @return: the persisted and updated CEDA_ObservationCollection element  
    84         """         
    85         ceda_observationCollection = self._getObservationCollection() 
    86         # TBD 
    87         #self._processCitation(ceda_observationCollection) 
    88         return ceda_observationCollection 
    89  
    90     def _update(self):  
    91         # ...and the data entity document has not changed 
    92         if hasAtomDocumentSameHash(self._dataEntityMigration): 
    93                 return self._getObservationCollection() 
    94         else: 
    95                 return self.updateObservationCollection()  
    96  
    97     def _create(self):  
     90    def _execute(self, ceda_observationCollection):  
    9891        """ 
    9992            Creates a new CEDA_ObservationCollection instance in the Moles3DB using the self._dataEntityMigration object. 
    … …  
    10194            @return: the persisted CEDA_ObservationCollection element  
    10295        """ 
    103         ceda_observationCollection = CEDA_ObservationCollection() 
    104         self._processCitation(ceda_observationCollection) 
    105         #self._processResult(ceda_observationCollection) 
    106         ceda_observationCollection.publicationState = MO_PublicationStateValue.cl_working 
    107                    
    108         EPBRepo.moles3EPB.persistInstance(ceda_observationCollection)         
    109         EPBRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, {'ceda_observation_coll_id': ceda_observationCollection.id}) 
     96        if not self._dataEntityHasSameHash: 
     97            self._processCitation(ceda_observationCollection) 
     98            #self._processResult(ceda_observationCollection) 
     99         
     100        #Is a first time process? 
     101        if not hasMOBeenProcessed(self._dataEntityMigration): 
     102            docHash = getAtomDocumentHashByMO(self._dataEntityMigration) 
     103            ceda_observationCollection.publicationState = MO_PublicationStateValue.cl_working 
     104            self.epbRepo.moles3EPB.persistInstance(ceda_observationCollection) 
     105            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \ 
     106                {'ceda_observation_coll_id': ceda_observationCollection.id, \ 
     107                 'doc_hash': docHash}) 
    110108 
     109        #Has to updated the hash? 
     110        if not self._dataEntityHasSameHash and hasMOBeenProcessed(self._dataEntityMigration): 
     111            docHash = getAtomDocumentHashByMO(self._dataEntityMigration) 
     112            self.epbRepo.migrationEPB.updateMigrationObject(self._dataEntityMigration, \ 
     113                {'doc_hash': docHash}) 
    111114         
    112         #Adds the CedaGUID 
    113         ceda_guid = CedaGUID() 
    114         ceda_guid.id = calculateHash(self._dataEntityMigration.data_ent_id) 
    115         ceda_guid.ceda_observationcollection = ceda_observationCollection.id 
    116         EPBRepo.moles3EPB.persistInstance(ceda_guid) 
    117         DataEntityProcessor.log.info("GUID for this ObservationCollection: %s" % (ceda_guid.id)) 
    118          
    119         return ceda_observationCollection 
    120  
    121     def _processResultAccumulation(self, ceda_observation):         
    122         ceda_observation = EPBRepo.moles3EPB.loadAttributes(ceda_observation, "resultAccumulation")         
    123         if ceda_observation.resultAccumulation is None: 
    124             updateFrequency = extractUpdateFrequency(self._dataEntityMigration) 
    125             if updateFrequency: 
    126                 resultAccumulation = MD_MaintenanceFrequencyCode.from_string(updateFrequency) 
    127                 EPBRepo.moles3EPB.updateCedaObject(ceda_observation, {'resultAccumulation': resultAccumulation}) 
     115        #Has a proper CEDAGUID? 
     116        if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observationCollection) is None: 
     117            #Adds the CedaGUID 
     118            ceda_guid = CedaGUID() 
     119            ceda_guid.id = calculateHash(self._dataEntityMigration.data_ent_id) 
     120            setattr(ceda_guid, 'ceda_observationcollection', ceda_observationCollection.id) 
     121            self.epbRepo.moles3EPB.persistInstance(ceda_guid) 
     122            DataEntityProcessor.log.info("GUID for this ObservationCollection: %s" % (ceda_guid.id)) 
    128123 
    129124    def _processDOI(self, deploymentMigration, ceda_observation, deProcessor, single_deployment):         
    … …  
    134129                    #collection_identifier = Moles3EPB.extractCollectionIdentifierByTitle(MD_CODE_MOLES2_CITATION, self.migrationSessions.molesSession) 
    135130                    #if collection_identifier.count()==1: 
    136                     #    ceda_observation.identifier.append(collection_identifier.first()) 
    137                  
     131                    #    ceda_observation.identifier.append(collection_identifier.first())                 
    138132        deProcessor.assignDOI(ceda_observation, doi) 
    139133 
    140134    def _processDeploymentMigration(self, deploymentMigration, single_deployment):                                       
    141         deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration) 
     135        deProcessor = DeploymentProcessor(self._dataEntityMigration, deploymentMigration, self.epbRepo) 
    142136        try: 
    143137            DataEntityProcessor.log.info("Processing deployment: %s" % (migrationObjectDescription(deploymentMigration))) 
    144             ceda_observation = deProcessor.process() 
    145             try:             
    146                 self._processResultAccumulation(ceda_observation) 
    147             except Exception as ex: 
    148                 pass     
     138            ceda_observation = deProcessor.process()     
    149139            try:                             
    150140                self._processDOI(deploymentMigration, ceda_observation, deProcessor, single_deployment) 
    151141            except Exception as ex: 
    152                 pass                 
    153  
    154                 if not hasAtomDocumentSameHash(deploymentMigration): 
    155                     doc_hash = getAtomDocumentHash(deploymentMigration.docStatus, deploymentMigration.docType, deploymentMigration.docOwner, deploymentMigration.docName) 
    156                     EPBRepo.migrationEPB.updateMigrationObject(deploymentMigration, {'doc_hash': doc_hash})                 
     142                pass                                 
    157143        except NoAssociatedAuthor as ex: 
    158144            raise ex                  
    … …  
    165151     
    166152    def process(self): 
    167         cedaObservationCollection = None 
     153        obsColl = None 
    168154        exs = [] 
    169155        DataEntityProcessor.log.info("Processing dataEntity: %s" % (migrationObjectDescription(self._dataEntityMigration))) 
    170156        try : 
    171157            if self._dataEntityMigration.ceda_observation_coll_id: 
    172                 cedaObservationCollection = self._update() 
     158                obsColl = self.epbRepo.moles3EPB.search(CEDA_ObservationCollection, self._dataEntityMigration.ceda_observation_coll_id)  
    173159            else: 
    174                 cedaObservationCollection = self._create() 
     160                obsColl = CEDA_ObservationCollection() 
     161            self._execute(obsColl)                  
    175162        except Exception as ex: 
    176163            exs.append(ex) 
    177164            return exs         
    178  
    179165         
    180166        #retrieves the associated deployment links from the data_entity 
    181         deploymentsLinks = findDeploymentsInDE(self._dataEntityMigration) 
    182          
     167        deploymentsLinks = findDeploymentsInDE(self._dataEntityMigration)         
    183168        #retrieves the DataEntityMigration sorted by creation date 
    184         deploymentMigrations = EPBRepo.migrationEPB.getAllDeploymentsMigrationByDataEntitySortedByDate( \ 
     169        deploymentMigrations = self.epbRepo.migrationEPB.getAllDeploymentsMigrationByDataEntitySortedByDate( \ 
    185170                                                self._dataEntityMigration, deploymentsLinks) 
    186171        howManydm = 0 
    … …  
    192177            try: 
    193178                ceda_observation = self._processDeploymentMigration(deploymentMigration, howManydm == 1) 
    194                 EPBRepo.moles3EPB.updateCedaObject(cedaObservationCollection, {'member': ceda_observation}) 
     179                #Is a first time process? 
     180                if not hasMOBeenProcessed(deploymentMigration): 
     181                    self.epbRepo.moles3EPB.updateCedaObject(obsColl, {'member': ceda_observation}) 
    195182                                       
    196183            except Exception as ex: 
    197                 exs.append(ex) 
    198             except RuntimeError as er: 
    199                 print er                 
    200         #self.migrationSessions.molesSession.commit()                         
    201                  
     184                exs.append(ex)   
    202185        return exs 

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py

    r8326 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 15 Nov 2011 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from libs.migration.processor.commons import findMolesLineage,\ 
    734    createMO_ResponsiblePartyInfo,\ 
    8     DO_BADC, DO_NEODC, findAuthorsInResource, CEDA,\ 
     35    DO_BADC, DO_NEODC, findAuthorsInResource, \ 
    936    createMD_Identifier, extractSummary, extractQuality, \ 
    10     hasAtomDocumentSameHash, getAtomDocumentHashByMO, extractTitle,\ 
     37    hasMOSameHash, getAtomDocumentHashByMO, extractTitle,\ 
    1138        createCEDA_Result,\ 
    1239    createEX_GeographicBoundingBox, extractGeographicExtentInMigrationDocument, findDownloadLinksInMigrationDocument,\ 
    13     findDOIInMigrationDocument, extractContent, createCI_Citation, createCI_Date, createDate,\ 
     40    extractContent, createCI_Citation, createCI_Date, createDate,\ 
    1441    createTM_Position, createTM_Instant, extractMolesCreationDate,\ 
    1542    createDateTime, isoDateTimeStringToTimeDate, extractMolesProviderID,\ 
    … …  
    2350    createCI_ResponsibleParty, extractUpdateFrequency,\ 
    2451    findLinksInMigrationDocument, findSubTypeInDPT, extractMolesPublishedDate,\ 
    25     createMD_Keywords 
     52    createMD_Keywords, hasMOBeenProcessed 
    2653from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue 
    2754from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 
    28 from libs.migration.MigrationEPB import MigrationEPB 
    2955from libs.epb import EPB 
    3056from libs.migration.processor.deployment_data import DeploymentDataProcessor 
    … …  
    3258from libs.migration.exception.exceptions import NoDataLineage,\ 
    3359    NoAssociatedAuthor 
    34 from MolesManager.moles3epb import Moles3EPB 
    3560from libs.migration.authors import authors 
    3661from logging import StreamHandler 
    … …  
    4267from copy import deepcopy 
    4368import datetime 
    44 from libs.migration.InfodbEPB import InfodbEPB 
    4569from ea_model.ceda_metadatamodel.ceda_result import ceda_curationvalue 
    4670from ea_model.ceda_metadatamodel.ceda_utilities.ceda_reviewfrequencyvalue import CEDA_ReviewFrequencyValue 
    … …  
    5074from ea_model.iso_19115_2006_metadata_corrigendum.maintenance_information.md_maintenancefrequencycode import MD_MaintenanceFrequencyCode 
    5175from ea_model.ceda_metadatamodel.ceda_result.ceda_curationvalue import CEDA_CurationValue 
    52 from libs.migration.processor.EPBRepo import EPBRepo 
    53  
     76 
     77MET_GEO_FEATURE = 'Meteorological geographical features' 
     78ORTHOIMAGERY = 'Orthoimagery' 
     79             
    5480class DeploymentProcessor(object): 
    5581    ''' 
    … …  
    6187    log.addHandler(StreamHandler()) 
    6288    log.setLevel(logging.INFO)     
    63     def __init__(self, dataEntityMigration, deploymentMigration): 
     89    def __init__(self, dataEntityMigration, deploymentMigration, epbRepo): 
    6490        ''' 
    6591            Initializes the class 
    6692            @param dataEntityMigration: a DataEntityMigration instance 
    67             @param deploymentMigration: the DeploymentMigration instance     
     93            @param deploymentMigration: the DeploymentMigration instance 
     94            @param epbRepo: an instance of EPBRepo               
    6895        '''               
    6996        self._dataEntityMigration = dataEntityMigration 
    7097        self._deploymentMigration = deploymentMigration 
    71         self._dataEntityHasSameHash = hasAtomDocumentSameHash(self._dataEntityMigration) and self._dataEntityMigration.doc_hash is not None 
    72         self._deploymentHasSameHash = hasAtomDocumentSameHash(self._deploymentMigration) and self._deploymentMigration.doc_hash is not None 
     98        self.epbRepo = epbRepo 
     99        self._dataEntityHasSameHash = hasMOSameHash(self._dataEntityMigration) 
     100        self._deploymentHasSameHash = hasMOSameHash(self._deploymentMigration) 
     101        self._deploymentHasBeenProcessed = hasMOBeenProcessed(self._deploymentMigration) 
    73102 
    74103    def _existsCEDAasPublisher(self): 
    … …  
    113142        
    114143 
    115     def _assignName(self, observation):         
    116         ''' 
    117             @param relatedPartyInfos: a MO_ResponsiblePartyInfo list 
    118             @return True if the documents changed, False otherwise 
    119         ''' 
    120         whereAreAuthors = self._deploymentMigration         
    121         doc_authors = findAuthorsInResource(self._deploymentMigration)         
    122         if doc_authors['authors'] in [DO_BADC, DO_NEODC]: 
    123             doc_authors = findAuthorsInResource(self._dataEntityMigration) 
    124             whereAreAuthors = self._dataEntityMigration  
    125                  
    126         ind_names = [] 
    127         org_names = [] 
    128         if authors.has_key(doc_authors['authors']): 
    129             tmp_auth = authors[doc_authors['authors']] 
    130             ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) 
    131  
    132             if len(ret['ind']) > 0: 
    133                 i_party = createCI_Individual(name = ret['ind'][0]) 
    134                 observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, [i_party]))                  
    135             if len(ret['org']) > 0: 
    136                 i_party = createMO_Organization(name = ret['org'][0]) 
    137                 observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, [i_party])) 
    138              
    139             if len(ret['ind']) > 1: 
    140                 ind_names.extend(ret['ind'][1:])                                                                         
    141             if len(ret['org']) > 1: 
    142                 org_names.extend(ret['org'][1:]) 
    143             if len(ret['co_ind']) > 0:                                 
    144                 ind_names.extend(ret['co_ind']) 
    145             if len(ret['co_org']) > 0:                                 
    146                 org_names.extend(ret['co_org'])                     
    147                          
    148         else: 
    149             raise NoAssociatedAuthor(doc_authors['authors'], migrationObject = whereAreAuthors) 
    150                        
    151         if doc_authors['contributors'] and authors.has_key(doc_authors['contributors']): 
    152             tmp_auth = authors[doc_authors['contributors']]             
    153             ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) 
    154             ind_names.extend(ret['ind']) 
    155             ind_names.extend(ret['co_ind']) 
    156             org_names.extend(ret['org']) 
    157             org_names.extend(ret['co_org']) 
    158              
    159         i_party = [] 
    160         for nm in ind_names: 
    161             i_party.append(createCI_Individual(name = nm)) 
    162                  
    163         for nm in org_names: 
    164             i_party.append(createMO_Organization(name = nm)) 
    165              
    166         if i_party: 
    167             observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_coinvestigator, i_party)) 
     144 
    168145 
    169146    def _extractIndividualsAndOrganizations(self, tmp_auth, whereAreAuthors): 
    … …  
    204181            print e 
    205182        return ret 
    206  
    207     def _assignLineage(self, observation): 
    208         data_lineage = findMolesLineage(self._dataEntityMigration) 
    209         if data_lineage is None: 
    210             raise NoDataLineage(self._dataEntityMigration) 
    211         observation.dataLineage = data_lineage         
    212          
    213     def _assignTitle(self, observation):        
    214         doc_title = extractTitle(self._deploymentMigration) 
    215         if doc_title is None: 
    216             doc_title = extractTitle(self._dataEntityMigration) 
    217          
    218         if doc_title.startswith('deployment_') or doc_title.startswith('Deployment_'): 
    219             links = findLinksInMigrationDocument(self._deploymentMigration) 
    220             dptList = links['DPT'] 
    221             if links.has_key('DPT'): 
    222                 doc_title = 'Data from ' + dptList[0]['title'] 
    223                 if len(dptList) > 2: 
    224                     for dpt in dptList[1:-2]: 
    225                         doc_title += ', ' + dpt['title'] 
    226                     if len(dptList) > 1: 
    227                         doc_title += ' and ' + dptList[-1] 
    228                                                          
    229             links = findLinksInDeployment(self._deploymentMigration) 
    230             if links.has_key('OBS'): 
    231                 obsList = [] 
    232                 for obs in links['OBS']: 
    233                     observationStation = EPBRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, obs + '.atom') 
    234                     obsList.append((extractTitle(observationStation), findSubTypeInDPT(observationStation))) 
    235                  
    236                 if obsList[0][1] in ['stationary platform' ,'moving platform', 'ship','aircraft','satellite','computer']: 
    237                     doc_title += ' on ' 
    238                 else :  
    239                     doc_title += ' at ' 
    240                     doc_title += obsList[0][0] 
    241                 if len(obsList) > 2: 
    242                     for obs in obsList[1:-2]: 
    243                         doc_title += ', ' + obs[0] 
    244                     if len(obsList) > 1: 
    245                         doc_title += ' and ' + obsList[-1][0] 
    246              
    247             if links.has_key('ACTIVITY'):              
    248                 for link in links['ACTIVITY']: 
    249                     activity = EPBRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, link + '.atom')                     
    250       
    251                     projSubType = findSubTypeInDPT(activity) 
    252                     doc_title += ' for the ' + extractTitle(activity) 
    253                     if projSubType[0:14] == 'dgActivityData': 
    254                         doc_title += ' ' + projSubType[14:] 
    255                     else : 
    256                         doc_title += ' ' + projSubType 
    257                                                         
    258         else:              
    259             if doc_title[0:10] != 'Data from' : 
    260                 doc_title = "Data from " + doc_title             
    261         auth = createCI_Citation(title = 'ceda_title')                   
    262         identifier = createMD_Identifier(code = doc_title, authority = auth) 
    263         observation.identifier.append(identifier) 
    264  
    265     def _assignDescription(self, observation): 
    266         description = extractSummary(self._deploymentMigration, self._dataEntityMigration) 
    267         if description: 
    268             observation.description = description 
    269          
    270     def _assignQuality(self, observation): 
    271         doc_quality = extractQuality(self._dataEntityMigration) 
    272         doc_date = findUpdatedDate(self._dataEntityMigration) 
    273         ci_dates = []             
    274         if doc_date: 
    275             i_date = createDate(isoDateTimeStringToTimeDate(doc_date))                 
    276             ci_dates.append(createCI_Date(CI_DateTypeCode.cl_revision, date = i_date))             
    277         else: 
    278             i_date = createDate(isoDateTimeStringToTimeDate(datetime.datetime.now())) 
    279             ci_dates.append(createCI_Date(CI_DateTypeCode.cl_creation, date = i_date)) 
    280  
    281         i_specification = createCI_Citation(title = "CEDA Data Quality Statement", date=ci_dates) 
    282         i_dq_result = createDQ_ConformanceResult(doc_quality, True, i_specification) 
    283         i_quality_element = createDQ_Element(i_dq_result)         
    284         observation.resultQuality.append(i_quality_element) 
    285          
     183         
     184        
    286185    def updateObservation(self): 
    287186        return EPB.searchOrCreate(CEDA_Observation, self._deploymentMigration.ceda_observation_id) 
    288          
    289  
    290     def _assignResult(self, observation): 
    291         # TDB - Check that if is an update or not!         
    292         download = findDownloadLinksInMigrationDocument(self._deploymentMigration) 
    293         content = None 
    294         if len(download) == 0: 
    295             download = findDownloadLinksInMigrationDocument(self._dataEntityMigration) 
    296             content = extractContent(self._dataEntityMigration) 
    297         else: 
    298             content = extractContent(self._deploymentMigration)  
    299         for dwn in download: 
    300             int_description = None 
    301             int_applicationProfile = None 
    302             if content.has_key('formats'): 
    303                 #int_applicationProfile = content['formats'] 
    304                 pass 
    305             if dwn['href'].startswith('http://badc.nerc.ac.uk/browse') or dwn['href'].startswith('http://neodc.nerc.ac.uk/browse'): 
    306                 int_description = "download directly from archive"    
    307             i_source = createMO_OnlineResource(linkage = dwn['href'], name = dwn['title'], \ 
    308                                                    function = CI_OnLineFunctionCode.cl_download, \ 
    309                                                    description = int_description, applicationProfile = int_applicationProfile) 
    310              
    311         dataentity_id = '%s__ATOM__%s' % (self._dataEntityMigration.doc_owner, self._dataEntityMigration.doc_name) 
    312         dataentity_id = dataentity_id.replace('.atom', '')            
    313         infodb_de = EPBRepo.infodbEPB.getCedaInfoApp_dataentityByDE_ID(dataentity_id) 
    314         i_logical_path = '/dummy' 
    315         if infodb_de is None: 
    316             i_logical_path = dwn['href'][dwn['href'].index('/browse/') + 7:] 
    317          
    318         if infodb_de and infodb_de.has_key('logical_path'): 
    319             i_logical_path = infodb_de['logical_path'] 
    320          
    321         i_category = ceda_curationvalue.CEDA_CurationValue.cl_a                             
    322         if infodb_de and infodb_de.has_key('category') and infodb_de['category']: 
    323             i_category = CEDA_CurationValue.from_string(infodb_de['category'].lower()) #self._getcurationcategory(infodb_de['category'])                 
    324         observation.result = createCEDA_Result(i_category, i_logical_path, source = [i_source])                                 
    325  
    326  
    327     def _assignGeographicExtent(self, observation): 
    328         # TDB - Check that if is an update or not! 
    329         ge = extractGeographicExtentInMigrationDocument(self._deploymentMigration) 
    330         if not ge: 
    331             ge = extractGeographicExtentInMigrationDocument(self._dataEntityMigration) 
    332         if ge: 
    333             observation.geographicExtent.append(createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south'])) 
    334  
    335     def _assignCreationDate(self, observation): 
    336         # TDB - Check that if is an update or not! 
    337         creation_date = extractMolesPublishedDate(self._deploymentMigration) 
    338         if creation_date is None: 
    339             creation_date = extractMolesCreationDate(self._deploymentMigration) 
    340         py_datetime = isoDateTimeStringToTimeDate(creation_date) 
    341         date_time = createDateTime(py_datetime)     
    342         tm_position = createTM_Position(dateTime8601 = date_time) 
    343         observation.resultTime = createTM_Instant(tm_position) 
    344187 
    345188    def assignDOI(self, observation, doi): 
    … …  
    348191             
    349192            #Check if a doi has been already assigned 
    350             observation = EPBRepo.moles3EPB.loadAttributes(observation, 'identifier') 
     193            observation = self.epbRepo.moles3EPB.loadAttributes(observation, 'identifier') 
    351194            obs_identifier = observation.identifier 
    352195            if obs_identifier: 
    … …  
    372215                i_authority = createCI_Citation("DOI", date = ci_date) 
    373216                identifier = createMD_Identifier(code = doi, authority=i_authority) 
    374                 EPBRepo.moles3EPB.updateCedaObject(observation, {'identifier': identifier}) 
     217                self.epbRepo.moles3EPB.updateCedaObject(observation, {'identifier': identifier}) 
    375218                DeploymentProcessor.log.info("DOI: %s" % (doi))                                   
    376219 
    377     def _assignPublisherCurator(self, observation):         
     220    def _assignKeywords(self, ceda_observation): 
     221        if self._deploymentHasSameHash: 
     222            return 
     223         
     224        provider_id = extractMolesProviderID(self._deploymentMigration) 
     225        i_keywords = [] 
     226        if provider_id == DO_BADC: 
     227            i_keywords.append(MET_GEO_FEATURE) 
     228        if provider_id == DO_NEODC: 
     229            i_keywords.append(ORTHOIMAGERY) 
     230        if len(i_keywords) > 0: 
     231            #Is a first time process? 
     232            if not self._deploymentHasBeenProcessed:  
     233                ceda_observation.keywords.append(createMD_Keywords(i_keywords)) 
     234            else: 
     235                ceda_observation.keywords.keyword = i_keywords           
     236 
     237    def _assignLineage(self, observation): 
     238        if self._deploymentHasSameHash: 
     239            return 
     240         
     241        data_lineage = findMolesLineage(self._dataEntityMigration) 
     242        if data_lineage is None: 
     243            raise NoDataLineage(self._dataEntityMigration) 
     244         
     245        if data_lineage != observation.dataLineage: 
     246            observation.dataLineage = data_lineage   
     247 
     248    def _assignResult(self, observation): 
     249        if self._deploymentHasSameHash and self._dataEntityHasSameHash: 
     250            return 
     251 
     252        i_sources = []                 
     253        download = findDownloadLinksInMigrationDocument(self._deploymentMigration) 
     254        content = None 
     255        if len(download) == 0: 
     256            download = findDownloadLinksInMigrationDocument(self._dataEntityMigration) 
     257            content = extractContent(self._dataEntityMigration) 
     258        else: 
     259            content = extractContent(self._deploymentMigration) 
     260        for dwn in download: 
     261            int_description = None 
     262            int_applicationProfile = None 
     263            if content.has_key('formats'): 
     264                #int_applicationProfile = content['formats'] 
     265                pass 
     266            if dwn['href'].startswith('http://badc.nerc.ac.uk/browse') or dwn['href'].startswith('http://neodc.nerc.ac.uk/browse'): 
     267                int_description = "download directly from archive"     
     268            i_sources.append(createMO_OnlineResource(linkage = dwn['href'], name = dwn['title'], \ 
     269                                                   function = CI_OnLineFunctionCode.cl_download, \ 
     270                                                   description = int_description, applicationProfile = int_applicationProfile)) 
     271             
     272        dataentity_id = '%s__ATOM__%s' % (self._dataEntityMigration.doc_owner, self._dataEntityMigration.doc_name) 
     273        dataentity_id = dataentity_id.replace('.atom', '')            
     274        infodb_de = self.epbRepo.infodbEPB.getCedaInfoApp_dataentityByDE_ID(dataentity_id) 
     275        i_logical_path = '/dummy' 
     276        if infodb_de is None: 
     277            i_logical_path = dwn['href'][dwn['href'].index('/browse/') + 7:] 
     278         
     279        if infodb_de and infodb_de.has_key('logical_path'): 
     280            i_logical_path = infodb_de['logical_path'] 
     281                 
     282        i_category = ceda_curationvalue.CEDA_CurationValue.cl_a                                     
     283        if infodb_de and infodb_de.has_key('category') and infodb_de['category']: 
     284            i_category = CEDA_CurationValue.from_string(infodb_de['category'].lower())         
     285 
     286        if not self._deploymentHasBeenProcessed:                      
     287            observation.result = createCEDA_Result(i_category, i_logical_path, source = i_sources) 
     288            return  
     289         
     290        if observation.result.internalPath != i_logical_path: 
     291            observation.result.internalPath = i_logical_path 
     292             
     293        if observation.result.curationCategory != i_category: 
     294            observation.result.curationCategory = i_category 
     295                         
     296        #Still have to update observation.result.source 
     297 
     298    def _assignPublisherCurator(self, observation): 
     299        if self._deploymentHasSameHash: 
     300            return 
     301            
    378302        provider_id = extractMolesProviderID(self._deploymentMigration) 
    379303        party = None 
    … …  
    403327            party = createMO_Organization(name = 'NERC - UK Solar System Data Centre', contactInfo = [contact]) 
    404328         
    405         if party: 
     329        if party and not self._deploymentHasBeenProcessed: 
    406330            observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_publisher, [party])) 
    407331            observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_custodian, deepcopy([party]))) 
    … …  
    409333            observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_pointofcontact, deepcopy([party]))) 
    410334            observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_metadataowner, deepcopy([party]))) 
    411             observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_curator, deepcopy([party])))                                                             
    412  
    413     def _assignPhenomenonTime(self, observation):         
    414         doc_phenomenon_time = extractMolesTemporalRange(self._deploymentMigration) 
    415          
     335            observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_curator, deepcopy([party]))) 
     336            return         
     337        #Still have to update observation.result.source 
     338 
     339    def _assignQuality(self, observation): 
     340        if self._dataEntityHasSameHash: 
     341            return 
     342                 
     343        doc_quality = extractQuality(self._dataEntityMigration) 
     344        doc_date = findUpdatedDate(self._dataEntityMigration) 
     345        ci_dates = []             
     346        if doc_date: 
     347            i_date = createDate(isoDateTimeStringToTimeDate(doc_date))                 
     348            ci_dates.append(createCI_Date(CI_DateTypeCode.cl_revision, date = i_date))             
     349        else: 
     350            i_date = createDate(isoDateTimeStringToTimeDate(datetime.datetime.now())) 
     351            ci_dates.append(createCI_Date(CI_DateTypeCode.cl_creation, date = i_date)) 
     352 
     353        i_specification = createCI_Citation(title = "CEDA Data Quality Statement", date=ci_dates) 
     354        i_dq_result = createDQ_ConformanceResult(doc_quality, True, i_specification) 
     355        i_quality_element = createDQ_Element(i_dq_result) 
     356         
     357        if not self._deploymentHasBeenProcessed:      
     358            observation.resultQuality.append(i_quality_element) 
     359            return                 
     360        #Still have to update observation.result.source 
     361 
     362    def _assignDescription(self, observation): 
     363        if self._dataEntityHasSameHash and self._deploymentHasSameHash: 
     364            return 
     365         
     366        description = extractSummary(self._deploymentMigration, self._dataEntityMigration) 
     367        if description: 
     368            observation.description = description 
     369 
     370    def _assignTitle(self, observation):   
     371        if self._dataEntityHasSameHash and self._deploymentHasSameHash: 
     372            return 
     373                      
     374        doc_title = extractTitle(self._deploymentMigration) 
     375        if doc_title is None: 
     376            doc_title = extractTitle(self._dataEntityMigration) 
     377         
     378        if doc_title.startswith('deployment_') or doc_title.startswith('Deployment_'): 
     379            links = findLinksInMigrationDocument(self._deploymentMigration) 
     380            dptList = links['DPT'] 
     381            if links.has_key('DPT'): 
     382                doc_title = 'Data from ' + dptList[0]['title'] 
     383                if len(dptList) > 2: 
     384                    for dpt in dptList[1:-2]: 
     385                        doc_title += ', ' + dpt['title'] 
     386                    if len(dptList) > 1: 
     387                        doc_title += ' and ' + dptList[-1] 
     388                                                         
     389            links = findLinksInDeployment(self._deploymentMigration) 
     390            if links.has_key('OBS'): 
     391                obsList = [] 
     392                for obs in links['OBS']: 
     393                    observationStation = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, obs + '.atom') 
     394                    obsList.append((extractTitle(observationStation), findSubTypeInDPT(observationStation))) 
     395                 
     396                if obsList[0][1] in ['stationary platform' ,'moving platform', 'ship','aircraft','satellite','computer']: 
     397                    doc_title += ' on ' 
     398                else :  
     399                    doc_title += ' at ' 
     400                    doc_title += obsList[0][0] 
     401                if len(obsList) > 2: 
     402                    for obs in obsList[1:-2]: 
     403                        doc_title += ', ' + obs[0] 
     404                    if len(obsList) > 1: 
     405                        doc_title += ' and ' + obsList[-1][0] 
     406             
     407            if links.has_key('ACTIVITY'):              
     408                for link in links['ACTIVITY']: 
     409                    activity = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, link + '.atom')                     
     410      
     411                    projSubType = findSubTypeInDPT(activity) 
     412                    doc_title += ' for the ' + extractTitle(activity) 
     413                    if projSubType[0:14] == 'dgActivityData': 
     414                        doc_title += ' ' + projSubType[14:] 
     415                    else : 
     416                        doc_title += ' ' + projSubType                                                     
     417        else:              
     418            if doc_title[0:10] != 'Data from' : 
     419                doc_title = "Data from " + doc_title             
     420        auth = createCI_Citation(title = 'ceda_title')                   
     421        identifier = createMD_Identifier(code = doc_title, authority = auth) 
     422         
     423        if not self._deploymentHasBeenProcessed:  
     424            observation.identifier.append(identifier) 
     425            return             
     426        #Still have to update observation.identifier           
     427 
     428    def _assignGeographicExtent(self, observation): 
     429        if self._dataEntityHasSameHash and self._deploymentHasSameHash: 
     430            return   
     431         
     432        ge = extractGeographicExtentInMigrationDocument(self._deploymentMigration) 
     433        if not ge: 
     434            ge = extractGeographicExtentInMigrationDocument(self._dataEntityMigration) 
     435        geographicExtent = createEX_GeographicBoundingBox(ge['east'], ge['north'], ge['west'], ge['south']) 
     436         
     437        if not self._deploymentHasBeenProcessed: 
     438            observation.geographicExtent.append(geographicExtent) 
     439            return           
     440        #Still have to update observation.geographicExtent 
     441             
     442    def _assignCreationDate(self, observation): 
     443        if self._deploymentHasSameHash: 
     444            return   
     445         
     446        creation_date = extractMolesPublishedDate(self._deploymentMigration) 
     447        if creation_date is None: 
     448            creation_date = extractMolesCreationDate(self._deploymentMigration) 
     449        py_datetime = isoDateTimeStringToTimeDate(creation_date) 
     450        date_time = createDateTime(py_datetime)     
     451        tm_position = createTM_Position(dateTime8601 = date_time) 
     452         
     453        if not self._deploymentHasBeenProcessed:  
     454            observation.resultTime = createTM_Instant(tm_position) 
     455            return         
     456        #Still have to update observation.geographicExtent 
     457 
     458    def _assignPhenomenonTime(self, observation):   
     459        if self._deploymentHasSameHash: 
     460            return   
     461               
     462        doc_phenomenon_time = extractMolesTemporalRange(self._deploymentMigration)         
    416463        if doc_phenomenon_time: 
     464            pt = None 
    417465            if '/' in doc_phenomenon_time: 
    418466                period = doc_phenomenon_time.split('/') 
    … …  
    425473                end_tm_instant = createTM_Instant(end_position) 
    426474                 
    427                 observation.phenomenonTime = createTM_Period(begin_tm_instant, end_tm_instant) 
     475                pt = createTM_Period(begin_tm_instant, end_tm_instant) 
    428476            else: 
    429                 observation.phenomenonTime = createTM_Position(date8601 = createDate(isoDateStringToTimeDate(doc_phenomenon_time)))          
    430  
     477                pt = createTM_Position(date8601 = createDate(isoDateStringToTimeDate(doc_phenomenon_time))) 
     478             
     479            if not self._deploymentHasBeenProcessed: 
     480                observation.phenomenonTime = pt 
     481        #Still have to update observation.phenomenonTime                 
     482                 
    431483    def _assignPermission(self, observation): 
     484        if self._deploymentHasSameHash and self._dataEntityHasSameHash: 
     485            return  
     486         
    432487        access_link = findAccessLinksInMigrationDocument(self._deploymentMigration) 
    433488        dwn_link = findDownloadLinksInMigrationDocument(self._deploymentMigration) 
    … …  
    438493        i_use_limitation = [] 
    439494         
     495        permission = None 
    440496        if len(access_link) == 0: 
    441497            if len(dwn_link) == 0: 
    … …  
    449505                i_use_limitation.append("Access to these data is restricted. To obtain access please apply for access at: %s" % (access_link[0]['href'])) 
    450506                i_accessConstraints.append(MD_RestrictionCode.cl_restricted) 
    451                 observation.permission = createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints)                 
     507                observation.permission = createMD_LegalConstraints(useLimitation = i_use_limitation, accessConstrains = i_accessConstraints) 
     508                 
     509        if not self._deploymentHasBeenProcessed: 
     510            observation.permission = permission        
     511        #Still have to update observation.permission 
    452512                                     
    453513        '''                                 
    … …  
    455515        if not contentDict.has_key('access-restricted'): 
    456516            contentDict = extractContent(self._dataEntityMigration) 
    457         ''' 
    458              
     517        '''             
     518 
    459519    def _assignMoles2Link(self, ceda_observation): 
     520        if self._deploymentHasSameHash: 
     521            return  
     522                 
    460523        i_code = 'http://badc.nerc.ac.uk/view/%s__ATOM__%s' % (self._deploymentMigration.doc_owner, self._deploymentMigration.doc_name) 
    461524        i_code = i_code.replace('.atom', '') 
    … …  
    463526        i_authority = createCI_Citation('moles2url') 
    464527        identifier = createMD_Identifier(code = i_code, authority = i_authority) 
    465         ceda_observation.identifier.append(identifier) 
    466      
     528        if not self._deploymentHasBeenProcessed:              
     529            ceda_observation.identifier.append(identifier) 
     530        #Still have to update observation.permission             
     531 
     532    def _assignInternalReview(self, ceda_observation): 
     533        if self._deploymentHasBeenProcessed:              
     534            return 
     535                
     536        i_party = createCI_Individual(name = 'Graham Parton') 
     537        i_reviewer = createMO_ResponsiblePartyInfo(MO_RoleValue.cl_processor, [i_party]) 
     538        ceda_observation.internalReview.append( \ 
     539            createCEDA_Review(reviewer=i_reviewer, reviewFrequency=CEDA_ReviewFrequencyValue.cl_yearly, \ 
     540                              reviewStatus=CEDA_ReviewStatusValue.cl_required))   
     541 
    467542    def _assignLanguage(self, ceda_observation): 
     543        if self._deploymentHasBeenProcessed:              
     544            return 
     545         
    468546        i_date_stamp = createDate(datetime.datetime.now()) 
    469547        #i_contact = createCI_Citation("", date = i_date_stamp) 
    470548        i_contact = createCI_ResponsibleParty(CI_RoleCode.cl_user) 
    471549        ceda_observation.metadata = createMD_Metadata(date_stamp=i_date_stamp, contact = [i_contact], language = "English") 
     550                         
     551    def _processResultAccumulation(self, ceda_observation):   
     552        if self._dataEntityHasSameHash:              
     553            return                       
     554 
     555            updateFrequency = extractUpdateFrequency(self._dataEntityMigration) 
     556            if updateFrequency: 
     557                resultAccumulation = MD_MaintenanceFrequencyCode.from_string(updateFrequency) 
     558                if not self._deploymentHasBeenProcessed: 
     559                    self.epbRepo.moles3EPB.updateCedaObject(ceda_observation, {'resultAccumulation': resultAccumulation})              
     560                    return 
     561        #Still have to update observation.permission 
     562                     
     563    def _assignName(self, observation):         
     564        ''' 
     565            @param relatedPartyInfos: a MO_ResponsiblePartyInfo list 
     566            @return True if the documents changed, False otherwise 
     567        ''' 
     568        if self._deploymentHasSameHash and self._dataEntityHasSameHash: 
     569            return  
     570         
     571        whereAreAuthors = self._deploymentMigration         
     572        doc_authors = findAuthorsInResource(self._deploymentMigration)         
     573        if doc_authors['authors'] in [DO_BADC, DO_NEODC]: 
     574            doc_authors = findAuthorsInResource(self._dataEntityMigration) 
     575            whereAreAuthors = self._dataEntityMigration  
     576                 
     577        ind_names = [] 
     578        org_names = [] 
     579        if authors.has_key(doc_authors['authors']): 
     580            tmp_auth = authors[doc_authors['authors']] 
     581            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) 
     582 
     583            if len(ret['ind']) > 0: 
     584                i_party = createCI_Individual(name = ret['ind'][0]) 
     585                observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, [i_party]))                  
     586            if len(ret['org']) > 0: 
     587                i_party = createMO_Organization(name = ret['org'][0]) 
     588                observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, [i_party])) 
     589             
     590            if len(ret['ind']) > 1: 
     591                ind_names.extend(ret['ind'][1:])                                                        
     592            if len(ret['org']) > 1: 
     593                org_names.extend(ret['org'][1:]) 
     594            if len(ret['co_ind']) > 0:                                 
     595                ind_names.extend(ret['co_ind']) 
     596            if len(ret['co_org']) > 0:                                 
     597                org_names.extend(ret['co_org'])                     
     598                         
     599        else: 
     600            raise NoAssociatedAuthor(doc_authors['authors'], migrationObject = whereAreAuthors) 
     601                       
     602        if doc_authors['contributors'] and authors.has_key(doc_authors['contributors']): 
     603            tmp_auth = authors[doc_authors['contributors']]             
     604            ret = self._extractIndividualsAndOrganizations(tmp_auth, whereAreAuthors) 
     605            ind_names.extend(ret['ind']) 
     606            ind_names.extend(ret['co_ind']) 
     607            org_names.extend(ret['org']) 
     608            org_names.extend(ret['co_org']) 
     609             
     610        i_party = [] 
     611        for nm in ind_names: 
     612            i_party.append(createCI_Individual(name = nm)) 
     613                 
     614        for nm in org_names: 
     615            i_party.append(createMO_Organization(name = nm)) 
     616             
     617        if i_party: 
     618            if not self._deploymentHasBeenProcessed: 
     619                observation.relatedParty.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_coinvestigator, i_party)) 
     620                return 
     621        #Still have to update observation.permission 
    472622     
    473     def _assignInternalReview(self, ceda_observation): 
    474         i_party = createCI_Individual(name = 'Graham Parton') 
    475         i_reviewer = createMO_ResponsiblePartyInfo(MO_RoleValue.cl_processor, [i_party]) 
    476         ceda_observation.internalReview.append( \ 
    477             createCEDA_Review(reviewer=i_reviewer, reviewFrequency=CEDA_ReviewFrequencyValue.cl_yearly, \ 
    478                               reviewStatus=CEDA_ReviewStatusValue.cl_required))    
    479      
    480     def _assignUpdateFrequency(self, ceda_observation): 
    481         updateFrequency = extractUpdateFrequency(self._deploymentMigration) 
    482         if updateFrequency: 
    483             ceda_observation.resultAccumulation = MD_MaintenanceFrequencyCode.from_string(updateFrequency) 
    484  
    485     def _assignKeywords(self, ceda_observation): 
    486         provider_id = extractMolesProviderID(self._deploymentMigration) 
    487         i_keywords = [] 
    488         if provider_id == DO_BADC: 
    489             i_keywords.append('Meteorological geographical features') 
    490         if provider_id == DO_NEODC: 
    491             i_keywords.append('Orthoimagery') 
    492         if len(i_keywords) > 0: 
    493             ceda_observation.keywords.append(createMD_Keywords(i_keywords))             
    494      
    495     def _create(self): 
    496         ceda_observation = CEDA_Observation() 
     623    def _execute(self, ceda_observation): 
    497624        self._assignKeywords(ceda_observation) 
    498625        self._assignLineage(ceda_observation) 
    499626        self._assignResult(ceda_observation) 
    500         self._assignPublisherCurator(ceda_observation) 
    501                          
    502         ceda_observation.publicationState = MO_PublicationStateValue.cl_working 
     627        self._assignPublisherCurator(ceda_observation)                         
    503628        self._assignQuality(ceda_observation)         
    504629        self._assignDescription(ceda_observation) 
    505630        self._assignTitle(ceda_observation)     
    506         self._assignGeographicExtent(ceda_observation)     
    507         #self._assignDOI(ceda_observation)         
     631        self._assignGeographicExtent(ceda_observation)                 
    508632        self._assignCreationDate(ceda_observation)         
    509633        self._assignPhenomenonTime(ceda_observation)         
    … …  
    512636        self._assignInternalReview(ceda_observation)         
    513637        self._assignLanguage(ceda_observation) 
    514         self._assignUpdateFrequency(ceda_observation)     
    515         self._assignName(ceda_observation)   
    516         EPBRepo.moles3EPB.persistInstance(ceda_observation)         
    517         EPBRepo.migrationEPB.updateMigrationObject(self._deploymentMigration, {'ceda_observation_id': ceda_observation.id}) 
    518          
    519         #Adds the CedaGUID         
    520         ceda_guid = CedaGUID() 
    521         ceda_guid.id = calculateHash(self._deploymentMigration.depl_id) 
    522         ceda_guid.ceda_observation = ceda_observation.id 
    523         EPBRepo.moles3EPB.persistInstance(ceda_guid) 
    524         DeploymentProcessor.log.info("GUID for this Observation: %s" % (ceda_guid.id)) 
    525                  
    526         #process the CEDA_Observation.procedure 
    527         deploymentDataProcessor = DeploymentDataProcessor(self._deploymentMigration) 
    528          
    529         links = findLinksInDeployment(self._deploymentMigration) 
    530         procedure = deploymentDataProcessor.createProcess(links) 
    531          
    532         #Temporary commented because CEDA_Project.subProject is not correctly mapped to the DB 
    533         project = deploymentDataProcessor.createProject(links)  
    534         EPBRepo.moles3EPB.updateCedaObject(ceda_observation, {'procedure': procedure, 'inSupportOf': project}) 
     638        self._processResultAccumulation(ceda_observation)             
     639        self._assignName(ceda_observation) 
     640        #self._assignDOI(ceda_observation)         
     641         
     642        #Is a first time process? 
     643        if not hasMOBeenProcessed(self._deploymentMigration): 
     644            ceda_observation.publicationState = MO_PublicationStateValue.cl_working             
     645            docHash = getAtomDocumentHashByMO(self._dataEntityMigration) 
     646            self.epbRepo.moles3EPB.persistInstance(ceda_observation)         
     647            self.epbRepo.migrationEPB.updateMigrationObject(self._deploymentMigration, \ 
     648                {'ceda_observation_id': ceda_observation.id, 
     649                 'doc_hash': docHash})                 
     650         
     651        #Has a proper CEDAGUID? 
     652        if self.epbRepo.moles3EPB.retrieveGUIDFromInstance(ceda_observation) is None:        
     653            ceda_guid = CedaGUID() 
     654            ceda_guid.id = calculateHash(self._deploymentMigration.depl_id) 
     655            ceda_guid.ceda_observation = ceda_observation.id 
     656            self.epbRepo.moles3EPB.persistInstance(ceda_guid) 
     657            DeploymentProcessor.log.info("GUID for this Observation: %s" % (ceda_guid.id)) 
     658 
     659 
     660        if not self._deploymentHasBeenProcessed:                 
     661            deploymentDataProcessor = DeploymentDataProcessor(self._deploymentMigration, self.epbRepo)                 
     662            procedure = deploymentDataProcessor.createProcess() 
     663            project = deploymentDataProcessor.createProject()  
     664            self.epbRepo.moles3EPB.updateCedaObject(ceda_observation, {'procedure': procedure, 'inSupportOf': project}) 
     665 
     666        #Still have to update observation.procedure 
     667        #Still have to update observation.project 
    535668         
    536669        return ceda_observation 
    537  
    538     def _getObservation(self): 
    539         return EPBRepo.moles3EPB.search(CEDA_Observation, self._deploymentMigration.ceda_observation_id) 
    540              
    541     def _update(self): 
    542         if hasAtomDocumentSameHash(self._deploymentMigration): 
    543             return self._getObservation() 
    544         else: 
    545             return self.updateObservation() 
    546670         
    547671    def process(self): 
    … …  
    549673        #Moles3 object exists...  
    550674        if self._deploymentMigration.ceda_observation_id: 
    551             ceda_observation = self._update() 
     675            ceda_observation = self.epbRepo.moles3EPB.search(CEDA_Observation, self._deploymentMigration.ceda_observation_id) 
    552676        else: 
    553677            #... does not exist so create it 
    554             ceda_observation = self._create() 
     678            ceda_observation =  ceda_observation = CEDA_Observation() 
    555679     
    556         if not self._deploymentHasSameHash: 
    557             self._deploymentMigration.doc_hash = getAtomDocumentHashByMO(self._deploymentMigration) 
    558             self._commitMigration() 
    559              
     680        self._execute(ceda_observation)     
    560681        return ceda_observation 

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment_data.py

    r8325 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 13 Feb 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from libs.migration.processor.commons import findSubTypeInDPT,\ 
    734    createCEDA_Processing, createCEDA_Instrument, createCEDA_Project,\ 
    835    findSummary, findDocumentationInMigrationDocument, createCI_Citation,\ 
    9     createMO_OnlineResource, hasAtomDocumentSameHash 
    10 from libs.migration.MigrationEPB import MigrationEPB 
    11 from MolesManager.moles3epb import Moles3EPB 
     36    createMO_OnlineResource, findLinksInDeployment 
    1237from ea_model.moles3_4.utilities.mo_publicationstatevalue import MO_PublicationStateValue 
    13 from libs.migration.processor.EPBRepo import EPBRepo 
    1438 
    1539class DeploymentDataProcessor(object): 
    1640     
    17     def __init__(self, deploymentMigration): 
     41    def __init__(self, deploymentMigration, epbRepo): 
     42        ''' 
     43            @param epbRepo: an instance of EPBRepo 
     44        '''         
    1845        self._deploymentMigration = deploymentMigration 
     46        self.epbRepo = epbRepo         
     47        self._links = findLinksInDeployment(self._deploymentMigration) 
    1948         
    2049    def _commitDeploymentMigration(self, associateWithCedaObservation, dataProductionTool, dataProductionToolField): 
    21         EPBRepo.moles3EPB.persistInstance(associateWithCedaObservation) 
    22         EPBRepo.migrationEPB.updateMigrationObject(dataProductionTool, {dataProductionToolField: associateWithCedaObservation.id}) 
     50        self.epbRepo.moles3EPB.persistInstance(associateWithCedaObservation) 
     51        self.epbRepo.migrationEPB.updateMigrationObject(dataProductionTool, {dataProductionToolField: associateWithCedaObservation.id}) 
    2352                                    
    2453         
    25     def createProject(self, links): 
     54    def createProject(self): 
    2655        project = None 
    27         if links.has_key('ACTIVITY'): 
    28             for link in links['ACTIVITY']: 
    29                 activity = EPBRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, link + '.atom') 
     56        if self._links.has_key('ACTIVITY'): 
     57            for link in self._links['ACTIVITY']: 
     58                activity = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, link + '.atom') 
    3059                i_abstract = findSummary(activity) 
    3160                doc_link = findDocumentationInMigrationDocument(activity) 
    … …  
    4069        return project 
    4170         
    42     def createProcess(self, links): 
     71    def createProcess(self): 
    4372        associateWithCedaObservation = None 
    4473        hasCedaProcessing = False 
    4574        hasCedaComposite = False 
    4675        hasCedaAcquisition = False 
    47         for dpt in links['DPT']: 
    48             dataProductionTool = EPBRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, dpt + '.atom') 
     76        for dpt in self._links['DPT']: 
     77            dataProductionTool = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, dpt + '.atom') 
    4978             
    5079            #has the document changed?             
    51             #if hasAtomDocumentSameHash(dataProductionTool): 
     80            #if hasMOSameHash(dataProductionTool): 
    5281            #    dataProductionTool.doc_hash = getAtomDocumentHashByMO(self._deploymentMigration) 
    5382            #    self._migrationSessions.migrationSession.commit() 
    … …  
    79108                 
    80109        #if not a DPT.subType == 'model' then.... 
    81         for obs in links['OBS']: 
    82             observationStation = EPBRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, obs + '.atom') 
     110        for obs in self._links['OBS']: 
     111            observationStation = self.epbRepo.migrationEPB.getDeploymentDataMigrationByName(self._deploymentMigration, obs + '.atom') 
    83112  
    84113            #has the document changed?             
    85             #if hasAtomDocumentSameHash(observationStation): 
     114            #if hasMOSameHash(observationStation): 
    86115            #    observationStation.doc_hash = getAtomDocumentHashByMO(self._deploymentMigration) 
    87116            #    self._migrationSessions.migrationSession.commit() 

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py

    r8325 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 15 Nov 2011 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from libs.migration.processor.commons import docStatus, getCollectionRefs,\ 
    … …  
    1138from libs.migration.db.classes import DeploymentDataMigration,\ 
    1239    DeploymentsMigration, DataEntityMigration 
    13 from libs.migration.processor.EPBRepo import EPBRepo 
    1440 
    1541class LoadResources(object): 
     42 
     43    def __init__(self, epbRepo): 
     44        ''' 
     45            @param epbRepo: an instance of EPBRepo 
     46        ''' 
     47        self.epbRepo = epbRepo 
    1648 
    1749    def process(self): 
    … …  
    2658        return ex 
    2759 
    28     def _createMigrationObject(self, migrationClass, doc_id, docName, docOwner, docStatus, docHash, docCreation): 
     60    def _createMigrationObject(self, migrationClass, doc_id, docName, docOwner, docStatus, docCreation): 
    2961        migrationObject = None 
    3062        if migrationClass == DeploymentsMigration: 
    … …  
    4274        migrationObject.doc_name = docName 
    4375        migrationObject.doc_owner = docOwner 
    44         migrationObject.doc_status = docStatus 
    45         migrationObject.doc_hash = docHash                       
     76        migrationObject.doc_status = docStatus                       
    4677        migrationObject.doc_creation = stringToTimestamp(docCreation) 
    4778        try: 
    48             EPBRepo.migrationEPB.persistInstance(migrationObject) 
     79            self.epbRepo.migrationEPB.persistInstance(migrationObject) 
    4980        except Exception as e: 
    5081            print e 
    … …  
    82113            @param docOwner: a common.docOwner item    
    83114        """ 
    84            
    85         docHash = getAtomDocumentHash(docStatus, docType, docOwner, docName) 
     115          
     116        #The docHash has to be set/check when the document is processed!   
     117        #docHash = getAtomDocumentHash(docStatus, docType, docOwner, docName) 
    86118        xmlDocument = getAtomDocumentAsElementtree(docStatus, docType, docOwner, docName) 
    87119        doc_id = findID(xmlDocument) 
    88120          
    89121        #The document has been already classified 
    90         if doc_id is None or EPBRepo.migrationEPB.search(migrationClass, doc_id) is not None: 
     122        if doc_id is None or self.epbRepo.migrationEPB.search(migrationClass, doc_id) is not None: 
    91123            return 
    92124         
    … …  
    94126        #The document shall be classified                                 
    95127        docCreation = self._extractCreationDate(xmlDocument)        
    96         self._createMigrationObject(migrationClass, doc_id, docName, docOwner, docStatus, docHash, docCreation) 
     128        self._createMigrationObject(migrationClass, doc_id, docName, docOwner, docStatus, docCreation) 
    97129 
    98130 

  • mauRepo/MolesManager/trunk/src/libs/migration/processor/migrationProcess.py

    r8333 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 10 Jan 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from libs.migration.processor.dataEntity import DataEntityProcessor 
    7 from libs.migration.MigrationEPB import MigrationEPB 
    834import logging 
    935from logging import StreamHandler 
    10 from libs.migration.processor.EPBRepo import EPBRepo 
    1136 
    1237class MigrationProcess(): 
    … …  
    1843        Manages the migration process from Moles2 to Moles3. 
    1944    """ 
    20     def __init__(self): 
    21         pass 
     45    def __init__(self, epbRepo): 
     46        ''' 
     47            @param epbRepo: an instance of EPBRepo 
     48        ''' 
     49        self.epbRepo = epbRepo 
    2250     
    2351    def process(self, dataEntityObject = None): 
    … …  
    3159        """ 
    3260        exs = [] 
    33         data_ents = EPBRepo.migrationEPB.getDataEntityMigrationbyPath(dataEntityObject) 
    34         """ 
    35         #Does the migrationObject exists? 
    36         if dataEntityObject: 
    37             data_ents.append(MigrationEPB.getDataEntityMigrationbyPath(dataEntityObject, session = self.migrationSessions.migrationSession)) 
    38         else: #then process all the DataEntities 
    39             data_ents = MigrationEPB.getAllDataEntityMigration(session = self.migrationSessions.migrationSession) 
    40         """ 
     61        data_ents = self.epbRepo.migrationEPB.getDataEntityMigrationbyPath(dataEntityObject) 
    4162             
    4263        #loops over the data entities 
    4364        for dataEntityMigration in data_ents:                          
    44             dep = DataEntityProcessor(dataEntityMigration)             
     65            dep = DataEntityProcessor(dataEntityMigration, self.epbRepo)             
    4566            exs.extend(dep.process()) 
    4667            MigrationProcess.log.info("Done")             
    47         return exs  
    48          
    49     def commitAll(self): 
    50         self._migrationSession.commit() 
    51         self._molesSession.commit() 
    52  
    53     def rollbackAll(self): 
    54         self._migrationSession.rollback() 
    55         self._molesSession.rollback() 
     68        return exs 

  • mauRepo/MolesManager/trunk/src/libs/migration/tests/dropAllTables.sql

    r8323 r8358  
    1 DROP TABLE IF EXISTS angle CASCADE; 
     1DROP TABLE IF EXISTS angle CASCADE; 
    22DROP TABLE IF EXISTS "any" CASCADE; 
    33DROP TABLE IF EXISTS cc_concatenatedoperation CASCADE; 
    … …  
    5454DROP TABLE IF EXISTS ci_series CASCADE; 
    5555DROP TABLE IF EXISTS ci_telephone CASCADE; 
    56 DROP TABLE IF EXISTS class08_4 CASCADE; 
    57 DROP TABLE IF EXISTS class09_1 CASCADE; 
    58 DROP TABLE IF EXISTS class09_2 CASCADE; 
    59 DROP TABLE IF EXISTS class09_3 CASCADE; 
    60 DROP TABLE IF EXISTS class09_4 CASCADE; 
    61 DROP TABLE IF EXISTS class10_1 CASCADE; 
    62 DROP TABLE IF EXISTS class12_1 CASCADE; 
    63 DROP TABLE IF EXISTS class12_2 CASCADE; 
    64 DROP TABLE IF EXISTS class12_3 CASCADE; 
    65 DROP TABLE IF EXISTS class12_4 CASCADE; 
    6656DROP TABLE IF EXISTS cl_classificationcollection CASCADE; 
    6757DROP TABLE IF EXISTS cl_classificationcollection_cv_surfacevaluepair CASCADE; 
    … …  
    552542DROP TABLE IF EXISTS uomscale CASCADE; 
    553543DROP TABLE IF EXISTS url CASCADE; 
    554 DROP TABLE IF EXISTS data_entity_migration CASCADE; 
    555 DROP TABLE IF EXISTS deployment_data_migration CASCADE; 
    556 DROP TABLE IF EXISTS deployments_migration CASCADE; 
    557544DROP TABLE IF EXISTS cv_solidvaluepair_cv_discretecoverage CASCADE; 
     545DROP TABLE IF EXISTS geometries CASCADE; 
     546DROP TABLE IF EXISTS geometry_columns CASCADE; 
     547DROP TABLE IF EXISTS spatial_ref_sys CASCADE; 
     548 
     549 
    558550DROP TABLE IF EXISTS ceda_guid CASCADE; 
     551UPDATE data_entity_migration  
     552        set ceda_observation_coll_id = NULL, doc_hash = NULL; 
    559553 
     554UPDATE deployments_migration  
     555        set ceda_observation_id = NULL, doc_hash = NULL; 
     556         
     557UPDATE deployment_data_migration  
     558        set ceda_processing_id = NULL, ceda_instrument_id = NULL, 
     559        ceda_compositeprocess_id = NULL, ceda_acquisition_id = NULL, 
     560        ceda_project_id = NULL, doc_hash = NULL; 
    560561 
    561562 

  • mauRepo/MolesManager/trunk/src/libs/migration/tests/migrationprocess.py

    r8351 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 7 Mar 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633from unittest import TestCase 
    … …  
    3865        moles3Factory = Moles3EPBFactory(molesDB) 
    3966        migrationFactory = MigrationEPBFactory(migrationDB) 
    40          
    41         EPBRepo.moles3EPB = moles3Factory.createEPB()  
    42         EPBRepo.migrationEPB = migrationFactory.createEPB() 
    43         EPBRepo.infodbEPB = infoFactory.createEPB() 
    44         ''' 
    45         lr = LoadResources() 
    46         ex = lr.process() 
    47         for e in ex: 
    48             print e 
    49         ''' 
     67 
     68        self.epbRepo = EPBRepo(moles3Factory.createEPB() , \ 
     69                               migrationFactory.createEPB(), \ 
     70                               infoFactory.createEPB()) 
     71 
     72        if False: 
     73            lr = LoadResources(self.epbRepo) 
     74            ex = lr.process() 
     75            for e in ex: 
     76                print e 
     77 
    5078    def testMigrationProcess(self):         
    51         mp = MigrationProcess() 
     79        mp = MigrationProcess(self.epbRepo) 
    5280 
    5381        dataEntity = MigrationObject() 

  • mauRepo/MolesManager/trunk/src/libs/migration/tests/testconfig.py

    r8323 r8358  
    11''' 
     2BSD Licence 
     3Copyright (c) 2012, Science & Technology Facilities Council (STFC) 
     4All rights reserved. 
     5 
     6Redistribution and use in source and binary forms, with or without modification,  
     7are permitted provided that the following conditions are met: 
     8 
     9    * Redistributions of source code must retain the above copyright notice,  
     10        this list of conditions and the following disclaimer. 
     11    * Redistributions in binary form must reproduce the above copyright notice, 
     12        this list of conditions and the following disclaimer in the documentation 
     13        and/or other materials provided with the distribution. 
     14    * Neither the name of the Science & Technology Facilities Council (STFC)  
     15        nor the names of its contributors may be used to endorse or promote  
     16        products derived from this software without specific prior written permission. 
     17 
     18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"  
     19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,  
     20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR  
     21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS 
     22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,  
     23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF  
     24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 
     25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 
     26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE  
     27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
     28 
    229Created on 7 Mar 2012 
    330 
    4 @author: mnagni 
     31@author: Maurizio Nagni 
    532''' 
    633MIGRATION_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3' 
    734MOLES3_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/Moles3' 
    835INFO_DB_CONNECTION = 'postgresql://cedainfo:ler239b@bora.badc.rl.ac.uk:5432/cedainfo' 
    9  
    10 #TESTMIGRATION_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/ceda_moles3_test' 
    11 #TESTMOLES3_DB_CONNECTION = 'postgresql://badc:rotyn217m@neptune.badc.rl.ac.uk:5432/ceda_moles3_test' 
    1236 
    1337from libs.migration.db.migrationTables import doTables as doMigration 
Note: See TracChangeset for help on using the changeset viewer.