Changeset 8083


Ignore:
Timestamp:
01/02/12 16:50:51 (8 years ago)
Author:
mnagni
Message:

Verified that ingests MO_RsponsiblePartyInfo
Better implementation for lists in Model2Form but has to solve the visualization problem (forms into forms?)
CEDA_Observation ingestion duplicates each time the same ingested record. To fix (tomorrow?)
Added some tests to migration
Now can the migration client ingest single data_entity

Location:
mauRepo/MolesManager/trunk
Files:
3 added
11 edited

Legend:

Unmodified
Added
Removed
  • mauRepo/MolesManager/trunk/resources/pylintrc

    r8052 r8083  
    299299# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 
    300300# tab). 
    301 indent-string='    ' 
     301indent-string="\t" 
  • mauRepo/MolesManager/trunk/src/MolesManager/forms/date.py

    r8082 r8083  
    1111from sqlalchemy.orm.util import class_mapper 
    1212from sqlalchemy.orm.collections import InstrumentedList 
     13from django.forms.formsets import formset_factory 
    1314     
    1415 
     
    1617    form = Model2Form(obj, mapper) 
    1718    return type('CoreForm', (CoreForm,), form._createFields())     
    18  
    19 def createFields(obj = None, mapper = None): 
    20     form = Model2Form(obj, mapper) 
    21     return form._createFields()   
    2219 
    2320class CoreForm(Form): 
     
    4037 
    4138    def _createFields(self): 
    42         instrList = [] 
    4339        if self._mapper is None: 
    4440            self._mapper = class_mapper(self._clazz) 
    4541        if self._mapper is None: 
    4642            raise Exception("Class not mapped %s" % (self._clazz)) 
    47         if not isinstance(self._instance, InstrumentedList): 
    48             self._loopOnProperties(self._instance) 
    49         else: 
    50             for item in self._instance: 
    51                 instrList.append(self._loopOnProperties(item)) 
    52  
     43        self._loopOnProperties(self._instance, self._mapper) 
    5344        return self._coreFields 
    5445     
    55     def _loopOnProperties(self, instance): 
    56         for clazz_property in self._mapper.iterate_properties: 
     46    def _loopOnProperties(self, instance, mapper): 
     47        for clazz_property in mapper.iterate_properties: 
    5748            try: 
    5849                if isinstance(clazz_property, ColumnProperty): 
     
    7263        if relationship_property.key == 'relatedParty': 
    7364            print relationship_property.key 
     65 
     66        prop = getattr(instance, relationship_property.key)                     
     67        isInstrumentList = isinstance(prop, InstrumentedList) 
     68 
    7469        if (relationship_property.direction is not None): 
    7570            if (relationship_property.direction.name == 'MANYTOONE'): 
    7671                pass 
    7772            elif (relationship_property.direction.name == 'ONETOMANY'): 
    78                 if relationship_property.key.startswith('_'): 
    79                     fields = createFields(getattr(instance, relationship_property.key), relationship_property.mapper) 
    80                     for key, value in fields.iteritems(): 
    81                         self._coreFields[key] = value 
    82                 else:                            
    83                     self._coreFields[relationship_property.key] = createForm(getattr(instance, relationship_property.key), relationship_property.mapper) 
     73                #is a subclass or property multiplicity? 
     74                if relationship_property.key.startswith('_'):                                 
     75                    self._loopOnProperties(prop, relationship_property.mapper) 
     76                else: 
     77                    if isInstrumentList: 
     78                        items = self._processInstrumentList(prop) 
     79                        self._coreFields[relationship_property.key] = items 
     80                    else:                            
     81                        self._coreFields[relationship_property.key] = createForm(prop, relationship_property.mapper) 
     82     
     83    def _processInstrumentList(self, items): 
     84        ret = [] 
     85        formset = None 
     86        if len(items) > 0 : 
     87            formset = formset_factory(createForm(items[0])) 
     88         
     89        for item in items: 
     90            ret.append(createForm(item)) 
     91        return ret 
    8492     
    8593    def _extractValue(self, property_name, instance): 
  • mauRepo/MolesManager/trunk/src/MolesManager/static/templates/cedaObservation.html

    r8054 r8083  
    1414 
    1515        <div class="fieldWrapper"> 
    16                 <label for="id_dataLineage">dataLineage</label> {{ coObs.dataLineage }}  
    17                 <label for="id_relatedParty">relatedParty</label> {{ coObs.relatedParty }} 
     16                <label for="id_dataLineage">dataLineage</label> {{ coObs.dataLineage }}                  
     17                <label for="id_relatedParty">relatedParty</label> 
     18                        <ul>     
     19                        {% for respInfo in coObs.relatedParty %}                         
     20                                <li>{{ respInfo.role }}</li> 
     21                        {% endfor %} 
     22                        </ul>            
    1823        </div> 
    1924</form> 
  • mauRepo/MolesManager/trunk/src/libs/migration/MigrationEPB.py

    r8082 r8083  
    2929 
    3030    @classmethod 
     31    def getDataEntityMigrationbyPath(self, migrationObject, session = None): 
     32        intSession = MigrationEPB._getSession(session) 
     33        return intSession.query(DataEntityMigration).filter(*[EPB.buildFilter('doc_status', migrationObject.doc_status)]).filter(*[EPB.buildFilter('doc_owner', migrationObject.doc_owner)]).filter(*[EPB.buildFilter('doc_name', migrationObject.doc_name)]).first() 
     34 
     35    @classmethod 
    3136    def getDeploymentsMigrationByName(self, migrationObject, doc_name, session = None): 
    3237        intSession = MigrationEPB._getSession(session) 
  • mauRepo/MolesManager/trunk/src/libs/migration/client.py

    r8082 r8083  
    1010import time 
    1111from threading import Timer 
     12from libs.migration.db.classes import MigrationObject 
     13from libs.migration.processor.commons import DS_pUBLISHED, DO_BADC 
    1214 
    1315 
     
    5456#t.stop() 
    5557 
    56 #lr = LoadResources() 
    57 #lr.process() 
     58''' 
     59lr = LoadResources() 
     60mp = MigrationProcess() 
    5861 
    59 #mp = MigrationProcess() 
    60 #mp.process() 
     62mo = MigrationObject() 
     63mo.doc_status = DS_pUBLISHED 
     64mo.doc_owner = DO_BADC 
     65mo.doc_name = 'dataent_csip.atom' 
    6166 
    62  
    63  
     67if True: 
     68    lr.process() 
     69    mp.process(mo) 
    6470''' 
    6571 
    66 ''' 
    67  
  • mauRepo/MolesManager/trunk/src/libs/migration/db/classes.py

    r8082 r8083  
    55''' 
    66 
    7 class DataEntityMigration(object): 
     7class MigrationObject(object): 
    88    def __init__(self): 
    99        self.doc_name = None 
    1010        self.doc_owner = None 
    11         self.doc_status = None                     
     11        self.doc_status = None 
     12        self.doc_creation = None         
     13 
     14class DataEntityMigration(MigrationObject): 
     15    def __init__(self):                    
    1216        self.data_ent_id = None 
    1317        self.ceda_observation_coll_id = None 
    14         self.doc_creation = None 
     18        super(DataEntityMigration, self).__init__() 
    1519 
    1620class DeploymentsMigration(object): 
    17     def __init__(self): 
    18         self.doc_name = None      
    19         self.doc_owner = None            
    20         self.doc_status = None         
     21    def __init__(self):         
    2122        self.depl_id = None 
    2223        self.ceda_observation_id = None 
    23         self.doc_creation = None 
     24        super(DeploymentsMigration, self).__init__() 
    2425         
    2526class DeploymentDataMigration(object): 
    26     def __init__(self): 
    27         self.doc_name = None 
    28         self.doc_owner = None   
    29         self.doc_status = None                       
     27    def __init__(self):                      
    3028        self.deployment_data_id = None 
    31         self.doc_creation = None         
     29        super(DeploymentDataMigration, self).__init__()                 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/commons.py

    r8082 r8083  
    3030DO_NEODC = 'neodc.nerc.ac.uk' 
    3131DO_UKSSDC = 'ukssdc.ac.uk' 
     32 
     33CEDA = 'Centre for Environmental Data Archive' 
    3234docOwners = (DO_BADC, DO_NEODC, DO_UKSSDC) 
    3335 
     
    114116    return lineage.text 
    115117 
    116 def findAuthorInResource(dataEntityMigration):     
    117     resourceXML = getAtomDocumentByMO(dataEntityMigration) 
     118def _extractAuthors(authorsCSV): 
     119    if authorsCSV is None: 
     120        return [] 
     121    authors = authorsCSV.split(',') 
     122    for index in range(len(authors)): 
     123        authors[index] = authors[index].strip() 
     124        if len(authors[index]) == 0: 
     125            authors.remove(authors[index]) 
     126    return authors 
     127 
     128def findAuthorsInResource(resourceMigration): 
     129    ''' 
     130        Returns a dictionary with the following keys: 
     131        'authors': a list of string representing the authors 
     132        'contributors': a list of string representing the contributors 
     133    ''' 
     134    ret = {}    
     135    resourceXML = getAtomDocumentByMO(resourceMigration) 
     136    ret['authors'] = _extractAuthors(findAuthorInResource(resourceXML)) 
     137    ret['contributors'] = _extractAuthors(findContributorInResource(resourceXML)) 
     138    return ret 
     139 
     140def findAuthorInResource(resourceXML):  
    118141    author = resourceXML.find('%sauthor/%sname' % (atomNS, atomNS)) 
    119142    return _returnNotNoneText(author) 
     143 
     144def findContributorInResource(resourceXML):  
     145    contributors = resourceXML.find('%scontributor/%sname' % (atomNS, atomNS)) 
     146    return _returnNotNoneText(contributors) 
    120147 
    121148def findID(dataEntXML): 
     
    220247    #return _createMO_ResponsiblePartyInfo(role, name) 
    221248 
    222 def createMO_ResponsiblePartyInfoAsCI_Organization(role, name): 
    223     ''' 
    224         @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
    225         @param names: the CI_Party name 
    226     ''' 
    227     return _createMO_ResponsiblePartyInfo(role, name, CI_Organisation) 
    228  
    229 def createMO_ResponsiblePartyInfoAsCI_Individual(role, name): 
    230     ''' 
    231         @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
    232         @param names: the CI_Party name 
    233     ''' 
    234     return _createMO_ResponsiblePartyInfo(role, name, CI_Individual) 
    235  
    236 def _createMO_ResponsiblePartyInfo(role, name, partyType = CI_Party): 
    237     """ 
    238         @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
    239         @param names: the CI_Party name 
     249def createMO_ResponsiblePartyInfoAsCI_Organization(role, names): 
     250    ''' 
     251        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
     252        @param names: the name assigned to each CI_Party 
     253    ''' 
     254    return _createMO_ResponsiblePartyInfo(role, names, CI_Organisation) 
     255 
     256def createMO_ResponsiblePartyInfoAsCI_Individual(role, names): 
     257    ''' 
     258        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
     259        @param names: the name assigned to each CI_Party 
     260    ''' 
     261    return _createMO_ResponsiblePartyInfo(role, names, CI_Individual) 
     262 
     263def _createMO_ResponsiblePartyInfo(role, names, partyType = CI_Party): 
     264    """ 
     265        @param role: a CI_RoleCode/MO_RoleValue assigned to this ResponsibleParty 
     266        @param names: the name assigned to each CI_Party 
    240267        @param partyType: the CI_Party implementation to use  
    241268    """ 
    242269    mo_responsableInfo = MO_ResponsiblePartyInfo() 
    243     mo_responsableInfo.party = []     
     270    mo_responsableInfo.party = [] 
    244271    mo_responsableInfo.role = role 
    245     ci_org = partyType() 
    246     ci_org.name = name 
    247     mo_responsableInfo.party.append(ci_org) 
    248      
    249     #--------------THIS HAS TO BE FIXED----------- 
    250     # MO_ResponsiblePartyInfo should correctly inherit the role attribute  
    251     # from CI_ResponsibleInfo!!!! 
    252     #mo_responsableInfo._ci_responsibility.role = role 
    253     #--------------THIS HAS TO BE FIXED----------- 
     272    for name in names: 
     273        ci_org = partyType() 
     274        ci_org.name = name 
     275        mo_responsableInfo.party.append(ci_org) 
    254276    return mo_responsableInfo 
    255  
    256  
    257 ''' 
    258 def createMO_ResponsiblePartyInfo(author): 
    259     respParty = MO_ResponsiblePartyInfo() 
    260     respParty.role = MO_RoleValue.cl_author 
    261     mo_organization = MO_Organization() 
    262     respParty.party = [] 
    263     respParty.party.append(mo_organization) 
    264     respParty.party.name = author 
    265     return respParty 
    266 ''' 
    267277 
    268278def createMD_Identifier(code): 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/dataEntity.py

    r8082 r8083  
    5252                new_ceda_obs = deProcessor.process() 
    5353                 
     54                #For the moment the CEDA_ObservationCollation is not correctly persisted in the DB 
     55                #The code below has to be activated when the DB will be fixed 
     56                ''' 
     57                #is the ceda_collection.member still empty? 
     58                if len(ceda_observationCollection.member) == 0: 
     59                    ceda_observationCollection.member.append(new_ceda_obs)                 
     60                    continue                 
     61                 
    5462                for old_ceda_obs in ceda_observationCollection.member: 
    55                     if new_ceda_obs.ceda_observation_id == old_ceda_obs.ceda_observation_id: 
    56                         ceda_observationCollection.member.remove(old_ceda_obs)                     
    57                         new_ceda_obs.append(new_ceda_obs) 
    58                     self._migrationProcess.commitAll()                 
     63                        if new_ceda_obs.id == old_ceda_obs.id: 
     64                            ceda_observationCollection.member.remove(old_ceda_obs)                     
     65                            new_ceda_obs.append(new_ceda_obs) 
     66                        self._migrationProcess.commitAll() 
     67                '''                 
    5968            except NoDataLineage as ex: 
    6069                    self._migrationProcess.rollbackAll() 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/deployment.py

    r8082 r8083  
    66from libs.migration.processor.commons import findMolesLineage,\ 
    77    findAuthorInResource, createMO_ResponsiblePartyInfo, getAtomDocumentByMO,\ 
    8     DO_BADC, DO_NEODC 
     8    DO_BADC, DO_NEODC, findAuthorsInResource, CEDA 
    99from ea_model.moles3_4.utilities.mo_rolevalue import MO_RoleValue 
    10 from libs.migration.exception.exceptions import NoDataLineage 
    1110from ea_model.ceda_metadatamodel.ceda_observation.ceda_observation import CEDA_Observation 
    12 import re 
    1311from MolesManager.moles3epb import Moles3EPB 
    1412from libs.migration.MigrationEPB import MigrationEPB 
     
    4240        return False 
    4341                     
    44     def _extractResponsiblePartyInfo(self, authors): 
     42    def _extractResponsiblePartyInfo(self, authors = None, parsedParties = []): 
    4543        ''' 
    46             @param authors: a list of authors (after the first are considered co-authors) 
     44            @param authors: the deployment's authors  
     45            @param parsedParties: a list to fill with the created MO_ResponsiblePartyInfos 
    4746        ''' 
    48         if authors is None: 
    49             return 
    50          
    51         parsedParties = [] 
    52         if (len(set(authors) & set([DO_BADC, DO_NEODC])) > 0): 
    53             parsedParties.append(self._createResponsibleParty('Centre for Environmental Data Archive', MO_RoleValue.cl_curator)) 
    54             authorsCSV = findAuthorInResource(self._dataEntityMigration) 
    55             authors = self._extractAuthors(authorsCSV) 
    56          
    57         #First is assumend to be the author  
    58         parsedParties.append(self._createResponsibleParty(authors[0], MO_RoleValue.cl_author)) 
    59         if (authors.remove(authors[0])): 
    60             for coauthor in authors.remove(authors[0]): 
    61                 parsedParties.append(self._createResponsibleParty(coauthor, MO_RoleValue.cl_author)) 
    62  
    63         return parsedParties             
    64  
    65     def _extractAuthors(self, authorsCSV): 
    66         newAuthors = [authorsCSV]  
    67         if re.match(',', authorsCSV): 
    68             newAuthors = authorsCSV.split(',') 
    69         return newAuthors 
    70  
    71     def _createResponsibleParty(self, name, role): 
    72         ''' 
    73             @param name: The CI_Party name 
    74             @param role: The assigned role             
    75         ''' 
    76         return createMO_ResponsiblePartyInfo(role, name) 
     47        #First is assumend to be the author 
     48        parsedParties.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, authors[:1]))     
     49        parsedParties.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_coinvestigator, authors[1:]))             
    7750 
    7851    def _addResponsiblePartyInfo(self, oldResponsiblePartyInfos, newResponsiblePartyInfo): 
     
    8154            if oldPartyInfo.role == newResponsiblePartyInfo.role: 
    8255                opi = oldPartyInfo 
    83         if len(oldResponsiblePartyInfos) == 0: 
     56                break 
     57             
     58        if len(oldResponsiblePartyInfos) == 0 or opi is None: 
    8459            oldResponsiblePartyInfos.append(newResponsiblePartyInfo) 
    8560         
     
    133108        ''' 
    134109            @param relatedPartyInfos: a MO_ResponsiblePartyInfo list 
    135         '''         
    136         authorsCSV = findAuthorInResource(self._deploymentMigration) 
    137         if authorsCSV is None: 
    138             return 
     110        '''        
     111        newResponsiblePartyInfos = [] 
    139112         
    140         newAuthors = self._extractAuthors(authorsCSV)  
     113        authors = findAuthorsInResource(self._deploymentMigration) 
    141114         
    142         newResponsiblePartyInfos = self._extractResponsiblePartyInfo(newAuthors)     
     115        if (len(set(authors['authors']) & set([DO_BADC, DO_NEODC])) > 0): 
     116            authors = findAuthorsInResource(self._dataEntityMigration) 
     117             
     118        newResponsiblePartyInfos.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_curator, [CEDA]))          
     119        newResponsiblePartyInfos.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_author, authors['authors'])) 
     120        newResponsiblePartyInfos.append(createMO_ResponsiblePartyInfo(MO_RoleValue.cl_coinvestigator, authors['contributors'])) 
     121                     
    143122        #This methods check prepare the insert/update of the responsiblePartyInfos 
    144123        self._checkResponsiblePartyExist(relatedPartyInfos, newResponsiblePartyInfos) 
    145124        for newResponsiblePartyInfo in newResponsiblePartyInfos: 
    146125            self._addResponsiblePartyInfo(relatedPartyInfos, newResponsiblePartyInfo) 
    147          
    148         ''' 
    149         if not self._existsCEDAasPublisher(): 
    150             rp = self._ceda_observation.relatedParty 
    151             rp.append(createMO_ResponsiblePartyInfoAsCI_Organization(MO_RoleValue.cl_publisher, DeploymentProcessor.publisherName)) 
    152         ''' 
    153126 
    154127    def _assignLineage(self): 
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py

    r8082 r8083  
    144144            Loads all the folders below the atoms ('published', 'working', 'Published') folders 
    145145        """ 
    146         ''' 
    147             docStatus = ('published', 'working', 'Published')  
    148         ''' 
     146         
    149147        for status in docStatus:               
    150148            self._loadDataEntities(status)             
  • mauRepo/MolesManager/trunk/src/libs/migration/processor/migrationProcess.py

    r8082 r8083  
    1414        self._molesSession = Moles3EPB.getNewMolesSession() 
    1515     
    16     def process(self): 
    17         sorted_data_ents = MigrationEPB.getDataEntityMigrationOrderByDate(session = self._migrationSession) 
     16    def process(self, migrationObject = None): 
    1817        exs = [] 
     18        sorted_data_ents = [] 
     19         
     20        if migrationObject: 
     21            sorted_data_ents.append(MigrationEPB.getDataEntityMigrationbyPath(migrationObject, session = self._migrationSession)) 
     22        else: 
     23            sorted_data_ents = MigrationEPB.getDataEntityMigrationOrderByDate(session = self._migrationSession) 
     24             
    1925        #loops over the sorted data entities 
    20         for dataEntityMigration in sorted_data_ents:                 
     26        for dataEntityMigration in [f for f in sorted_data_ents if f != None]:                 
    2127            dep = DataEntityProcessor(dataEntityMigration, self) 
    2228            exs.append(dep.process()) 
Note: See TracChangeset for help on using the changeset viewer.