source: mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py @ 8358

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/mauRepo/MolesManager/trunk/src/libs/migration/processor/loadResources.py@8358
Revision 8358, 6.4 KB checked in by mnagni, 7 years ago (diff)

Major refactoring about migration (now handles better create/update, even if the single updates have to be quite fully implemented)
Added the connection pool from SQLAlchemy

Line 
1'''
2BSD Licence
3Copyright (c) 2012, Science & Technology Facilities Council (STFC)
4All rights reserved.
5
6Redistribution and use in source and binary forms, with or without modification,
7are permitted provided that the following conditions are met:
8
9    * Redistributions of source code must retain the above copyright notice,
10        this list of conditions and the following disclaimer.
11    * Redistributions in binary form must reproduce the above copyright notice,
12        this list of conditions and the following disclaimer in the documentation
13        and/or other materials provided with the distribution.
14    * Neither the name of the Science & Technology Facilities Council (STFC)
15        nor the names of its contributors may be used to endorse or promote
16        products derived from this software without specific prior written permission.
17
18THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
20THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS
22BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
23OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
24SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
26OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29Created on 15 Nov 2011
30
31@author: Maurizio Nagni
32'''
33from libs.migration.processor.commons import docStatus, getCollectionRefs,\
34    findMolesCreationDate, findID, stringToTimestamp, buildExistTypePath,\
35    buildExistDocPath, DT_DATA_ENTITIES, DT_DEPLOYMENTS, DT_DEPLOYMENT_DATA,\
36    getOwnerRefs, getAtomDocumentAsElementtree, getAtomDocumentHash
37from libs.migration.exception.exceptions import NoCreationDate, NoDocumentID
38from libs.migration.db.classes import DeploymentDataMigration,\
39    DeploymentsMigration, DataEntityMigration
40
41class LoadResources(object):
42
43    def __init__(self, epbRepo):
44        '''
45            @param epbRepo: an instance of EPBRepo
46        '''
47        self.epbRepo = epbRepo
48
49    def process(self):
50        """
51            Loads all the folders below the atoms ('published', 'working', 'Published') folders
52        """ 
53        ex = []       
54        for status in docStatus:       
55            ex.extend(self._loadCollections(status, DT_DATA_ENTITIES, DataEntityMigration))
56            ex.extend(self._loadCollections(status, DT_DEPLOYMENTS, DeploymentsMigration))
57            ex.extend(self._loadCollections(status, DT_DEPLOYMENT_DATA, DeploymentDataMigration))                                               
58        return ex
59
60    def _createMigrationObject(self, migrationClass, doc_id, docName, docOwner, docStatus, docCreation):
61        migrationObject = None
62        if migrationClass == DeploymentsMigration:
63            migrationObject = DeploymentsMigration()
64            migrationObject.depl_id = doc_id
65        elif  migrationClass == DataEntityMigration:
66            migrationObject = DataEntityMigration()
67            migrationObject.data_ent_id = doc_id
68        elif migrationClass == DeploymentDataMigration:
69            migrationObject = DeploymentDataMigration()
70            migrationObject.deployment_data_id = doc_id
71         
72        if migrationObject is None:
73            raise Exception("migrationObject is None")
74        migrationObject.doc_name = docName
75        migrationObject.doc_owner = docOwner
76        migrationObject.doc_status = docStatus                     
77        migrationObject.doc_creation = stringToTimestamp(docCreation)
78        try:
79            self.epbRepo.migrationEPB.persistInstance(migrationObject)
80        except Exception as e:
81            print e
82
83    def _loadCollections(self, docStatus, docType, migrationClass):
84        ex = []
85        refs = getCollectionRefs(buildExistTypePath(docStatus, docType))           
86        for ref in refs:
87            docOwner = ref.get('name') 
88            if docOwner.startswith('.'):
89                continue                   
90            ex.extend(self._loadMigrationDocs(docStatus, docOwner, docType, migrationClass))
91
92        return ex
93
94    def _loadMigrationDocs(self, docStatus, docOwner, docType, migrationClass):
95        ex = []
96        refs = getOwnerRefs(docStatus, docType, docOwner)                   
97        for ref in refs:
98            docName = ref.get('name')
99            try:
100                self._processMigrationDoc(docName, docStatus, docOwner, docType, migrationClass)
101            except Exception as e:
102                if hasattr(e, 'value'):
103                    e.value = buildExistDocPath(docStatus, docType, docOwner, docName)
104                ex.append(e)
105        return ex
106
107    def _processMigrationDoc(self, docName, docStatus, docOwner, docType, migrationClass):
108        """
109            Ingests documents like DataEntity, Deployment, Deployment_Data. The document is register in a DB table of the appropriate
110            type (DataEntityMigration, DeploymentsMigration, DeploymentDataMigration).
111            @param docName: the document name
112            @param docStatus: a common.docStatus item
113            @param docOwner: a common.docOwner item   
114        """
115         
116        #The docHash has to be set/check when the document is processed! 
117        #docHash = getAtomDocumentHash(docStatus, docType, docOwner, docName)
118        xmlDocument = getAtomDocumentAsElementtree(docStatus, docType, docOwner, docName)
119        doc_id = findID(xmlDocument)
120         
121        #The document has been already classified
122        if doc_id is None or self.epbRepo.migrationEPB.search(migrationClass, doc_id) is not None:
123            return
124       
125                         
126        #The document shall be classified                               
127        docCreation = self._extractCreationDate(xmlDocument)       
128        self._createMigrationObject(migrationClass, doc_id, docName, docOwner, docStatus, docCreation)
129
130
131    def _extractID(self, xmlDocument):
132        doc_id = findID(xmlDocument) 
133        if doc_id is None:
134            raise NoDocumentID(doc_id)
135        return doc_id
136
137    def _extractCreationDate(self, xmlDocument):
138        creationDate = findMolesCreationDate(xmlDocument)           
139        if creationDate is None:
140            raise NoCreationDate(creationDate)
141        return creationDate
142           
143
144       
Note: See TracBrowser for help on using the repository browser.