Changeset 3821 for TI01-discovery

Timestamp:

23/04/08 17:24:55 (11 years ago)

Author:

cbyrom

Message:

Fix a few problems - including referencing the xquery libraries; these
have now been added to the ndgUtils egg and are extracted locally and
referenced directly. Also add functionality to deal with the moles -> other
transforms + add new utility methods and tidy up code and add more logging.

Location:

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch

Files:

• FileUtilities.py (modified) (4 diffs)
• PostgresRecord.py (modified) (13 diffs)
• SpaceTimeIngestFromMOLES.py (modified) (1 diff)
• oai_ingest_new2.py (modified) (5 diffs)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/FileUtilities.py

@@ -1 +1 @@
 import os, sys, logging
+from os import makedirs
+from os.path import normpath,dirname,exists,abspath
 
 class FileUtilities:
@@ -10 +12 @@
         Constructor
         '''
+
+    def createFile(self, fileName, content):
+        '''
+        Create a file with the specified name and content
+        @param fileName: name of file to create
+        @param content: content of file to create  
+        '''
+        try:
+            f = open(fileName,'w')
+            f.write(content)
+            f.close()
+        except:
+            sys.exit("ERROR: Problem encountered when creating file, %s" %fileName)
 
 
@@ -25 +40 @@
         from holger@trillke.net 2002/03/18
         '''
-        from os import makedirs
-        from os.path import normpath,dirname,exists,abspath
-    
         logging.info("Creating dir: " + path) 
         dpath = normpath(dirname(path))
@@ -45 +57 @@
         else:
             try:
-                makepath(dir)
+                self.makepath(dir)
             except:
                 sys.exit("Failed at setting up directory, %s" %dir)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

@@ -19 +19 @@
 from ETxmlView import loadET, nsdumb
 import molesReadWrite as MRW
+from ndgUtils.ndgObject import ndgObject
+from FileUtilities import FileUtilities
 
 class PostgresRecord:
@@ -28 +30 @@
     documentTypes = ['MOLES', 'DIF', 'DC', 'MDIP', 'ISO19139']
         
-    def __init__(self, filename, datacentre_groups, datacentre_namespace, discovery_id, xq, docType):
+    def __init__(self, filename, ndg_dataprovider, datacentre_groups, datacentre_namespace, discovery_id, xq, docType):
         logging.info("Setting up Postgres record for file, " + filename)
         self.filename = filename
+    
+        # NB, if we're dealing with an NDG data provider, the details are slightly different
+        if ndg_dataprovider:
+            discObj=ndgObject(discovery_id)
+            self._local_id = discObj.localID
+            self._repository_local_id = discObj.repository
+        else:
+            self._local_id = discovery_id
+            self._repository_local_id = datacentre_namespace
+            
         self._datacentre_groups = datacentre_groups
         self._repository = datacentre_namespace
@@ -40 +52 @@
         self._allDocs = []  # array to store all the transformed docs - for easy retrieval by the DAO
 
+        self._fileUtils = FileUtilities()
+
+        # get the dir of the file - needed by the xquery to use as the target collection
+        tmp = filename.split('/')
+        self._dir = '/'.join(tmp[0:len(tmp)-1])
+        self._shortFilename = tmp[len(tmp)-1]
+        
+        # dir to store a temp copy of the moles file, when produced - for use by other transforms
+        self._molesDir = None
+
         # firstly load contents of file
         self.originalFormat = file(filename).read()
@@ -46 +68 @@
         # DO WE NEED TO DO THIS??
         self.correctedFormat = loadET(self.originalFormat)
-        
-
-        #debugging stuff
-#        self.logger.printOutput("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
-#        print self.correctedFormat
-#        print self.originalFormat.keys()
-#        for i in self.originalFormat: print i.tag
-#        print dir(self.originalFormat)
-#        self.logger.printOutput("vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv")
-        #we use nsdumb in case the namespace causes difficulties ...
-#        helper=nsdumb(self.originalFormat)
-        #print helper
-#        self.id=helper.getText(self.originalFormat,'DatasetIdentifier')
 
         
@@ -69 +78 @@
         
         # do some initial setting up of record
-#        self.setUpRecord()
         self.doRecordTransforms()
         self.getSpatioTemporalData()
@@ -82 +90 @@
         for docType in self.documentTypes:
             self.getDocumentFormat(docType)
+            
         logging.info("Transforms complete")
 
+
+    def createMolesFile(self):
+        '''
+        Check if a moles file exists on the system; if not, assume the moles transform has not
+        been ran and then produce this file - to allow for use in the various xqueries
+        '''
+        logging.info("Creating moles file on system - for use with other xquery transforms")
+        self._molesDir = self._dir + "/moles/"
+        self._fileUtils.setUpDir(self._molesDir)
+        
+        if self._molesFormat is None:
+            self.doMolesTransform()
+            
+        self._fileUtils.createFile(self._molesDir + self._shortFilename, self._molesFormat)
+        logging.info("Moles file created - at %s" %self._molesDir)
+            
 
     def doTransform(self, xQueryType):
@@ -93 +118 @@
         logging.info("Running XQuery transform, " + xQueryType + " to create transformed document")
 
+        # firstly, check if this is a moles -> something else query; if so, ensure there is a valid
+        # moles file available for the transform - and use the correct dir for the xquery collection
+        dir = self._dir
+        if xQueryType.find('moles2') > -1:
+            if self._molesDir is None:
+                self.createMolesFile()
+                
+            dir = self._molesDir
+            
         # get the query and set this up to use properly
-        xquery = self._xq.actual(xQueryType, self.filename, self._repository, self.discovery_id)
+        xquery = self._xq.actual(xQueryType, dir, self._repository_local_id, self._local_id)
 
         # sort out the input ID stuff
@@ -100 +134 @@
         xquery=xquery.replace('repository_localid', self._repository)
 
+        # strip out the eXist reference to the libraries; these files should be available in the
+        # running dir - as set up by oai_ingest.py
+        xquery=xquery.replace('xmldb:exist:///db/xqueryLib/Vocabs/', '')
+        xquery=xquery.replace('xmldb:exist:///db/xqueryLib/Utilities/', '')
+
         # write the query to file, to make it easier to input
         # NB, running directly at the command line leads to problems with the interpretation of $ characters
         xqFile = "currentQuery.xq"
-        f=open(xqFile,'w')
-        f.write(xquery)
-        f.close()
+        self._fileUtils.createFile(xqFile, xquery)
 
         # Now do the transform
-#        xqCommand = "java -cp /home/users/cbyrom/opt/saxonsa/saxon9sa.jar:/home/users/cbyrom/opt/saxonsa net.sf.saxon.Query " + \
         os.putenv ('PATH', ':/usr/java/jdk1.5.0_03/bin:/usr/java/jdk1.5.0_03:/usr/java/jdk1.5.0_03/lib/tools.jar:/usr/local/WSClients/OAIBatch:/usr/local/exist-client/bin:/bin:/usr/bin:.')
         xqCommand = "java -cp saxon9.jar net.sf.saxon.Query " + xqFile
@@ -117 +153 @@
 
         print output
-        print "ss,", status
         if status is not None:
             sys.exit("Failed at running the XQuery")
@@ -163 +198 @@
         tmpDir = os.getcwd() + "/"
         tmpKeywordsDir = os.getcwd() + "/kewordsAdded/"
-        fileUtils = FileUtilities(self.logger)
-        fileUtils.setUpDir(tmpDir)
-        fileUtils.setUpDir(tmpKeywordsDir)
+        self._fileUtils.setUpDir(tmpDir)
+        self._fileUtils.setUpDir(tmpKeywordsDir)
         tmpFile = 'tmpFile.xml'
-        f=open(tmpDir + "/" + tmpFile,'w')
-        f.write(self._molesFormat)
-        f.close()
+        self._fileUtils.createFile(tmpDir + "/" + tmpFile, self._molesFormat)
 
         keywordAdder.main(tmpDir, tmpKeywordsDir, self.datacentre_groups)
@@ -179 +211 @@
         
         # Finally, tidy up temp dirs
-        fileUtils.cleanDir(tmpDir)
-        fileUtils.clearDir(tmpKeywordsDir)
+        self._fileUtils.cleanDir(tmpDir)
+        self._fileUtils.clearDir(tmpKeywordsDir)
         logging.info("Completed adding keywords")
         
@@ -197 +229 @@
         if self._molesFormat is None:
             self.doMolesTransform()
+            self.createMolesFile()
         
         # check the document isn't already defined
@@ -205 +238 @@
                 return doc
         except:
-            logging.info("Creating new transformed document")
+            logging.info("Document not available - creating new transformed document")
 
         # the doc type doesn't exist - so run the xquery

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/SpaceTimeIngestFromMOLES.py

@@ -25 +25 @@
 import db_funcs
 import os
-import PostgresDBUtils
 
 #connect to db (in separate db functions module)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_ingest_new2.py

@@ -34 +34 @@
 import ndgUtils
 from ndgUtils.ndgXqueries import ndgXqueries
-from ndgUtils.ndgObject import ndgObject
 from FileUtilities import FileUtilities
 from PostgresRecord import PostgresRecord
@@ -65 +64 @@
         Add a file to the postgres DB - extracting and storing all the required
         data in the process
+        @param filename: full path of file to add to postgres DB 
         '''
         logging.info("Adding file, " + filename + ", to postgres DB")
         discoveryID = getID(filename)
         
-        # NB, if we're dealing with an NDG data provider, the details are slightly different
-        if NDG_dataProvider:
-                discObj=ndgObject(discoveryID)
-        discoveryID = discObj.localID
-        datacentre_namespace = discObj.repository
-        
         # first of all create a PostgresRecord - this object represents all the data required
         # for a DB entry
-        record = PostgresRecord(filename, datacentre_groups, datacentre_namespace, discoveryID, xq, datacentre_format)
+        record = PostgresRecord(filename, NDG_dataProvider, datacentre_groups, datacentre_namespace, discoveryID, xq, datacentre_format)
 
         # Now create the data access object to interface to the DB
@@ -145 +139 @@
         
         logging.info("datacentre namespace: " + datacentre_namespace)
+        
+        if NDG_dataProvider:
+                logging.info("Datacentre classified as an NDG data provider")
+        else:
+                logging.info("Datacentre is not classificied as an NDG data provider")
         print lineSeparator
         
@@ -160 +159 @@
 print lineSeparator
 print "RUNNING: oai_ingest.py"          
-
-verboseMode = False
-
 
 # check for verbose option
@@ -276 +272 @@
 
 # now set up the required XQueries
+# - NB, extract the xquery libraries locally for easy reference
 xq=ndgXqueries()
+for libFile in xq.xqlib:
+        fileUtils.createFile(libFile, xq.xqlib[libFile])
 
 # Process the resulting files and put the data into the postgres DB