Changeset 3862

Timestamp:

07/05/08 17:16:18 (11 years ago)

Author:

cbyrom

Message:

Fix workflow so that transforms are only ran when actually required

• i.e. when the data is not already in the DB + fix a few small bugs

+ add extra logging + wrapper spatiotemporal data in record via get
methods.

Location:

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch

Files:

• PostgresDAO.py (modified) (1 diff)
• PostgresRecord.py (modified) (3 diffs)
• oai_document_ingester.py (modified) (8 diffs)
• run_all_ingest.py (modified) (2 diffs)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

@@ -190 +190 @@
         
         # Work out the relationship between the spatial and temporal data and handle appropriately
-        timeData = self._record.stData.getTemporalData()
-        spatialData = self._record.stData.getSpatialData()
+        timeData = self._record.getTemporalData()
+        spatialData = self._record.getSpatialData()
         
         # check if we have any spatiotemporal data to add; escape if not

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

@@ -74 +74 @@
         self.scn = 1    # system change number - keeps track of number of mods to a particular row
         
-        # do some initial setting up of record
-        self.doRecordTransforms()
-        self.getSpatioTemporalData()
-
+        # spatiotemporal data object
+        self.stData = None
 
     def escapeSpecialCharacters(self, inputString):
@@ -251 +249 @@
 
         # the doc type doesn't exist - so run the xquery
-        setattr(self, attributeName, self.doTransform(xqName))
+        transformedDoc = self.doTransform(xqName)
+        setattr(self, attributeName, transformedDoc)
+        return transformedDoc
         
     
@@ -258 +258 @@
         Return a list of all the available doc types in the record
         '''
-        if len(self._allDocs) > 0:
+        # if the stored docs array is the same size as the array of all doc types
+        # assume all transforms have been done - and just return these
+        if len(self._allDocs) == len(self.documentTypes):
             return self._allDocs
         
         for docType in self.documentTypes:
             self._allDocs.append([docType, self.getDocumentFormat(docType)])
+
         return self._allDocs
+        
+    
+    def getTemporalData(self):
+        '''
+        Retrieves the temporal data for the record; if this hasn't been discovered yet,
+        do the necessary parsing
+        @return: TimeRange object array with temporal data
+        '''
+        if self.stData is None:
+            self.getSpatioTemporalData()
+        
+        return self.stData.getTemporalData()
+        
+    
+    def getSpatialData(self):
+        '''
+        Retrieves the spatial data for the record; if this hasn't been discovered yet,
+        do the necessary parsing
+        @return: Coords object array with spatial data
+        '''
+        if self.stData is None:
+            self.getSpatioTemporalData()
+        
+        return self.stData.getSpatialData()
         
 

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

@@ -139 +139 @@
                         logging.info("Datacentre classified as an NDG data provider")
                 else:
-                        logging.info("Datacentre is not classificied as an NDG data provider")
-                print self.lineSeparator
+                        logging.info("Datacentre is not classified as an NDG data provider")
+                logging.info(self.lineSeparator)
 
                 
@@ -165 +165 @@
                 Display input params for the script
                 '''
-                print "Usage: python oai_document_ingester.py -v|d<datacentre>"
-                print " - where:\n   <datacentre> is the data centre to ingest data from; and"
-                print " -v - verbose mode for output logging"
-                print " -d - debug mode for output logging"
+                logging.info("Usage: python oai_document_ingester.py [OPTION] <datacentre>")
+                logging.info(" - where:\n   <datacentre> is the data centre to ingest data from; and options are:")
+                logging.info(" -v - verbose mode for output logging")
+                logging.info(" -d - debug mode for output logging")
                 sys.exit(2)
 
@@ -177 +177 @@
                 '''
                 self.lineSeparator = "-----------------------------"
-                print self.lineSeparator
-                print "RUNNING: oai_document_ingester.py"               
+                logging.info(self.lineSeparator)
+                logging.info("RUNNING: oai_document_ingester.py")
                 
                 # check for verbose option
@@ -185 +185 @@
                 except getopt.GetoptError, err:
                     # print help information and exit:
-                    print str(err) # will print something like "option -a not recognized"
-                    self.usage()
+                    logging.info(str(err)) # will print something like "option -a not recognized"
                     
                 loggingLevel = logging.WARNING
                 for o, a in opts:
                     if o == "-v":
-                        print " - Verbose mode ON"
+                        logging.info(" - Verbose mode ON")
                         loggingLevel = logging.INFO
                     elif o == "-d":
-                        print " - Debug mode ON"
+                        logging.info(" - Debug mode ON")
                         loggingLevel = logging.DEBUG
                 
@@ -200 +199 @@
                                                     format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
                 
-                print self.lineSeparator
+                logging.info(self.lineSeparator)
                 
                 if datacentre is None:
@@ -230 +229 @@
                 
                 # check harvest dir exists and that there are any records to harvest?
-                dpath = os.path.normpath(os.path.dirname(self._harvest_home))
-                if not os.path.exists(dpath):
+                if not os.path.exists(self._harvest_home):
                         logging.info("Harvest directory for datacentre %s (%s) could not be found - exiting" \
                                                  %(datacentre, self._harvest_home))
@@ -238 +236 @@
                         logging.info("Nothing to harvest this time from %s" %datacentre)
                         return
-                
                 
                 # The directory to put things for a tape backup (should already exist)
@@ -318 +315 @@
                 #fileUtils.cleanDir(self._harvest_home)
                 
-                print self.lineSeparator
-                print "INFO: Number of files processed = %s" %numfilesproc
-                print "INFO: Number of files ingested = %s" %self._no_files_ingested
+                logging.info(self.lineSeparator)
+                logging.info("INFO: Number of files processed = %s" %numfilesproc)
+                logging.info("INFO: Number of files ingested = %s" %self._no_files_ingested)
                 if status == 0:
-                    print "INFO: Procedure oai_document_ingester.py completed"
+                    logging.info("INFO: Procedure oai_document_ingester.py completed")
                 else:
-                    print "ERROR: Procedure oai_document_ingester.py FAILED with status %s" %status
-                print self.lineSeparator
+                    logging.error("ERROR: Procedure oai_document_ingester.py FAILED with status %s" %status)
+                logging.info(self.lineSeparator)
                 
         
 if __name__=="__main__":
+        logging.basicConfig(level=logging.INFO,
+                                                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
         opts, args = getopt.getopt(sys.argv[1:], '-vd')
         if len(args) < 1:

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

@@ -9 +9 @@
 
 lineSeparator = "-----------------------------"
-print lineSeparator
-print "RUNNING: run_all_ingest.py"
-print lineSeparator
 
 # configure logging
 logging.basicConfig(level=logging.INFO,
                     format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
+logging.info(lineSeparator)
+logging.info("RUNNING: run_all_ingest.py")
+logging.info(lineSeparator)
 
 
@@ -42 +42 @@
                 error_counter +=1
                 
-print "run_all_ingest.py complete - processed %s config files" %run_counter
+logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
 if error_counter > 0:           
-    print "WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter
+    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter)