Changeset 3869

Timestamp:

08/05/08 17:18:16 (11 years ago)

Author:

cbyrom

Message:

Add code to delete records if they weren't properly ingested - to allow
clean reruns of the ingest + fix some problems with the keyword adding
code.

Location:

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch

Files:

• PostgresDAO.py (modified) (4 diffs)
• PostgresRecord.py (modified) (3 diffs)
• run_all_ingest.py (modified) (1 diff)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

@@ -146 +146 @@
         '''
         logging.info("Deleting existing spatiotemporal data for record")
-        sqlCmd = "DELETE FROM SPATIAL_TEMPORAL_DATA WHERE original_record_id = " + \
+        sqlCmd = "DELETE FROM SPATIAL_TEMPORAL_DATA WHERE original_document_id = " + \
             str(self._record.db_id) + ";"      
 
@@ -237 +237 @@
         self._record.db_id = id[0][0] 
         logging.info("Original document inserted in Postgres DB")
+            
+    
+    def deleteOriginalRecord(self):
+        '''
+        Delete the original metadata doc from the postgres DB
+        '''
+        logging.info("Deleting original document from Postgres DB")
+        sqlCmd = "SELECT delete_document('" + str(self._record.db_id) + "');" 
+
+        db_funcs.runSQLCommand(self._connection, sqlCmd)
+        logging.info("Original document deleted from Postgres DB")
         
     
@@ -266 +277 @@
         for docType, doc in self._record.getAllDocs():
             sqlCmd = "INSERT INTO TRANSFORMED_DOCUMENT (transformed_document_id, " \
-                "original_record_id, transformed_format, " \
+                "original_document_id, transformed_format, " \
                 "transformed_document, create_date, scn) VALUES (" \
                 "DEFAULT, '" + str(self._record.db_id) + "', '" + \
@@ -287 +298 @@
         for docType, doc in self._record.getAllDocs():
             sqlCmd = "UPDATE TRANSFORMED_DOCUMENT SET transformed_document = '" + doc + \
-                "', update_date = current_timestamp WHERE original_record_id = " + \
+                "', update_date = current_timestamp WHERE original_document_id = " + \
                 str(self._record.db_id) + " AND transformed_format = '" + docType + "';"
 

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

@@ -19 +19 @@
 from FileUtilities import FileUtilities
 from SpatioTemporalData import SpatioTemporalData
+import keywordAdder
 
 class PostgresRecord:
@@ -184 +185 @@
                      %self.docType)
 
+        # now run the appropriate transform and set the attribute
+        setattr(self, "_molesFormat", self.doTransform(xqName))
+
         # add keywords, if required
         if self._datacentre_groups != "":
-            addKeywords()
-
-        # now run the appropriate transform and set the attribute
-        setattr(self, "_molesFormat", self.doTransform(xqName))
+            self.addKeywords()
         
         # escape any apostrophes
@@ -203 +204 @@
         '''
         logging.info("Adding datacentre keywords to moles file")
+
         # NB, use temporary directories to do the keyword additions
-        tmpDir = os.getcwd() + "/"
-        tmpKeywordsDir = os.getcwd() + "/kewordsAdded/"
+        tmpDir = os.getcwd() + "/tmp/"
+        tmpKeywordsDir = os.getcwd() + "/keywordsAdded/"
         self._fileUtils.setUpDir(tmpDir)
         self._fileUtils.setUpDir(tmpKeywordsDir)
         tmpFile = 'tmpFile.xml'
-        self._fileUtils.createFile(tmpDir + "/" + tmpFile, self._molesFormat)
-
-        keywordAdder.main(tmpDir, tmpKeywordsDir, self.datacentre_groups)
-
+        self._fileUtils.createFile(tmpDir + tmpFile, self._molesFormat)
+
+        keywordAdder.main(tmpDir, tmpKeywordsDir, self._datacentre_groups)
+
+        sys.exit()
         # Now load in the converted file
         f=open(tmpKeywordsDir + "/" + tmpFile, 'r')

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

@@ -40 +40 @@
                 oai_document_ingester(datacentre)
             except:
+                logging.error("Exception thrown - detail: ")
+                logging.error(sys.exc_info())
+                logging.info("Continue processing next datacentre config file...")
                 error_counter +=1