Changeset 3869


Ignore:
Timestamp:
08/05/08 17:18:16 (11 years ago)
Author:
cbyrom
Message:

Add code to delete records if they weren't properly ingested - to allow
clean reruns of the ingest + fix some problems with the keyword adding
code.

Location:
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresDAO.py

    r3862 r3869  
    146146        ''' 
    147147        logging.info("Deleting existing spatiotemporal data for record") 
    148         sqlCmd = "DELETE FROM SPATIAL_TEMPORAL_DATA WHERE original_record_id = " + \ 
     148        sqlCmd = "DELETE FROM SPATIAL_TEMPORAL_DATA WHERE original_document_id = " + \ 
    149149            str(self._record.db_id) + ";"       
    150150 
     
    237237        self._record.db_id = id[0][0]  
    238238        logging.info("Original document inserted in Postgres DB") 
     239             
     240     
     241    def deleteOriginalRecord(self): 
     242        ''' 
     243        Delete the original metadata doc from the postgres DB 
     244        ''' 
     245        logging.info("Deleting original document from Postgres DB") 
     246        sqlCmd = "SELECT delete_document('" + str(self._record.db_id) + "');"  
     247 
     248        db_funcs.runSQLCommand(self._connection, sqlCmd) 
     249        logging.info("Original document deleted from Postgres DB") 
    239250         
    240251     
     
    266277        for docType, doc in self._record.getAllDocs(): 
    267278            sqlCmd = "INSERT INTO TRANSFORMED_DOCUMENT (transformed_document_id, " \ 
    268                 "original_record_id, transformed_format, " \ 
     279                "original_document_id, transformed_format, " \ 
    269280                "transformed_document, create_date, scn) VALUES (" \ 
    270281                "DEFAULT, '" + str(self._record.db_id) + "', '" + \ 
     
    287298        for docType, doc in self._record.getAllDocs(): 
    288299            sqlCmd = "UPDATE TRANSFORMED_DOCUMENT SET transformed_document = '" + doc + \ 
    289                 "', update_date = current_timestamp WHERE original_record_id = " + \ 
     300                "', update_date = current_timestamp WHERE original_document_id = " + \ 
    290301                str(self._record.db_id) + " AND transformed_format = '" + docType + "';" 
    291302 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/PostgresRecord.py

    r3862 r3869  
    1919from FileUtilities import FileUtilities 
    2020from SpatioTemporalData import SpatioTemporalData 
     21import keywordAdder 
    2122 
    2223class PostgresRecord: 
     
    184185                     %self.docType) 
    185186 
     187        # now run the appropriate transform and set the attribute 
     188        setattr(self, "_molesFormat", self.doTransform(xqName)) 
     189 
    186190        # add keywords, if required 
    187191        if self._datacentre_groups != "": 
    188             addKeywords() 
    189  
    190         # now run the appropriate transform and set the attribute 
    191         setattr(self, "_molesFormat", self.doTransform(xqName)) 
     192            self.addKeywords() 
    192193         
    193194        # escape any apostrophes 
     
    203204        ''' 
    204205        logging.info("Adding datacentre keywords to moles file") 
     206 
    205207        # NB, use temporary directories to do the keyword additions 
    206         tmpDir = os.getcwd() + "/" 
    207         tmpKeywordsDir = os.getcwd() + "/kewordsAdded/" 
     208        tmpDir = os.getcwd() + "/tmp/" 
     209        tmpKeywordsDir = os.getcwd() + "/keywordsAdded/" 
    208210        self._fileUtils.setUpDir(tmpDir) 
    209211        self._fileUtils.setUpDir(tmpKeywordsDir) 
    210212        tmpFile = 'tmpFile.xml' 
    211         self._fileUtils.createFile(tmpDir + "/" + tmpFile, self._molesFormat) 
    212  
    213         keywordAdder.main(tmpDir, tmpKeywordsDir, self.datacentre_groups) 
    214  
     213        self._fileUtils.createFile(tmpDir + tmpFile, self._molesFormat) 
     214 
     215        keywordAdder.main(tmpDir, tmpKeywordsDir, self._datacentre_groups) 
     216 
     217        sys.exit() 
    215218        # Now load in the converted file 
    216219        f=open(tmpKeywordsDir + "/" + tmpFile, 'r') 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

    r3862 r3869  
    4040                oai_document_ingester(datacentre) 
    4141            except: 
     42                logging.error("Exception thrown - detail: ") 
     43                logging.error(sys.exc_info()) 
     44                logging.info("Continue processing next datacentre config file...") 
    4245                error_counter +=1 
    4346                 
Note: See TracChangeset for help on using the changeset viewer.