Changeset 5243


Ignore:
Timestamp:
30/04/09 17:36:03 (10 years ago)
Author:
cbyrom
Message:

Adjust logging and output of error in ingest scripts.

Location:
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/abstractdocumentingester.py

    r5218 r5243  
    6969                # outputing data. The initial call to logging needs to be tracked down 
    7070                # and configured correctly, so this can be avoided... 
    71                 self.logger = logging.getLogger() 
    72                 self.logger.setLevel(loggingLevel) 
     71#               self.logger = logging.getLogger() 
     72#               self.logger.setLevel(loggingLevel) 
    7373 
    7474                # create console handler and set level to debug 
    75                 ch = logging.StreamHandler() 
    76                 ch.setLevel(loggingLevel) 
     75#               ch = logging.StreamHandler() 
     76#               ch.setLevel(loggingLevel) 
    7777                # create formatter 
    78                 formatter = logging.Formatter('%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 
     78#               formatter = logging.Formatter('%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 
    7979                # add formatter to ch 
    80                 ch.setFormatter(formatter) 
     80#               ch.setFormatter(formatter) 
    8181                # add ch to logger 
    82                 self.logger.addHandler(ch) 
     82#               self.logger.addHandler(ch) 
    8383                return args 
    8484 
     
    123123                except: 
    124124                        logging.error("Exception thrown - detail: ") 
    125                         logging.error(sys.exc_info()) 
     125                        errors = sys.exc_info() 
     126                        logging.error(errors) 
     127                        self._error_messages += str(errors) 
    126128                         
    127129                        if dao: 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/feeddocumentingester.py

    r4948 r5243  
    247247                self._no_files_ingested = 0 
    248248                self._no_problem_files = 0 
     249                self._error_messages = '' 
    249250                 
    250251                self.deName = VTD().TERM_DATA[VTD.DE_TERM].title 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

    r5153 r5243  
    77''' 
    88import os, sys, logging 
     9# annoyingly, an import (CSML file, I think) sets the logging config during imports - so set this 
     10# here to get there first - since you can only set the config once 
     11logging.basicConfig(level=logging.DEBUG, 
     12                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 
    913from time import strftime 
    1014import ndg.common.src.lib.fileutilities as FileUtilities 
     
    1620        - including running the various transforms and parsings to get all doc types and spatiotemporal  
    1721        data in the correct form in the DB 
     22        @return outMessage: string summary of ingest outcome 
    1823        ''' 
    19          
    2024 
    2125        def processDataCentre(self, dataCentre): 
     
    2327                self._no_files_ingested = 0 
    2428                self._no_problem_files = 0 
     29                self._error_messages = '' 
    2530                self.dataCentre = dataCentre 
    2631                 
     
    6065                numfilesproc = self._convertAndIngestFiles(self.originals_dir, self.discovery_dir) 
    6166                 
     67                outMessage = "OAI Document ingest processing complete:" 
    6268                logging.info("oai_document_ingest processing complete:") 
    6369                if self._no_problem_files == 0: 
     
    7076                logging.info(self.lineSeparator) 
    7177                logging.info("INFO: Number of files processed = %s" %numfilesproc) 
     78                outMessage += 'Number of files processed = %s' %numfilesproc 
    7279                logging.info("INFO: Number of files ingested = %s" %self._no_files_ingested) 
     80                outMessage += 'Number of files ingestest = %s' %self._no_files_ingested 
     81                if self._error_messages: 
     82                        outMessage += 'Errors: %s' %self._error_messages  
    7383                logging.info(self.lineSeparator) 
    7484                print "Script finished running." 
     85                return outMessage 
    7586 
    7687         
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

    r4980 r5243  
    33""" 
    44import os, sys, logging,time,string 
     5 
     6# configure logging 
     7logging.basicConfig(level=logging.INFO, 
     8                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 
     9logging.info(lineSeparator) 
     10logging.info("RUNNING: run_all_ingest.py") 
     11logging.info(lineSeparator) 
     12 
    513from oai_document_ingester import oai_document_ingester 
    614 
     
    1018 
    1119lineSeparator = "-----------------------------" 
    12  
    13 # configure logging 
    14 logging.basicConfig(level=logging.INFO, 
    15                     format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') 
    16 logging.info(lineSeparator) 
    17 logging.info("RUNNING: run_all_ingest.py") 
    18 logging.info(lineSeparator) 
    1920 
    2021current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from 
Note: See TracChangeset for help on using the changeset viewer.