Changeset 4980


Ignore:
Timestamp:
16/02/09 10:59:58 (11 years ago)
Author:
cbyrom
Message:

Fix ingester to use new object structure correctly and more efficiently.
Remove log checking as this is no longer applicable to current logging.

File:
1 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

    r4711 r4980  
    1818logging.info(lineSeparator) 
    1919 
    20  
    2120current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from 
    2221logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir) 
     
    3029filenames = os.listdir(current_dir + 'datacentre_config/') 
    3130 
     31ingester = oai_document_ingester() 
    3232for filename in filenames: 
    3333        if filename.endswith(config_suffix): 
     
    4646            try: 
    4747                run_counter += 1 
    48                 oai_document_ingester(datacentre) 
    49              
    50              
     48                ingester.processDataCentre(datacentre) 
    5149            except: 
    5250                logging.error("Exception thrown - detail: ") 
     
    5452                logging.info("Continue processing next datacentre config file...") 
    5553                error_counter +=1 
    56                  
    57             #summarise harvest info from report doc now produced by ingester program 
    58             recOpFileName = "data/" + datacentre + "_ingestSummary.txt" 
    59              
    60             try: 
    61                 ingest_report_file = open(recOpFileName, "r") 
    62                                       
    63                 for line in ingest_report_file.readlines():                     
    64                     words  = string.split(line)                                     
    65                     if len(words) == 0: 
    66                         continue 
    67                     if words[0] == 'PROCESSED':  
    68                         summaryFile.write("Number of processed files: " + words[1] + "\n") 
    69                     if words[0] == 'INGESTED': 
    70                         summaryFile.write("Number of ingested files: " + words[1] + "\n") 
    71                     if words[0] == 'PROBLEM_NUM': 
    72                         summaryFile.write("Number of problem files: " + words[1] + "\n") 
    73                     if words[0] == 'PROBLEM_FILE': 
    74                         summaryFile.write("Problem file name: " + words[1] + "\n") 
    75                                            
    76                     ingest_report_file.close() 
    77                     
    78             except: 
    79                 summaryFile.write("Could not extract summary info for " + datacentre + " ingest!!\n") 
    80              
    81              
    8254             
    8355summaryFile.close() 
Note: See TracChangeset for help on using the changeset viewer.