Changeset 4711


Ignore:
Timestamp:
23/12/08 16:55:23 (11 years ago)
Author:
sdonegan
Message:

Updated to collate information from each data centre ingest report and email collated info to s.j.donegan@…

File:
1 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

    r3869 r4711  
    22"""  
    33""" 
    4 import os, sys, logging 
     4import os, sys, logging,time,string 
    55from oai_document_ingester import oai_document_ingester 
     6 
    67 
    78run_counter = 0 
     
    2122logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir) 
    2223 
     24#create file for summary report on all ingests 
     25summaryFileName = "data/runAllIngestSummary.txt"         
     26summaryFile = open(summaryFileName,'w')             
    2327 
    24 # iterate over all files in the current directory and process any config ones 
     28#iterate over all files in the current directory and process any config ones 
    2529config_suffix = '_config.properties' 
    2630filenames = os.listdir(current_dir + 'datacentre_config/') 
     31 
    2732for filename in filenames: 
    2833        if filename.endswith(config_suffix): 
    2934            logging.info("Found config file: %s" %filename) 
    30              
     35           
    3136            # get the datacentre from the filename 
    3237            datacentre = filename.replace(config_suffix, '') 
     38            summaryFile.write("\n=====================================================================\n") 
     39            summaryFile.write("Ingest report for data centre: " + datacentre + " at " + str(time.asctime()) + "\n") 
     40                     
    3341            if datacentre.find('backup') > -1: 
    3442                continue 
     
    3947                run_counter += 1 
    4048                oai_document_ingester(datacentre) 
     49             
     50             
    4151            except: 
    4252                logging.error("Exception thrown - detail: ") 
     
    4454                logging.info("Continue processing next datacentre config file...") 
    4555                error_counter +=1 
    46                  
     56                 
     57            #summarise harvest info from report doc now produced by ingester program 
     58            recOpFileName = "data/" + datacentre + "_ingestSummary.txt" 
     59             
     60            try: 
     61                ingest_report_file = open(recOpFileName, "r") 
     62                                      
     63                for line in ingest_report_file.readlines():                     
     64                    words  = string.split(line)                                     
     65                    if len(words) == 0: 
     66                        continue 
     67                    if words[0] == 'PROCESSED':  
     68                        summaryFile.write("Number of processed files: " + words[1] + "\n") 
     69                    if words[0] == 'INGESTED': 
     70                        summaryFile.write("Number of ingested files: " + words[1] + "\n") 
     71                    if words[0] == 'PROBLEM_NUM': 
     72                        summaryFile.write("Number of problem files: " + words[1] + "\n") 
     73                    if words[0] == 'PROBLEM_FILE': 
     74                        summaryFile.write("Problem file name: " + words[1] + "\n") 
     75                                           
     76                    ingest_report_file.close() 
     77                    
     78            except: 
     79                summaryFile.write("Could not extract summary info for " + datacentre + " ingest!!\n") 
     80             
     81             
     82             
     83summaryFile.close() 
     84 
     85if os.path.exists(summaryFileName): 
     86    commandline = "cat " + summaryFileName + " | mail -s 'ignis discovery ingest report' s.j.donegan@rl.ac.uk" 
     87    os.system(commandline) 
     88 
    4789logging.info("run_all_ingest.py complete - processed %s config files" %run_counter) 
     90logging.info("ingest report at: "  + summaryFileName) 
     91 
    4892if error_counter > 0:            
    4993    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter) 
Note: See TracChangeset for help on using the changeset viewer.