source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py @ 5243

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py@5243
Revision 5243, 2.5 KB checked in by cbyrom, 11 years ago (diff)

Adjust logging and output of error in ingest scripts.

  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2"""
3"""
4import os, sys, logging,time,string
5
6# configure logging
7logging.basicConfig(level=logging.INFO,
8                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
9logging.info(lineSeparator)
10logging.info("RUNNING: run_all_ingest.py")
11logging.info(lineSeparator)
12
13from oai_document_ingester import oai_document_ingester
14
15
16run_counter = 0
17error_counter = 0
18
19lineSeparator = "-----------------------------"
20
21current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
22logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir)
23
24#create file for summary report on all ingests
25summaryFileName = "data/runAllIngestSummary.txt"       
26summaryFile = open(summaryFileName,'w')           
27
28#iterate over all files in the current directory and process any config ones
29config_suffix = '_config.properties'
30filenames = os.listdir(current_dir + 'datacentre_config/')
31
32ingester = oai_document_ingester()
33for filename in filenames:
34        if filename.endswith(config_suffix):
35            logging.info("Found config file: %s" %filename)
36         
37            # get the datacentre from the filename
38            datacentre = filename.replace(config_suffix, '')
39            summaryFile.write("\n=====================================================================\n")
40            summaryFile.write("Ingest report for data centre: " + datacentre + " at " + str(time.asctime()) + "\n")
41                   
42            if datacentre.find('backup') > -1:
43                continue
44           
45            # now invoke the ingest script
46            logging.info("Running the ingest script for datacentre: %s" %datacentre)
47            try:
48                run_counter += 1
49                ingester.processDataCentre(datacentre)
50            except:
51                logging.error("Exception thrown - detail: ")
52                logging.error(sys.exc_info())
53                logging.info("Continue processing next datacentre config file...")
54                error_counter +=1
55           
56summaryFile.close()
57
58if os.path.exists(summaryFileName):
59    commandline = "cat " + summaryFileName + " | mail -s 'ignis discovery ingest report' s.j.donegan@rl.ac.uk"
60    os.system(commandline)
61
62logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
63logging.info("ingest report at: "  + summaryFileName)
64
65if error_counter > 0:           
66    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter)
67
Note: See TracBrowser for help on using the repository browser.