source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py @ 4980

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py@4980
Revision 4980, 2.5 KB checked in by cbyrom, 11 years ago (diff)

Fix ingester to use new object structure correctly and more efficiently.
Remove log checking as this is no longer applicable to current logging.

  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2"""
3"""
4import os, sys, logging,time,string
5from oai_document_ingester import oai_document_ingester
6
7
8run_counter = 0
9error_counter = 0
10
11lineSeparator = "-----------------------------"
12
13# configure logging
14logging.basicConfig(level=logging.INFO,
15                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
16logging.info(lineSeparator)
17logging.info("RUNNING: run_all_ingest.py")
18logging.info(lineSeparator)
19
20current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
21logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir)
22
23#create file for summary report on all ingests
24summaryFileName = "data/runAllIngestSummary.txt"       
25summaryFile = open(summaryFileName,'w')           
26
27#iterate over all files in the current directory and process any config ones
28config_suffix = '_config.properties'
29filenames = os.listdir(current_dir + 'datacentre_config/')
30
31ingester = oai_document_ingester()
32for filename in filenames:
33        if filename.endswith(config_suffix):
34            logging.info("Found config file: %s" %filename)
35         
36            # get the datacentre from the filename
37            datacentre = filename.replace(config_suffix, '')
38            summaryFile.write("\n=====================================================================\n")
39            summaryFile.write("Ingest report for data centre: " + datacentre + " at " + str(time.asctime()) + "\n")
40                   
41            if datacentre.find('backup') > -1:
42                continue
43           
44            # now invoke the ingest script
45            logging.info("Running the ingest script for datacentre: %s" %datacentre)
46            try:
47                run_counter += 1
48                ingester.processDataCentre(datacentre)
49            except:
50                logging.error("Exception thrown - detail: ")
51                logging.error(sys.exc_info())
52                logging.info("Continue processing next datacentre config file...")
53                error_counter +=1
54           
55summaryFile.close()
56
57if os.path.exists(summaryFileName):
58    commandline = "cat " + summaryFileName + " | mail -s 'ignis discovery ingest report' s.j.donegan@rl.ac.uk"
59    os.system(commandline)
60
61logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
62logging.info("ingest report at: "  + summaryFileName)
63
64if error_counter > 0:           
65    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter)
66
Note: See TracBrowser for help on using the repository browser.