source: TI01-discovery-Ingest/trunk/v4.3.0/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py @ 7186

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI01-discovery-Ingest/trunk/v4.3.0/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py@7186
Revision 7186, 4.4 KB checked in by sdonegan, 9 years ago (diff)

Adding files from v4.2.0 tag to form basis for v4.3.0 version

Line 
1#!/usr/bin/env python
2"""
3"""
4import os, sys, logging,time,string
5
6lineSeparator = "-----------------------------"
7
8# configure logging
9logging.basicConfig(level=logging.INFO,
10                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
11
12logging.info(lineSeparator)
13logging.info("RUNNING: run_all_ingest.py")
14logging.info(lineSeparator)
15
16from oai_document_ingester import oai_document_ingester
17
18#pick up path info from processing config file
19configFilePath = '/disks/glue1/sdonegan/NDG3_workspace/buildouts/oai_document_ingester_MEDIN/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.config'
20
21if not os.path.isfile(configFilePath):
22        sys.exit("ERROR: Could not find the processing config file")
23                   
24processingConfig = {}
25                   
26processing_config_file = open(configFilePath, "r")
27               
28for line in processing_config_file.readlines():
29        words  = string.split(line)
30        if len(words) == 0:
31                continue
32        elif words[0] == 'code_directory':
33                processingConfig['code_directory'] = words[1]                                                           
34        elif words[0] == 'base_directory':
35                processingConfig['base_directory'] = words[1]
36        elif words[0] == 'reporting_directory':
37                processingConfig['reporting_directory'] = words[1]
38        elif words[0] == 'passwords_file':
39                processingConfig['passwords_file'] = words[1]
40        elif words[0] == 'datcentre_configs':
41                processingConfig['datcentre_configs'] = words[1]
42        elif words[0] == 'ingest_name':
43                processingConfig['ingest_name'] = words[1]             
44
45run_counter = 0
46error_counter = 0
47
48#current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
49#current_dir = '/home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/' # NOTE this is changed to the production buildout directory
50current_dir = processingConfig['code_directory']
51logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir)
52
53#create file for summary report on all ingests
54#reportingDir = '/home/badc/discovery_docs/ingestDocs/data/' # base dir where production reporting and output should go
55reportingDir = processingConfig['reporting_directory']
56summaryFileName = reportingDir + 'runAllIngestSummary.txt'       
57summaryFile = open(summaryFileName,'w')           
58
59#iterate over all files in the current directory and process any config ones
60config_suffix = '_config.properties'
61filenames = os.listdir(current_dir + 'datacentre_config/')
62
63ingester = oai_document_ingester()
64for filename in filenames:
65        if filename.endswith(config_suffix):
66            logging.info("Found config file: %s" %filename)
67           
68            # get the datacentre from the filename
69            datacentre = filename.replace(config_suffix, '')
70            summaryFile.write("\n=====================================================================\n")
71            summaryFile.write("Ingest report for data centre: " + datacentre + " at " + str(time.asctime()) + "\n")
72                   
73            if datacentre.find('backup') > -1:
74                continue
75           
76            # now invoke the ingest script
77            logging.info("Running the ingest script for datacentre: %s" %datacentre)
78            try:
79                run_counter += 1
80                success,report = ingester.processDataCentre(datacentre,None, None, configFilePath)
81               
82                #add report to file
83                #summaryFile.write('***************** %s *********************\n' %datacentre)
84                summaryFile.write(report)
85                #summaryFile.write('\n******************************************************************\n\n')
86           
87            except:
88                logging.error("Exception thrown - detail: ")
89                logging.error(sys.exc_info())
90                logging.info("Continue processing next datacentre config file...")
91                error_counter +=1
92                       
93           
94summaryFile.close()
95
96if os.path.exists(summaryFileName):
97    commandline = "cat " + summaryFileName + " | mail -s %s steve.donegan@stfc.ac.uk" %processingConfig['ingest_name']
98    os.system(commandline)
99
100logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
101logging.info("ingest report at: "  + summaryFileName)
102
103if error_counter > 0:
104    errorTxt = "WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter
105    logging.error(errorTxt)
106    commandline = "echo " + errorTxt + " | mail -s %s_PROBLEM steve.donegan@stfc.ac.uk" %processingConfig['ingest_name']
107
Note: See TracBrowser for help on using the repository browser.