source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py @ 3869

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py@3869
Revision 3869, 1.8 KB checked in by cbyrom, 12 years ago (diff)

Add code to delete records if they weren't properly ingested - to allow
clean reruns of the ingest + fix some problems with the keyword adding
code.

  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2"""
3"""
4import os, sys, logging
5from oai_document_ingester import oai_document_ingester
6
7run_counter = 0
8error_counter = 0
9
10lineSeparator = "-----------------------------"
11
12# configure logging
13logging.basicConfig(level=logging.INFO,
14                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
15logging.info(lineSeparator)
16logging.info("RUNNING: run_all_ingest.py")
17logging.info(lineSeparator)
18
19
20current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
21logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir)
22
23
24# iterate over all files in the current directory and process any config ones
25config_suffix = '_config.properties'
26filenames = os.listdir(current_dir + 'datacentre_config/')
27for filename in filenames:
28        if filename.endswith(config_suffix):
29            logging.info("Found config file: %s" %filename)
30           
31            # get the datacentre from the filename
32            datacentre = filename.replace(config_suffix, '')
33            if datacentre.find('backup') > -1:
34                continue
35           
36            # now invoke the ingest script
37            logging.info("Running the ingest script for datacentre: %s" %datacentre)
38            try:
39                run_counter += 1
40                oai_document_ingester(datacentre)
41            except:
42                logging.error("Exception thrown - detail: ")
43                logging.error(sys.exc_info())
44                logging.info("Continue processing next datacentre config file...")
45                error_counter +=1
46               
47logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
48if error_counter > 0:           
49    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter)
50
Note: See TracBrowser for help on using the repository browser.