source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py @ 3862

#!/usr/bin/env python
""" 
"""
import os, sys, logging
from oai_document_ingester import oai_document_ingester

run_counter = 0
error_counter = 0

lineSeparator = "-----------------------------"

# configure logging
logging.basicConfig(level=logging.INFO,
                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
logging.info(lineSeparator)
logging.info("RUNNING: run_all_ingest.py")
logging.info(lineSeparator)


current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir)


# iterate over all files in the current directory and process any config ones
config_suffix = '_config.properties'
filenames = os.listdir(current_dir + 'datacentre_config/')
for filename in filenames:
        if filename.endswith(config_suffix):
            logging.info("Found config file: %s" %filename)
            
            # get the datacentre from the filename
            datacentre = filename.replace(config_suffix, '')
            if datacentre.find('backup') > -1:
                continue
            
            # now invoke the ingest script
            logging.info("Running the ingest script for datacentre: %s" %datacentre)
            try:
                run_counter += 1
                oai_document_ingester(datacentre)
            except:
                error_counter +=1
                
logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
if error_counter > 0:           
    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter)