1 | #!/usr/bin/env python |
---|
2 | """ |
---|
3 | """ |
---|
4 | import os, sys, logging |
---|
5 | from oai_document_ingester import oai_document_ingester |
---|
6 | |
---|
7 | run_counter = 0 |
---|
8 | error_counter = 0 |
---|
9 | |
---|
10 | lineSeparator = "-----------------------------" |
---|
11 | |
---|
12 | # configure logging |
---|
13 | logging.basicConfig(level=logging.INFO, |
---|
14 | format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s') |
---|
15 | logging.info(lineSeparator) |
---|
16 | logging.info("RUNNING: run_all_ingest.py") |
---|
17 | logging.info(lineSeparator) |
---|
18 | |
---|
19 | |
---|
20 | current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from |
---|
21 | logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir) |
---|
22 | |
---|
23 | |
---|
24 | # iterate over all files in the current directory and process any config ones |
---|
25 | config_suffix = '_config.properties' |
---|
26 | filenames = os.listdir(current_dir + 'datacentre_config/') |
---|
27 | for filename in filenames: |
---|
28 | if filename.endswith(config_suffix): |
---|
29 | logging.info("Found config file: %s" %filename) |
---|
30 | |
---|
31 | # get the datacentre from the filename |
---|
32 | datacentre = filename.replace(config_suffix, '') |
---|
33 | if datacentre.find('backup') > -1: |
---|
34 | continue |
---|
35 | |
---|
36 | # now invoke the ingest script |
---|
37 | logging.info("Running the ingest script for datacentre: %s" %datacentre) |
---|
38 | try: |
---|
39 | run_counter += 1 |
---|
40 | oai_document_ingester(datacentre) |
---|
41 | except: |
---|
42 | error_counter +=1 |
---|
43 | |
---|
44 | logging.info("run_all_ingest.py complete - processed %s config files" %run_counter) |
---|
45 | if error_counter > 0: |
---|
46 | logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter) |
---|
47 | |
---|