source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py @ 3862

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py@3862
Revision 3862, 1.6 KB checked in by cbyrom, 12 years ago (diff)

Fix workflow so that transforms are only ran when actually required

  • i.e. when the data is not already in the DB + fix a few small bugs

+ add extra logging + wrapper spatiotemporal data in record via get
methods.

  • Property svn:executable set to *
Line 
1#!/usr/bin/env python
2"""
3"""
4import os, sys, logging
5from oai_document_ingester import oai_document_ingester
6
7run_counter = 0
8error_counter = 0
9
10lineSeparator = "-----------------------------"
11
12# configure logging
13logging.basicConfig(level=logging.INFO,
14                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
15logging.info(lineSeparator)
16logging.info("RUNNING: run_all_ingest.py")
17logging.info(lineSeparator)
18
19
20current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
21logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir)
22
23
24# iterate over all files in the current directory and process any config ones
25config_suffix = '_config.properties'
26filenames = os.listdir(current_dir + 'datacentre_config/')
27for filename in filenames:
28        if filename.endswith(config_suffix):
29            logging.info("Found config file: %s" %filename)
30           
31            # get the datacentre from the filename
32            datacentre = filename.replace(config_suffix, '')
33            if datacentre.find('backup') > -1:
34                continue
35           
36            # now invoke the ingest script
37            logging.info("Running the ingest script for datacentre: %s" %datacentre)
38            try:
39                run_counter += 1
40                oai_document_ingester(datacentre)
41            except:
42                error_counter +=1
43               
44logging.info("run_all_ingest.py complete - processed %s config files" %run_counter)
45if error_counter > 0:           
46    logging.error("WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter)
47
Note: See TracBrowser for help on using the repository browser.