source: TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py @ 1121

#!/usr/bin/env python
""" Script oai_ingest.py takes parameter <datacentre>.
The /usr/local/WSClients/OAIBatch directory contains this python script, a config file
and some java which handle difs after harvesting. The pre-processed files are then ingested
to the eXist XML db.

 Under this directory the following structure should be maintained:

 ./data
 - /DATACENTRE/
                - discovery/:           Records with namespace, schema declaration deleted. After having run the script.
                                       Ready to ingest in the discovery service.
                - oai/difYYYYMMDD/      Records as harvested from OAI

 Where  /DATACENTRE  varies for the different data providers

"""
#History:
# 12/05/06 SEL spelling correction
# 30/05/06 SEL cope with many files for processing."Argument list too long" problem.
# 31/05/06 SEL need to use passwords now. (replace xxxxxx in real version).

import os
import sys
import commands
import string

status = 0
numfilesproc = 0
harvest_home = ""

if (len(sys.argv) < 2):
    print "<datacentre>  parameter not supplied."
    sys.exit()
else:
    datacentre = sys.argv[1]

# Other settings and constants
date_string = commands.getoutput ("date +'%y%m%d_%H%M'")
os.putenv ('EXIST_HOME', '/usr/local/eXist')
os.putenv ('PATH', ':/usr/java/j2sdk1.4.2_04/lib/tools.jar:/usr/java/j2sdk1.4.2_04/bin:/usr/local/WSClients/OAIBatch:/usr/local/eXist/bin:/bin:/usr/bin:.')
os.putenv ('CLASSPATH','.:/usr/java/j2sdk1.4.2_04/bin:/usr/local/WSClients/OAIBatch')

# Get the harvested records directory for this datacentre from the config file for that data centre
# The harvested records directory depends on the datacentres OAI base url, the set and  format. These have to be know up-front.
datacentre_config_filename = "/usr/local/WSClients/OAIBatch/" + datacentre + "_config.properties"
print "Datacentre config file = %s" %datacentre_config_filename
datacentre_config_file = open(datacentre_config_filename, "r")

for line in datacentre_config_file.readlines():
    words  = string.split(line)
    if len(words) == 0:
        continue
    if words[0] == 'host_path':
        harvest_home = string.rstrip(words[1])
        break

if harvest_home == "":
    sys.exit("Failed at getting harvested records directory stage. datacentre config file tried = %s" %datacentre_config_filename)
datacentre_config_file.close()

if len( os.listdir(harvest_home)) == 0:
    print "Nothing to harvest this time from %s" %datacentre
    sys.exit()

# The directory to put things for a tape backup (should already exist)
backupdir = '/disks/glue1/oaiBackup/'

# Create/clear the directory for a pristine copy of the difs in case the script rewrites something wrong
if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy"):
    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/ | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/{\}"
    print "Executing : " + commandline
    status = os.system(commandline)
else:   
    commandline = "mkdir /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy"
    print "Executing : " + commandline
    status= os.system(commandline)

if status != 0:
    sys.exit("Failed at creating copy dir stage")

# make the pristine copy. Cope with there being lots of files in the directory.

commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/oai/difcopy"
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at making pristine copy stage")

# Create/clear the directory for the processing copy of the difs.
if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"):
    commandline = "rm -f /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/*"
    print "Executing : " + commandline
    status = os.system(commandline)
else:
    commandline = "mkdir /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"
    print "Executing : " + commandline
    status= os.system(commandline)

# make the processing copy
commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/discovery"
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at making processing copy stage")

# The file config.properties contains the name=value pair to parse the filename in java oaiProc.jar.
# Copy the datacentre specific version of config to config.properties file. 

commandline = "cp /usr/local/WSClients/OAIBatch/" + datacentre +"_config.properties /usr/local/WSClients/OAIBatch/config.properties"
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at copying config file stage")

#Change os directory to that with the java.jar in it.    
os.chdir('/usr/local/WSClients/OAIBatch')


#Execute the script which processes the files
filenames = os.listdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery")
for filename in filenames:
        if filename.find('.xml') != -1:
                full_filename = "/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/" + filename
                print "Processing : " + full_filename
                commandline = "java -jar /usr/local/WSClients/OAIBatch/oai_Proc.jar %s " %(full_filename)
                print commandline
                status= os.system(commandline)
                if status!=0:
                    break
                numfilesproc += 1
        else:
                print 'File %s is not xml format. Not processed'  %(full_filename)
if status!=0:
    sys.exit("Failed at processing file %s with java oai_Proc.jar stage with status %s" %(full_filename, status))

#The script reads the files from OAIBatch/data/datacentre/discovery and outputs within the same directory the files.
#The result will get rid of the "oai%3Aucar.ncar.scd.cdp%3A" type of thing that oai adds to
#the filenames and it will leave <DIF> as the root element.
#
#Once the pre-processing has finished remove the originals from the discovery directory:
commandline = "ls -1 usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/oai* | xargs -i rm usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/{\}"
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at removing original oai style records from discovery directory")

# ingest the datacentres records into eXist db (backups of exist happen nightly).
commandline = "client.sh -c /db/dif/" + datacentre + " -u admin -P xxxxxx -p /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/"
#print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status))

#Make copies of discovery and oai/difcopy areas to backup area for tape backups
this_backupdir = backupdir + datacentre + "_" + date_string + "_difcopy"
commandline = "mkdir " + this_backupdir 
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at creating backup directory %s" %this_backupdir)

commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/ | xargs -i cp /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/{\} " + this_backupdir
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at copying to backup directory %s" %this_backupdir)

this_backupdir = backupdir + datacentre + "_" + date_string + "_discovery"
commandline = "mkdir " + this_backupdir
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at creating backup directory %s" %this_backupdir)
commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/ | xargs -i cp /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/{\} " + this_backupdir
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at copying to backup directory %s" %this_backupdir)
        
#Clear out the original harvest records area
commandline = "ls -1 " + harvest_home + " | xargs -i rm -f {\}"
print "Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at clearing out original harvest records area %s" %harvest_home)


print "======================================================"
print "No. of files pre-processed = %s" %numfilesproc
if status == 0:
    print " Procedure oai_ingest.py ran to end"
else:
    print "Procedure oai_ingest.py FAILED with status %s" %status 
    
print "======================================================"