source: TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py @ 3803

#!/usr/bin/env python
""" Script oai_ingest.py takes parameters <datacentre> <dbinfofile> <existhost> <backupdir> <javahome>.
NOTE:might have to change the location of tomcat if it's not in the usual place (/usr/local/tomcat).

The /usr/local/WSClients/OAIBatch directory contains:-
 - this python script, plus some other modules eg ndgUtils for parts of the process.
 - a DataProvider specific config file,
 - the python module for extracting spatiotemporal information and adding to postgres db.
Under this directory the following structure should be maintained:
 ./data
 - /DATACENTRE/
        - discovery/:          Re-named documents.
        - discovery_corrected  Documents with schema namespaces corrected, ready to ingest in the discovery service.
        - oai/difYYYYMMDD/    Documents as harvested from OAI
 Where  /DATACENTRE  varies for the different data providers
"""
#History:
# 12/05/06 SEL spelling correction
# 30/05/06 SEL cope with many files for processing."Argument list too long" problem.
# 31/05/06 SEL need to use passwords now. (replace xxxxxx in real version).
# 16/10/06 SEL Changed to using python oaiClean.py module instead of java code.
# 16/10/06 SEL exist db upgraded and deployed at different location, java upgrade.
# 17/10/06 SEL cope with different discovery formats - not just DIF.
# 23/10/06 SEL keywords not mandatory in config file.
# 24/10/06 SEL fix bug where 'in' directory not being cleared initially. display more information when running.
#  December 2007 SEL rewrite to use Bryans' python XQuery stuff to create mini-moles instead of java.
#                    Also extracted hard coded pwds into a file.
# April 2008 SEL use exist interface that comes with the .war rather than the old-hat stuff in exist-client

import os
import sys
import commands
import string
import SpaceTimeIngestFromMOLES
import keywordAdder
from SchemaNameSpace import SchemaNameSpace
from DIF import DIF
from MDIP import MDIP
import ndgUtils
from ndgUtils.ndgXqueries import ndgXqueries
from ndgUtils.eXistInterface import ndg_eXist
from ndgUtils.ndgObject import ndgObject
import ConfigParser
from xmlrpclib import Fault
from ndgUtils.elementtree import ElementTree as ET
from ndgUtils.ndgDirectory import ndgDirectory

def getID(filename):
        ''' Gets the identifier out of an input metadata xml record. Copes with DIF and MDIP currently.'''
        xml=file(filename).read()
        if datacentre_format == "DIF":
            d=DIF(xml)
            ID=d.entryID
        elif datacentre_format == "MDIP":
            d=MDIP(xml)
            ID=d.id
        else:
            sys.exit("Only handles DIF or MDIP here.")
        return ID

status = 0
numfilesproc = 0
harvest_home = ""
datacentre_groups = ""
datacentre_format = ""
datacentre_namespace = ""
NDG_dataProvider = False

if (len(sys.argv) < 6):
    print "ERROR: <datacentre> or <db info file> or <existhost> or <backupdir> or <java_home> parameter not supplied."
    sys.exit()
else:
    datacentre = sys.argv[1]
    dbinfoname = sys.argv[2]
    existhost = sys.argv[3]
    # The directory to put things for a tape backup (should already exist)
    backupdir = sys.argv[4]
    #backupdir = '/disks/glue1/oaiBackup/'
    java_home = sys.argv[5]

#Change os directory to that with the code in it.
os.chdir('/usr/local/WSClients/OAIBatch')

# Other settings and constants
date_string = commands.getoutput ("date +'%y%m%d_%H%M'")
#os.putenv ('EXIST_HOME', '/usr/local/exist-client')
#os.putenv ('JAVA_HOME', '/usr/java/jdk1.5.0_03')
#os.putenv ('PATH', ':/usr/java/jdk1.5.0_03/bin:/usr/java/jdk1.5.0_03:/usr/java/jdk1.5.0_03/lib/tools.jar:/usr/local/WSClients/OAIBatch:/usr/local/exist-client/bin:/bin:/usr/bin:.')
#os.putenv ('CLASSPATH','.:/usr/java/jdk1.5.0_03/lib/tools.jar')

#Xquery settings
xq=ndgXqueries()
xmldb=ndg_eXist(db=existhost)

# Get the harvested records directory and groups for this datacentre from the datacentre specific config file
# The harvested records directory depends on the datacentres OAI base url, the set and  format. These have to be know up-front.
# The groups denote which 'portal groups' they belong to - for limiting searches to say NERC-only datacentres records.
# Groups are added to the intermediate MOLES when it is created.
datacentre_config_filename = "/usr/local/WSClients/OAIBatch/" + datacentre + "_config.properties"
print "INFO: Datacentre config file = %s" %datacentre_config_filename
datacentre_config_file = open(datacentre_config_filename, "r")

for line in datacentre_config_file.readlines():
    words  = string.split(line)
    if len(words) == 0:
        continue
    if words[0] == 'host_path':
        harvest_home = string.rstrip(words[1])
    if words[0] == 'groups':
        datacentre_groups = words[1:]
    if words[0] == 'format':
        datacentre_format = words[1]
    if words[0] == 'namespace':
        datacentre_namespace = words[1]
    if words[0] == 'NDG_dataProvider':
        NDG_dataProvider = True

datacentre_config_file.close()

if harvest_home == "":
    sys.exit("ERROR: Failed at getting harvested records directory stage. datacentre config file tried = %s" %datacentre_config_filename)
else:
    print "INFO: harvested records are in %s" %harvest_home

if datacentre_groups == "":
    print "INFO: No groups/keywords set for datacentre %s" %datacentre
else:
    print "INFO: datacentre groups/keywords = %s" %datacentre_groups

if datacentre_format == "":
    sys.exit("ERROR: Failed at stage: getting datacentre format. datacentre config file tried = %s" %datacentre_config_filename)
else:
    print "INFO: format being harvested = %s" %datacentre_format

if datacentre_namespace == "":
    sys.exit("ERROR: Failed at stage: getting datacentre namespace. datacentre config file tried = %s" %datacentre_config_filename)
else:
    print "INFO: datacentre namespace = %s" %datacentre_namespace

#any records to harvest?
if len( os.listdir(harvest_home)) == 0:
    print "INFO: Nothing to harvest this time from %s" %datacentre
    sys.exit()

# get the exist db access info
#host ='glue.badc.rl.ac.uk'
dbaccess={}
dbinfo_file=open(dbinfoname,"r")
for line in dbinfo_file.readlines():
    words  = string.split(line)
    if len(words) < 2:
        continue
    dbaccess[(words[0],words[1])] = [words[2]]
dbinfo_file.close()
#print dbaccess
db_admin = dbaccess[(existhost,'admin')][0]
#print db_admin

# Create/clear the 'in' directory pristine copy of the discovery records
if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals"):
    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals/ | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals/{\}"
    print "INFO: Executing : " + commandline
    status = os.system(commandline)
else:
    commandline = "mkdir /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals"
    print "INFO: Executing : " + commandline
    status= os.system(commandline)

if status != 0:
    sys.exit("ERROR: Failed at creating copy dir stage")

# make the 'in' pristine copy. Cope with there being lots of files in the directory.

commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/oai/originals"
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at making pristine copy stage")

# Create/clear the directory for the 'out' processed copy of the discovery records.
if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"):
    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/ | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/{\}"
    print "INFO: Executing : " + commandline
    status = os.system(commandline)
else:
    commandline = "mkdir /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"
    print "INFO: Executing : " + commandline
    status= os.system(commandline)

# Create/clear the directory for the 'out' namespace corrected copy of the discovery records.
if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected"):
    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected/ | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected/{\}"
    print "INFO: Executing : " + commandline
    status = os.system(commandline)
else:
    commandline = "mkdir /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected"
    print "INFO: Executing : " + commandline
    status= os.system(commandline)

# The file config.properties contains the location of the particular datacentres harvested records.
# Copy the datacentre specific version of config to config.properties file.
commandline = "cp /usr/local/WSClients/OAIBatch/" + datacentre +"_config.properties /usr/local/WSClients/OAIBatch/config.properties"
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at copying config file stage")

#Execute the script which processes/renames the files (changed 08/01/07 to get id from inside file)
indir="/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals"
outdir="/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"
#wrapFlag=False
filenames = os.listdir(indir)
for filename in filenames:
        if filename.find('.xml') != -1:
                original_filename = indir + "/" + filename
                ident=getID(original_filename)
                print "INFO: ID extracted from the discovery record = %s" %ident
                if NDG_dataProvider:
                    new_filename = outdir + "/"+ ident.replace(":","__")+".xml"
                else:
                    ident = ident.replace(":","-")
                    ident = ident.replace("/","-")
                    new_filename = outdir + "/" +datacentre_namespace+ "__"+datacentre_format+ "__"+ ident +".xml"
                print "INFO: original file = %s, newfile = %s" %(original_filename, new_filename)
                commandline = "cp "+original_filename+ " " +new_filename
                #print "Executing : " + commandline
                status = os.system(commandline)
                if status !=0:
                    sys.exit("ERROR: Failed at re-naming file stage")
                numfilesproc += 1
        else:
                print 'WARNING: File %s is not xml format. Not processed'  %(filename)

#replace any namespace declarations with a standard one which we know works in NDG
indir="/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"
outdir="/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected"
filenames = os.listdir(indir)
for filename in filenames:
        if filename.find('.xml') != -1:
                    in_filename = indir + "/" + filename
                    corrected_filename = outdir + "/" + filename
                    try:
                       SchemaNameSpace(in_filename, corrected_filename,datacentre_format)
                    except:
                       print "ERROR: SchemaNameSpace failed on file %s"%in_filename

# ingest the datacentres records into eXist db (backups of exist happen nightly).
commandline = java_home +" -jar -Dexist.home=/usr/local/tomcat/webapps/exist/WEB-INF /usr/local/tomcat/webapps/exist/WEB-INF/lib/start.jar client -c /db/discovery/original/"+datacentre_format+ "/" +datacentre_namespace+ " -u admin -P "+db_admin+" -p "+outdir+ " -ouri=xmldb:exist://"+existhost+":8080/exist/xmlrpc"
#commandline = "$EXIST_HOME/bin/client.sh -c /db/discovery/original/"+datacentre_format+ "/" +datacentre_namespace+ " -u admin -P "+db_admin+" -p "+outdir
print "INFO: Executing : actual command to ingest originals into exist db."
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status))

#are there any old moles records hanging around.If so, remove.
try:
   os.stat("./DIF2MOLES")
except:
   print "INFO: No old moles records hanging around"
else:
   commandline = "ls -1 ./DIF2MOLES | xargs -i rm ./DIF2MOLES/{\}"
   print "INFO: Executing : " + commandline
   status = os.system(commandline)
   if status !=0:
       sys.exit("ERROR: Failed at clearing out DIF2MOLES area.")

# Then run the minimum moles creator for each discovery record
# Put records in ./DIF2MOLES with same filename

# First get the list of discovery record ids from the db collection
targetCollection = "/db/discovery/original/"+datacentre_format+ "/" +datacentre_namespace
if datacentre_format == 'DIF':
    ndgDir=ndgDirectory(targetCollection,existhost,docType='DIF')
    #print ndgDir.members
else:
    print 'ERROR: mini-moles creation does not handle MDIP yet! So this WILL FAIL (probably)'


#create the mini-moles for each Discovery record in the collection
for member in ndgDir.members:
    #print member
    filename= member['fileName']
    disc_id = member['EntryID']
    print "INFO: internal id = %s" %disc_id
    print "INFO: discovery filename = %s" %filename
    # now create the xquery
    # sort out the output ID stuff ...
    if NDG_dataProvider:
        discObj=ndgObject(disc_id)
        xquery=xq.actual('dif2moles',targetCollection,discObj.repository,discObj.localID)
    else:
        xquery=xq.actual('dif2moles',targetCollection,datacentre_namespace,disc_id)
    # and then sort out the input ID stuff
    xquery=xquery.replace('Input_Entry_ID',disc_id)
    xquery=xquery.replace('repository_localid', datacentre_namespace )
    #print xq.help('dif2moles')
    molesid,s=xmldb.executeQuery(xquery)
    moles_from_dif=xmldb.retrieve(molesid,0)
    #print moles_from_dif
    # now write out xml to file
    outdir= './DIF2MOLES'
    f=open(outdir+"/"+filename,'w')
    f.write(moles_from_dif)
    f.close()

#There should be some records now
try:
    os.stat("./DIF2MOLES")
except:
    print "ERROR: couldn't create any minimum moles records for %s" %datacentre
    sys.exit()

#Add keywords if necessary
if datacentre_groups == "":
    commandline = "ls -1 ./DIF2MOLES/ | xargs -i mv ./DIF2MOLES/{\} ./FINALMOLES/"
    print "INFO: Executing : " + commandline
    status = os.system(commandline)
    if status !=0:
        sys.exit("ERROR: Failed at moving MOLES to FINAL directory")
else:
    keywordAdder.main('./DIF2MOLES', './FINALMOLES', datacentre_groups)

# ingest the created discovery minimum molesrecords into eXist db.
commandline = java_home +" -jar -Dexist.home=/usr/local/tomcat/webapps/exist/WEB-INF /usr/local/tomcat/webapps/exist/WEB-INF/lib/start.jar client -c /db/discovery/moles -u admin -P "+db_admin+" -p ./FINALMOLES -ouri=xmldb:exist://"+existhost+":8080/exist/xmlrpc"
#commandline = "$EXIST_HOME/bin/client.sh -c /db/discovery/moles -u admin -P "+db_admin+" -p ./FINALMOLES"
print "INFO: Executing : actual command to ingest mini-moles into exist db."
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status))

#Extract the spatiotemporal info from created moles and put in Postgres db
SpaceTimeIngestFromMOLES.main("./FINALMOLES")

#Make copies of discovery and oai/originals and DIF2MOLES areas to backup area for tape backups
this_backupdir = backupdir + datacentre + "_" + date_string + "_originals"
commandline = "mkdir " + this_backupdir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at creating backup directory %s" %this_backupdir)

commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals/ | xargs -i cp /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/originals/{\} " + this_backupdir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at copying to backup directory %s" %this_backupdir)

this_backupdir = backupdir + datacentre + "_" + date_string + "_discovery"
commandline = "mkdir " + this_backupdir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at creating backup directory %s" %this_backupdir)

commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/ | xargs -i cp /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/{\} " + this_backupdir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at copying to backup directory %s" %this_backupdir)

this_backupdir = backupdir + datacentre + "_" + date_string + "_FINALMOLES"
commandline = "mkdir " + this_backupdir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at creating backup directory %s" %this_backupdir)

commandline = "ls -1 ./FINALMOLES | xargs -i cp ./FINALMOLES/{\} " + this_backupdir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at copying to backup directory %s" %this_backupdir)

#Clear out the original harvest records area and FINALMOLES
commandline = "ls -1 ./FINALMOLES | xargs -i rm ./FINALMOLES/{\}"
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at clearing out FINALMOLES area %s" %harvest_home)

commandline = "ls -1 " + harvest_home + " | xargs -i rm " + harvest_home + "/{\}"
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("ERROR: Failed at clearing out original harvest records area %s" %harvest_home)

print "======================================================"
print "No. of files pre-processed = %s" %numfilesproc
if status == 0:
    print " Procedure oai_ingest.py ran to end"
else:
    print "ERROR: Procedure oai_ingest.py FAILED with status %s" %status

print "======================================================"