source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_ingest_new.py @ 3800

#!/usr/bin/env python
""" Script oai_ingest.py takes parameters <datacentre> <dbinfofile>.
The /usr/local/WSClients/OAIBatch directory contains:-
 - this python script, plus some other modules eg ndgUtils for parts of the process.
 - a DataProvider specific config file,
 - the python module for extracting spatiotemporal information and adding to postgres db.
Under this directory the following structure should be maintained:
 ./data
 - /DATACENTRE/
                - discovery/:         Re-named documents.
        - discovery_corrected Documents with schema namespaces corrected, ready to ingest in the discovery service.
                - oai/difYYYYMMDD/    Documents as harvested from OAI
 Where  /DATACENTRE  varies for the different data providers
"""
#History:
# 12/05/06 SEL spelling correction
# 30/05/06 SEL cope with many files for processing."Argument list too long" problem.
# 31/05/06 SEL need to use passwords now. (replace xxxxxx in real version).
# 16/10/06 SEL Changed to using python oaiClean.py module instead of java code.
# 16/10/06 SEL exist db upgraded and deployed at different location, java upgrade.
# 17/10/06 SEL cope with different discovery formats - not just DIF.
# 23/10/06 SEL keywords not mandatory in config file.
# 24/10/06 SEL fix bug where 'in' directory not being cleared initially. display more information when running.
#  December 2007 SEL rewrite to use Bryans' python XQuery stuff to create mini-moles instead of java.
#                    Also extracted hard coded pwds into a file.
# 11/04/08 CByrom Tidy up script by organising code into reusable functions + variables 
# + remove dependency on eXist DB

import os
import sys
import commands
import string
import keywordAdder
from SchemaNameSpace import SchemaNameSpace
from DIF import DIF
from MDIP import MDIP
import ndgUtils
from ndgUtils.ndgXqueries import ndgXqueries
from ndgUtils.ndgObject import ndgObject
import ConfigParser
from ndgUtils.ndgDirectory import ndgDirectory
from ndgUtils.eXistInterface import ndg_eXist
from FileUtilities import setUpDir, cleanDir, makeBackUp
from PostgresRecord import PostgresRecord
import PostgresDAL

def getID(filename):
        ''' 
        Gets the identifier out of an input metadata xml record. 
        Copes with DIF and MDIP currently.
        @param filename - name of document file being processed
        @return: ID - id to use to refer to the document
        '''
        xml=file(filename).read()
        if datacentre_format == "DIF":
            d=DIF(xml)
            ID=d.entryID
        elif datacentre_format == "MDIP":
            d=MDIP(xml)
            ID=d.id
        else:
            sys.exit("Only handles DIF or MDIP here.")
        return ID

def addFileToPostgresDB(filename):
        '''
        Add a file to the postgres DB - extracting and storing all the required
        data in the process
        '''
        print "Adding file, %s, to postgres DB" %filename
        
        # first of all create a PostgresRecord - this object represents all the data required
        # for a DB entry
        record = PostgresRecord(filename, NDG_dataProvider)
        dal = PostgresDAL(record)
        dal.createOrUpdateRecord()

def getConfigDetails(datacentre):
        '''
        Get the harvested records directory and groups for this datacentre from the 
        datacentre specific config file.  The harvested records directory depends on the 
        datacentres OAI base url, the set and format. These have to be know up-front.
        The groups denote which 'portal groups' they belong to - for limiting searches to 
        say NERC-only datacentres records.
        Groups are added to the intermediate MOLES when it is created.
        @param datacentre: datacentre to use when looking up config file 
        '''
        # set the variables to use the global copies, not the local ones
        global harvest_home, datacentre_groups, datacentre_format, datacentre_namespace
        global datacentre_config_filename, NDG_dataProvider
        datacentre_config_filename = base_dir + datacentre + "_config.properties"
        print "INFO: Retrieving data from datacentre config file = %s" %datacentre_config_filename
        
        # Check this file exists; if not, assume an invalid datacentre has been specified
        if not os.path.isfile(datacentre_config_filename):
            sys.exit("ERROR: Could not find the config file; either this doesn't exist or the datacentre " \
                "specified (%s) is invalid\n" %datacentre)
            
        datacentre_config_file = open(datacentre_config_filename, "r")
        
        for line in datacentre_config_file.readlines():
            words  = string.split(line)
            if len(words) == 0:
                continue
            if words[0] == 'host_path':
                harvest_home = string.rstrip(words[1])
            if words[0] == 'groups':
                datacentre_groups = words[1:]
            if words[0] == 'format':
                datacentre_format = words[1]
            if words[0] == 'namespace':
                datacentre_namespace = words[1]
            if words[0] == 'NDG_dataProvider':
                NDG_dataProvider = True
        
        datacentre_config_file.close()
        
        if harvest_home == "":
            sys.exit("Failed at getting harvested records directory stage. datacentre config file tried = %s" %datacentre_config_filename)
        
        print "INFO: harvested records are in %s" %harvest_home
        
        if datacentre_groups == "":
            print "INFO: No groups/keywords set for datacentre %s" %datacentre
        else:
            print "INFO: datacentre groups/keywords = %s" %datacentre_groups
        
        if datacentre_format == "":
            sys.exit("Failed at stage: getting datacentre format. datacentre config file tried = %s" %datacentre_config_filename)
        
        print "INFO: format being harvested = %s" %datacentre_format
        
        if datacentre_namespace == "":
            sys.exit("Failed at stage: getting datacentre namespace. datacentre config file tried = %s" %datacentre_config_filename)
        
        print "INFO: datacentre namespace = %s" %datacentre_namespace
        print lineSeparator
        

def usage():
        '''
        Display input params for the script
        '''
        print "Usage: python oai_ingest.py <datacentre> (<dbInfoFile>)"
        print " - where:\n   <datacentre> is the data centre to ingest data from; and"
        print "   <dbInfoFile> provides info on the eXist DB to use to do XQuery transforms"
        sys.exit()


def ingestOriginalsToeXist():
        '''
        Put original docs into eXist DB - NB, this allows for easy transform via XQuery
        into the other required doc types
        '''
        print "INFO: Putting original docs in eXist..."
        # this command creates the targetCollection in eXist from the contents of the discovery_corrected_dir
        commandline = "$EXIST_HOME/bin/client.sh -c " + targetCollection + " -u admin -P " + \
                db_admin + " -p " + discovery_corrected_dir
        print lineSeparator
        print "INFO: Executing : actual command to ingest into exist db", commandline
        status = os.system(commandline)
        if status !=0:
            sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status))
        print "INFO: Original docs now in eXist"
        print lineSeparator


def createEXistMiniMoles():
        '''
        Transform the docs in eXist into the mini moles format - to allow use of the various
        existing xqueries to do the transforms from this format to others
        '''
        # First get the list of discovery record ids from the db collection
        print "INFO: Creating minimoles documents from original docs in eXist"
        if datacentre_format == 'DIF':
            ndgDir=ndgDirectory(targetCollection,host, datacentre_format.upper())
        else:
            print 'ERROR: mini-moles creation does not handle MDIP yet! So this WILL FAIL (probably)'
        
        #create the mini-moles for each Discovery record in the collection
        for member in ndgDir.members:
            filename= member['fileName']
            disc_id = member['EntryID']
            print "INFO: internal id = %s" %disc_id
            print "INFO: discovery filename = %s" %filename
            # now create the xquery
            # sort out the output ID stuff ...
            if NDG_dataProvider:
                discObj=ndgObject(disc_id)
                xquery=xq.actual('dif2moles',targetCollection,discObj.repository,discObj.localID)
            else:
                xquery=xq.actual('dif2moles',targetCollection,datacentre_namespace,disc_id)

            # and then sort out the input ID stuff
            xquery=xquery.replace('Input_Entry_ID',disc_id)
            xquery=xquery.replace('repository_localid', datacentre_namespace )

            # Now do the transform
            print "INFO: Running XQuery transform to create minimoles document"
            molesid,s=xmldb.executeQuery(xquery)
            moles_from_dif=xmldb.retrieve(molesid,0)

            minimoles_file = minimoles_dir + "/" + filename
            print "INFO: Creating new minimoles file: ", minimoles_file  
            f=open(minimoles_file,'w')
            f.write(moles_from_dif)
            f.close()
            
            filecount = 0
            for root, dirs, files in os.walk(minimoles_dir):
                filecount += len(files)
                
                if filecount == 0:
                    print "ERROR: couldn't create any minimum moles records for %s" %datacentre
                    sys.exit()
                
                #Add keywords if necessary
                addKeywords()
                
                # ingest the created discovery minimum molesrecords into eXist db.
                commandline = "$EXIST_HOME/bin/client.sh -c /db/discovery/moles -u admin -P " + db_admin + \
                " -p " + finalmoles_dir
                print "INFO: Executing : actual command to ingest into exist db."
                status = os.system(commandline)
                if status !=0:
                    sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status))


def createEXistCollections():
        '''
        Ensure all the required collections are set up in the eXist DB
        '''
        commandline = "$EXIST_HOME/bin/client.sh -c /db " + \
                " -u admin -P " + db_admin + " -m "
        collections = ['/db/xqueryLib', '/db/discovery/moles']
        print lineSeparator
        for collection in collections:
                cmd = commandline + collection + " -p"
                print "INFO: Executing : actual command to create DB collection ", cmd
                status = os.system(cmd)
                print status
                if status !=0:
                        sys.exit("Failed to create DB collection. Status = %s" %(status))
        

        
def addKeywords():
        if datacentre_groups == "":
            commandline = "ls -1 " + minimoles_dir + " | xargs -i mv " + minimoles_dir + \
                "{\} " + finalmoles_dir
            print "INFO: Executing : " + commandline
            status = os.system(commandline)
            if status !=0:
                sys.exit("Failed at moving MOLES to FINAL directory")
        else:
            keywordAdder.main(minimoles_dir, finalmoles_dir, datacentre_groups)

                        
def addXQueriesToeXist():
        '''
        The current set of XQueries files refer to some xquery libraries - which are stored in eXist
        - add them to the DB for use elsewhere
        '''
        commandline = "$EXIST_HOME/bin/client.sh -c /db " + \
                " -u admin -P " + db_admin + " -p " + base_dir + 'xquery'
        print lineSeparator
        print "INFO: Executing : actual command to ingest into exist db", commandline
        status = os.system(commandline)
        print status
        if status !=0:
            sys.exit("Failed at ingesting into exist db. Datacentre =  %s. Status = %s" %(datacentre,status))
        

        
lineSeparator = "-----------------------------"
print lineSeparator
print "RUNNING: oai_ingest.py"          
print lineSeparator

if (len(sys.argv) < 2 or len(sys.argv) > 3):
        usage()
else:
    datacentre = sys.argv[1]

# set the default password file
dbinfoname = "ingest.txt"
if (len(sys.argv) == 3):
        dbinfoname = sys.argv[2]

status = 0
numfilesproc = 0
base_dir = "/usr/local/WSClients/OAIBatch/" # this is the base dir that the script is ran from
#base_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
        
data_dir = base_dir + "data/" + datacentre  # dir relating to the specified datacentre docs

#Change os directory to that with the harvested documents in it.
os.chdir(base_dir)

# Other settings and constants
date_string = commands.getoutput("date +'%y%m%d_%H%M'")
#os.putenv('EXIST_HOME', '/usr/local/exist-client')
os.putenv('EXIST_HOME', '/home/users/cbyrom/opt/eXist')
os.putenv('PATH', ':/usr/java/jdk1.5.0_03/bin:/usr/java/jdk1.5.0_03:/usr/java/jdk1.5.0_03/lib/tools.jar:/usr/local/WSClients/OAIBatch:/usr/local/exist-client/bin:/bin:/usr/bin:.')

# set the global variables to retrieve from the config file
harvest_home = ""
datacentre_groups = ""
datacentre_format = ""
datacentre_namespace = ""
NDG_dataProvider = False
getConfigDetails(datacentre)

#any records to harvest?
if len( os.listdir(harvest_home)) == 0:
    print "INFO: Nothing to harvest this time from %s" %datacentre
    sys.exit()

# get the db access info from the specified input file
dbaccess={}
dbinfo_file=open(dbinfoname,"r")
for line in dbinfo_file.readlines():
    words  = string.split(line)
    if len(words) < 2:
        continue
    dbaccess[(words[0],words[1])] = [words[2]]
dbinfo_file.close()
#host ='glue.badc.rl.ac.uk'
host ='localhost'
try:
        db_admin = dbaccess[(host,'admin')][0]
except:
        sys.exit("ERROR: Could not find details for %s DB user 'admin' in DB config file\n" %host)

# The directory to put things for a tape backup (should already exist)
backupdir = '/disks/glue1/oaiBackup/'

# the following dirs define where the specific documents should go
originals_dir = data_dir +"/oai/originals/"
discovery_dir = data_dir +"/discovery/"
discovery_corrected_dir = data_dir +"/discovery_corrected/"

# Create/clear the 'in' directory pristine copy of the discovery records
setUpDir(originals_dir)
commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} " + originals_dir
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at making pristine copy stage")

# Create/clear the directory for the 'out' processed copy of the discovery records.
setUpDir(discovery_dir)
    
# Create/clear the directory for the 'out' namespace corrected copy of the discovery records.
setUpDir(discovery_corrected_dir)

# The file config.properties contains the location of the particular datacentres harvested records.
# Copy the datacentre specific version of config to config.properties file.
commandline = "cp " + datacentre_config_filename + " " + base_dir + "config.properties"
print "INFO: Executing : " + commandline
status = os.system(commandline)
if status !=0:
    sys.exit("Failed at copying config file stage")

#Execute the script which processes/renames the files (changed 08/01/07 to get id from inside file)
# NB, this copies files from the original dir to the discovery dir
print lineSeparator
print "INFO: Renaming files:"
for filename in os.listdir(originals_dir):
        if filename.find('.xml') != -1:
                original_filename = originals_dir + filename
                ident=getID(original_filename)
                print "INFO: ID extracted from the discovery record = %s" %ident
                if NDG_dataProvider:
                    new_filename = discovery_dir + ident.replace(":","__")+".xml"
                else:
                    ident = ident.replace(":","-")
                    ident = ident.replace("/","-")
                    new_filename = discovery_dir + "/" +datacentre_namespace+ "__"+datacentre_format+ "__"+ ident +".xml"
 
                print "original file = %s\nnewfile = %s" %(original_filename, new_filename)
                
                try:
                        os.rename(original_filename, new_filename)
                except:
                        sys.exit("ERROR: Failed to rename file %s to %s" %(original_filename, new_filename))
                numfilesproc += 1
        else:
                print 'WARNING: File %s is not xml format. Not processed'  %(filename)

#replace any namespace declarations with a standard one which we know works in NDG
# NB, this copies files from the discovery dir to the discovery corrected dir
print lineSeparator
print "INFO: Correcting namespaces of files (corrected files in %s)" %discovery_corrected_dir
for filename in os.listdir(discovery_dir):
        if filename.find('.xml') != -1:
                    in_filename = discovery_dir + filename
                    corrected_filename = discovery_corrected_dir + filename
                    try:
                       SchemaNameSpace(in_filename, corrected_filename,datacentre_format)
                    except:
                       print "ERROR: SchemaNameSpace failed on file %s"%in_filename

# put the original docs in eXist - as the targetCollection
# NB, firstly ensure the appropriate collections are set up
createEXistCollections()

# now set up the required XQueries and check xqueries libs are available from eXist
targetCollection = "/db/discovery/original/" +datacentre_format+ "/" +datacentre_namespace
xq=ndgXqueries()
xmldb=ndg_eXist(db='' + host + '')

addXQueriesToeXist()

ingestOriginalsToeXist()

# now transform these to minimoles format
minimoles_dir = base_dir + 'MINIMOLES/'
finalmoles_dir = base_dir + 'FINALMOLES/'
setUpDir(minimoles_dir)
setUpDir(finalmoles_dir)
createEXistMiniMoles()

# TODO: need to skip the eXist steps below and just populate the postGres DB
# 1. using the discovery_corrected_dir - (do we need the non corrected files?) - import into original column
# 2. use the correct xquery to transform the original doc directly and store t


# Process the resulting files and put the data into the postgres DB
filenames = os.listdir(discovery_corrected_dir)
for filename in filenames:
        addFileToPostgresDB(discovery_corrected_dir + filename)

#SpaceTimeIngestFromMOLES.main("./FINALMOLES")

#Extract the spatiotemporal info and put into NEW postgis tables
#SpaceTimeIngestPostgisFromMOLES.main("./FINALMOLES")

#Make copies of discovery and oai/originals and DIF2MOLES areas to backup area for tape backups
backupdir_base = backupdir + datacentre + "_" + date_string
this_backupdir = backupdir_base + "_originals"
makeBackUp(originals_dir, this_backupdir)

this_backupdir = backupdir_base + "_discovery"
makeBackUp(discovery_dir, this_backupdir)

this_backupdir = backupdir_base + "_FINALMOLES"
makeBackUp("./FINALMOLES", this_backupdir)

#Clear out the original harvest records area and FINALMOLES
cleanDir("./FINALMOLES")
cleanDir(harvest_home)

print "======================================================"
print "No. of files pre-processed = %s" %numfilesproc
if status == 0:
    print " Procedure oai_ingest.py ran to end"
else:
    print "ERROR: Procedure oai_ingest.py FAILED with status %s" %status

print "======================================================"