source: TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_ingest_new2.py @ 3845

#!/usr/bin/env python
""" Script oai_ingest.py takes parameters <datacentre> <dbinfofile>.
The /usr/local/WSClients/OAIBatch directory contains:-
 - this python script, plus some other modules eg ndgUtils for parts of the process.
 - a DataProvider specific config file,
 - the python module for extracting spatiotemporal information and adding to postgres db.
Under this directory the following structure should be maintained:
 ./data
 - /DATACENTRE/
                - discovery/:         Re-named documents.
        - discovery_corrected Documents with schema namespaces corrected, ready to ingest in the discovery service.
                - oai/difYYYYMMDD/    Documents as harvested from OAI
 Where  /DATACENTRE  varies for the different data providers
"""
#History:
# 12/05/06 SEL spelling correction
# 30/05/06 SEL cope with many files for processing."Argument list too long" problem.
# 31/05/06 SEL need to use passwords now. (replace xxxxxx in real version).
# 16/10/06 SEL Changed to using python oaiClean.py module instead of java code.
# 16/10/06 SEL exist db upgraded and deployed at different location, java upgrade.
# 17/10/06 SEL cope with different discovery formats - not just DIF.
# 23/10/06 SEL keywords not mandatory in config file.
# 24/10/06 SEL fix bug where 'in' directory not being cleared initially. display more information when running.
#  December 2007 SEL rewrite to use Bryans' python XQuery stuff to create mini-moles instead of java.
#                    Also extracted hard coded pwds into a file.
# 11/04/08 CByrom Tidy up script by organising code into reusable functions + variables 
# + remove dependency on eXist DB

import os, sys, string, getopt, logging
from time import strftime
from SchemaNameSpace import SchemaNameSpace
from DIF import DIF
from MDIP import MDIP
import ndgUtils
from ndgUtils.ndgXqueries import ndgXqueries
from FileUtilities import FileUtilities
from PostgresRecord import PostgresRecord
from PostgresDAO import PostgresDAO
import db_funcs

class oai_ingest:
        '''
        Class to handle the ingest of files from the OAI harvester to the discovery service postgres DB
        - including running the various transforms and parsings to get all doc types and spatiotemporal data
        in the correct form in the DB
        '''

        def getID(self, filename):
                ''' 
                Gets the identifier out of an input metadata xml record. 
                Copes with DIF and MDIP currently.
                @param filename - name of document file being processed
                @return: ID - id to use to refer to the document
                '''
                logging.info("Retrieving identifier for metadata record " + filename)
                xml=file(filename).read()
                if self._datacentre_format == "DIF":
                    d=DIF(xml)
                    ID=d.entryID
                elif self._datacentre_format == "MDIP":
                    d=MDIP(xml)
                    ID=d.id
                else:
                    sys.exit("Only handles DIF or MDIP here.")
        
                logging.info("Found identifier: " + ID)
                return ID
        
        
        def addFileToPostgresDB(self, filename):
                '''
                Add a file to the postgres DB - extracting and storing all the required
                data in the process
                @param filename: full path of file to add to postgres DB 
                '''
                if not os.path.isfile(filename):
                        logging.info("Skipping, %s - not a valid file" %filename)
                        return
                
                logging.info("Adding file, " + filename + ", to postgres DB")
                discoveryID = self.getID(filename)
                
                # first of all create a PostgresRecord - this object represents all the data required
                # for a DB entry
                record = PostgresRecord(filename, self._NDG_dataProvider, \
                                                            self._datacentre_groups, self._datacentre_namespace, \
                                                            discoveryID, self._xq, self._datacentre_format)
        
                # Now create the data access object to interface to the DB
                dao = PostgresDAO(record, self._dbConnection)
                
                # Finally, write the new record
                dao.createOrUpdateRecord()
        
        
        def getConfigDetails(self, datacentre):
                '''
                Get the harvested records directory and groups for this datacentre from the 
                datacentre specific config file.  The harvested records directory depends on the 
                datacentres OAI base url, the set and format. These have to be know up-front.
                The groups denote which 'portal groups' they belong to - for limiting searches to 
                say NERC-only datacentres records.
                Groups are added to the intermediate MOLES when it is created.
                @param datacentre: datacentre to use when looking up config file 
                '''
                self._datacentre_config_filename = self._base_dir + 'datacentre_config/' + datacentre + "_config.properties"
                logging.info("Retrieving data from datacentre config file, " + self._datacentre_config_filename)
                
                # Check this file exists; if not, assume an invalid datacentre has been specified
                if not os.path.isfile(self._datacentre_config_filename):
                    sys.exit("ERROR: Could not find the config file; either this doesn't exist or the datacentre " \
                        "specified (%s) is invalid\n" %datacentre)
                    
                datacentre_config_file = open(self._datacentre_config_filename, "r")
                
                for line in datacentre_config_file.readlines():
                    words  = string.split(line)
                    if len(words) == 0:
                        continue
                    if words[0] == 'host_path':
                        self._harvest_home = string.rstrip(words[1])
                    if words[0] == 'groups':
                        self._datacentre_groups = words[1:]
                    if words[0] == 'format':
                        self._datacentre_format = words[1]
                    if words[0] == 'namespace':
                        self._datacentre_namespace = words[1]
                    if words[0] == 'self._NDG_dataProvider':
                        self._NDG_dataProvider = True
                
                datacentre_config_file.close()
                
                if self._harvest_home == "":
                    sys.exit("Failed at getting harvested records directory stage. datacentre config file tried = %s" %self._datacentre_config_filename)
                
                logging.info("harvested records are in " + self._harvest_home)
                
                if self._datacentre_groups == "":
                    logging.info("No groups/keywords set for datacentre " + datacentre)
                else:
                    logging.info("datacentre groups/keywords: " + self._datacentre_groups)
                
                if self._datacentre_format == "":
                    sys.exit("Failed at stage: getting datacentre format. datacentre config file tried = %s" %self._datacentre_config_filename)
                
                logging.info("format being harvested: " + self._datacentre_format)
                
                if self._datacentre_namespace == "":
                    sys.exit("Failed at stage: getting datacentre namespace. datacentre config file tried = %s" %self._datacentre_config_filename)
                
                logging.info("datacentre namespace: " + self._datacentre_namespace)
                
                if self._NDG_dataProvider:
                        logging.info("Datacentre classified as an NDG data provider")
                else:
                        logging.info("Datacentre is not classificied as an NDG data provider")
                print self.lineSeparator

                
        def _getDBConnection(self):
                '''
                Get the default DB connection - by reading in data from the db config file
                '''
                logging.info("Setting up connection to postgres DB")
                dbinfo_file=open('ingest.config', "r")
                dbinfo = dbinfo_file.read().split()
                if len(dbinfo) != 4:
                        raise ValueError, 'Incorrect data in config file'
                
                self._dbConnection = db_funcs.db_connect(dbinfo[0], dbinfo[1], dbinfo[2], dbinfo[3])
                logging.info("Postgres DB connection now set up")

        
        def usage(self):
                '''
                Display input params for the script
                '''
                print "Usage: python -v oai_ingest.py <datacentre>"
                print " - where:\n   <datacentre> is the data centre to ingest data from; and"
                print " -v - verbose mode for output logging"
                sys.exit(2)

                
        def __init__(self, datacentre=None):
                '''
                Main entry point for script
                '''
                self.lineSeparator = "-----------------------------"
                print self.lineSeparator
                print "RUNNING: oai_ingest.py"          
                
                # check for verbose option
                try:
                    opts, args = getopt.getopt(sys.argv[1:], "v")
                except getopt.GetoptError, err:
                    # print help information and exit:
                    print str(err) # will print something like "option -a not recognized"
                    self.usage()
                    
                loggingLevel = logging.WARNING
                for o, a in opts:
                    if o == "-v":
                        print " - Verbose mode ON"
                        loggingLevel = logging.DEBUG
                
                logging.basicConfig(level=loggingLevel,
                                                    format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
                
                print self.lineSeparator
                
                if datacentre is None:
                        self.usage()
                
                # create file utils object to do various file related stuff
                fileUtils = FileUtilities()
                
                status = 0
                numfilesproc = 0
                self._base_dir = os.getcwd() + "/"# this is the base dir that the script is ran from
                        
                data_dir = self._base_dir + "data/" + datacentre  # dir relating to the specified datacentre docs
                
                #Change os directory to that with the harvested documents in it.
                os.chdir(self._base_dir)
                
                # - to run on Windows under cygwin, use the following
                #os.putenv('PATH', 'C:\\opt\\cygwin\\bin')
                
                # set the global variables to retrieve from the config file
                self._harvest_home = ""
                self._datacentre_groups = ""
                self._datacentre_format = ""
                self._datacentre_namespace = ""
                self._NDG_dataProvider = False
                self.getConfigDetails(datacentre)
                
                #any records to harvest?
                if len( os.listdir(self._harvest_home)) == 0:
                    logging.info("Nothing to harvest this time from " + datacentre)
                    sys.exit()
                
                # The directory to put things for a tape backup (should already exist)
                #backupdir = '/disks/glue1/oaiBackup/'
                # TODO: uncomment above on live system
                backupdir = data_dir + "/backups/"
                
                # the following dirs define where the specific documents should go
                originals_dir = data_dir + "/oai/originals/"
                discovery_dir = data_dir + "/discovery/"
                
                # Create/clear the 'in' directory pristine copy of the discovery records
                fileUtils.setUpDir(originals_dir)
                commandline = "ls -1 " + self._harvest_home + "/ | xargs -i cp " + self._harvest_home + "/{\} " + originals_dir
                #commandline = "find " + self._harvest_home + " -type f -print | xargs -i cp \{\} " + originals_dir
                logging.info("Executing : " + commandline)
                status = os.system(commandline)
                
                if status !=0:
                    sys.exit("Failed at making pristine copy stage")
                
                # Create/clear the directory for the 'out' processed copy of the discovery records.
                fileUtils.setUpDir(discovery_dir)
                    
                #Execute the script which processes/renames the files (changed 08/01/07 to get id from inside file)
                # - also replace any namespace declarations with a standard one which we know works in NDG
                # NB, this copies files from the original dir to the discovery dir
                logging.info(self.lineSeparator)
                logging.info("Renaming files:")
                for filename in os.listdir(originals_dir):
                        if filename.endswith('.xml'):
                                original_filename = originals_dir + filename
                                ident=self.getID(original_filename)
                                
                                if self._NDG_dataProvider:
                                        new_filename = discovery_dir + ident.replace(":","__")+".xml"
                                else:
                                                ident = ident.replace(":","-")
                                                ident = ident.replace("/","-")
                                                new_filename = discovery_dir + "/" +self._datacentre_namespace+ "__"+self._datacentre_format+ "__"+ ident +".xml"
                                                logging.info("original file = " + original_filename)
                                                logging.info("newfile = " + new_filename)
                                
                                # now correct any namespace issues
                                try:
                                    SchemaNameSpace(original_filename, new_filename, self._datacentre_format)
                                except:
                                        sys.exit("ERROR: SchemaNameSpace failed on file %s" %original_filename)
                                numfilesproc += 1
                        else:
                                logging.warning('File %s is not xml format. Not processed'  %(filename))
                
                logging.info(self.lineSeparator)
                
                # now set up the required XQueries
                # - NB, extract the xquery libraries locally for easy reference
                self._xq=ndgXqueries()
                for libFile in self._xq.xqlib:
                        fileUtils.createFile(libFile, self._xq.xqlib[libFile])
                
                # Process the resulting files and put the data into the postgres DB
                # - firstly set up a db connection to use
                self._dbConnection = None
                self._getDBConnection()
                
                filenames = os.listdir(discovery_dir)
                for filename in filenames:
                        self.addFileToPostgresDB(discovery_dir + filename)
                
                #Make copies of discovery and oai/originals and DIF2MOLES areas to backup area for tape backups
                backupdir_base = backupdir + datacentre + "_" + strftime("%y%m%d_%H%M")
                
                this_backupdir = backupdir_base + "_originals/"
                fileUtils.makeBackUp(originals_dir, this_backupdir)
                
                #Clear out the original harvest records area and FINALMOLES
                fileUtils.cleanDir(originals_dir)
                fileUtils.cleanDir(discovery_dir)
                # TODO: uncomment following line when live on system
                #fileUtils.cleanDir(self._harvest_home)
                
                print self.lineSeparator
                print "INFO: No. of files pre-processed = %s" %numfilesproc
                if status == 0:
                    print "INFO: Procedure oai_ingest.py completed"
                else:
                    print "ERROR: Procedure oai_ingest.py FAILED with status %s" %status
                print self.lineSeparator
                
        
if __name__=="__main__":
        opts, args = getopt.getopt(sys.argv[1:], "v")
        if len(args) < 1:
                oai_ingest()
        
        oai_ingest(args[0])