Changeset 3861 for TI01-discovery

Timestamp:

07/05/08 15:54:12 (13 years ago)

Author:

cbyrom

Message:

Clear out redundant code and rename main script, oai_document_ingester

• change references to this accordingly.

Location:

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch

Files:

• RecordCollection.py (deleted)
• SpaceTimeIngestFromMOLES.py (deleted)
• SpaceTimeIngestPostgisFromMOLES.py (deleted)
• oai_document_ingester.py (moved) (moved from TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_ingest_new2.py) (9 diffs)
• oai_ingest.py (deleted)
• oai_ingest_new.py (deleted)
• retired (deleted)
• run_all_ingest.py (modified) (3 diffs)

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

@@ -1 +1 @@
 #!/usr/bin/env python
-""" Script oai_ingest.py takes parameters <datacentre> <dbinfofile>.
-The /usr/local/WSClients/OAIBatch directory contains:-
- - this python script, plus some other modules eg ndgUtils for parts of the process.
- - a DataProvider specific config file,
- - the python module for extracting spatiotemporal information and adding to postgres db.
-Under this directory the following structure should be maintained:
- ./data
- - /DATACENTRE/
-                - discovery/:         Re-named documents.
-        - discovery_corrected Documents with schema namespaces corrected, ready to ingest in the discovery service.
-                - oai/difYYYYMMDD/    Documents as harvested from OAI
- Where  /DATACENTRE  varies for the different data providers
-"""
-#History:
-# 12/05/06 SEL spelling correction
-# 30/05/06 SEL cope with many files for processing."Argument list too long" problem.
-# 31/05/06 SEL need to use passwords now. (replace xxxxxx in real version).
-# 16/10/06 SEL Changed to using python oaiClean.py module instead of java code.
-# 16/10/06 SEL exist db upgraded and deployed at different location, java upgrade.
-# 17/10/06 SEL cope with different discovery formats - not just DIF.
-# 23/10/06 SEL keywords not mandatory in config file.
-# 24/10/06 SEL fix bug where 'in' directory not being cleared initially. display more information when running.
-#  December 2007 SEL rewrite to use Bryans' python XQuery stuff to create mini-moles instead of java.
-#                    Also extracted hard coded pwds into a file.
-# 11/04/08 CByrom Tidy up script by organising code into reusable functions + variables 
-# + remove dependency on eXist DB
-
+'''
+ Main script to do the document ingest from the OAI harvested files to the 
+ discovery postgres DB.  NB, can be ran for all datacentres using the run_all_ingest.py script
+ or can specify an individual datacentre to run the ingester on.
+ As well as doing the ingest, a backup directory is created to store the created moles files.
+'''
 import os, sys, string, getopt, logging
 from time import strftime
@@ -39 +18 @@
 import db_funcs
 
-class oai_ingest:
+class oai_document_ingester:
         '''
         Class to handle the ingest of files from the OAI harvester to the discovery service postgres DB
-        - including running the various transforms and parsings to get all doc types and spatiotemporal data
-        in the correct form in the DB
+        - including running the various transforms and parsings to get all doc types and spatiotemporal 
+        data in the correct form in the DB
         '''
 
@@ -83 +62 @@
                 # first of all create a PostgresRecord - this object represents all the data required
                 # for a DB entry
-                record = PostgresRecord(filename, self._NDG_dataProvider, \
+                try:
+                        record = PostgresRecord(filename, self._NDG_dataProvider, \
                                                             self._datacentre_groups, self._datacentre_namespace, \
                                                             discoveryID, self._xq, self._datacentre_format)
         
-                # Now create the data access object to interface to the DB
-                dao = PostgresDAO(record, self._dbConnection)
-                
-                # Finally, write the new record
-                if dao.createOrUpdateRecord():
-                        self._no_files_ingested += 1
+                        # Now create the data access object to interface to the DB
+                        dao = PostgresDAO(record, self._dbConnection)
+                
+                        # Finally, write the new record
+                        if dao.createOrUpdateRecord():
+                                self._no_files_ingested += 1
+                except:
+                        logging.error("Exception thrown - detail: ")
+                        logging.error(sys.exc_info())
                         
         
@@ -141 +124 @@
                     logging.info("No groups/keywords set for datacentre " + datacentre)
                 else:
-                    logging.info("datacentre groups/keywords: " + self._datacentre_groups)
+                    logging.info("datacentre groups/keywords: " + str(self._datacentre_groups))
                 
                 if self._datacentre_format == "":
@@ -182 +165 @@
                 Display input params for the script
                 '''
-                print "Usage: python -v oai_ingest.py <datacentre>"
+                print "Usage: python oai_document_ingester.py -v|d<datacentre>"
                 print " - where:\n   <datacentre> is the data centre to ingest data from; and"
                 print " -v - verbose mode for output logging"
@@ -195 +178 @@
                 self.lineSeparator = "-----------------------------"
                 print self.lineSeparator
-                print "RUNNING: oai_ingest.py"          
+                print "RUNNING: oai_document_ingester.py"               
                 
                 # check for verbose option
@@ -246 +229 @@
                 self.getConfigDetails(datacentre)
                 
-                #any records to harvest?
-                if len( os.listdir(self._harvest_home)) == 0:
-                    logging.info("Nothing to harvest this time from " + datacentre)
-                    sys.exit()
+                # check harvest dir exists and that there are any records to harvest?
+                dpath = os.path.normpath(os.path.dirname(self._harvest_home))
+                if not os.path.exists(dpath):
+                        logging.info("Harvest directory for datacentre %s (%s) could not be found - exiting" \
+                                                 %(datacentre, self._harvest_home))
+                        return
+                elif len(os.listdir(self._harvest_home)) == 0:
+                        logging.info("Nothing to harvest this time from %s" %datacentre)
+                        return
+                
                 
                 # The directory to put things for a tape backup (should already exist)
@@ -262 +251 @@
                 # Create/clear the 'in' directory pristine copy of the discovery records
                 fileUtils.setUpDir(originals_dir)
-                commandline = "ls -1 " + self._harvest_home + "/ | xargs -i cp " + self._harvest_home + "/{\} " + originals_dir
-                #commandline = "find " + self._harvest_home + " -type f -print | xargs -i cp \{\} " + originals_dir
+                commandline = "find " + self._harvest_home + " -type f -print | xargs -i cp \{\} " + originals_dir
                 logging.info("Executing : " + commandline)
                 status = os.system(commandline)
-                
+
                 if status !=0:
                     sys.exit("Failed at making pristine copy stage")
@@ -334 +322 @@
                 print "INFO: Number of files ingested = %s" %self._no_files_ingested
                 if status == 0:
-                    print "INFO: Procedure oai_ingest.py completed"
+                    print "INFO: Procedure oai_document_ingester.py completed"
                 else:
-                    print "ERROR: Procedure oai_ingest.py FAILED with status %s" %status
+                    print "ERROR: Procedure oai_document_ingester.py FAILED with status %s" %status
                 print self.lineSeparator
                 
         
 if __name__=="__main__":
-        opts, args = getopt.getopt(sys.argv[1:], "v")
+        opts, args = getopt.getopt(sys.argv[1:], '-vd')
         if len(args) < 1:
-                oai_ingest()
-        
-        oai_ingest(args[0])
+                oai_document_ingester()
+        
+        oai_document_ingester(args[0])

TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

@@ -3 +3 @@
 """
 import os, sys, logging
-from oai_ingest_new2 import oai_ingest
+from oai_document_ingester import oai_document_ingester
 
 run_counter = 0
+error_counter = 0
 
 lineSeparator = "-----------------------------"
@@ -13 +14 @@
 
 # configure logging
-logging.basicConfig(level=logging.DEBUG,
+logging.basicConfig(level=logging.INFO,
                     format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
 
@@ -35 +36 @@
             # now invoke the ingest script
             logging.info("Running the ingest script for datacentre: %s" %datacentre)
-            oai_ingest(datacentre)
-            run_counter += 1
+            try:
+                run_counter += 1
+                oai_document_ingester(datacentre)
+            except:
+                error_counter +=1
                 
-print "run_all_ingest.py complete - processed %s config files" %run_counter             
+print "run_all_ingest.py complete - processed %s config files" %run_counter
+if error_counter > 0:           
+    print "WARNING: %s errors were encountered during the run - check logfiles for more details" %error_counter
 
-