Changeset 1889 for TI01-discovery

Timestamp:

18/12/06 19:16:44 (14 years ago)

Author:

selatham

Message:

getting auto ingest working

Location:

TI01-discovery/trunk/ingestAutomation/OAIBatch

Files:

• bodc_config.properties (modified) (1 diff)
• neodc_config.properties (modified) (1 diff)
• nocs_config.properties (modified) (1 diff)
• oai_ingest.py (modified) (7 diffs)
• pml_config.properties (modified) (1 diff)

TI01-discovery/trunk/ingestAutomation/OAIBatch/bodc_config.properties

@@ -8 +8 @@
 #
 #Define groups - portal groups for limiting searches by 'group of datacentres'.
-groups NERC-DDC NERC MDIP
+groups NERC-DDC MDIP
 #
 #Define which format is harvested from the data centre (one only)
-format dif
+format DIF
 #
 #Define the data providers namespace

TI01-discovery/trunk/ingestAutomation/OAIBatch/neodc_config.properties

@@ -8 +8 @@
 #
 #Define groups - portal groups for limiting searches by 'group of datacentres'.
-groups NERC NERC-DDC
+#
 #
 #Define which format is harvested from the data centre (one only)
-format dif
+format DIF
 #
 #Define the data providers namespace

TI01-discovery/trunk/ingestAutomation/OAIBatch/nocs_config.properties

@@ -8 +8 @@
 #
 #Define groups - portal groups for limiting searches by 'group of datacentres'.
-groups NERC
+#
 #
 #Define which format is harvested from the data centre (one only)
-format dif
+format DIF
 #
 #Define the data providers namespace

TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

@@ -1 +1 @@
 #!/usr/bin/env python
 """ Script oai_ingest.py takes parameter <datacentre>.
-The /usr/local/WSClients/OAIBatch directory contains this python script, a DataProvider specific config file
-and the oaiClean.py class which cleans up discovery records after harvesting.
-The pre-processed files are then ingested to the eXist XML db.
-
- Under this directory the following structure should be maintained:
-
+The /usr/local/WSClients/OAIBatch directory contains:-
+ - this python script,
+ - a DataProvider specific config file,
+ - the d2b.jar moles creator class which creates moles discovery records,
+ - the python module for extracting spatiotemporal information and adding to postgres db.
+Under this directory the following structure should be maintained:
  ./data
  - /DATACENTRE/
-                - discovery/:         Records with namespace, schema declaration deleted - after having run
-                                      the oaiClean script. Ready to ingest in the discovery service.
-                - oai/difYYYYMMDD/    Records as harvested from OAI
-
+                - discovery/:         Re-named documents ready to ingest in the discovery service.
+                - oai/difYYYYMMDD/    Documents as harvested from OAI
  Where  /DATACENTRE  varies for the different data providers
-
 """
 #History:
@@ -48 +45 @@
 date_string = commands.getoutput ("date +'%y%m%d_%H%M'")
 os.putenv ('EXIST_HOME', '/usr/local/exist-client')
-os.putenv ('PATH', ':/usr/java/jdk1.5.0_03/jre:/usr/java/jdk1.5.0_03:/usr/java/jdk1.5.0_03/lib/tools.jar:/usr/local/WSClients/OAIBatch:/usr/local/exist-client/bin:/bin:/usr/bin:.')
-os.putenv ('CLASSPATH','.:/usr/java/j2sdk1.4.2_04/bin:/usr/local/WSClients/OAIBatch')
+os.putenv ('JAVA_HOME', '/usr/java/jdk1.5.0_03')
+os.putenv ('PATH', ':/usr/java/jdk1.5.0_03/bin:/usr/java/jdk1.5.0_03:/usr/java/jdk1.5.0_03/lib/tools.jar:/usr/local/WSClients/OAIBatch:/usr/local/exist-client/bin:/bin:/usr/bin:.')
+os.putenv ('CLASSPATH','.:/usr/java/jdk1.5.0_03/lib/tools.jar')
 
 # Get the harvested records directory and groups for this datacentre from the datacentre specific config file
@@ -140 +138 @@
     sys.exit("Failed at copying config file stage")
 
-#Change os directory to that with the other code in it. (need this?)
+#Change os directory to that with the code in it.
 os.chdir('/usr/local/WSClients/OAIBatch')
 
@@ -156 +154 @@
                 #print "original file = %s, newfile = %s" %(original_filename, new_filename)
                 commandline = "cp "+original_filename+ " " +new_filename
-                print "Executing : " + commandline
+                #print "Executing : " + commandline
                 status = os.system(commandline)
                 if status !=0:
@@ -174 +172 @@
 # Then run the minimum moles creator  which will run over all records in the supplied collection
 # creates a directory ./DIF2MOLES to pass back records with original filename
-commandline = "java -jar d2b.jar repositoryID " +datacentre_namespace+" repositoryLocalID "+datacentre+" format "+datacentre_format+" repository xmldb:exist://glue.badc.rl.ac.uk:8080/exist/xmlrpc userpw xxxxxx targetCollection /db/discovery/original/"+datacentre_format+"/"+datacentre_namespace
-print commandline
+commandline = "java -jar D2B/d2b.jar repositoryID " +datacentre_namespace+" repositoryLocalID "+datacentre+" format "+datacentre_format+" repository xmldb:exist://glue.badc.rl.ac.uk:8080/exist/xmlrpc userpw xxxxxx targetCollection /db/discovery/original/"+datacentre_format+"/"+datacentre_namespace
+print "Executing command to run d2b.jar"
 status= os.system(commandline)
 if status!=0:
@@ -182 +180 @@
 
 # ingest the created discovery minimum molesrecords into eXist db.
-commandline = "$EXIST_HOME/bin/client.sh -c ./DIF2MOLES -u admin -P xxxxxx -p ./DIF2MOLES"
+commandline = "$EXIST_HOME/bin/client.sh -c /db/discovery/moles -u admin -P xxxxxx -p ./DIF2MOLES"
 print "Executing : actual command to ingest into exist db"
 status = os.system(commandline)
@@ -229 +227 @@
     sys.exit("Failed at creating backup directory %s" %this_backupdir)
 
-commandline = "ls -1 ./DIF2MOLES | xargs -i cp ./DIF2MOLES{\} " + this_backupdir
+commandline = "ls -1 ./DIF2MOLES | xargs -i cp ./DIF2MOLES/{\} " + this_backupdir
 print "Executing : " + commandline
 status = os.system(commandline)

TI01-discovery/trunk/ingestAutomation/OAIBatch/pml_config.properties

@@ -10 +10 @@
 #
 #Define which format is harvested from the data centre (one only)
-format dif
+format DIF
 #
 #Define the data providers namespace