Changeset 2067 for TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

Timestamp:

26/01/07 20:38:15 (13 years ago)

Author:

selatham

Message:

Plugging in new d2boneoff.

File:

• TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py (modified) (6 diffs)

TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

@@ -161 +161 @@
                     y=loadET(xml)
                     d=DIF(xml)
-                    print d.entryID
+                    #print "ID extracted from the DIF = %s" %d.entryID
                     if NDG_dataProvider:
                         new_filename = outdir + "/"+d.entryID.replace(":","__")+".xml"
@@ -168 +168 @@
                 else:
                     sys.exit("Doesn't handle anything else but DIF here.")
-                #new_filename = outdir + "/" +datacentre_namespace+ "__"+datacentre_format+ "__"+filename.split('%3A')[-1]
-                print "original file = %s, newfile = %s" %(original_filename, new_filename)
+                #print "original file = %s, newfile = %s" %(original_filename, new_filename)
                 commandline = "cp "+original_filename+ " " +new_filename
                 #print "Executing : " + commandline
@@ -187 +186 @@
 
 #are there any old records hanging around.If so, copy away
-outdir = "./DIF2MOLES"
 try:
-    os.stat(outdir)
+    os.stat("./DIF2MOLES")
 except:
     print "No old moles records hanging around"
@@ -198 +196 @@
     if status !=0:
         sys.exit("Failed at clearing out DIF2MOLES area.")
-    commandline = "rmdir ./DIF2MOLES"
-    print "Executing : " + commandline
-    status = os.system(commandline)
-    if status !=0:
-        sys.exit("Failed at removing DIF2MOLES directory.")
-
-# Then run the minimum moles creator  which will run over all records in the supplied collection
-# creates a directory ./DIF2MOLES to pass back records with original filename
-commandline = "java -jar D2B/d2b.jar repositoryID " +datacentre_namespace+" repositoryLocalID "+datacentre+" format "+datacentre_format+" repository xmldb:exist://glue.badc.rl.ac.uk:8080/exist/xmlrpc userpw xxxxxx targetCollection /db/discovery/original/"+datacentre_format+"/"+datacentre_namespace
-print "Executing command to run d2b.jar"
-status= os.system(commandline)
-if status!=0:
-    print "ERROR: couldn't create the minimum moles records"
-    sys.exit
+    #commandline = "rmdir ./DIF2MOLES"
+    #print "Executing : " + commandline
+    #status = os.system(commandline)
+    i#f status !=0:
+     #   sys.exit("Failed at removing DIF2MOLES directory.")
+
+# Then run the minimum moles creator for each discovery record
+# Put records in ./DIF2MOLES with original filename
+filenames = os.listdir(outdir)
+for filename in filenames:
+        if filename.find('.xml') != -1:
+                if datacentre_format == "DIF":
+                    original_filename = outdir + "/" + filename
+                    from DIF import DIF
+                    from ETxmlView import loadET
+                    xml=file(original_filename).read()
+                    y=loadET(xml)
+                    d=DIF(xml)
+                    print "ID extracted from the DIF = %s" %d.entryID
+                    if NDG_dataProvider:
+                        molesLocalID = d.entryID.split(":",2)[2]
+                    else:
+                        molesLocalID = d.entryID
+                    print "molesLocalID is %s" %molesLocalID
+                    commandline = "java -jar D2B/d2boneoff.jar repositoryID " +datacentre_namespace+" repositoryLocalID "+datacentre+" format "+ \
+                    datacentre_format+" repository xmldb:exist://glue.badc.rl.ac.uk:8080/exist/xmlrpc userpw xxxxxx targetCollection /db/discovery/original/"+ \
+                    datacentre_format+"/"+datacentre_namespace +" inputRecordID "+d.entryID+ " outputLocalID "+molesLocalID+ " > ./DIF2MOLES/"+filename
+                    print "Executing command to run d2boneoff.jar"
+                    status= os.system(commandline)
+                    if status==10:
+                        print "WARNING: couldn't find the record"
+                    elif status!=0:
+                        print "ERROR: couldn't create the minimum moles records"
+                        sys.exit
 #There should be some records now
 try:
-    os.stat(outdir)
+    os.stat("./DIF2MOLES")
 except:
     print "ERROR: couldn't create any minimum moles records for %s" %datacentre
@@ -227 +245 @@
 
 #Extract the spatiotemporal info from created moles and put in Postgres db
-SpaceTimeIngestFromMOLES.main(outdir)
+SpaceTimeIngestFromMOLES.main("./DIF2MOLES")
 
 #Make copies of discovery and oai/originals and DIF2MOLES areas to backup area for tape backups
@@ -283 +301 @@
 
 #remove the DIF2MOLES directory
-commandline = "rmdir ./DIF2MOLES"
-print "Executing : " + commandline
-status = os.system(commandline)
-if status !=0:
-    sys.exit("Failed at removing DIF2MOLES directory %s" %harvest_home)
+##commandline = "rmdir ./DIF2MOLES"
+#print "Executing : " + commandline
+#status = os.system(commandline)
+#if status !=0:
+#    sys.exit("Failed at removing DIF2MOLES directory %s" %harvest_home)
 
 print "======================================================"