Changeset 2252

Timestamp:

09/03/07 17:33:03 (13 years ago)

Author:

selatham

Message:

Copes with MDIP format. Some new config file

Location:

TI01-discovery/trunk/ingestAutomation/OAIBatch

Files:

• MDIP.py (added)
• SpaceTimeIngestFromMOLES.py (modified) (1 diff)
• bodc_config.properties (modified) (1 diff)
• dassh_config.properties (added)
• oai_ingest.py (modified) (3 diffs)
• ukho_config.properties (added)

TI01-discovery/trunk/ingestAutomation/OAIBatch/SpaceTimeIngestFromMOLES.py

@@ -88 +88 @@
                 print "INFO: XML moles document %s does not contain a bounding box." %full_filename
                 no_bbox=True
+            if bbox_list[0] =='None':
+                no_bbox=True
+
             try:
                 dates=dgMeta.dgMetadataRecord.dgDataEntity.dgDataSummary.dgDataCoverage.dgTemporalCoverage.DateRange

TI01-discovery/trunk/ingestAutomation/OAIBatch/bodc_config.properties

@@ -5 +5 @@
 #
 #Define host_path as the full directory name where this data centre's records will be harvested into.
-host_path /usr/local/jakarta-tomcat/webapps/oai/WEB-INF/harvested_records/grid-bodc-nerc-ac-uk-dif-ndg_bodc
+host_path /usr/local/jakarta-tomcat/webapps/oai/WEB-INF/harvested_records/grid-bodc-nerc-ac-uk-dif
 #
 #Define groups - portal groups for limiting searches by 'group of datacentres'.
-groups NERC-DDC MDIP
+#groups NERC_DDC MDIP
 #
 #Define which format is harvested from the data centre (one only)

TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

@@ -29 +29 @@
 import SpaceTimeIngestFromMOLES
 import keywordAdder
-#import oaiClean
+from DIF import DIF
+from MDIP import MDIP
+
+def getID(filename):
+        ''' Gets the identifier out of an input metadata xml record. Copes with DIF and MDIP currently.'''
+        xml=file(filename).read()
+        if datacentre_format == "DIF":
+            d=DIF(xml)
+            ID=d.entryID
+        elif datacentre_format == "MDIP":
+            d=MDIP(xml)
+            ID=d.id
+        else:
+            sys.exit("Only handles DIF or MDIP here.")
+        return ID
 
 status = 0
@@ -156 +170 @@
         if filename.find('.xml') != -1:
                 original_filename = indir + "/" + filename
-                if datacentre_format == "DIF":
-                    from DIF import DIF
-                    from ETxmlView import loadET
-                    xml=file(original_filename).read()
-                    y=loadET(xml)
-                    d=DIF(xml)
-                    #print "ID extracted from the DIF = %s" %d.entryID
-                    if NDG_dataProvider:
-                        new_filename = outdir + "/"+d.entryID.replace(":","__")+".xml"
-                    else:
-                        new_filename = outdir + "/" +datacentre_namespace+ "__"+datacentre_format+ "__"+d.entryID+".xml"
+                ident=getID(original_filename)
+                print "ID extracted from the DIF = %s" %ident
+                if NDG_dataProvider:
+                    new_filename = outdir + "/"+ ident.replace(":","__")+".xml"
                 else:
-                    sys.exit("Doesn't handle anything else but DIF here.")
-                #print "original file = %s, newfile = %s" %(original_filename, new_filename)
+                    new_filename = outdir + "/" +datacentre_namespace+ "__"+datacentre_format+ "__"+ ident +".xml"
+                print "original file = %s, newfile = %s" %(original_filename, new_filename)
                 commandline = "cp "+original_filename+ " " +new_filename
                 #print "Executing : " + commandline
@@ -203 +210 @@
 for filename in filenames:
         if filename.find('.xml') != -1:
-                if datacentre_format == "DIF":
                     original_filename = outdir + "/" + filename
-                    from DIF import DIF
-                    from ETxmlView import loadET
-                    xml=file(original_filename).read()
-                    y=loadET(xml)
-                    d=DIF(xml)
-                    print "ID extracted from the DIF = %s" %d.entryID
+                    ident=getID(original_filename)
                     if NDG_dataProvider:
-                        molesLocalID = d.entryID.split(":",2)[2]
+                        molesLocalID = ident.split(":",2)[2]
                     else:
-                        molesLocalID = d.entryID
+                        molesLocalID = ident
                     print "molesLocalID is %s" %molesLocalID
                     commandline = "java -jar D2B/d2boneoff.jar repositoryID " +datacentre_namespace+" repositoryLocalID "+datacentre+" format "+ \
                     datacentre_format+" repository xmldb:exist://glue.badc.rl.ac.uk:8080/exist/xmlrpc userpw xxxxxx targetCollection /db/discovery/original/"+ \
-                    datacentre_format+"/"+datacentre_namespace +" inputRecordID "+d.entryID+ " outputLocalID "+molesLocalID+ " > ./DIF2MOLES/"+filename
+                    datacentre_format+"/"+datacentre_namespace +" inputRecordID "+ ident+ " outputLocalID "+molesLocalID+ " > ./DIF2MOLES/"+filename
                     print "Executing command to run d2boneoff.jar"
                     status= os.system(commandline)