Changeset 2508 for TI01-discovery

Timestamp:

22/05/07 18:10:10 (12 years ago)

Author:

selatham

Message:

various improvements

Location:

TI01-discovery/trunk/ingestAutomation/OAIBatch

Files:

• SchemaNameSpace.py (modified) (1 diff)
• SpaceTimeIngestFromMOLES.py (modified) (3 diffs)
• oai_ingest.py (modified) (4 diffs)
• ukho_config.properties (modified) (1 diff)

TI01-discovery/trunk/ingestAutomation/OAIBatch/SchemaNameSpace.py

@@ -11 +11 @@
         #print dir(self.ff)
         #print dir(self.ww)
-        print "format: %s" %self.format
+        #print "format: %s" %self.format
         self.lines=self.ff.readlines()
         for self.line in self.lines:
             if self.format== "DIF" and self.line.startswith('<DIF'):
-                print "changing line"
+                print "INFO: changing line for %s. output to %s" %(infile,outfile)
                 self.line='<DIF xmlns="http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">\n'
             self.ww.write(self.line)

TI01-discovery/trunk/ingestAutomation/OAIBatch/SpaceTimeIngestFromMOLES.py

@@ -37 +37 @@
             sql=sql.replace("'"+startdate+"',"," ")
         if str(enddate)=='noenddate':
+            sql=sql.replace(", enddate"," ")
             sql=sql.replace("enddate"," ")
+            #print sql
             sql=sql.replace(",   'no '"," ")
-            sql=sql.replace("'"+enddate+"',"," ")
+            sql=sql.replace(", '"+enddate+"'"," ")
+            #print sql
         if str(west)=='null' or str(south)=='null' or str(east)=='null' or str(north)=='null':
             sql=sql.replace("coordinates,"," ")
@@ -117 +120 @@
                 print "INFO: XML moles document %s does not contain any spatiotemporal info." %full_filename
                 continue
+
             if no_dates:
                 pass
@@ -122 +126 @@
                 startdate=dates.DateRangeStart
                 enddate= dates.DateRangeEnd
-                if startdate==None:
+                if startdate==None or startdate=='None':
                     startdate="nostartdate"
-                if enddate==None:
+                if enddate==None or enddate=='None':
                     enddate="noenddate"
 

TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

@@ -154 +154 @@
 # Create/clear the directory for the 'out' namespace corrected copy of the discovery records.
 if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected"):
-    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/ | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected/{\}"
+    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected/ | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery_corrected/{\}"
     print "Executing : " + commandline
     status = os.system(commandline)
@@ -193 +193 @@
                 status = os.system(commandline)
                 if status !=0:
-                    sys.exit("Failed at re-naming file stage")
+                    sys.exit("ERROR: Failed at re-naming file stage")
                 numfilesproc += 1
         else:
@@ -209 +209 @@
                        SchemaNameSpace(in_filename, corrected_filename,datacentre_format)
                     except:
-                       print "SchemaNameSpace failed on file %s"%in_filename
+                       print "ERROR: SchemaNameSpace failed on file %s"%in_filename
 
 # ingest the datacentres records into eXist db (backups of exist happen nightly).
@@ -228 +228 @@
     status = os.system(commandline)
     if status !=0:
-        sys.exit("Failed at clearing out DIF2MOLES area.")
+        sys.exit("ERROR: Failed at clearing out DIF2MOLES area.")
 
 # Then run the minimum moles creator for each discovery record

TI01-discovery/trunk/ingestAutomation/OAIBatch/ukho_config.properties

@@ -5 +5 @@
 #
 #Define host_path as the full directory name where this data centre's records will be harvested into.
-host_path /usr/local/jakarta-tomcat/webapps/oai/WEB-INF/harvested_records/www-dassh-ac-uk-8080-MDIP-ukho
+host_path /usr/local/jakarta-tomcat/webapps/oai/WEB-INF/harvested_records/62-172-108-26-8080-MDIP
 #
 #Define groups - portal groups for limiting searches by 'group of datacentres'.