Changeset 1066 for TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

Timestamp:

30/05/06 19:25:37 (14 years ago)

Author:

selatham

Message:

needs to cope with large numbers of files

File:

• TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py (modified) (6 diffs)

TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

@@ -18 +18 @@
 #History:
 # 12/05/06 SEL spelling correction
-#
+# 30/05/06 SEL cope with many files for processing."Argument list too long" problem.
 #
 
@@ -69 +69 @@
 # Create/clear the directory for a pristine copy of the difs in case the script rewrites something wrong
 if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy"):
-    commandline = "rm -f /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/*"
+    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/* | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/{\}"
     print "Executing : " + commandline
     status = os.system(commandline)
@@ -80 +80 @@
     sys.exit("Failed at creating copy dir stage")
 
-# make the pristine copy
+# make the pristine copy. Cope with there being lots of files in the directory.
 
-commandline = "ls -1 " + harvest_home + "/ | xargs -i cp \{\} " /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy"
+commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/oai/difcopy"
 print "Executing : " + commandline
 status = os.system(commandline)
@@ -99 +99 @@
 
 # make the processing copy
-commandline = "cp " + harvest_home + "/*.xml /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery"
+commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/discovery"
 print "Executing : " + commandline
 status = os.system(commandline)
@@ -139 +139 @@
 #the filenames and it will leave <DIF> as the root element.
 #
-#Once the pre-processing has finished remove the originals from the discovery directory: 
-commandline = "rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/oai*"
+#Once the pre-processing has finished remove the originals from the discovery directory:
+commandline = "ls -1 usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/oai* | xargs -i rm usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/{\}"
 print "Executing : " + commandline
 status = os.system(commandline)
@@ -160 +160 @@
 if status !=0:
     sys.exit("Failed at creating backup directory %s" %this_backupdir)
-commandline = "cp " + "/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/* " + this_backupdir
+commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/ | xargs -i cp /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/{\}" + this_backupdir
 print "Executing : " + commandline
 status = os.system(commandline)