Ignore:
Timestamp:
30/05/06 19:25:37 (14 years ago)
Author:
selatham
Message:

needs to cope with large numbers of files

File:
1 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/trunk/ingestAutomation/OAIBatch/oai_ingest.py

    r1045 r1066  
    1818#History: 
    1919# 12/05/06 SEL spelling correction 
    20 # 
     20# 30/05/06 SEL cope with many files for processing."Argument list too long" problem. 
    2121# 
    2222 
     
    6969# Create/clear the directory for a pristine copy of the difs in case the script rewrites something wrong 
    7070if os.path.isdir("/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy"): 
    71     commandline = "rm -f /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/*" 
     71    commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/* | xargs -i rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/{\}" 
    7272    print "Executing : " + commandline 
    7373    status = os.system(commandline) 
     
    8080    sys.exit("Failed at creating copy dir stage") 
    8181 
    82 # make the pristine copy 
     82# make the pristine copy. Cope with there being lots of files in the directory. 
    8383 
    84 commandline = "ls -1 " + harvest_home + "/ | xargs -i cp \{\} " /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy" 
     84commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/oai/difcopy" 
    8585print "Executing : " + commandline 
    8686status = os.system(commandline) 
     
    9999 
    100100# make the processing copy 
    101 commandline = "cp " + harvest_home + "/*.xml /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery" 
     101commandline = "ls -1 " + harvest_home + "/ | xargs -i cp " + harvest_home + "/{\} /usr/local/WSClients/OAIBatch/data/" + datacentre + "/discovery" 
    102102print "Executing : " + commandline 
    103103status = os.system(commandline) 
     
    139139#the filenames and it will leave <DIF> as the root element. 
    140140# 
    141 #Once the pre-processing has finished remove the originals from the discovery directory:  
    142 commandline = "rm /usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/oai*" 
     141#Once the pre-processing has finished remove the originals from the discovery directory: 
     142commandline = "ls -1 usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/oai* | xargs -i rm usr/local/WSClients/OAIBatch/data/" + datacentre +"/discovery/{\}" 
    143143print "Executing : " + commandline 
    144144status = os.system(commandline) 
     
    160160if status !=0: 
    161161    sys.exit("Failed at creating backup directory %s" %this_backupdir) 
    162 commandline = "cp " + "/usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/* " + this_backupdir 
     162commandline = "ls -1 /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/ | xargs -i cp /usr/local/WSClients/OAIBatch/data/" + datacentre +"/oai/difcopy/{\}" + this_backupdir 
    163163print "Executing : " + commandline 
    164164status = os.system(commandline) 
Note: See TracChangeset for help on using the changeset viewer.