Changeset 6186


Ignore:
Timestamp:
18/12/09 09:44:49 (10 years ago)
Author:
sdonegan
Message:

Added ability to take processing info from a config file (not in svn for obvious reasons!).

Location:
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
Files:
3 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/abstractdocumentingester.py

    r6135 r6186  
    203203                         
    204204         
     205        def getProcessingConfig(self,configFilePath): 
     206                 
     207                ''' 
     208                Fed up with messing about with hardcoded directory paths.   
     209                This method to get relevant values out of the oai_document_ingester.config file 
     210                 
     211                Returns a dictionary of values with keys: 
     212                 
     213                #directory in which the code resides 
     214                code_directory /home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/ 
     215 
     216                #base directory in which metadata is extracted to and converted to 
     217                base_directory /home/badc/discovery_docs/ingestDocs/ 
     218 
     219                #directory in which to write reports to 
     220                reporting_directory /home/badc/discovery_docs/ingestDocs/data/ 
     221 
     222                #path to the passwords file 
     223                passwords_file /home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/passwords.txt 
     224 
     225                #datacentre config file directory path 
     226                datcentre_configs /home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/datacentre_config/ 
     227                ''' 
     228                 
     229                # Check this file exists; if not, assume an invalid datacentre has been specified 
     230                if not os.path.isfile(configFilePath): 
     231                    sys.exit("ERROR: Could not find the processing config file") 
     232                     
     233                processingConfig = {} 
     234                     
     235                processing_config_file = open(configFilePath, "r") 
     236                 
     237                for line in processing_config_file.readlines(): 
     238                        words  = string.split(line) 
     239                        if len(words) == 0: 
     240                                continue 
     241                        elif words[0] == 'code_directory': 
     242                                processingConfig['code_directory'] = words[1]                                                            
     243                        elif words[0] == 'base_directory': 
     244                                processingConfig['base_directory'] = words[1]) 
     245                        elif words[0] == 'reporting_directory': 
     246                                processingConfig['reporting_directory'] = words[1] 
     247                        elif words[0] == 'passwords_file': 
     248                                processingConfig['passwords_file'] = words[1] 
     249                        elif words[0] == 'datcentre_configs': 
     250                                processingConfig['datcentre_configs'] = words[1]                                 
     251                         
     252                return processingConfig 
     253         
     254         
    205255        def getConfigDetails(self, datacentre): 
    206256                ''' 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

    r6130 r6186  
    4646                #self._base_dir = os.getcwd() + "/" # this is the base dir that the script is ran from 
    4747 
    48                 #this is the absolute path to the production ingest dir for ndg3 
    49                 self._code_dir = "/home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/" # this is the base dir that the script is ran from 
    50                 self._base_dir = "/home/badc/discovery_docs/ingestDocs/" 
     48                #extract relevant directories etc from processing config gile 
     49                self.processingDict = self.getProcessingConfig('/home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.config') 
     50                #self._code_dir = "/home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/" # this is the base dir that the script is ran from 
     51                #self._base_dir = "/home/badc/discovery_docs/ingestDocs/" 
     52                 
     53                self._code_dir = self.processingDict['code_directory'] 
     54                self._base_dir = self.processingDict['base_directory'] 
    5155                 
    5256                self._setupDataCentreDirs() 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/run_all_ingest.py

    r6154 r6186  
    1616from oai_document_ingester import oai_document_ingester 
    1717 
     18#pick up path info from processing config file 
     19if not os.path.isfile(configFilePath): 
     20        sys.exit("ERROR: Could not find the processing config file") 
     21                     
     22        processingConfig = {} 
     23                     
     24        processing_config_file = open(configFilePath, "r") 
     25                 
     26        for line in processing_config_file.readlines(): 
     27                words  = string.split(line) 
     28                if len(words) == 0: 
     29                        continue 
     30                elif words[0] == 'code_directory': 
     31                        processingConfig['code_directory'] = words[1]                                                            
     32                elif words[0] == 'base_directory': 
     33                        processingConfig['base_directory'] = words[1]) 
     34                elif words[0] == 'reporting_directory': 
     35                        processingConfig['reporting_directory'] = words[1] 
     36                elif words[0] == 'passwords_file': 
     37                        processingConfig['passwords_file'] = words[1] 
     38                elif words[0] == 'datcentre_configs': 
     39                        processingConfig['datcentre_configs'] = words[1] 
     40 
     41 
    1842 
    1943run_counter = 0 
     
    2145 
    2246#current_dir = os.getcwd() + "/"# this is the base dir that the script is ran from 
    23 current_dir = '/home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/' # NOTE this is changed to the production buildout directory 
     47#current_dir = '/home/badc/buildouts/oai_document_ingester/ingestAutomation-upgrade/OAIBatch/' # NOTE this is changed to the production buildout directory 
     48current_dir = processingConfig['code_directory'] 
    2449logging.info("Running ingest script for all config files in the current run directory (%s)" %current_dir) 
    2550 
    2651#create file for summary report on all ingests 
    27 reportingDir = '/home/badc/discovery_docs/ingestDocs/data/' # base dir where production reporting and output should go 
     52#reportingDir = '/home/badc/discovery_docs/ingestDocs/data/' # base dir where production reporting and output should go 
     53reportingDir = processingConfig['reporting_directory'] 
    2854summaryFileName = reportingDir + 'runAllIngestSummary.txt'         
    2955summaryFile = open(summaryFileName,'w')             
Note: See TracChangeset for help on using the changeset viewer.