Ignore:
Timestamp:
31/07/09 15:08:01 (10 years ago)
Author:
sdonegan
Message:

Adjust messaging so FULL report is made available to info editor for email

Location:
TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch
Files:
9 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/Utilities.py

    r5537 r5542  
    260260        et=loadET(xml) 
    261261        helper=nsdumb(et) 
    262         if self._datacentre_format=='DIF': 
     262        if ((self._datacentre_format=='DIF') or (self._datacentre_format=='dif')): 
    263263            #return helper.getText(et,'Entry_ID') 
    264264            self.datasetID=helper.getText(et,'Entry_ID') 
     
    269269            self.datasetEndDateNom = helper.getText(et,'Temporal_Coverage/Stop_Date') 
    270270             
     271            #Fudge to get around some DC's using "entry_title" and others "dataset_title".  grrr. 
     272            if self.datasetName == '': 
     273             self.datasetName == helper.getText(et,'Entry_Title') 
     274                 
    271275            #TODO amend this - just a fudge to ingest records from crappy badc/neodc whilst pipeline down;.. 
    272276            if self.datasetEndDateNom == '': 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/abstractdocumentingester.py

    r5537 r5542  
    358358                 
    359359                logging.debug("Setting up connection to postgres DB") 
    360                 self.pgc = pgc(configFile = 'ingest.config') 
     360                self.pgc = pgc(configFile = '/usr/local/ndg-oai-info-editor/ingest.config') 
    361361                logging.info("Postgres DB connection now set up") 
    362362 
     
    376376                #Clear out the original harvest records area and discovery dir 
    377377                #FileUtilities.cleanDir(self.originals_dir) 
    378                 FileUtilities.cleanDir(self.discovery_dir) 
     378                #FileUtilities.cleanDir(self.discovery_dir) 
    379379                logging.info("Ingest directories cleared") 
    380380 
     
    458458                logging.info("oai_document_ingest processing complete:") 
    459459                 
    460                 recOpFile.write("Ingest report for data centre: " + datacentre + "\n") 
     460                #Update log file details op on ingest to s atring which is THEN wrtten to a file, so as can return and email for info editor 
     461                message = "Ingest report for data centre: " + datacentre + "\n" 
     462                message = message + "Ingest date: " + str(datetime.datetime.now()) + "\n" 
     463                message = message + "Original metadata directory: " + self._harvest_home + "\n\n" 
     464                message = message + "TOTAL PROCESSED " + str(numfilesproc) + "\n" 
     465                message = message + "INGESTED (Created) " + str(self._no_files_changed)  + "\n" 
     466                message = message + "INGESTED (Updated) " + str(self._no_files_ingested)  + "\n" 
     467                message = message + "DELETED " + str(self._no_files_deleted)  + "\n" 
     468                message = message + "PROBLEM FILES " + str(self._no_problem_files)  + "\n" 
     469                 
     470                '''recOpFile.write("Ingest report for data centre: " + datacentre + "\n") 
    461471                recOpFile.write("Ingest date: " + str(datetime.datetime.now()) + "\n") 
    462472                recOpFile.write("Original metadata directory: " + self._harvest_home + "\n\n") 
     
    465475                recOpFile.write("INGESTED (Updated) " + str(self._no_files_ingested)  + "\n") 
    466476                recOpFile.write("DELETED " + str(self._no_files_deleted)  + "\n") 
    467                 recOpFile.write("PROBLEM FILES " + str(self._no_problem_files)  + "\n") 
     477                recOpFile.write("PROBLEM FILES " + str(self._no_problem_files)  + "\n")''' 
     478                 
    468479                 
    469480                for badFile in self.updateFailList: 
    470                         recOpFile.write("PROBLEM_FILE " + badFile + "\n") 
    471                  
    472                 return numfilesproc 
     481                        #recOpFile.write("PROBLEM_FILE " + badFile + "\n") 
     482                        message = message +"PROBLEM_FILE " + badFile + "\n" 
     483                          
     484                recOpFile.write(message) 
     485                                 
     486                return numfilesproc, message 
    473487 
    474488 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/datacentre_config/badc_config.properties

    r4208 r5542  
    55# 
    66#Define host_path as the full directory name where this data centre's records will be harvested into. 
    7 host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/badc-nerc-ac-uk-dif 
    8 # 
     7host_path /home/users/sdonegan/workspace/sampleIngest/badc-nerc-ac-uk-dif 
     8 
    99#Define groups - portal groups for limiting searches by 'group of datacentres' 
    1010#(if not already in the records). 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/datacentre_config/bgs_config.properties

    r4208 r5542  
    55# 
    66#Define host_path as the full directory name where this data centre's records will be harvested into. 
    7 host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/bgs-nerc-ac-uk-dif 
     7#host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/bgs-nerc-ac-uk-dif 
     8host_path /home/users/sdonegan/workspace/sampleIngest/www-geodata-bgs-ac-uk-dif 
    89#host_path /home/users/sdonegan/workspace/oai_ingest2/xml/bgs_DPPP 
    910# 
    1011#Define groups - portal groups for limiting searches by 'group of datacentres'. 
    11 groups NERC_DDC http://vocab.ndg.nerc.ac.uk/term/N010/current NDGO0003 
     12#groups NERC_DDC http://vocab.ndg.nerc.ac.uk/term/N010/current NDGO0003 
    1213# 
    1314#Define which format is harvested from the data centre (one only) 
     
    1819# 
    1920#Say whether this is a full NDG data Provider who uses NDG identifiers etc. 
    20  
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/datacentre_config/bodc_config.properties

    r4208 r5542  
    55# 
    66#Define host_path as the full directory name where this data centre's records will be harvested into. 
    7 host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/grid-bodc-nerc-ac-uk-dif 
     7#host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/grid-bodc-nerc-ac-uk-dif 
     8host_path /home/users/sdonegan/workspace/sampleIngest/grid-bodc-nerc-ac-uk-dif 
    89# 
    910#Define groups - portal groups for limiting searches by 'group of datacentres'. 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/datacentre_config/neodc_config.properties

    r4208 r5542  
    55# 
    66#Define host_path as the full directory name where this data centre's records will be harvested into. 
    7 host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/neodc-nerc-ac-uk-dif 
     7#host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/neodc-nerc-ac-uk-dif 
     8host_path /home/users/sdonegan/workspace/sampleIngest/neodc-nerc-ac-uk-dif 
    89# 
    910#Define groups - portal groups for limiting searches by 'group of datacentres'. 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/datacentre_config/nocs_config.properties

    r4208 r5542  
    66# 
    77#Define host_path as the full directory name where this data centre's records will be harvested into. 
    8 host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/oai-noc-soton-ac-uk-dif 
     8#host_path /usr/local/apache/apache-tomcat-6.0.18/webapps/oai/WEB-INF/harvested_records/oai-noc-soton-ac-uk-dif 
     9host_path /home/users/sdonegan/workspace/sampleIngest/oai-noc-soton-ac-uk-dif 
    910# 
    1011#Define groups - portal groups for limiting searches by 'group of datacentres'. 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/ingest.config

    r3855 r5542  
    1 calum glue calum boybear 51000 
     1discovery localhost badc hwk298t 5432 
  • TI01-discovery/branches/ingestAutomation-upgrade/OAIBatch/oai_document_ingester.py

    r5509 r5542  
    9898                self._getPostgresDBConnection() 
    9999 
    100                 numfilesproc = self._convertAndIngestFiles(self.originals_dir, self.discovery_dir, dataCentre) 
     100                numfilesproc, processingReport = self._convertAndIngestFiles(self.originals_dir, self.discovery_dir, dataCentre) 
    101101                 
    102102                outMessage = "OAI Document ingest processing complete:\n" 
     
    118118                logging.info(message) 
    119119                outMessage += message 
     120                 
     121                #Changed message to include more detail (SJD) but also now add any errors 
    120122                if self._error_messages: 
    121123                        outMessage += 'Errors: %s' %self._error_messages  
     124                        processingReport += 'Errors: %s' %self._error_messages 
    122125                logging.info(self.lineSeparator) 
     126                 
    123127                print "Script finished running." 
    124                 return isSuccess, outMessage 
     128                return isSuccess, processingReport 
    125129 
    126130        def setIndFileToIngest(self, indFileToIngest): 
Note: See TracChangeset for help on using the changeset viewer.