Changeset 5245 for TI01-discovery


Ignore:
Timestamp:
05/05/09 10:19:21 (10 years ago)
Author:
cbyrom
Message:

Extend harvest functionality to also include ingest of harvested docs
+ improve error handling and catch special cases when creating
providers with names that already exist + improve UI look and feel.

Location:
TI01-discovery/trunk/OAIInfoEditor/oai_info_editor
Files:
9 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/controllers/edit.py

    r5242 r5245  
    1111import oai_info_editor.model.formattype as format 
    1212from oai_info_editor.model.repositoryinfo import RepositoryInfo as RI, \ 
    13     RI_DELIMITER, NAME_ELEMENT_NAME, SPLIT_BY_SET_NAME 
     13    RI_DELIMITER, NAME_ELEMENT_NAME, SPLIT_BY_SET_NAME, URL_ELEMENT_NAME 
    1414import oai_info_editor.model.providerinfo as PI 
    1515  
     
    4444        # NB, we always need the provider name specified - this is effectively the 
    4545        # primary key for the data 
     46        c.errors = {} 
    4647        if not self.providerName: 
    47             c.errors = {'name': 'Please enter a value'} 
     48            c.errors[PI.NAME_ATT_NAME] = 'Please enter a value' 
    4849         
    4950        i = 0 
     
    5354            if not ri.isEmpty(): 
    5455                self.repositoryInfos.append(ri) 
     56            else: 
     57                # remove any errors associated with empty records 
     58                for key in c.errors.keys(): 
     59                    if key.startswith(str(i)): 
     60                        del c.errors[key] 
    5561            i += 1 
    5662             
     
    7682            if val: 
    7783                inputs[att] = val 
     84            elif att == NAME_ELEMENT_NAME or att == URL_ELEMENT_NAME: 
     85                c.errors[prefix + att] = 'Please enter a value' 
    7886 
    7987        if not inputs: 
     
    136144 
    137145        # now create or update the data 
    138         if providerName == const.NEW_PROVIDER_INFO_NAME: 
    139             pi, fileName = g.dao.createProviderInfo(pi) 
    140         else: 
    141             # update to ensure that the records are returned 
    142             pi = g.dao.updateProviderInfo(pi) 
     146        try: 
     147            if providerName == const.NEW_PROVIDER_INFO_NAME: 
     148                pi, fileName = g.dao.createProviderInfo(pi) 
     149            else: 
     150                # update to ensure that the records are returned 
     151                pi = g.dao.updateProviderInfo(pi) 
     152        except Exception, e: 
     153            c.errors = {'System error': e} 
     154            return render('genshi', 'error') 
    143155 
    144156        # now, return to the view page 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/controllers/harvest.py

    r5239 r5245  
    2727                             %(repositoryName, providerName)) 
    2828 
    29         outMessage = "Repository data, %s, for provider with name, %s, successfully harvested" \ 
    30             %(repositoryName, providerName) 
    31         status, result = g.harvester.harvestRepository(ri) 
     29        try: 
     30            status, result = g.harvester.runHarvestAndIngest(providerName, ri) 
     31        except Exception, e: 
     32            c.errors = {'System error': e} 
     33            return render('genshi', 'error') 
     34 
     35        outMessage = "Repository data, %s, for provider with name, %s, successfully harvested. (Details: %s)" \ 
     36            %(repositoryName, providerName, result) 
    3237        if not status: 
    3338            outMessage = "A problem (%s) occurred whilst harvesting the data - please retry later" \ 
    3439                %result 
    35              
     40 
    3641        # now return to the home page - and display pop up 
    3742        session[const.POP_UP_MESSAGE_SESSION_KEY] = outMessage 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/dal/editorfileclient.py

    r5242 r5245  
    201201        self.pic, fileName = self.__getProviderInfoData() 
    202202         
     203        # check for case when a name has been changed and this now overlaps with another 
     204        # existing entry 
     205        if providerInfo.name != providerInfo.oldName: 
     206            pi = self.getProviderInfo(providerInfo.name) 
     207            if pi: 
     208                raise ValueError("Cannot change provider name - a provider with the name, '%s' already exists" \ 
     209                                 %providerInfo.name)             
     210         
    203211        self.pic.addProviderInfo(providerInfo, raiseErrorIfNotFound = True) 
    204212        self.pic = self.__createOrUpdatePICollection(self.pic) 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/lib/harvester.py

    r5241 r5245  
    88from oai_info_editor.dal.providerinfodao import * 
    99from ndg.common.src.models.myconfig import myConfig 
     10from OAIBatch.oai_document_ingester import oai_document_ingester 
    1011 
    1112 
     
    3839            if outDir: 
    3940                self.outDir = outDir 
    40          
     41 
    4142        self.dao = None # data access object for retrieving all providers info 
     43        self.ingester = None # ingester script for adding the harvested data to the Discovery service 
     44 
    4245        logging.info("ProviderInfoDAO initialised") 
    4346     
     
    5154        @raise ValueError: if repositoryInfo is not a RepositoryInfo object 
    5255        @return status, outMessage: Status = True, if successful, False otherwise 
    53         outMessage = any error message received if harvesting fails  
     56        outMessage = summary of harvest outcome  
    5457        ''' 
    5558        logging.info("Running data harvest") 
     59         
     60        # check we're not running on windows - this doesn't work since the commands 
     61        # library for running system commands is only unix systems compatible 
     62        # NB, this would be better placed in the constructor - since this would stop 
     63        # the app from running in the first place; put it here temporarily to allow 
     64        # testing of the app on windows 
     65        if sys.platform.lower().startswith('win'): 
     66            raise SystemError("Harvest functionality does not work on Windows machines " + \ 
     67                              "- application should be running on a Unix system.") 
     68 
    5669        localDir = self.outDir 
    5770        if outDir: 
     
    7285         
    7386        status, message = commands.getstatusoutput(harvestCMD) 
     87         
     88        logging.debug("Harvest output: '%s'" %message) 
    7489         
    7590        if status or message.find('cannotDisseminateFormat') > -1: 
     
    92107                 
    93108                status, message = commands.getstatusoutput(harvestCMD) 
     109                logging.debug("Harvest output: '%s'" %message) 
    94110                if status or message.find('cannotDisseminateFormat') > -1: 
    95111                    logging.error("Problem occurred whilst running harvest: %s" %message) 
    96112 
     113        data = message.split('\n') 
    97114        if status or message.find('cannotDisseminateFormat') > -1: 
    98115            # retrieve pertinent part of error message to return to user 
    99             data = message.split('\n') 
    100116            # NB, usually the penulimate line has the clearest digest of the error on it 
    101117            # - just incase there are exceptions to this, return just the last line 
     
    108124            outMessage = outMessage.replace('\'', '') 
    109125            return False, outMessage 
    110                  
    111126 
    112127        logging.info("- harvest completed successfully") 
    113         return True, message 
    114      
     128        # NB, the last line has the summary of the harvest - so just return this 
     129        return True, data[-1] 
     130 
     131 
     132    def ingestDocuments(self, providerName): 
     133        ''' 
     134        Ingest harvested documents into the discovery service 
     135        @param providerName: Name of provider whose documents should be ingested 
     136        @return status, outMessage: Status = True, if successful, False otherwise 
     137        outMessage = summary of ingest outcome  
     138        ''' 
     139        logging.debug("Running ingest of documents") 
     140        if not self.ingester: 
     141            self.ingester = oai_document_ingester() 
     142         
     143        result = "" 
     144        isSuccess = False 
     145        try: 
     146            result = ingester.processDataCentre(datacentre) 
     147            isSuccess = True 
     148        except: 
     149            logging.error("Exception thrown - detail: ") 
     150            result = sys.exc_info() 
     151            logging.error(result) 
     152            logging.info("Continue processing next datacentre config file...") 
     153 
     154        logging.debug("Document ingest complete") 
     155        return isSuccess, result 
     156 
     157 
     158    def runHarvestAndIngest(self, providerName, repositoryInfo, outDir = None): 
     159        ''' 
     160        Harvest and ingest documents into the discovery service 
     161        @param providerName: Name of provider whose documents should be ingested 
     162        @param repositoryInfo: RepositoryInfo object with data on the repository to 
     163        be harvested 
     164        @keyword outDir: directory to harvest files to 
     165        @return status, outMessage: Status = True, if successful, False otherwise 
     166        outMessage = summary of ingest outcome  
     167        ''' 
     168        logging.debug("Running harvest then ingest") 
     169        isSuccess, outMessage = self.harvestRepository(repositoryInfo, outDir = outDir) 
     170        if isSuccess: 
     171            logging.debug("- harvest successful, so running ingest") 
     172            isSuccess, ingestMessage = self.ingestDocuments(providerName) 
     173             
     174            outMessage += ingestMessage 
     175        else: 
     176            logging.debug(" - harvest failed so avoid document ingest") 
     177         
     178        return isSuccess, outMessage 
     179         
    115180 
    116181    def __getLocalRepositoryDir(self, repositoryInfo, baseDir): 
     
    175240            for ri in pi.repositoryInfos: 
    176241                logging.info(" - harvesting repository info, '%s'" %ri.name) 
    177                 self.harvestRepository(ri) 
     242                self.runHarvestAndIngest(pi.name, ri) 
    178243                 
    179244        logging.info("- harvesting complete") 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/public/layout/style/oai_editor_style.css

    r5242 r5245  
    134134#contentsRight  {border-left: 1px solid #3c78b5; margin-left:250px;} 
    135135 
    136 .error {display:block;text-align:center;font-size:150%;background-color:red; padding:10px;} 
     136.error  
     137{ 
     138display:block; 
     139text-align:center; 
     140font-size:150%; 
     141background-color:red; 
     142/*padding:10px;*/ 
     143} 
    137144.javascriptWarning  
    138145{ 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/edit_provider_infos.html

    r5239 r5245  
    1313            <div class="metadata"> 
    1414                            <h3>$c.title</h3> 
     15                    <div py:if="c.errors"> 
     16                                <div py:replace="displayErrors()"/> 
     17                        </div> 
    1518                ${Markup(h.form(c.saveLink, method='post'))} 
    1619                                <div style="overflow:auto; height:450px;"> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/error.html

    r5226 r5245  
    1212        <div id="contents"> 
    1313            <div class="metadata"> 
    14                             <a py:if="c.editMode" href="${h.url_for(controller='home',action='index')}">Home</a> 
    1514                        <div py:replace="displayErrors()"/> 
    1615                </div> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/layout.html

    r5242 r5245  
    3737                            </td></tr> 
    3838                    </table> 
    39 <div class="javascriptWarning"><script type="text/javascript"><!-- 
    40 document.write("<div style='display:none' id='moreinfo'>"); 
    41 //--> 
    42 </script>WARNING: You currently have javascript disabled.  For these pages to work correctly, please enable javascript.<script type="text/javascript"><!-- 
    43 document.write("</div>"); 
    44 //--></script></div> 
    4539                <div id="loading" class='hidden'><img src="$g.loadingIcon" alt="Loading" />Please wait...</div> 
    4640                    ${select('*[local-name()!="panelTab" and local-name()!="includeCorrectionLink"]')} 
    4741      </div> 
     42                <div class="javascriptWarning"><script type="text/javascript"><!-- 
     43                document.write("<div style='display:none' id='moreinfo'>"); 
     44                //--> 
     45                </script>WARNING: You currently have javascript disabled.  For these pages to work correctly, please enable javascript.<script type="text/javascript"><!-- 
     46                document.write("</div>"); 
     47                //--></script></div> 
    4848      <div py:replace="footer()"/> 
    4949    </div> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/utils.html

    r5242 r5245  
    11<?python  
    22        import oai_info_editor.lib.constants as constants 
     3        from oai_info_editor.model.repositoryinfo import RI_DELIMITER 
    34?> 
    45<html xmlns="http://www.w3.org/1999/xhtml" xmlns:py="http://genshi.edgewall.org/" 
     
    910        <table align="center"> 
    1011        <tr py:for="type, error in c.errors.items()"> 
     12                <?python 
     13                type = type.split(RI_DELIMITER)[-1] 
     14                ?> 
    1115                <td>${type}:</td> <td align="left">'${error}'</td> 
    1216        </tr> 
Note: See TracChangeset for help on using the changeset viewer.