Changeset 5239 for TI01-discovery


Ignore:
Timestamp:
29/04/09 17:21:08 (10 years ago)
Author:
cbyrom
Message:

Add Harvester class to run the jOAI Harvester API and run repository
harvests. Add code throughout the app to allow harvests to be
ran from the UI. Add new test data and tests to exercise the new
functionality and adjust the config file to allow input of required
harvest data. Add confirmation pop up when running deletes.

Location:
TI01-discovery/trunk/OAIInfoEditor
Files:
9 added
13 edited

Legend:

Unmodified
Added
Removed
  • TI01-discovery/trunk/OAIInfoEditor/editor.config

    r5237 r5239  
    1616[DATA_STORE] 
    1717# specify where the persisted data, and backups of this, should be stored 
    18 appDataFileName: providerInfo 
     18appDataFileName: oai_info_data 
    1919appDataFileDir: oaiInfoEditorData 
    2020backupFileDir: oaiInfoEditorDataBackups 
     21harvestDir: harvestData 
    2122 
    2223[USER_LIST] 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/config/oiemiddleware.py

    r5226 r5239  
    99from oai_info_editor.model.user import * 
    1010from oai_info_editor.dal.providerinfodao import * 
     11from oai_info_editor.lib.harvester import Harvester  
    1112     
    1213class OIEMiddleware(object): 
     
    4647        self.globals.dataDir = cf.get('DATA_STORE', 'appDataFileDir') 
    4748        self.globals.backupDir = cf.get('DATA_STORE', 'backupFileDir') 
     49        harvestDir = cf.get('DATA_STORE', 'harvestDir') 
     50        if not harvestDir: 
     51            raise ValueError("No harvest directory config data has been specified - so cannot harvest files.") 
     52         
     53        self.globals.harvester = Harvester(outDir = harvestDir) 
     54 
    4855 
    4956        self.globals.users = {} 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/config/routing.py

    r5235 r5239  
    2626    map.connect('deleteProviderInfo', 'deleteProviderInfo/:providerName', controller = 'edit', action='deleteProviderInfo') 
    2727    map.connect('createProviderInfo', 'createProviderInfo', controller = 'edit', action='createProviderInfo') 
     28    map.connect('harvestProviderInfo', 'harvestProviderInfo/:providerName/:repositoryName', controller = 'harvest', action='harvestProviderInfo') 
    2829 
    2930    map.connect('deleteRepositoryInfo', 'deleteProviderInfo/:providerName/:repositoryName',  
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/controllers/edit.py

    r5237 r5239  
    2727        c.saveLink = h.url_for('saveProviderInfo', providerName = providerName) 
    2828        c.deleteLink = h.url_for('deleteProviderInfo', providerName = providerName) 
     29        c.harvestLink = '' 
    2930         
    3031        c.title = const.NEW_PAGE_TITLE 
     
    110111        pi = g.dao.deleteProviderInfo(providerName) 
    111112 
     113        # prepare pop up message to display result 
    112114        session[const.POP_UP_MESSAGE_SESSION_KEY] = "Data for provider with name, %s, successfully deleted" %providerName 
    113115        session.save() 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/controllers/view.py

    r5237 r5239  
    2929                     
    3030        c.editLink = h.url_for('editProviderInfo', providerName = '') 
     31        # NB, this is a slight hack - we need to build this link gradually, dynamically 
     32        # - if no info is specified for the providerName it messes up what is 
     33        # returned by url_for 
     34        c.harvestLink = h.url_for('harvestProviderInfo',  
     35                                  providerName = const.PROVIDER_REPLACEMENT,  
     36                                  repositoryName = const.REPOSITORY_REPLACEMENT) 
    3137        if c.user.isAdmin: 
    3238            c.createLink = h.url_for('createProviderInfo') 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/editor.config

    r5237 r5239  
    1616[DATA_STORE] 
    1717# specify where the persisted data, and backups of this, should be stored 
    18 appDataFileName: current 
     18appDataFileName: oai_info_data 
    1919appDataFileDir: oaiInfoEditorData 
    2020backupFileDir: oaiInfoEditorDataBackups 
     21harvestDir: harvestData 
    2122 
    2223[USER_LIST] 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/lib/constants.py

    r5235 r5239  
    3333NEW_PROVIDER_INFO_NAME = "newPI" 
    3434 
     35# dummy strings to use when constructing the harvest urls - these should be 
     36# replaced with the real provider and repository names 
     37PROVIDER_REPLACEMENT = "xyz321" 
     38REPOSITORY_REPLACEMENT = "123zyx" 
    3539 
    3640# different types of text input fields - used by utils.EditTextField 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/edit_provider_infos.html

    r5229 r5239  
    1616                                <div style="overflow:auto; height:450px;"> 
    1717                                        <div py:for="pi in c.providerInfos" py:strip=""> 
    18                                                 <div py:replace="ProviderInfoTable(pi, True, None)" /> 
     18                                                <div py:replace="ProviderInfoTable(pi, True, None, c.harvestLink)" /> 
    1919                                        </div> 
    2020                                </div> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/home.html

    r5236 r5239  
    2828                              Select the provider info data to view/edit  
    2929                              <table> 
     30                              <tr> 
     31                                <td py:if="c.user.isAdmin"> 
     32                                        <a href="${h.url_for('viewProviderInfo', providerName = constants.ALL_PROVIDER_IDS)}">All data</a> 
     33                                </td> 
     34                              </tr> 
    3035                              <tr py:for="pi in c.providerInfos"> 
    3136                                <td> 
    3237                                        <a href="${h.url_for('viewProviderInfo', providerName = urllib.quote(pi.name))}">${Markup(pi.name)}</a> 
    33                                 </td> 
    34                               </tr> 
    35                               <tr> 
    36                                 <td py:if="c.user.isAdmin"> 
    37                                         <a href="${h.url_for('viewProviderInfo', providerName = constants.ALL_PROVIDER_IDS)}">All data</a> 
    3838                                </td> 
    3939                              </tr> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/provider_info.html

    r5237 r5239  
    1111                import urllib 
    1212        ?> 
     13       <script py:if="c.user.isAdmin" type="text/javascript"> 
     14function confirmProviderInfoDelete()  
     15{ 
     16        var r=confirm('Do you really want to delete this provider info data?'); 
     17        if (r==true) 
     18        { 
     19                Div_show("loading"); 
     20                return true;   
     21        } 
     22        return false; 
     23} 
     24        </script> 
    1325 
    14         <div class="providerInfoTable" py:def="ProviderInfoTable(records, isEditable, editLink)"> 
     26        <div class="providerInfoTable" py:def="ProviderInfoTable(records, isEditable, editLink, harvestLink)"> 
    1527        <table width="100%"> 
    1628                <tr class="headerRow"> 
     
    1931                        <th align="left">Provider Name</th> 
    2032                </tr> 
    21                 <div py:replace="ProviderInfoRecord(records, isEditable, editLink)" /> 
    22                  
     33                <div py:replace="ProviderInfoRecord(records, isEditable, editLink, harvestLink)" /> 
    2334        </table> 
    2435    </div> 
    2536 
    26         <div py:def="ProviderInfoRecord(record, isEditable, editLink)"> 
     37        <div py:def="ProviderInfoRecord(record, isEditable, editLink, harvestLink)"> 
    2738        <tr> 
    2839                <span py:if="editLink">  
    2940                        <td> 
    30                         <a href="${'%s%s' %(c.editLink, urllib.quote(record.name))}" onclick="Div_show('loading');">Edit</a> 
     41                        <a href="${'%s%s' %(editLink, urllib.quote(record.name))}" onclick="Div_show('loading');">Edit</a> 
    3142                        </td> 
    3243                        <td py:if="c.user.isAdmin"> 
    33                         <a href="${'%s%s' %(c.deleteLink, urllib.quote(record.name))}" onclick="Div_show('loading');">Delete</a> 
     44                        <a href="${'%s%s' %(c.deleteLink, urllib.quote(record.name))}" onclick="return confirmProviderInfoDelete();">Delete</a> 
    3445                        </td> 
    3546                        <td py:if="not c.user.isAdmin" /> 
     
    4859                <th>Repository Info</th> 
    4960                <td colspan="$colSpan"> 
    50                 <div py:replace="RepositoryInfoTable(record.repositoryInfos, isEditable, editLink, None)" /> 
     61                <div py:replace="RepositoryInfoTable(record.repositoryInfos, isEditable, editLink, harvestLink.replace(constants.PROVIDER_REPLACEMENT, urllib.quote(record.name)))" /> 
    5162                </td> 
    5263        </tr> 
    53          
    5464        </div> 
    5565</html> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/repository_info.html

    r5236 r5239  
    1111                import oai_info_editor.model.formattype as format 
    1212        ?> 
    13         <div py:def="RepositoryInfoTable(records, isEditable, editLink, editAtt)"> 
     13        <div py:def="RepositoryInfoTable(records, isEditable, editLink, harvestLink)"> 
    1414        <div class="yui-skin-sam sortDiv"> 
    1515                <table id="generalRecordTable" class="sortTable"> 
     
    2626            <div py:for="i in range(len(records))" py:strip=""> 
    2727                    <tr py:if="not records[i].isEmpty()"> 
    28                         <div py:replace="RepositoryInfoRecord(records[i], isEditable, str(i) + ri.RI_DELIMITER)" /> 
     28                        <div py:replace="RepositoryInfoRecord(records[i], isEditable, str(i) + ri.RI_DELIMITER, harvestLink)" /> 
    2929                    </tr> 
    3030            </div> 
     
    3333                        ?> 
    3434            <div py:if="not records" py:strip=""> 
    35                 <div py:replace="RepositoryInfoRecord(newRecord, isEditable, '0' + ri.RI_DELIMITER)" /> 
     35                <div py:replace="RepositoryInfoRecord(newRecord, isEditable, '0' + ri.RI_DELIMITER, None)" /> 
    3636            </div> 
    3737                        <span py:if="isEditable and records" py:strip=""> 
    3838                        <tr id="addRecord" class="hidden"> 
    39                                 <div py:replace="RepositoryInfoRecord(newRecord, isEditable, str(len(records)) + ri.RI_DELIMITER)" /> 
     39                                <div py:replace="RepositoryInfoRecord(newRecord, isEditable, str(len(records)) + ri.RI_DELIMITER, None)" /> 
    4040                        </tr> 
    4141                    <tr id="updateLink" class="shown"> 
     
    5252     </div> 
    5353 
    54         <div py:def="RepositoryInfoRecord(record, isEditable, prefix)" py:strip=""> 
     54        <div py:def="RepositoryInfoRecord(record, isEditable, prefix, harvestLink)" py:strip=""> 
    5555                <div py:replace="EditTextField(prefix + ri.NAME_ELEMENT_NAME, record.name, isEditable, constants.TEXT_FIELD, width='35%')"/> 
    5656                <div py:replace="EditTextField(prefix + ri.URL_ELEMENT_NAME, record.url, isEditable, constants.TEXT_FIELD, width='35%')"/> 
     
    6767                        ${Markup(h.check_box(prefix + ri.SPLIT_BY_SET_NAME, checked = record.splitBySet, disabled = not isEditable))} 
    6868                </td> 
     69                <td> 
     70                <form py:if="harvestLink" action="${harvestLink.replace(constants.REPOSITORY_REPLACEMENT, urllib.quote(record.name))}" method="post"> 
     71              ${Markup(h.submit('Harvest', **{'onclick': 'Div_show("loading");'}))} 
     72                </form> 
     73                </td> 
    6974        </div> 
    7075</html> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/templates/view_provider_infos.html

    r5233 r5239  
    2323                                <div style="overflow:auto; height:450px;"> 
    2424                                        <div py:for="pi in c.providerInfos" py:strip=""> 
    25                                                 <div py:replace="ProviderInfoTable(pi, False, c.editLink)" /> 
     25                                                <div py:replace="ProviderInfoTable(pi, False, c.editLink, c.harvestLink)" /> 
    2626                                        </div> 
    2727                                </div> 
  • TI01-discovery/trunk/OAIInfoEditor/oai_info_editor/tests/testdata/testconstants.py

    r5226 r5239  
    3131except: 
    3232    TEST_DATA_DIR = "C:/Users/byrc/5509/NewEclipseWorkspace/OAIInfoEditor/oai_info_editor/tests/testdata/" 
    33      
     33 
     34LIB_DATA_DIR = "" 
     35try: 
     36    LIB_DATA_DIR = getTestDataDir('oai_info_editor' + os.sep + 'lib' + os.sep) 
     37except: 
     38    LIB_DATA_DIR = "C:/Users/byrc/5509/NewEclipseWorkspace/OAIInfoEditor/oai_info_editor/lib/" 
     39 
     40HARVEST_COMMAND = 'java %s -cp %sDLESETools.jar:%sjdom-b7.jar:%sxercesImpl.jar:%sxml-apis.jar org.dlese.dpc.oai.harvester.Harvester %s %s %s'\ 
     41    %('%s', LIB_DATA_DIR, LIB_DATA_DIR, LIB_DATA_DIR, LIB_DATA_DIR, '%s', '%s', '%s') 
     42 
    3443VALID_REP_INFO_FILE = TEST_DATA_DIR + 'test_repository_info.xml' 
    3544f = open(VALID_REP_INFO_FILE, "r") 
     
    5665VALID_REP_NAME_2 = 'PROVIDER2 REP2' 
    5766VALID_REP_NAME_3 = 'PROVIDER2 REP3' 
     67VALID_REAL_REP_NAME = 'DLESE' 
     68# NB, the commented out provider is a test one provided by DLESE - but it doesn't work! 
     69VALID_REAL_REP_URL = 'http://badc.nerc.ac.uk/badc_oai/provider'#'http://dlese.org/oai/provider' 
     70VALID_REAL_REP_FORMAT = 'dif'#'adn' 
    5871VALID_REP_URL_0 = 'http://rep1.ac.uk' 
    5972VALID_REP_URL = 'http://rep2.ac.uk' 
     
    6881VALID_HARVEST_TIME = '09:00' 
    6982VALID_SPLIT_BY_SET = 'True' 
     83 
     84VALID_REAL_REPOSITORY_INFO = createRepositoryInfoWithData(VALID_REAL_REP_NAME,  
     85                                                          VALID_REAL_REP_URL, 
     86                                                          VALID_REAL_REP_FORMAT, "",) 
    7087 
    7188REPOSITORY_INFO_0 = createRepositoryInfoWithData(VALID_REP_NAME_0, VALID_REP_URL_0, 
     
    113130VALID_ADMIN_USER = User(VALID_USER_ID, ['admin']) 
    114131INVALID_USER = User('frida', []) 
     132 
     133VALID_HARVEST_DIR = 'tmp/' 
Note: See TracChangeset for help on using the changeset viewer.