Ignore:
Timestamp:
16/02/09 10:56:10 (11 years ago)
Author:
cbyrom
Message:

Add new utility methods to determine default proxies to use - checking
environment variables to establish this - and checking the url to ensure the proxy is needed for it (i.e. excluding no_proxy conforming urls). Adjust codebase to use this - when doing simple url lookups and when opening normal urls.

Location:
ndgCommon/trunk/ndg/common/src
Files:
2 edited

Legend:

Unmodified
Added
Removed
  • ndgCommon/trunk/ndg/common/src/lib/utilities.py

    r4970 r4977  
    77import csml.csmlscan as CsmlScan 
    88 
     9# environment variables to check when looking up default proxy set ups 
     10PROXY_KEY = 'http_proxy' 
     11NO_PROXY_KEY = 'no_proxy' 
    912 
    1013ISO8601_RE = "([0-9]{4})(-([0-9]{2})(-([0-9]{2})([T\s]?([0-9]{2}):([0-9]{2})(:([0-9]{2})(\.([0-9]+))?)?" + \ 
     
    2023esc_chars = {'\xb0':'°','°':'°'} 
    2124     
    22 # Define proxies required when using urllib 
    23 PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'} 
    24  
    2525URLIB2_INITIALISED = False 
    2626 
     
    8989 
    9090 
    91 def openURLWithProxy(uri): 
    92     ''' 
    93     Open a simple url connection using the standard proxy, and retrieve the contents 
     91def getDefaultProxy(url): 
     92    ''' 
     93    Checks the OS environment to see if any proxies are specified; if so, 
     94    return the proxy details in a dict 
     95    @param url: url that is being looked up; NB, there may be a no_proxy env 
     96    variable set - and this may affect how urls for certain machines will be 
     97    proxied 
     98    @return dict: with format key = proxy type, val = proxy value  
     99    - e.g. 'http':'http://wwwcache.rl.ac.uk:8080' 
     100    ''' 
     101    proxies = {} 
     102    proxy = os.getenv(PROXY_KEY) 
     103    if proxy: 
     104        proxies['http'] = proxy 
     105        logging.debug("Found proxy setting - checking if this applies to the url being accessed") 
     106        noProxyHosts = os.getenv(NO_PROXY_KEY) 
     107        if noProxyHosts: 
     108            for host in noProxyHosts.split(';'): 
     109                if url.lower().find(host.lower().strip()) > -1: 
     110                    logging.debug("Proxies disabled for communication with uri machine ('%s')" 
     111                                  %host) 
     112                    proxies = {} 
     113    return proxies 
     114 
     115 
     116def openURLWithDefaultProxy(url): 
     117    ''' 
     118    Checks the OS environment to see if any proxies are specified; if so, 
     119    use these when opening a URL 
     120    @param url: url to open 
     121    ''' 
     122    logging.debug("Checking environment variables for proxies") 
     123    proxy = getDefaultProxy(url) 
     124    return openURLWithProxy(url, proxy) 
     125         
     126 
     127def openURLWithProxy(uri, proxy): 
     128    ''' 
     129    Open a simple url connection using the specified proxy, and retrieve the contents 
    94130    @param uri: uri to read from 
     131    @param proxy: dict with proxy info in format key = protocol, val = proxy host 
    95132    @return pageData: data read from uri  
    96133    '''     
    97134    logging.debug("Reading info from uri, '%s'" %uri) 
    98     f = urllib.urlopen(uri, proxies = PROXIES) 
     135    f = urllib.urlopen(uri, proxies = proxy) 
    99136    pageData = f.read() 
    100137    f.close() 
     
    118155        socket.setdefaulttimeout(120) 
    119156     
    120         proxy_support = urllib2.ProxyHandler(PROXIES) 
     157        proxy = getDefaultProxy(uri) 
     158        proxy_support = urllib2.ProxyHandler(proxy) 
    121159     
    122160        # build a new opener that adds authentication and caching FTP handlers 
  • ndgCommon/trunk/ndg/common/src/models/vocabtermdata.py

    r4844 r4977  
    66''' 
    77import logging, time, re, urllib 
    8 from ndg.common.src.lib.utilities import openURLWithProxy 
     8from ndg.common.src.lib.utilities import openURLWithDefaultProxy 
    99     
    1010def isValidTermURI(uri): 
     
    1919    logging.debug("Checking vocab term uri, '%s'" %uri) 
    2020    try: 
    21         pageData = openURLWithProxy(uri) 
     21        pageData = openURLWithDefaultProxy(uri) 
    2222        if pageData.find('<rdf:RDF') > -1: 
    2323            logging.debug("- found valid term") 
     
    395395        ''' 
    396396        logging.info("Retrieving vocab server data page") 
    397         self.VOCAB_DATA_PAGE = openURLWithProxy(self.VOCAB_SERVER_URL) 
     397        self.VOCAB_DATA_PAGE = openURLWithDefaultProxy(self.VOCAB_SERVER_URL) 
    398398        self.REFRESH_TIME = time.time() + self.ONE_HOUR 
    399399        logging.info("Vocab server data retrieved") 
Note: See TracChangeset for help on using the changeset viewer.