Ignore:
Timestamp:
19/01/09 16:52:53 (11 years ago)
Author:
cbyrom
Message:

Improve url check, by creating new method which sets the default
proxies to use with the lookup + adjust CSML creation to have the
dataset ID as a keyword - to allow this to be randomly generated if not
set.

Location:
ndgCommon/trunk/ndg/common/src
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • ndgCommon/trunk/ndg/common/src/lib/atomvalidator.py

    r4830 r4844  
    340340                error = e.faultString.split(':')[-1]  
    341341            elif isinstance(e, socket.error): 
    342                 error = " ".join(e.args) 
     342                if hasattr(e, 'args'): 
     343                    args = e.args 
     344                    if not isinstance(e.args, list): 
     345                        args = [e.args] 
     346                     
     347                    for arg in args: 
     348                        error += " ".join(str(arg)) 
    343349                 
    344350            errorMessage = "Problem experienced when validating against schema:%s'%s'" \ 
  • ndgCommon/trunk/ndg/common/src/lib/granulite.py

    r4836 r4844  
    484484        logging.info("Create CSML file from the CDML file - NB, this will be stored in eXist too " + \ 
    485485                     "and will be used to extract dataset information from") 
    486         csmlFileName = utilities.createCSMLFile(fn, self._datasetID, self._cdmlTimeAxis) 
     486        csmlFileName = utilities.createCSMLFile(fn, self._cdmlTimeAxis, datasetID = self._datasetID) 
    487487        os.remove(fn) 
    488488         
  • ndgCommon/trunk/ndg/common/src/lib/utilities.py

    r4793 r4844  
    33@author: C Byrom 
    44''' 
    5 import os, sys, logging, re, cgi, urlparse, httplib, time, urllib2, socket 
     5import os, sys, logging, re, cgi, urlparse, httplib, time, urllib2, urllib, socket, uuid 
    66from ndg.common.src.models.ndgObject import ndgObject 
    77import csml.csmlscan as CsmlScan 
     
    1919 
    2020esc_chars = {'\xb0':'°','°':'°'} 
     21     
     22# Define proxies required when using urllib 
     23PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'} 
     24 
     25URLIB2_INITIALISED = False 
     26 
     27def openURLWithProxy(uri): 
     28    ''' 
     29    Open a simple url connection using the standard proxy, and retrieve the contents 
     30    @param uri: uri to read from 
     31    @return pageData: data read from uri  
     32    '''     
     33    logging.debug("Reading info from uri, '%s'" %uri) 
     34    f = urllib.urlopen(uri, proxies = PROXIES) 
     35    pageData = f.read() 
     36    f.close() 
     37    logging.debug("- returning info from uri") 
     38    return pageData 
     39 
    2140     
    2241def simpleURLCheck(uri): 
     
    3049    ''' 
    3150    logging.debug("Checking validity of uri, '%s'" %uri) 
    32     # set the socket timeout period 
    33     socket.setdefaulttimeout(40) 
     51     
     52    if not URLIB2_INITIALISED: 
     53        # set the socket timeout period 
     54        socket.setdefaulttimeout(40) 
     55     
     56        proxy_support = urllib2.ProxyHandler(PROXIES) 
     57     
     58        # build a new opener that adds authentication and caching FTP handlers 
     59        opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler) 
     60     
     61        # install it 
     62        urllib2.install_opener(opener) 
    3463     
    3564    try: 
     
    216245 
    217246         
    218 def createCSMLFile(CDMLFilePath, datasetID, timeAxis): 
     247def createCSMLFile(CDMLFilePath, timeAxis, datasetID = None): 
    219248    ''' 
    220249    Create a CSML file by running csmlscan.py against the specified CDML file 
    221250    @param CDMLFilePath: string path to CDML file 
    222     @param datasetID: string dataset ID to use in CSMLfile 
    223251    @param timeAxis: string name of time axis to use in CDML file 
     252    @keyword datasetID: string dataset ID to use in CSMLfile - if not set, a random 
     253    name will be generated instead 
    224254    @return: CSMLFileName: name of CSML file produced 
    225255    ''' 
    226256    logging.info("Creating CSML file from CDML file by running csmlscan") 
     257     
     258    if not datasetID: 
     259        datasetID = str(uuid.uuid1()) 
    227260    CSMLFileName = datasetID + "_csml.xml" 
    228261    logging.debug("Inputs specified: datasetID = %s, timeAxis = %s" %(datasetID, timeAxis)) 
  • ndgCommon/trunk/ndg/common/src/models/vocabtermdata.py

    r4832 r4844  
    66''' 
    77import logging, time, re, urllib 
     8from ndg.common.src.lib.utilities import openURLWithProxy 
    89     
    910def isValidTermURI(uri): 
     
    1819    logging.debug("Checking vocab term uri, '%s'" %uri) 
    1920    try: 
    20         page = urllib.urlopen(uri) 
    21         pageData = page.read() 
     21        pageData = openURLWithProxy(uri) 
    2222        if pageData.find('<rdf:RDF') > -1: 
    2323            logging.debug("- found valid term") 
     
    269269     
    270270    BADC_BROWSE_ROOT = 'http://badc.nerc.ac.uk/browse' 
    271      
    272     # Define proxies required when using urllib 
    273     PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'} 
    274271 
    275272    def __init__(self): 
     
    398395        ''' 
    399396        logging.info("Retrieving vocab server data page") 
    400         f = urllib.urlopen(self.VOCAB_SERVER_URL, proxies = self.PROXIES) 
    401         self.VOCAB_DATA_PAGE = f.read() 
    402         f.close() 
     397        self.VOCAB_DATA_PAGE = openURLWithProxy(self.VOCAB_SERVER_URL) 
    403398        self.REFRESH_TIME = time.time() + self.ONE_HOUR 
    404399        logging.info("Vocab server data retrieved") 
Note: See TracChangeset for help on using the changeset viewer.