Changeset 4844

Timestamp:

19/01/09 16:52:53 (12 years ago)

Author:

cbyrom

Message:

Improve url check, by creating new method which sets the default
proxies to use with the lookup + adjust CSML creation to have the
dataset ID as a keyword - to allow this to be randomly generated if not
set.

Location:

ndgCommon/trunk/ndg/common

Files:

• src/lib/atomvalidator.py (modified) (1 diff)
• src/lib/granulite.py (modified) (1 diff)
• src/lib/utilities.py (modified) (4 diffs)
• src/models/vocabtermdata.py (modified) (4 diffs)
• unittests/clients/xmldb/eXist/testexistdbclient.py (modified) (3 diffs)
• unittests/lib/testutilities.py (modified) (1 diff)

ndgCommon/trunk/ndg/common/src/lib/atomvalidator.py

@@ -340 +340 @@
                 error = e.faultString.split(':')[-1] 
             elif isinstance(e, socket.error):
-                error = " ".join(e.args)
+                if hasattr(e, 'args'):
+                    args = e.args
+                    if not isinstance(e.args, list):
+                        args = [e.args]
+                    
+                    for arg in args:
+                        error += " ".join(str(arg))
                 
             errorMessage = "Problem experienced when validating against schema:%s'%s'" \

ndgCommon/trunk/ndg/common/src/lib/granulite.py

@@ -484 +484 @@
         logging.info("Create CSML file from the CDML file - NB, this will be stored in eXist too " + \
                      "and will be used to extract dataset information from")
-        csmlFileName = utilities.createCSMLFile(fn, self._datasetID, self._cdmlTimeAxis)
+        csmlFileName = utilities.createCSMLFile(fn, self._cdmlTimeAxis, datasetID = self._datasetID)
         os.remove(fn)
         

ndgCommon/trunk/ndg/common/src/lib/utilities.py

@@ -3 +3 @@
 @author: C Byrom
 '''
-import os, sys, logging, re, cgi, urlparse, httplib, time, urllib2, socket
+import os, sys, logging, re, cgi, urlparse, httplib, time, urllib2, urllib, socket, uuid
 from ndg.common.src.models.ndgObject import ndgObject
 import csml.csmlscan as CsmlScan
@@ -19 +19 @@
 
 esc_chars = {'\xb0':'°','°':'°'}
+    
+# Define proxies required when using urllib
+PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'}
+
+URLIB2_INITIALISED = False
+
+def openURLWithProxy(uri):
+    '''
+    Open a simple url connection using the standard proxy, and retrieve the contents
+    @param uri: uri to read from
+    @return pageData: data read from uri 
+    '''    
+    logging.debug("Reading info from uri, '%s'" %uri)
+    f = urllib.urlopen(uri, proxies = PROXIES)
+    pageData = f.read()
+    f.close()
+    logging.debug("- returning info from uri")
+    return pageData
+
     
 def simpleURLCheck(uri):
@@ -30 +49 @@
     '''
     logging.debug("Checking validity of uri, '%s'" %uri)
-    # set the socket timeout period
-    socket.setdefaulttimeout(40)
+    
+    if not URLIB2_INITIALISED:
+        # set the socket timeout period
+        socket.setdefaulttimeout(40)
+    
+        proxy_support = urllib2.ProxyHandler(PROXIES)
+    
+        # build a new opener that adds authentication and caching FTP handlers
+        opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler)
+    
+        # install it
+        urllib2.install_opener(opener)
     
     try:
@@ -216 +245 @@
 
         
-def createCSMLFile(CDMLFilePath, datasetID, timeAxis):
+def createCSMLFile(CDMLFilePath, timeAxis, datasetID = None):
     '''
     Create a CSML file by running csmlscan.py against the specified CDML file
     @param CDMLFilePath: string path to CDML file
-    @param datasetID: string dataset ID to use in CSMLfile
     @param timeAxis: string name of time axis to use in CDML file
+    @keyword datasetID: string dataset ID to use in CSMLfile - if not set, a random
+    name will be generated instead
     @return: CSMLFileName: name of CSML file produced
     '''
     logging.info("Creating CSML file from CDML file by running csmlscan")
+    
+    if not datasetID:
+        datasetID = str(uuid.uuid1())
     CSMLFileName = datasetID + "_csml.xml"
     logging.debug("Inputs specified: datasetID = %s, timeAxis = %s" %(datasetID, timeAxis))

ndgCommon/trunk/ndg/common/src/models/vocabtermdata.py

@@ -6 +6 @@
 '''
 import logging, time, re, urllib
+from ndg.common.src.lib.utilities import openURLWithProxy
     
 def isValidTermURI(uri):
@@ -18 +19 @@
     logging.debug("Checking vocab term uri, '%s'" %uri)
     try:
-        page = urllib.urlopen(uri)
-        pageData = page.read()
+        pageData = openURLWithProxy(uri)
         if pageData.find('<rdf:RDF') > -1:
             logging.debug("- found valid term")
@@ -269 +269 @@
     
     BADC_BROWSE_ROOT = 'http://badc.nerc.ac.uk/browse'
-    
-    # Define proxies required when using urllib
-    PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'}
 
     def __init__(self):
@@ -398 +395 @@
         '''
         logging.info("Retrieving vocab server data page")
-        f = urllib.urlopen(self.VOCAB_SERVER_URL, proxies = self.PROXIES)
-        self.VOCAB_DATA_PAGE = f.read()
-        f.close()
+        self.VOCAB_DATA_PAGE = openURLWithProxy(self.VOCAB_SERVER_URL)
         self.REFRESH_TIME = time.time() + self.ONE_HOUR
         logging.info("Vocab server data retrieved")

ndgCommon/trunk/ndg/common/unittests/clients/xmldb/eXist/testexistdbclient.py

@@ -129 +129 @@
 
     def testSetUpEXistDB(self):
-        molesSchema = self.VALID_COLLECTION_PATH + \
-                ndgXqueries.ATOM_MOLES_SCHEMA  + '.xsd'
-        self.utils.dbc.deleteEXistFile(molesSchema)
-        self.assertEquals(None, self.utils.dbc._eXistDBClient__lookupEXistFile(molesSchema))
+        fileName = self.utils.dbc.xmldb.xq.xsd.keys()[0] + '.xsd'
+        pathName = ec.BASE_COLLECTION_PATH + fileName
+        self.utils.dbc.deleteEXistFile(pathName)
+        self.assertEquals(None, self.utils.dbc._eXistDBClient__lookupEXistFile(pathName))
 
         newClient = dbc.eXistDBClient(configFile = self.confFile, setUpDB = True)
-        self.assertNotEquals(None, newClient._eXistDBClient__lookupEXistFile(molesSchema))
+        self.assertNotEquals(None, newClient._eXistDBClient__lookupEXistFile(pathName))
 
 
@@ -242 +242 @@
         atom = self.utils.createAtomInEXist(tc.xmlString)
         atom = self.utils.createAtomInEXist(tc.xmlString)
+        time.sleep(5)
         backupPath = self.utils.dbc.backupName
         self.utils.dbc.deleteAtomInExist(atom)
@@ -259 +260 @@
         self.createdAtoms.append(atom.getFullPath())
         self.utils.dbc.createDIFDocumentFromAtom(atom)
+
+    def testTransformAtomIntoDIF(self):
+        atom = self.utils.createAtom(tc.xmlString)
+        atom.atomTypeID = VTD.DE_TERM
+        atom = self.utils.dbc.createAtomInExist(atom)
+        self.createdAtoms.append(atom.getFullPath())
+        doc = self.utils.dbc.transformAtomIntoDIF(atom)
+        self.assertNotEquals(None, doc)
+        self.assertTrue(doc.find('DIF') > -1)
+        self.assertTrue(doc.find(atom.datasetID) > -1)
         
     def tearDown(self):

ndgCommon/trunk/ndg/common/unittests/lib/testutilities.py

@@ -10 +10 @@
 class testUtilities(unittest.TestCase):
     
+    VALID_URL = 'http://bbc.co.uk'
+    INVALID_URL = 'http://bbc.co.ukblah'
+    
     def setUp(self):
         '''
         set up data used in the tests.
         '''
+
+    def testOpenURLWithProxy(self):
+        pageData = utilities.openURLWithProxy(self.VALID_URL)
+        self.assertNotEqual(None, pageData)
+
+    def testInvalidOpenURLWithProxy(self):
+        pageData = utilities.openURLWithProxy(self.INVALID_URL)
+        self.assertTrue(pageData.find('The requested URL could not be retrieved') > -1)
+
+    def testSimpleURLCheck(self):
+        self.assertEquals(1, utilities.simpleURLCheck(self.VALID_URL))
+
+    def testInvalidSimpleURLCheck(self):
+        self.assertEquals(0, utilities.simpleURLCheck(self.INVALID_URL))
 
     def testGetISO8601Date(self):