Changeset 4844
- Timestamp:
- 19/01/09 16:52:53 (12 years ago)
- Location:
- ndgCommon/trunk/ndg/common
- Files:
-
- 6 edited
Legend:
- Unmodified
- Added
- Removed
-
ndgCommon/trunk/ndg/common/src/lib/atomvalidator.py
r4830 r4844 340 340 error = e.faultString.split(':')[-1] 341 341 elif isinstance(e, socket.error): 342 error = " ".join(e.args) 342 if hasattr(e, 'args'): 343 args = e.args 344 if not isinstance(e.args, list): 345 args = [e.args] 346 347 for arg in args: 348 error += " ".join(str(arg)) 343 349 344 350 errorMessage = "Problem experienced when validating against schema:%s'%s'" \ -
ndgCommon/trunk/ndg/common/src/lib/granulite.py
r4836 r4844 484 484 logging.info("Create CSML file from the CDML file - NB, this will be stored in eXist too " + \ 485 485 "and will be used to extract dataset information from") 486 csmlFileName = utilities.createCSMLFile(fn, self._ datasetID, self._cdmlTimeAxis)486 csmlFileName = utilities.createCSMLFile(fn, self._cdmlTimeAxis, datasetID = self._datasetID) 487 487 os.remove(fn) 488 488 -
ndgCommon/trunk/ndg/common/src/lib/utilities.py
r4793 r4844 3 3 @author: C Byrom 4 4 ''' 5 import os, sys, logging, re, cgi, urlparse, httplib, time, urllib2, socket5 import os, sys, logging, re, cgi, urlparse, httplib, time, urllib2, urllib, socket, uuid 6 6 from ndg.common.src.models.ndgObject import ndgObject 7 7 import csml.csmlscan as CsmlScan … … 19 19 20 20 esc_chars = {'\xb0':'°','°':'°'} 21 22 # Define proxies required when using urllib 23 PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'} 24 25 URLIB2_INITIALISED = False 26 27 def openURLWithProxy(uri): 28 ''' 29 Open a simple url connection using the standard proxy, and retrieve the contents 30 @param uri: uri to read from 31 @return pageData: data read from uri 32 ''' 33 logging.debug("Reading info from uri, '%s'" %uri) 34 f = urllib.urlopen(uri, proxies = PROXIES) 35 pageData = f.read() 36 f.close() 37 logging.debug("- returning info from uri") 38 return pageData 39 21 40 22 41 def simpleURLCheck(uri): … … 30 49 ''' 31 50 logging.debug("Checking validity of uri, '%s'" %uri) 32 # set the socket timeout period 33 socket.setdefaulttimeout(40) 51 52 if not URLIB2_INITIALISED: 53 # set the socket timeout period 54 socket.setdefaulttimeout(40) 55 56 proxy_support = urllib2.ProxyHandler(PROXIES) 57 58 # build a new opener that adds authentication and caching FTP handlers 59 opener = urllib2.build_opener(proxy_support, urllib2.HTTPHandler) 60 61 # install it 62 urllib2.install_opener(opener) 34 63 35 64 try: … … 216 245 217 246 218 def createCSMLFile(CDMLFilePath, datasetID, timeAxis):247 def createCSMLFile(CDMLFilePath, timeAxis, datasetID = None): 219 248 ''' 220 249 Create a CSML file by running csmlscan.py against the specified CDML file 221 250 @param CDMLFilePath: string path to CDML file 222 @param datasetID: string dataset ID to use in CSMLfile223 251 @param timeAxis: string name of time axis to use in CDML file 252 @keyword datasetID: string dataset ID to use in CSMLfile - if not set, a random 253 name will be generated instead 224 254 @return: CSMLFileName: name of CSML file produced 225 255 ''' 226 256 logging.info("Creating CSML file from CDML file by running csmlscan") 257 258 if not datasetID: 259 datasetID = str(uuid.uuid1()) 227 260 CSMLFileName = datasetID + "_csml.xml" 228 261 logging.debug("Inputs specified: datasetID = %s, timeAxis = %s" %(datasetID, timeAxis)) -
ndgCommon/trunk/ndg/common/src/models/vocabtermdata.py
r4832 r4844 6 6 ''' 7 7 import logging, time, re, urllib 8 from ndg.common.src.lib.utilities import openURLWithProxy 8 9 9 10 def isValidTermURI(uri): … … 18 19 logging.debug("Checking vocab term uri, '%s'" %uri) 19 20 try: 20 page = urllib.urlopen(uri) 21 pageData = page.read() 21 pageData = openURLWithProxy(uri) 22 22 if pageData.find('<rdf:RDF') > -1: 23 23 logging.debug("- found valid term") … … 269 269 270 270 BADC_BROWSE_ROOT = 'http://badc.nerc.ac.uk/browse' 271 272 # Define proxies required when using urllib273 PROXIES = {'http':'http://wwwcache.rl.ac.uk:8080'}274 271 275 272 def __init__(self): … … 398 395 ''' 399 396 logging.info("Retrieving vocab server data page") 400 f = urllib.urlopen(self.VOCAB_SERVER_URL, proxies = self.PROXIES) 401 self.VOCAB_DATA_PAGE = f.read() 402 f.close() 397 self.VOCAB_DATA_PAGE = openURLWithProxy(self.VOCAB_SERVER_URL) 403 398 self.REFRESH_TIME = time.time() + self.ONE_HOUR 404 399 logging.info("Vocab server data retrieved") -
ndgCommon/trunk/ndg/common/unittests/clients/xmldb/eXist/testexistdbclient.py
r4810 r4844 129 129 130 130 def testSetUpEXistDB(self): 131 molesSchema = self.VALID_COLLECTION_PATH + \132 ndgXqueries.ATOM_MOLES_SCHEMA + '.xsd'133 self.utils.dbc.deleteEXistFile( molesSchema)134 self.assertEquals(None, self.utils.dbc._eXistDBClient__lookupEXistFile( molesSchema))131 fileName = self.utils.dbc.xmldb.xq.xsd.keys()[0] + '.xsd' 132 pathName = ec.BASE_COLLECTION_PATH + fileName 133 self.utils.dbc.deleteEXistFile(pathName) 134 self.assertEquals(None, self.utils.dbc._eXistDBClient__lookupEXistFile(pathName)) 135 135 136 136 newClient = dbc.eXistDBClient(configFile = self.confFile, setUpDB = True) 137 self.assertNotEquals(None, newClient._eXistDBClient__lookupEXistFile( molesSchema))137 self.assertNotEquals(None, newClient._eXistDBClient__lookupEXistFile(pathName)) 138 138 139 139 … … 242 242 atom = self.utils.createAtomInEXist(tc.xmlString) 243 243 atom = self.utils.createAtomInEXist(tc.xmlString) 244 time.sleep(5) 244 245 backupPath = self.utils.dbc.backupName 245 246 self.utils.dbc.deleteAtomInExist(atom) … … 259 260 self.createdAtoms.append(atom.getFullPath()) 260 261 self.utils.dbc.createDIFDocumentFromAtom(atom) 262 263 def testTransformAtomIntoDIF(self): 264 atom = self.utils.createAtom(tc.xmlString) 265 atom.atomTypeID = VTD.DE_TERM 266 atom = self.utils.dbc.createAtomInExist(atom) 267 self.createdAtoms.append(atom.getFullPath()) 268 doc = self.utils.dbc.transformAtomIntoDIF(atom) 269 self.assertNotEquals(None, doc) 270 self.assertTrue(doc.find('DIF') > -1) 271 self.assertTrue(doc.find(atom.datasetID) > -1) 261 272 262 273 def tearDown(self): -
ndgCommon/trunk/ndg/common/unittests/lib/testutilities.py
r4793 r4844 10 10 class testUtilities(unittest.TestCase): 11 11 12 VALID_URL = 'http://bbc.co.uk' 13 INVALID_URL = 'http://bbc.co.ukblah' 14 12 15 def setUp(self): 13 16 ''' 14 17 set up data used in the tests. 15 18 ''' 19 20 def testOpenURLWithProxy(self): 21 pageData = utilities.openURLWithProxy(self.VALID_URL) 22 self.assertNotEqual(None, pageData) 23 24 def testInvalidOpenURLWithProxy(self): 25 pageData = utilities.openURLWithProxy(self.INVALID_URL) 26 self.assertTrue(pageData.find('The requested URL could not be retrieved') > -1) 27 28 def testSimpleURLCheck(self): 29 self.assertEquals(1, utilities.simpleURLCheck(self.VALID_URL)) 30 31 def testInvalidSimpleURLCheck(self): 32 self.assertEquals(0, utilities.simpleURLCheck(self.INVALID_URL)) 16 33 17 34 def testGetISO8601Date(self):
Note: See TracChangeset
for help on using the changeset viewer.