Changeset 4175

Timestamp:

04/09/08 14:40:55 (13 years ago)

Author:

cbyrom

Message:

Fix handling of ndgObject config object + refactor common code in
ndgSearch into re-usable methods + add extra logging and error
handling + extend test suite.

Location:

exist/trunk/python/ndgUtils

Files:

• ndgObject.py (modified) (3 diffs)
• ndgSearch.py (modified) (9 diffs)
• test.config (modified) (1 diff)
• test_ndgRetrieve.py (modified) (1 diff)
• test_ndgSearch.py (modified) (5 diffs)
• testconstants.py (modified) (1 diff)

exist/trunk/python/ndgUtils/ndgObject.py

@@ -97 +97 @@
         # set default return format - if not set, just return in original format
         fmt=''
-        if self.config.has_option('DISCOVERY','formatDefault'):
-            fmt=self.config.get('DISCOVERY','formatDefault')
+        if self.config.config.has_option('DISCOVERY','formatDefault'):
+            fmt=self.config.config.get('DISCOVERY','formatDefault')
             logging.info("Default discovery format set to: %s" %fmt)
             
@@ -104 +104 @@
         # We'll build the following even if it can't be used (as would be the case for
         # a non data entity B record or an A record) because it's a useful template.
-        if self.schema!=fmt: qs=('outputSchema',fmt)
+        if self.schema!=fmt: 
+            qs=('outputSchema',fmt)
         self.discoveryURL=self.__buildURL(discoveryBASE,[qs])
 
@@ -110 +111 @@
         # - if not, default to 'unknown'
         servicehost = 'unknown'
-        if self.config.has_option('NDG_B_SERVICE',self.repository):
+        if self.config.config.has_option('NDG_B_SERVICE',self.repository):
             servicehost=self.config.get('NDG_B_SERVICE',self.repository)
             

exist/trunk/python/ndgUtils/ndgSearch.py

@@ -2 +2 @@
 from DiscoveryService_services_types import *
 from DiscoveryService_services import *
-import os.path
+import os.path, logging
 
 #normal imports
@@ -20 +20 @@
 from xml.dom import expatbuilder
 
+class NDGSearchError(Exception):
+    """Exception handling for NDG Search class."""
+    def __init__(self, msg):
+        logging.error(msg)
+        Exception.__init__(self, msg)
+
 class ExpatReaderClass:
       fromString = staticmethod(expatbuilder.parseString)
@@ -33 +39 @@
     ''' Provides a client to the NDG discovery services methods exposed by the Discovery.wsdl '''
     
-    def __init__(self,logger=None,tracefile=None,HostAndPort=default_HostAndPort):
+    def __init__(self, tracefile=None,HostAndPort=default_HostAndPort):
         '''Get an instance of the NDG Discovery Service. 
             Optional Keywords are: 
-                logger -  a python logging instance, for collecting stats
                 tracefile - for collecting debug output
                 HostAndPort - the location of the SOAP web service (default is ndg Glue)
@@ -45 +50 @@
                                     readerclass=ExpatReaderClass,tracefile=tracefile)
         self.serverSessionID=None
-        self.logger=logger
-        if self.logger: 
-            logger.info('Discovery web service connection to %s'%HostAndPort)
+        logging.info('Discovery web service connection to %s'%HostAndPort)
         self.__reset()
         
@@ -60 +63 @@
     def search(self,term,start=1,howmany=20,target='All',scope=None,bbox=None,dateRange=None,geoSearchType='overlaps'):
         ''' Get a list of documents for a specific term using a free text search'''
-        
+        logging.info("Running search - with term, '%s'" %term)
         self.__reset()
         #if target=='Authors':
@@ -91 +94 @@
             request.DateRange=dRange
 
-        if self.logger: itime=time.time()
+        itime=time.time()
         response=self.server.doSearch(request)
-            
-        if self.logger:
-            etime=time.time()-itime
-            self.logger.info('Search Request [%s] took [%ss]'%(term,etime))
+        etime=time.time()-itime
+        logging.debug('Search Request [%s] took [%ss]'%(term,etime))
+        
         if response._status:
             self.serverSessionID=response._resultId
@@ -103 +105 @@
         else:
             self.hits=response.Hits
+        
+        logging.info("Search returned %s results: %s" %(self.hits, self.documents))
         self.status=[response._statusMessage,]
         self.response=response
+        logging.info("Search complete")
         return self.documents
 
+
+    def __setUpRequestObject(self, documents, format):
+        '''
+        Set up a request object and set it up with the appropriate
+        Documents holder class and format
+        @param documents: documents to hold in request
+        @type documents: list
+        @param format: document format to use
+        @type format: str
+        @return: request object with Documents holder and format set up
+        '''
+        logging.debug("Setting up request object")
+        request=doPresentRequest()
+        request.Documents = request.new_documents()
+        request.Documents.Document = documents
+        request.Format = format
+        logging.debug("Request object set up")
+        return request
+
+
+    def __runDocumentPresent(self, request):
+        '''
+        Run a document retrieval, provided the request to use
+        @param request: request object to run the doPresent call with
+        @type request: request
+        @return: result - array of documents returned from call
+        @raise ValueError if document not retrieved properly
+        '''
+        logging.debug("Running 'doPresent()'")
+        itime=time.time()
+        searchResult=self.server.doPresent(request)
+        if searchResult._status:
+            result=searchResult.Documents.Document
+        else:
+            raise ValueError('Error during document retrieval: %s' \
+                             %searchResult._statusMessage)
+        etime=time.time()-itime
+        logging.debug('Document Load took [%ss]' %etime)
+        logging.debug("'doPresent() completed")
+        return result
+         
+
     def getDoc(self,document,format='original'):
-        ''' Return a single document from the backend database '''
-        
-        #create a request object
-        request=doPresentRequest()
-        #get an instance of the Documents holder class
-        DocList=request.new_documents()
-        request.Documents=DocList
-        DocList.Document=[document,]
-        request.Format=format
-        #try:
-        searchResult=self.server.doPresent(request)
-        #except Exception,e:
-        #   raise Exception,e
-        if searchResult._status:
-            result=searchResult.Documents.Document[0]
-        else:
-            raise ValueError('Error retrieving [%s] was [%s]'%(document,searchResult._statusMessage))
-            #result=searchResult._statusMessage
-        return result
+        ''' 
+        Return a single document from the backend database 
+        '''
+        logging.info("Retrieving document, '%s' in %s format" %(document, format))
+
+        request = self.__setUpRequestObject([document], format)
+
+        docs = self.__runDocumentPresent(request)
+        logging.info("Document retrieved successfully")
+        return docs[0]
+
                    
     def getAllDocs(self,format='original'):            
-        ''' Parse the list of documents, and retrieve them directly '''
-        
-        if self.documents is None: return []
-        if self.logger: itime=time.time()
-         
-        #create a request object
-        request=doPresentRequest()
-        #get an instance of the Documents holder class
-        DocList=request.new_documents()
-        request.Documents=DocList
-        DocList.Document=self.documents
-        request.Format=format
-        result=self.server.doPresent(request)
-        if result._status:
-            docs=result.Documents.Document
-        else:
-            raise ValueError('Error retrieving [%s] was [%s]'%
-                        (self.documents,result._statusMessage))
-        if self.logger:
-            etime=time.time()-itime
-            self.logger.info('Document Load [n=%s] took [%ss]'%(len(self.documents),etime))
-
+        ''' 
+        Parse the list of documents, and retrieve them directly 
+        '''
+        logging.info("Retrieving all documents specified")
+        logging.debug("- %s" %self.documents)
+        if self.documents is None:
+            logging.info("No documents specified to retrieve - returning")
+            return []
+        
+        request = self.__setUpRequestObject(self.documents, format)
+        docs = self.__runDocumentPresent(request)
+        logging.info("Documents retrieved successfully")
         return docs
             
@@ -173 +202 @@
             
     def getAllDocsAsElements(self):
-        ''' Get all the documents and load them into a list of ET instances '''
+        ''' 
+        Get all the documents and load them into a list of ET instances 
+        '''
         result=[]
         docs=self.getAllDocs()
@@ -186 +217 @@
             
     def getLabelledDocs(self,format='original'):
-        ''' Returns all the documents in sequence in a labelled list of strings'''
+        ''' 
+        Returns all the documents in sequence in a labelled list of strings
+        '''
+        logging.info("Retrieving all labelled documents")
         if self.hits==0: return []
         #filenames=self.documents.Document
         #so we know that the following call is the problem ...
         responses=self.getAllDocs(format)
-        filenames=self.documents
-        i=len(filenames)
+        i=len(self.documents)
         j=len(responses)
         if i!=j:
-            print filenames
-            raise ValueError,'Internal inconsistency in search return [hits:%s!=responses:%s]'%(i,j)
+            logging.debug("Requested docs: %s\nActual results: %s" \
+                          %(self.documents, responses))
+            raise NDGSearchError('Internal inconsistency in search \
+                return [hits:%s!=responses:%s]'%(i,j))
         indices=range(i)
         results=[]
         for i in indices:
-            ####results.append((filenames[i].strip('.xml'),responses[i]))
-            ####Wow, 'higem.xml'.strip('.xml') = hige ... and that's what split is supposed to do!
-            ff=os.path.splitext(filenames[i])
+            ff=os.path.splitext(self.documents[i])
             results.append((ff[0],responses[i]))
+        logging.info("Documents retrieved")
         return results
         

exist/trunk/python/ndgUtils/test.config

@@ -5 +5 @@
 [NDG_B_SERVICE]
 badc.nerc.ac.uk:chinook.badc.rl.ac.uk
-www.npm.ac.uk:www.npm.ac.uk
+www.npm.ac.uk:pgsql.npm.ac.uk
 instance:chinook.badc.rl.ac.uk
 

exist/trunk/python/ndgUtils/test_ndgRetrieve.py

@@ -62 +62 @@
         self.assertNotEqual(xml.find(tc.VALID_MOLES_DOC_ID), -1)
 
-    def offtestRemoteGet(self):
+    def OFFtestRemoteGet(self):
         ''' Attempt a remote get '''
         # TODO: not sure what remote repository to use for this?

exist/trunk/python/ndgUtils/test_ndgSearch.py

@@ -1 @@
-import unittest
+import unittest, logging
+import testconstants as tc
+import ndgObject as no
 from ndgSearch import ndgSearch
+
+logging.basicConfig(level=logging.DEBUG,
+                        format='%(asctime)s %(filename)s:%(lineno)d %(levelname)s %(message)s')
 
 class TestCase(unittest.TestCase):
@@ -14 +19 @@
     def testgetLabelledDocs(self):
         ''' Test returning of labelled documents '''
-        term='CD97'
+        term='dppp'
         s=ndgSearch()
         r=s.search(term)
         output=s.getLabelledDocs()
-        nocsfound=0
-        for i in output: 
-            if 'noc.soton' in i[0]: nocsfound=1
-        self.assertEqual(nocsfound,1)
+        self.assertNotEqual(len(output), 0)
         
     def testNoReturn(self):
@@ -32 +34 @@
     def testGetDoc(self):
         ''' Test obtaining a specific document which had better exist '''
-        doc='badc.nerc.ac.uk__DIF__dataent_COAPEC.xml'
-        s=ndgSearch()
-        r=s.getDoc(doc)
-        self.assertEqual('<DIF',r[0:4])
+        doc= tc.VALID_DISCOVERY_PROVIDER_ID + "__" + no.DIF_DOC_TYPE + \
+            "__" + tc.VALID_DISCOVERY_DOC_ID + ".xml"
+        s=ndgSearch()
+        r=s.getDoc(doc)
+        self.assertNotEqual(r.find('<DIF'), -1)
         
     def testSequence(self):
@@ -55 +58 @@
         ''' Tests getting via uri components '''
         
-        (r,s,l)='badc.nerc.ac.uk','DIF','dataent_COAPEC'
+        (r,s,l)=tc.VALID_DISCOVERY_PROVIDER_ID, \
+            no.DIF_DOC_TYPE, tc.VALID_DISCOVERY_DOC_ID
         ss=ndgSearch()
         x=ss.get(r,s,l,format='DC')
+        self.assertNotEqual(x.find('<oai_dc'), -1)
         
     def testFailedGet(self):
@@ -65 +70 @@
             r=s.getDoc(doc)
         except ValueError,e:
-            self.assertEqual(str(e),'Error retrieving [abc] was [Error retrieving all documents]')
+            self.assertEqual(str(e),\
+                'Error during document retrieval: Document retrieval was successful but generated no results.')
         except Exception,e:
             self.fail(str(e))

exist/trunk/python/ndgUtils/testconstants.py

@@ -19 +19 @@
 VALID_NUMSIM_DOC_ID = 'HadGEM1_CodeBase'
 VALID_ATOM_DOC_ID = 'dataent_11679938403412067'
-        
+
+VALID_DISCOVERY_PROVIDER_ID = "csl.gov.uk"
+VALID_DISCOVERY_DOC_ID = "csl.gov.uk__1"