source: TI07-MOLES/tags/wsgi-beta/ndgSearch.py @ 2364

# these imports are autogenerated by wsdl2py ...
from DiscoveryService_services_types import *
from DiscoveryService_services import *

#normal imports
import sys, time
try: #python 2.5
    from xml.etree import ElementTree as ET
except ImportError:
    try:
        # if you've installed it yourself it comes this way
        import ElementTree as ET
    except ImportError:
        # if you've egged it this is the way it comes
        from elementtree import ElementTree as ET
    
from ETxmlView import loadET,et2text

from xml.dom import expatbuilder

class ExpatReaderClass:
      fromString = staticmethod(expatbuilder.parseString)
      fromStream = staticmethod(expatbuilder.parse)
 

class ndgSearch:
    ''' Provides a client to the NDG discovery services methods exposed by the Discovery.wsdl '''
    
    def __init__(self,logger=None,tracefile=None):
        '''get an instance of the service'''
        #how do we get a different backend provider?
        loc=DiscoveryServiceLocator()
        #self.server=loc.getDiscovery(readerclass=ExpatReaderClass,
        self.server=loc.getDiscoveryServicePortType(readerclass=ExpatReaderClass,tracefile=tracefile)
        self.serverSessionID=None
        self.logger=logger
        self.__reset()
        
    def __reset(self):
        ''' Initialise and/or making sure we have no hangovers from a previous call '''
        self.documents=None
        self.hits=None
        self.error=None
        self.response=None
        self.status=None
    
    def search(self,term,start=1,howmany=20,target='All',scope=None,bbox=None,dateRange=None,):
        ''' Get a list of documents for a specific term using a free text search'''
        
        self.__reset()
        #if target=='Authors':
        #    request=doAuthorSearchRequest()
        #elif target=='Params':
        #    request=doParameterSearchRequest()
        #else:
        #    request=doFullTextSearchRequest()
        request=doSearchRequest()
        request.Term=term
        request.Start=start
        request.HowMany=howmany
        request.TermType={'Authors':'author','Params':'parameter','All':'fullText'}[target]
        if bbox is not None:
            box=request.new_BoundingBox()
            box.LimitNorth,box.LimitSouth,box.LimitWest,box.LimitEast=map(float,bbox)
            request.BoundingBox=box
        
        if scope is not None:
            request.Scope=scope
        
        if dateRange is not None:
            start=int(dateRange[0][2]),int(dateRange[0][1]),int(dateRange[0][0]),0,0,0,0
            end=int(dateRange[1][2]),int(dateRange[1][1]),int(dateRange[1][0]),0,0,0,0
            
            dRange=request.new_DateRange()
            dRange.DateRangeStart=start
            dRange.DateRangeEnd=end
            request.DateRange=dRange

        if self.logger: itime=time.time()
        response=self.server.doSearch(request)
            
        if self.logger:
            etime=time.time()-itime
            self.logger.info('Search Request [%s] took [%ss]'%(term,etime))
        if response._status:
            self.serverSessionID=response._resultId
            self.hits=response.Hits
            self.documents=response.Documents.Document
        else:
            self.hits=response.Hits
        self.status=[response._statusMessage,]
        self.response=response
        return self.documents

    def getDoc(self,document,format='original'):
        ''' Return a single document from the backend database '''
        
        #create a request object
        request=doPresentRequest()
        #get an instance of the Documents holder class
        DocList=request.new_documents()
        request.Documents=DocList
        DocList.Document=[document,]
        request.Format=format
        #try:
        searchResult=self.server.doPresent(request)
        #except Exception,e:
        #   raise Exception,e
        if searchResult._status:
            result=searchResult.Documents.Document[0]
        else:
            raise ValueError('Error retrieving [%s] was [%s]'%(document,searchResult._statusMessage))
            #result=searchResult._statusMessage
        return result
                   
    def getAllDocs(self,format='original'):            
        ''' Parse the list of documents, and retrieve them directly '''
        
        if self.documents is None: return []
        if self.logger: itime=time.time()
         
        #create a request object
        request=doPresentRequest()
        #get an instance of the Documents holder class
        DocList=request.new_documents()
        request.Documents=DocList
        DocList.Document=self.documents
        request.Format=format
        result=self.server.doPresent(request)
        if result._status:
            docs=result.Documents.Document
        else:
            raise ValueError('Error retrieving [%s] was [%s]'%
                        (self.documents,result._statusMessage))
        if self.logger:
            etime=time.time()-itime
            self.logger.info('Document Load [n=%s] took [%ss]'%(len(self.documents),etime))

        return docs
            
    def __xmlerror(self,docmessage):
        print docmessage
        return ET.fromstring('<Error><Document>%s</Document><Message>%s</Message></Error>'%docmessage)
            
    def getDocElement(self,document):
        ''' Takes a document path (maybe from a previous call to ndgSearch) and extracts that document
        into an ElementTree instance '''
        #we stick it straight into element tree because we need to use et to get the actual document
        #we want, not the envelope xml elements 
        
        doc=self.getDoc(document)
        path=document.rstrip('.xml')
        try:
            r=loadET(doc)
            #return r.find(path)
            return r
        except:
            return self.__xmlerror((path,doc))
            
    def getAllDocsAsElements(self):
        ''' Get all the documents and load them into a list of ET instances '''
        result=[]
        docs=self.getAllDocs()
        for doc in docs: 
            try:
                r=loadET(doc)
                result.append(r)#result.append(r.find(path))
            except:
                result.append(self.__xmlerror(doc))
        return result
            
            
    def getLabelledDocs(self,format='original'):
        ''' Returns all the documents in sequence in a labelled list of strings'''
        if self.hits==0: return []
        #filenames=self.documents.Document
        #so we know that the following call is the problem ...
        responses=self.getAllDocs(format)
        filenames=self.documents
        i=len(filenames)
        j=len(responses)
        if i!=j:
            print filenames
            raise ValueError,'Internal inconsistency in search return [hits:%s!=responses:%s]'%(i,j)
        indices=range(i)
        results=[]
        for i in indices:
            results.append((filenames[i].strip('.xml'),responses[i]))
        return results
        
    def get(self,repository,schema,localID,format='original',targetCollection=None):
        ''' Obtain a document via it's NDG id split up '''
        #nb argument targetCollection is here to provide same API as exist xmlrpc interface
        uri='%s__%s__%s'%(repository,schema,localID)
        fileName=uri+'.xml'
        return self.getDoc(fileName,format)
        
        
import unittest

class TestCase(unittest.TestCase):
    
    def testSearch(self):
        ''' Test fundamental search capability '''
        term='temperature'
        s=ndgSearch()
        howmany=10
        docs=s.search(term,start=1,howmany=howmany)
        print 'Asked for ',howmany,' documents (there were %s hits):'%s.hits
        print s.documents
       
    def testgetLabelledDocs(self):
        ''' Test returning of labelled documents '''
        term='CD97'
        s=ndgSearch()
        r=s.search(term)
        print s.status
        print s.documents
        output=s.getLabelledDocs()
        
    def testNoReturn(self):
        ''' Tests a search return with (hopefully nothing to be found)'''
        term='xpabnl'
        s=ndgSearch()
        r=s.search(term)
        print 'Hopefully this is zero: if not, expect the NoReturn test to fail:',s.hits
        output=s.getLabelledDocs()
        self.assertEqual(len(output),0)
        
    def testGetDoc(self):
        ''' Test obtaining a specific document which had better exist '''
        doc='noc.soton.ac.uk__DIF__NOCSDAT100.xml'
        #doc='ndg.noc.soton.ac.uk__DIF__NOCSDAT274.xml'
        s=ndgSearch()
        r=s.getDoc(doc)
        
    def testSequence(self):
        ''' Tests that repeated searches work and can support zero responses in the middle '''
        # this was a bug we found and needed a test case for
        s=ndgSearch()
        term='CD97' # hopefully just get a couple of hits for now.
        r=s.search(term)
        print s.hits
        #if we uncomment this it used to break, and not if uncommented, now it seems ok...
        output=s.getLabelledDocs() 
        #this failed as well, and we isolated the dependency in the previous call to this one.
        #res=s.getAllDocs()
        term='xpabnl'
        r=s.search(term)
        print s.hits
        print 'Hopefully this is also zero: if not, expect the Sequence test to fail:',s.hits
        
    def testGet(self):
        ''' Tests getting via uri components '''
        (r,s,l)='neodc.nerc.ac.uk','DIF','NEODC_ARSF_ATM_DAED'
        ss=ndgSearch()
        x=ss.get(r,s,l,format='DC')

if __name__=="__main__":
    unittest.main()