Changeset 2860

Timestamp:

30/08/07 11:42:47 (12 years ago)

Author:

lawrence

Message:

First step to remote ndg document retrieval

Location:

TI05-delivery/ows_framework/trunk/ows_server

Files:

• development.ini (modified) (1 diff)
• ndgDiscovery.config (modified) (4 diffs)
• ows_server/models/DocumentRetrieve.py (modified) (2 diffs)
• ows_server/models/ndgRetrieve.py (modified) (6 diffs)

TI05-delivery/ows_framework/trunk/ows_server/development.ini

@@ -33 +33 @@
 configfile = %(here)s/ndgDiscovery.config
 #the password file needs to be outside the egg:
-passwordFile =  %(here)s/passwords.txt  
-
+#passwordFile =  %(here)s/passwords.txt  
+#but it's gone back to ndgDiscoveryq
 
 # If you'd like to fine-tune the individual locations of the cache data dirs

TI05-delivery/ows_framework/trunk/ows_server/ndgDiscovery.config

@@ -13 +13 @@
 #
 # the following is the server on which the NDG discovery service is running! (Not to be confused with
-# the server on which the NDG discover web service is running)
+# the server on which the NDG discovery web service is running). This can and probably should be the local
+# server (i.e. dont change it!)
 #
 ndgServer:      %(server)s
@@ -28 +29 @@
 metadataMaintainer: b.n.lawrence@rl.ac.uk
 repository:       http://localhost:8080
+tbrecipient:      b.n.lawrence@rl.ac.uk
 
+# The following should only be needed for debugging some parts of the code when running on sandboxes behind a firewall
+proxyServer:      http://wwwcache3.rl.ac.uk:8080/
 
 [SEARCH]
@@ -81 +85 @@
 neodc.nerc.ac.uk: %(server)s
 badc.nerc.ac.uk: %(server)s
-npm.ac.uk: wwwdev.npm.ac.uk/rsdas/projects/ndg
-#grid.bodc.nerc.ac.uk: glue.badc.rl.ac.uk
-grid.bodc.nerc.ac.uk: localhost:8001
-ndg.noc.soton.ac.uk: ndg.noc.soton.ac.uk:8001
+www.npm.ac.uk: http://wwwdev.neodaas.ac.uk/projects/ndg
+grid.bodc.nerc.ac.uk: http://grid.bodc.nerc.ac.uk
+ndg.noc.soton.ac.uk: http://ndg.noc.soton.ac.uk:8001
 icon: %(layout)s/B.gif
 icon_alt: B Service
@@ -104 +107 @@
 grid.bodc.nerc.ac.uk: grid.bodc.nerc.ac.uk
 ndg.noc.soton.ac.uk: ndg.noc.soton.ac.uk
-npm.ac.uk: pgsql.npm.ac.uk
+www.npm.ac.uk: pgsql.npm.ac.uk
+passwordFile: /home/bnl/sandboxes/ndg/TI05-delivery/ows_framework/trunk/ows_server/passwords.txt
 
 [NDG_SECURITY]

TI05-delivery/ows_framework/trunk/ows_server/ows_server/models/DocumentRetrieve.py

@@ -2 +2 @@
 from ndg_xqueries import *
 from xml.dom import minidom 
+import urllib2
 #from stripped_xqueries import strip_de_xquery
 
@@ -114 +115 @@
         self.sessionRelease(id)
         return r
+            
+            
+class RestfulGet:
+    ''' Provides a get method to obtain an xml document from a remote NDG repository '''
+    def __init__(self,remoteHost,proxyServer=None):
+        self.remoteHost=remoteHost
+        if proxyServer is None:
+            proxyHandler=urllib2.ProxyHandler({})
+        else:
+            proxy=proxyServer
+            if proxy[0:4]!='http':proxy='http://'+proxy
+            proxyHandler=urllib2.ProxyHandler({'http':proxy})
+        self.opener=urllib2.build_opener(proxyHandler)
+
+    def get(self,repository,schema,localID,format='NDG-B0',targetCollection=None):
+        #TODO what about security? Probably means we need to get the headers of our responses sorted ...
+        url='%s/view/%s__%s__%s?format=raw&outputSchema=%s'%(self.remoteHost,repository,schema,localID,format)
+        #http://docs.python.org/lib/urllib2-examples.html
+        request=urllib2.Request(url)
+        response='Error obtaining remote file: '
+        try:
+            f = self.opener.open(request)
+            response=''
+        except urllib2.URLError,e:
+            if hasattr(e,'reason'):
+                response+='No access to server [%s]'%e.reason
+            elif hasattr(e,'code'):
+                response+='Response code [%s]'%e.code
+        except socket.error:
+            response+='Network Socket problem'
+        except Exception,e:
+            response+='[%s]'%str(e)
+        if response=='':
+            return f.read()
+        else:
+            raise IOError(response)
+    

TI05-delivery/ows_framework/trunk/ows_server/ows_server/models/ndgRetrieve.py

@@ -5 +5 @@
 
 
-def ndgRetrieve(uri,config,logger=0,output=''):
+def ndgRetrieve(uri,config,logger=0,output='',remote=0):
     
-    ''' Given an ndgObject, uri, retrieve it, with security in place '''
+    ''' Given an ndgObject, uri, retrieve it, with security in place.
+            If logger is non-zero it should be a python log class
+            If output is not '' it should be desired output schema (normally
+        to be used for discovery documents in a variety of outputs or
+        to indicate that an original document is required.
+            If remote is non zero, then attempt to get the document
+        from a remote site via a (secured) restful http call'''
     
-    
-    def getws(config,uri,):
+    def getws(config,uri,remote):
         ''' Get a ws connection to the local exist database '''
         # The logic here is that
@@ -20 +25 @@
         r=config.get('NDG_B_SERVICE',uri.repository)
         sr=config.get('DEFAULT','repository')
+        er=config.get('NDG_EXIST',uri.repository)
         if r<> sr:
-            return 0,'The uri [%s] is not available on [%s]'%(uri,server)
-        er=config.get('NDG_EXIST',uri.repository)
-        
-        #Changes by Dom:
-        #pwf=config.get('DEFAULT','passwordFile')  
-        #passwordFile is now in development.ini, moved from ndgDiscovery.config
-        pwf=cf= CONFIG['app_conf']['passwordFile']
-        ws=DocumentRetrieve.DocumentRetrieve(er,pwfile=pwf)
+            if not remote:
+                return 0,'The uri [%s] is not available on [%s]'%(uri,server)
+            else:
+                ps=config.get('DEFAULT','proxyServer')
+                ws=DocumentRetrieve.RestfulGet(r,proxyServer=ps)
+        else:
+            pwf=config.get('NDG_EXIST','passwordFile')
+            ws=DocumentRetrieve.DocumentRetrieve(er,pwfile=pwf)
         return 1,ws
     
@@ -34 +40 @@
     if uri.schema in ['DIF','MDIP']:
         r=server
-        ws=ndgSearch.ndgSearch()
+        status,ws=1,ndgSearch.ndgSearch()
     elif uri.schema[0:5]=='NDG-B' or uri.schema=='NumSim':
-        status,ws=getws(config,uri)
-        if not status: return status,ws
+        status,ws=getws(config,uri,remote)
     elif uri.schema=='NDG-A0':
-        status,ws=getws(config,uri)
-        if not status: return status,ws
+        status,ws=getws(config,uri,remote)
+     
+    if not status: return status,ws
+           
     #try:
     if uri.schema=='NDG-A0':
@@ -48 +55 @@
     else:
         target='/db/ndg_B_metadata'
-    
-    print uri.uri, 'Output format [%s]'%output
      
     if output=='': output=uri.schema 
    
-   
- 
     #do the actual retrieve:
     e=None
@@ -90 +93 @@
     
     def setup(self):
-        self.c=myConfig('../config/ndgDiscovery.config')
+        self.c=myConfig('../../ndgDiscovery.config')
     
     def testSOAP(self):
@@ -150 +153 @@
         if status: s=str(xml)
 
+    def testRemoteGet(self):
+        ''' Attempt a remote get '''
+        self.setup()
+        doc='www.npm.ac.uk__NDG-B1__data_entity.692'
+        uri=ndgObject.ndgObject(doc)
+        status,xml=ndgRetrieve(uri,self.c,remote=1)
+        if not status: print xml
+        self.assertEqual(status,1)