source: TI05-delivery/ows_framework/trunk/ows_server/ows_server/lib/ndgInterface.py @ 4237

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI05-delivery/ows_framework/trunk/ows_server/ows_server/lib/ndgInterface.py@4237
Revision 4237, 8.3 KB checked in by cbyrom, 12 years ago (diff)

Add 'create atom' page - to create the 'header' info for the input data +
create home page + start a new controller for list/search purposes.
Extend ndgInterface to allow specification of whether cache is used or
not - NB, this causes problems when editing/updating docs since the
original was always being reloaded from the cache.

Line 
1# Copyright (C) 2007 STFC & NERC (Science and Technology Facilities Council).
2# This software may be distributed under the terms of the
3# Q Public License, version 1.0 or later.
4# http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
5"""
6Manages interface to NDG documents and data, including caching
7"""
8
9from cache import Cache
10import os, tempfile
11import csml
12try:
13    import cdms2 as cdms
14except:
15    import cdms
16from ndgUtils import ndgRetrieve, ndgObject, xmlHandler2
17from pylons import request,session
18import logging
19
20#from ows_server.models.ndgSecurity import HandleSecurity
21from pylons import g # for handle to access control PEP interface
22try:
23    from ndg.security.common.authz.pdp import PDPError
24    from ndg.security.common.authz.pep import PEPError
25except ImportError, e:
26    from warnings import warn
27    warn(__name__ + ": access control is disabled: %s" % e, RuntimeWarning)
28   
29   
30class CDMSEntry(object):
31    """
32    A reference to a CDMS file that will delete the file when it is
33    garbage collected.
34
35    Instances of this class are used as Cache entry values.  When entries are
36    automatically removed from the cache the CDMSEntry object will be
37    garbage collected unless it is being accessed by another thread.
38
39    """
40
41    __slots__ = ['_f', 'var']
42   
43    def __init__(self, filename, varname):
44        logging.info('Caching file variable %s in %s' % (varname, filename))
45        self._f = cdms.open(filename)
46        self.var = self._f[varname]
47    def __del__(self):
48        filename = self._f.id
49        logging.info('Removing file %s' % filename)
50        self._f.close()
51        os.remove(filename)
52 
53
54class CSMLExtractCache(Cache):
55   
56    def __init__(self, cache_dir, max_size=0):
57        super(CSMLExtractCache, self).__init__(max_size)
58        self._cache_dir = cache_dir
59
60    def _extract(self, feature, sel):
61        (fd, filename) = tempfile.mkstemp('.nc', 'csml_wxs_', self._cache_dir)
62        os.close(fd)
63        (outputdir, ncname) = os.path.split(filename)
64
65        # Workarround until ticket:778 (TestExtractAll) is fixed
66        sel.update(longitude=(-180, 180), latitude=(-90, 90))
67
68        feature.subsetToGridSeries(ncname=ncname, outputdir=outputdir, **sel)
69
70        return filename
71   
72    def key(self, name):
73        """
74        Cache provides this method to map externally visible entry names
75        to internal keys.  We use it here to turn selectors into hashable
76        values.
77
78        """
79        (feature, sel) = name
80       
81        return (feature.id, tuple(sorted(sel.items())))
82
83    def build(self, key, name, opened, entry):
84        (feature, sel) = name
85        filename = self._extract(feature, sel)
86
87        return CDMSEntry(filename, feature.name.CONTENT)
88
89    def __getitem__(self, name):
90        """
91        Returns the opened CDMS object.
92
93        """
94        return super(CSMLExtractCache, self).__getitem__(name).var
95
96class ndgInterface:
97   
98    def __init__(self):
99        ''' Instantiate with three level cache:
100                - a file cache
101                - a xmlhandler object cache, and
102                - a parsed CSML object cache.'''
103
104        self.CSMLDataCache=CSMLExtractCache(
105                request.environ['paste.config']['app_conf']['tmp_dir'],max_size=10)
106        self.CSMLDocCache=Cache(max_size=10)
107        self.XMLHCache=Cache(max_size=10)
108       
109
110    def GetXML(self,uri,outputSchema='', useCache=True):
111        '''
112        This method provides a secure interface to the server
113        document cache and a remote NDG exist. It is assumed that
114        the local filesystem is protected in that you can't get to
115        files except via the CSML api
116        @param uri: ndg format uri to locate doc from
117        @keyword outputSchema: format to return doc in
118        @keyword useCache: check for data in the cache and use this, if set to
119        True (the default)   
120        '''
121        #    Note that this method should not be used to obtain
122        #unsecured discovery documents, these are called directly
123        #in the retrieve controller!
124
125        logging.info("Getting XML from uri, '%s' (outputschema: '%s')" \
126                     %(uri, outputSchema))
127        try:
128            ndgO=ndgObject(uri)
129            localFile=0
130        except ValueError:
131            ''' It's a local file not an ndg identifier '''
132            logging.info("File appears to be local - look for it there...")
133            ndgO=uri
134            localFile=1
135       
136        if session and 'ndgCleared' in session:
137            cleared=session['ndgCleared']
138        else:
139            cleared=None
140       
141        if outputSchema or not useCache:
142            #bypass the cache ...
143            status,xmlh=ndgRetrieve(ndgO,
144                                    request.environ['ndgConfig'],
145                                    output=outputSchema,
146                                    discovery=g.standalone)
147        else:
148            try:
149                xmlh=self.XMLHCache[uri]
150                status=1
151                logging.info('XMLH Cache hit for [%s]'%uri)
152            except:
153                logging.info('XMLH Cache miss for [%s]'%uri)
154                if localFile:
155                    status,xmlH=self.__getLocal(uri)
156                else:   
157                    status,xmlh=ndgRetrieve(ndgO,
158                                            request.environ['ndgConfig'],
159                                            output=outputSchema,
160                                            discovery=g.standalone)
161                if status: 
162                    self.XMLHCache[uri]=xmlh
163           
164        if not status: return status,xmlh
165       
166        # convert doc to an XML tree
167        xmlh=xmlHandler2.xmlHandler(xmlh,string=1)
168       
169        # valid values of the return objects SHOULD BE
170        #   ok:         status=1, xmlh=an xml handler instance.
171        #   exceptions, status=0, xmlh='Exception(e)'
172       
173        status,xmlh=self.__gatekeep(ndgO,xmlh)
174        if status:
175            if cleared is None:
176                session['ndgCleared']=[uri]
177            else:
178                session['ndgCleared'].append(uri)
179            session.save()
180       
181        return status,xmlh
182           
183    def GetParsedCSML(self,uri):
184       
185        ''' This method gets a parsed CSML object corresponding to the URI '''
186       
187        # do we need an xml handler instance to test the security?
188        if 'ndgCleared' not in session:
189            status,xmlh=self.GetXML(uri)
190        else: 
191            if uri not in session['ndgCleared']: 
192                status,xmlh=self.GetXML(uri)
193            else: status=1
194        if not status: return status,xmlh
195       
196        try:
197            d=self.CSMLDocCache[uri]
198            logging.info('CSML Cache hit for [%s]'%uri)
199        except:
200            logging.info('CSML Cache miss for [%s]'%uri)
201            status,xmlh=self.GetXML(uri)
202            if not status: return status,xmlh   
203            d=csml.parser.Dataset()
204            d.parseElemTree(xmlh.tree)
205            self.CSMLDocCache[uri]=d
206        status=1
207        return status,d
208       
209    def __gatekeep(self,uri,x):
210        ''' This is the NDG gatekeeper '''
211        if 'ndgSec' in session:
212            securityTokens=session['ndgSec']
213        else:
214            securityTokens=None
215
216        if not hasattr(g, 'pep'):
217            if not g.standalone:
218                raise PEPError(\
219                "Security is disabled but the standalone flag is set to False")
220               
221            logging.info("__gatekeep: access control is disabled - standalone " + \
222                     "config flag is set")
223           
224        try:
225            # Arguments are: a handle to the resource and a handle to the users
226            # security tokens
227            g.pep(dict(uri=uri, doc=x), securityTokens, None)
228            return True, x
229       
230        except PDPError, e:
231            # Caught a known access control condition
232            return False, 'Access Denied for %s %s' % (uri, e)
233               
234    def __getLocal(self,uri):
235        ''' Returns a local csml file (used for testing) '''
236        csml_dir = request.environ['paste.config']['app_conf']['csml_dir']
237        path = os.path.join(csml_dir, file)
238        if os.path.exists(path+'.csml'):
239            f = path+'.csml'
240        elif os.path.exists(path+'.xml'):
241            f = path +'.xml'
242        else:
243            return 0, '<p>Cannot find CSML file %s</p>' % file
244        r=f.read()
245        return 1,r     
246       
247
248
249interface=ndgInterface()
Note: See TracBrowser for help on using the repository browser.