source: TI05-delivery/ows_framework/trunk/ows_server/ows_server/lib/csml_cache.py @ 2631

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI05-delivery/ows_framework/trunk/ows_server/ows_server/lib/csml_cache.py@2631
Revision 2631, 3.0 KB checked in by spascoe, 13 years ago (diff)

Caching both NetCDF to disk and the parsed CSML object in memory.
This gives a substantial speed improvement on test_openlayers..html.

Line 
1# Copyright (C) 2007 STFC & NERC (Science and Technology Facilities Council).
2# This software may be distributed under the terms of the
3# Q Public License, version 1.0 or later.
4# http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
5"""
6Manages a Cache of a fixed number of GridSeries extracts.
7
8Extracts are cached to disk as NetCDF and opened on demand.
9@todo: More info on how this works
10
11"""
12
13from cache import Cache
14import os, tempfile
15import cdms
16from ows_server.lib.csml_util import get_csml_doc
17
18import logging
19logger = logging.getLogger('csml_cache')
20
21class CDMSEntry(object):
22    """
23    A reference to a CDMS file that will delete the file when it is
24    garbage collected.
25
26    Instances of this class are used as Cache entry values.  When entries are
27    automatically removed from the cache the CDMSEntry object will be
28    garbage collected unless it is being accessed by another thread.
29
30    """
31
32    __slots__ = ['_f', 'var']
33   
34    def __init__(self, filename, varname):
35        logger.info('Caching file variable %s in %s' % (varname, filename))
36        self._f = cdms.open(filename)
37        self.var = self._f[varname]
38    def __del__(self):
39        filename = self._f.id
40        logger.info('Removing file %s' % filename)
41        self._f.close()
42        os.remove(filename)
43 
44
45class CSMLExtractCache(Cache):
46   
47    def __init__(self, cache_dir, max_size=0):
48        super(CSMLExtractCache, self).__init__(max_size)
49        self._cache_dir = cache_dir
50
51    def _extract(self, feature, sel):
52        (fd, filename) = tempfile.mkstemp('.nc', 'csml_wxs_', self._cache_dir)
53        os.close(fd)
54        (outputdir, ncname) = os.path.split(filename)
55
56        # Workarround until ticket:778 (TestExtractAll) is fixed
57        sel.update(longitude=(-180, 180), latitude=(-90, 90))
58
59        feature.subsetToGridSeries(ncname=ncname, outputdir=outputdir, **sel)
60
61        return filename
62   
63    def key(self, name):
64        """
65        Cache provides this method to map externally visible entry names
66        to internal keys.  We use it here to turn selectors into hashable
67        values.
68
69        """
70        (feature, sel) = name
71       
72        return (feature.id, tuple(sorted(sel.items())))
73
74    def build(self, key, name, opened, entry):
75        (feature, sel) = name
76        filename = self._extract(feature, sel)
77
78        return CDMSEntry(filename, feature.name.CONTENT)
79
80    def __getitem__(self, name):
81        """
82        Returns the opened CDMS object.
83
84        """
85        return super(CSMLExtractCache, self).__getitem__(name).var
86
87
88class CSMLDocumentCache(Cache):
89    """
90    There is considerable overhead for parsing CSML documents.
91    This cache stores the csml.parser.Dataset objects in memory.
92
93    """
94    def build(self, key, name, opened, entry):
95        return get_csml_doc(name)
96
97#---------------------------------------------------------------------------
98
99from pylons import request
100extractCache = CSMLExtractCache(request.environ['paste.config']['app_conf']['tmp_dir'], max_size=3)
101csmlCache = CSMLDocumentCache(max_size=5)
Note: See TracBrowser for help on using the repository browser.