Ignore:
Timestamp:
22/06/07 10:19:07 (12 years ago)
Author:
spascoe
Message:

Caching both NetCDF to disk and the parsed CSML object in memory.
This gives a substantial speed improvement on test_openlayers..html.

Location:
TI05-delivery/ows_framework/trunk/ows_server
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • TI05-delivery/ows_framework/trunk/ows_server/ows_server.egg-info/PKG-INFO

    r2596 r2631  
    11Metadata-Version: 1.0 
    22Name: ows-server 
    3 Version: 0.0.0dev-r2594 
     3Version: 0.0.0dev-r2630 
    44Summary: UNKNOWN 
    55Home-page: UNKNOWN 

  • TI05-delivery/ows_framework/trunk/ows_server/ows_server.egg-info/SOURCES.txt

    r2596 r2631  
    2222ows_server/config/environment.py 
    2323ows_server/config/middleware.py 
     24ows_server/config/ndgDiscovery.config 
     25ows_server/config/ndgMiddleware.py 
    2426ows_server/config/routing.py 
    2527ows_server/controllers/__init__.py 
     28ows_server/controllers/abc.py 
    2629ows_server/controllers/csml_api.py 
    2730ows_server/controllers/csml_wcs.py 
    2831ows_server/controllers/csml_wms.py 
     32ows_server/controllers/discovery.py 
    2933ows_server/controllers/error.py 
     34ows_server/controllers/login.py 
     35ows_server/controllers/logout.py 
     36ows_server/controllers/retrieve.py 
    3037ows_server/controllers/template.py 
     38ows_server/controllers/trackback.py 
    3139ows_server/docs/index.txt 
    3240ows_server/i18n/__init__.py 
    … …  
    3442ows_server/lib/app_globals.py 
    3543ows_server/lib/base.py 
     44ows_server/lib/cache.py 
     45ows_server/lib/csml_cache.py 
    3646ows_server/lib/csml_util.py 
    3747ows_server/lib/decorators.py 
    … …  
    4050ows_server/lib/render.py 
    4151ows_server/lib/validators.py 
     52ows_server/lib/ndg/001deploy.sh 
     53ows_server/lib/ndg/00deploy.sh 
     54ows_server/lib/ndg/AccessControl.py 
     55ows_server/lib/ndg/DC.py 
     56ows_server/lib/ndg/DIF.py 
     57ows_server/lib/ndg/Date.py 
     58ows_server/lib/ndg/DeploymentHandling.py 
     59ows_server/lib/ndg/DiscoveryGUI.py 
     60ows_server/lib/ndg/DiscoveryService_services.py 
     61ows_server/lib/ndg/DiscoveryService_services_types.py 
     62ows_server/lib/ndg/DiscoveryState.py 
     63ows_server/lib/ndg/DiscoveryTemplate.py 
     64ows_server/lib/ndg/DocumentRetrieve.py 
     65ows_server/lib/ndg/ETxmlView.py 
     66ows_server/lib/ndg/MDIP.py 
     67ows_server/lib/ndg/People.py 
     68ows_server/lib/ndg/People2.py 
     69ows_server/lib/ndg/ServiceBinding.py 
     70ows_server/lib/ndg/cookieEG.py 
     71ows_server/lib/ndg/eXistInterface.py 
     72ows_server/lib/ndg/geoUtilities.py 
     73ows_server/lib/ndg/htmlUtilities.py 
     74ows_server/lib/ndg/ndg-wsdl2py 
     75ows_server/lib/ndg/ndg-wsdl2py-url 
     76ows_server/lib/ndg/ndg.ini 
     77ows_server/lib/ndg/ndgBrowse.py 
     78ows_server/lib/ndg/ndgCookie.py 
     79ows_server/lib/ndg/ndgDiscovery.config 
     80ows_server/lib/ndg/ndgDiscovery.py 
     81ows_server/lib/ndg/ndgGateKeeper.py 
     82ows_server/lib/ndg/ndgLog.py 
     83ows_server/lib/ndg/ndgLogin.py 
     84ows_server/lib/ndg/ndgLogout.py 
     85ows_server/lib/ndg/ndgObject.py 
     86ows_server/lib/ndg/ndgPageLayout.py 
     87ows_server/lib/ndg/ndgRetrieve.py 
     88ows_server/lib/ndg/ndgSearch.py 
     89ows_server/lib/ndg/ndgWrapDiscovery.py 
     90ows_server/lib/ndg/ndg_xqueries.py 
     91ows_server/lib/ndg/renderDiscoverySet.py 
     92ows_server/lib/ndg/renderEntity.py 
     93ows_server/lib/ndg/renderParameters.py 
     94ows_server/lib/ndg/renderService.py 
     95ows_server/lib/ndg/stubB.py 
     96ows_server/lib/ndg/sub_orphan.py 
     97ows_server/lib/ndg/wsgiEnv.py 
     98ows_server/lib/ndg/wsgiEnvTest.ini 
     99ows_server/lib/ndg/wsgiEnvTest.sh 
     100ows_server/lib/ndg/xmlHandler.py 
     101ows_server/lib/ndg/examples/badc.nerc.ac.uk__NDG-B1__dataent_chablis.xml 
     102ows_server/lib/ndg/examples/badc.nerc.ac.uk__NDG-B1__dpt_11634276941110630.xml 
     103ows_server/lib/ndg/examples/bodc.eg1.edmed.dif 
     104ows_server/lib/ndg/examples/bodc.eg2.edmed.dif 
     105ows_server/lib/ndg/examples/bodc.eg3.difChange.dif 
     106ows_server/lib/ndg/examples/example.config 
     107ows_server/lib/ndg/examples/ndg.noc.soton.ac.uk__DIF__NOCSDAT162.xml 
     108ows_server/lib/ndg/examples/neodc.eg1.dif 
     109ows_server/lib/ndg/examples/trouble.moles 
     110ows_server/lib/ndg/examples/ukho.eg1.mdip 
     111ows_server/models/DiscoveryService_services.py 
     112ows_server/models/DiscoveryService_services_types.py 
     113ows_server/models/DocumentRetrieve.py 
     114ows_server/models/ETxmlView.py 
     115ows_server/models/Utilities.py 
    42116ows_server/models/__init__.py 
     117ows_server/models/eXistInterface.py 
     118ows_server/models/ndgObject.py 
     119ows_server/models/ndgObject.py~ 
     120ows_server/models/ndgRetrieve.py 
     121ows_server/models/ndgSearch.py 
     122ows_server/models/ndgSearch.py~ 
     123ows_server/models/ndg_xqueries.py 
     124ows_server/models/ndg_xqueries.py~ 
     125ows_server/models/xmlHandler.py 
    43126ows_server/public/index.html 
     127ows_server/public/test_openlayers.html 
     128ows_server/public/layout/A.gif 
     129ows_server/public/layout/AIcon.gif 
     130ows_server/public/layout/AIconsRd.gif 
     131ows_server/public/layout/B.gif 
     132ows_server/public/layout/BIcon.gif 
     133ows_server/public/layout/BIconsRd.gif 
     134ows_server/public/layout/C.gif 
     135ows_server/public/layout/D.gif 
     136ows_server/public/layout/F.gif 
     137ows_server/public/layout/O.gif 
     138ows_server/public/layout/R.gif 
     139ows_server/public/layout/RIconsRd.png 
     140ows_server/public/layout/X.gif 
     141ows_server/public/layout/bodc_logo.gif 
     142ows_server/public/layout/browse_header4.gif 
     143ows_server/public/layout/help.html 
     144ows_server/public/layout/icon_print.gif 
     145ows_server/public/layout/icon_searchsimilar.gif 
     146ows_server/public/layout/key.gif 
     147ows_server/public/layout/keyG.gif 
     148ows_server/public/layout/metadata.css 
     149ows_server/public/layout/ndg.css 
     150ows_server/public/layout/ndgJavascript.js 
     151ows_server/public/layout/ndg_logo.gif 
     152ows_server/public/layout/print.css 
     153ows_server/public/layout/printer.gif 
     154ows_server/public/layout/record_bottom_left_corner.gif 
     155ows_server/public/layout/record_bottom_right_corner.gif 
     156ows_server/public/layout/record_top_left_corner.gif 
     157ows_server/public/layout/record_top_right_corner.gif 
     158ows_server/public/layout/searched_header.gif 
     159ows_server/public/layout/selected_header.gif 
     160ows_server/public/layout/style.css 
     161ows_server/public/layout/tick.png 
     162ows_server/public/layout/xml.css 
    44163ows_server/templates/__init__.py 
    45164ows_server/templates/autohandler 
     165ows_server/templates/content.kid 
    46166ows_server/templates/csml_describe.kid 
    47167ows_server/templates/csml_describe_feature.kid 
    48168ows_server/templates/exception_report.kid 
     169ows_server/templates/ndgPage.kid 
     170ows_server/templates/wayf.kid 
     171ows_server/templates/wcs_asynchronous.kid 
    49172ows_server/templates/ows/__init__.py 
    50173ows_server/templates/ows/get_capabilities.kid 
    … …  
    54177ows_server/tests/exampleCapabilities1.xml 
    55178ows_server/tests/make_data.py 
    56 ows_server/tests/test_extract.py 
    57179ows_server/tests/test_models.py 
    58180ows_server/tests/functional/__init__.py 
     181ows_server/tests/functional/test_abc.py 
    59182ows_server/tests/functional/test_csml_api.py 
    60183ows_server/tests/functional/test_csml_wms.py 
     184ows_server/tests/functional/test_retrieve.py 

  • TI05-delivery/ows_framework/trunk/ows_server/ows_server/controllers/csml_wms.py

    r2630 r2631  
    1212from ows_server.lib.decorators import * 
    1313from ows_server.lib import grid_util, render 
    14 from ows_server.lib.csml_util import get_csml_doc, extract_cache 
     14from ows_server.lib.csml_cache import csmlCache, extractCache 
    1515import ows_server.lib.validators as V 
    1616 
    … …  
    110110         
    111111        # Populate the context object with information required by the template 
    112         c.dataset = get_csml_doc(file) 
     112        c.dataset = csmlCache[file] 
    113113 
    114114        return self._renderCapabilities('ows/wms_capabilities') 
    … …  
    132132 
    133133        # Retrieve dataset and selected feature 
    134         dataset = get_csml_doc(file) 
     134        dataset = csmlCache[file] 
    135135        feature = dataset.getFeature(layers) 
    136136        if feature is None: 
    … …  
    161161                raise OWS_E.MissingParameterValue('%s dimension not specified' % dim, dim) 
    162162 
    163         # Get the entire lat/lon slice from the extract_cache 
    164         var = extract_cache[feature, sel] 
     163        # Get the entire lat/lon slice from the extractCache 
     164        var = extractCache[feature, sel] 
    165165 
    166166        # Do geographical subsetting in cdms 

  • TI05-delivery/ows_framework/trunk/ows_server/ows_server/lib/csml_cache.py

    r2630 r2631  
    1414import os, tempfile 
    1515import cdms 
     16from ows_server.lib.csml_util import get_csml_doc 
    1617 
    1718import logging 
    1819logger = logging.getLogger('csml_cache') 
    1920 
    20 class CSMLExtract(object): 
     21class CDMSEntry(object): 
    2122    """ 
    2223    A reference to a CDMS file that will delete the file when it is 
    … …  
    2425 
    2526    Instances of this class are used as Cache entry values.  When entries are 
    26     automatically removed from the cache the CSMLExtract object will be 
     27    automatically removed from the cache the CDMSEntry object will be 
    2728    garbage collected unless it is being accessed by another thread. 
    2829 
    … …  
    4243  
    4344 
    44 class CSMLCache(Cache): 
     45class CSMLExtractCache(Cache): 
     46     
    4547    def __init__(self, cache_dir, max_size=0): 
    46         super(CSMLCache, self).__init__(max_size) 
     48        super(CSMLExtractCache, self).__init__(max_size) 
    4749        self._cache_dir = cache_dir 
    4850 
    … …  
    7476        filename = self._extract(feature, sel) 
    7577 
    76         return CSMLExtract(filename, feature.name.CONTENT) 
     78        return CDMSEntry(filename, feature.name.CONTENT) 
    7779 
    7880    def __getitem__(self, name): 
    … …  
    8183 
    8284        """ 
    83         return super(CSMLCache, self).__getitem__(name).var 
     85        return super(CSMLExtractCache, self).__getitem__(name).var 
     86 
     87 
     88class CSMLDocumentCache(Cache): 
     89    """ 
     90    There is considerable overhead for parsing CSML documents. 
     91    This cache stores the csml.parser.Dataset objects in memory. 
     92 
     93    """ 
     94    def build(self, key, name, opened, entry): 
     95        return get_csml_doc(name) 
     96 
     97#--------------------------------------------------------------------------- 
     98 
     99from pylons import request 
     100extractCache = CSMLExtractCache(request.environ['paste.config']['app_conf']['tmp_dir'], max_size=3) 
     101csmlCache = CSMLDocumentCache(max_size=5) 

  • TI05-delivery/ows_framework/trunk/ows_server/ows_server/lib/csml_util.py

    r2630 r2631  
    1414import zipfile 
    1515import tempfile 
    16  
    17 from ows_server.lib import csml_cache 
    18 extract_cache = csml_cache.CSMLCache(request.environ['paste.config']['app_conf']['tmp_dir'], max_size=3) 
    1916 
    2017 
Note: See TracChangeset for help on using the changeset viewer.