source: TI03-DataExtractor/branches/old_stuff/dx/products.py @ 793

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI03-DataExtractor/branches/old_stuff/dx/products.py@793
Revision 793, 6.5 KB checked in by astephen, 13 years ago (diff)

Put all the old code in the old_stuff branch.

  • Property svn:executable set to *
Line 
1"""
2products.py
3===========
4
5Procucts module for the extractor package.
6
7This module holds the Product, Plot, DataFile and other classes
8used to generate products from the data extractor.
9
10"""
11
12# Import required modules
13import os
14import cdms
15import vcs
16import sys
17from exceptions import *
18
19# Import package modules including global variables
20from common import *
21from config import *
22from localRules import *
23from errorHandler import *
24from extractMetadata import *
25from requestDict import *
26from datasetdb import Datasetdb
27
28
29class Product(MetadataExtractor):
30    """
31    Base class for extracting data from input files. Other sub-classes actually
32    write output to files.
33    """ 
34
35    def __init__(self, filenames, request):
36        self.filenames=filenames
37        self.files=[]
38        self.request=RequestDict(request)
39        self.variables={}
40        self.user=request["user"]
41        self.numDatasets=self.request.getNumDatasets()
42           
43        for n in range(1, self.numDatasets+1):
44            self.files.append(cdms.open(self.filenames[n-1]))
45
46    def getParam(self):
47        """
48        getParam method - uses the requests information and data/metadata files
49        to extract the data objects requested by the user. These are then stored in
50        the list self.variables.
51        """
52       
53        # Copy standard out to local variable ready for redirection
54        stdout=sys.stdout
55
56        # Define a list to put variable names in
57        varnames=[]
58       
59        # Redirect standard output so output is not sent to screen
60        sys.stdout=RedirectStdout()
61
62        # Begin looping through for each dataset
63        for n in range(1, self.numDatasets+1):
64
65            # Define values for start and end for this request - these are in CDMS compatible format
66            (startTime, endTime, timeInterval)=self.request.getTemporalDimension(n)
67            startTimeString="%s-%s-%s %s:00" % tuple(startTime[:4]) 
68            endTimeString="%s-%s-%s %s:00" % tuple(endTime[:4]) 
69            (interval_value, interval_units)=timeInterval
70               
71            # Define horizontal spatial bounds
72            (northernExtent, westernExtent, southernExtent, easternExtent)=self.request.getHorizontalBounds()
73
74            if self.request.has_key("vertical_domain_%s" % n):
75                (verticalDomain, verticalUnits)=self.request.getVerticalDimension(n)
76                # If a vertical domain is defined then get from request
77                if verticalDomain!=None:
78                    try:
79                        levels=float(verticalDomain)
80                    except:
81                        verticalDomain=map(lambda x: float(x), verticalDomain)
82                        levels=(min(verticalDomain), max(verticalDomain))
83                else:
84                    levels=None
85            else:
86                levels=None
87
88            # Get the variable id from the variable name string in the request
89            variableName=self.request.getVariableID(n)
90            varnames.append(variableName)
91               
92            # If there is a vertical dimension then select user options
93            if levels:
94                self.variables[n]=self.files[n-1](variableName, 
95                   time=(startTimeString, endTimeString), 
96                   longitude=(westernExtent, easternExtent), 
97                   latitude=(southernExtent, northernExtent), level=levels)
98            # Else do not need to define levels in the call
99            else:
100                self.variables[n]=self.files[n-1](variableName, 
101                   time=(startTimeString, endTimeString),
102                   longitude=(westernExtent, easternExtent), 
103                   latitude=(southernExtent, northernExtent))
104
105        # Get back standard out
106        sys.stdout=stdout                 
107                   
108        # If the user has requested 2 datasets then subtract variable 2 from variable 1 and
109        # define the result as variable 0.
110
111        if self.numDatasets==2:
112            # Do we need to interpolate one dataset?
113            if self.variables[1].getGrid()!=self.variables[2].getGrid():
114                # Use the standard CDAT regridder to interpolate variable 2 to variable 1 grid.
115                # This might be replaced by the SCRIP regridder (or other) in future.
116                import regrid  # from CDMS
117                # Regrid variable 2 to new variable
118                var2=self.variables[2].regrid(self.variables[1].getGrid())     
119            else:
120                var2=self.variables[2]
121               
122            print "<P><B>Differencing datasets...<B><P>"
123
124            # Subtract var2 from var1
125            self.variables[0]=self.variables[1]-var2
126            # Create some metadata for the new variable name
127            self.variables[0].long_name="Differenced dataset: %s - %s" % (varnames[0],                  varnames[1])
128            self.variables[0].id="new_var" 
129            try:
130                del self.variables[0].units
131            except:
132                pass
133        else:
134            # If just one dataset then define that variable as variable 0
135            # (the one we write to a file).
136            self.variables[0]=self.variables[1]
137
138        return
139
140
141class DataFile(Product):
142    """
143    DataFile class - sub-class of Product. Holds the process method to actually write the
144    output to a file.
145    """
146   
147    def process(self):
148        """
149        process method - creates an output NetCDF file and writes to it. Also checks if
150        it can pick up any CF-Compliant Global Attributes to put them in the file.
151        """ 
152        self.ext=".nc"
153        pid=os.getpid()
154        filename="output_"+str(pid)+self.ext
155        # Make sure the output sub-directory exists
156        outputDir=checkSubDirectory(self.user)
157        outfile=os.path.join(outputDir, filename)
158
159        if os.path.isfile(outfile): os.unlink(outfile)
160        fout=cdms.open(outfile, "w")
161        # Write output variables
162        fout.write(self.variables[0])
163        # Write global attributes
164        dset=Datasetdb()
165        gattsToWrite={}
166
167        # If two lots then keep as ((metadata_1) - (metadata_2))
168        for n in range(1, self.request["num_datasets"]+1):
169            globalAtts=dset.getCFGlobalAttributes(self.filenames[n-1])
170            for gatt in globalAtts.keys():
171                if n==1: 
172                    gattsToWrite[gatt]=globalAtts[gatt]
173                elif n==2:
174                    gattsToWrite[gatt]="(("+gattsToWrite[gatt]+") - ("+globalAtts[gatt]+"))."
175
176        for gtw in gattsToWrite.keys():
177            fout.__setattr__(gtw, gattsToWrite[gtw])
178
179        fout.close()
180        os.chmod(outfile, OUTPUT_FILE_PERMISSION)
181        os.system("chown %s.%s %s" % (OUTPUT_FILE_USER, OUTPUT_FILE_GROUP, outfile))
182
183        # Remove preceeding forward slash if present in outfile path
184        httpOutputPath=outfile.split("/output/")[-1]
185        if httpOutputPath[0]=="/": 
186            fileByBrowser=os.path.join(HTTP_OUTPUT_DIR, httpOutputPath[1:])
187        else:
188            fileByBrowser=os.path.join(HTTP_OUTPUT_DIR, httpOutputPath)
189
190        return (fileByBrowser, outfile)
191       
192 
Note: See TracBrowser for help on using the repository browser.