source: nappy/trunk/nappy/nc_interface/na_to_cdms.py @ 3398

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/na_to_cdms.py@3398
Revision 3398, 16.1 KB checked in by astephen, 14 years ago (diff)
Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6na_to_cdms.py
7=============
8
9Container module for mixin class NAToCdms that is sub-classed
10by NAFile classes.
11
12"""
13
14# Imports from python standard library
15import sys
16import re
17import time
18
19# Import from nappy package
20from nappy.na_error import na_error
21import nappy.utils
22import nappy.utils.common_utils
23
24na_to_nc_map = utils.getConfigDict()["na_to_nc_map"]
25
26# Import external packages (if available)
27if sys.platform.find("win") > -1:
28    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
29try:
30    import cdms, Numeric
31except:
32    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
33cdms.setAutoBounds("off") 
34
35# Define global variables
36safe_nc_id = re.compile("[\/\s\[\(\)\]\=\+\-\?\#\~\@\&\$\%\!\*\{\}\^]+")
37time_units_pattn = re.compile("\w+\s+since\s+\d{4}-\d{1,2}-\d{1,2}\s+\d+:\d+:\d+")
38max_id_length = 40
39special_comment_known_strings = ("###NASA Ames Special Comments follow###", 
40                                     "###NASA Ames Special Comments end###", 
41                                  "Additional Variable Attributes defined in the source file and not translated elsewhere:",
42                                  "Additional Global Attributes defined in the source file and not translated elsewhere:", "\n")
43
44normal_comment_known_strings = ("###NASA Ames Normal Comments follow###", 
45                                "###NASA Ames Normal Comments end###", 
46                                "###Data Section begins on the next line###",
47                                            "Additional Variable Attributes defined in the source file and not translated elsewhere:",
48                                                "Additional Global Attributes defined in the source file and not translated elsewhere:", 
49                                "\n")
50
51time_units_warning_message = """\nWARNING: Could not recognise time units. For true NetCDF compability
52please insert the correct time unit string below in the format:
53   
54    <units> since <YYYY>-<MM>-<DD> <hh>-<mm>-<ss>
55   
56Where:
57    <units> is a known time interval such as years, months, days, etc.
58    <YYYY> is the year, <MM> is the month, <DD> is the day,
59    <hh> is the hour, <mm> is minutes, <ss> is seconds.
60"""
61
62DEBUG = nappy.utils.common_utils.getDebug() 
63
64class NADictToCdmsObjects:
65    """
66    Converts a NA File instance to a tuple of CDMS objects.
67    """
68   
69    def __init__(self, na_file_obj, variables="all", aux_variables="all",
70                 global_attributes={"Conventions":"CF-1.0"},
71                 time_units=None, time_warning=True, 
72                 rename_variables={}):
73        """
74        Sets up instance variables.
75        """
76        self.na_file_obj = na_file_obj     
77        self.variables = variables
78        self.aux_variables = aux_variables
79        self.global_attributes = global_attributes
80        self.time_units = time_units
81        self.time_warning = time_warning
82        self.rename_variables = rename_variables
83
84        # Check if we have capability to convert this FFI
85        if self.na_file_obj.ffi in (2110, 2160, 2310): 
86                raise Exception("Cannot convert NASA Ames File Format Index (FFI) " + `self.na_file_obj.ffi` + " to CDMS objects. No mapping implemented yet.")
87
88        self.output_message = []  # for output printing message
89        self.converted = False
90
91    def convert(self):
92        """
93        Reads the NASA Ames file object and converts to CDMS objects.
94        Returns (variable_list, aux_variable_list, global_attribute_dict).
95        All these can be readily written to a CDMS File object.
96        """
97        if self.converted == True:
98            print "Already converted to CDMS objects."
99            return (self.cdms_variables, self.cdms_aux_variables, self.global_attributes)
100
101        self.na_file_obj.readData()
102        #self.cdms_filename = filename
103        #self.cdms_file = cdms.open(self.cdms_filename, 'w')
104       
105        # at file level: write global attributes
106        #for key in global_attributes.keys():
107        #    setattr(self.cdms_file, key, global_attributes[key])
108
109        # Convert global attribute
110        self._mapNACommentsToGlobalAttributes()
111
112        # Convert axes
113        if not hasattr(self, 'cdms_axes'):  self._convertCdmsAxes()
114
115        # Convert main variables
116        if not hasattr(self, 'cdms_variables'):  self._convertCdmsVariables()
117        #for var in self.cdms_variables:
118        #    self.cdms_file.write(var)
119
120        # Then do auxiliary variables
121        if hasattr(self, "NAUXV") and (type(self.NAUXV) == type(1) and self.NAUXV > 0):   # Are there any auxiliary variables?
122            if not hasattr(self, 'cdms_aux_variables'): 
123                self._convertCdmsAuxVariables()
124           
125            #for avar in self.cdms_aux_variables:
126            #    self.cdms_file.write(avar)
127
128        #self.cdms_file.close()
129        #rt = "cdms_file '%s' written successfully." % self.cdms_filename
130        self.converted = True
131        return (self.cdms_variables, self.cdms_aux_variables, self.global_attributes)
132
133
134    def _mapNACommentsToGlobalAttributes(self):
135        """
136        Maps the NASA Ames comments section to global attributes and append them to the
137        self.global_attributes dictionary.
138        """
139       
140        for key in na_to_nc_map.keys():
141
142            if type(key) == type((1,2)):
143
144                if key == ("SCOM", "NCOM"):
145                    # Map special and normal comments into the global comments section
146                    comment_line = ""
147
148                    # Add Special comments first
149                    if self.na_file_obj.NSCOML > 0:
150                        comment_line += "###NASA Ames Special Comments follow###\n"
151
152                        for i in self.na_file_obj.SCOM: 
153                           
154                            if i.strip() not in special_comment_known_strings:
155                                comment_line += ("\n" + i)
156                           
157                        comment_line += "\n###NASA Ames Special Comments end###\n"
158
159                    # Now add normal comments
160                    if self.na_file_obj.NNCOML > 0:
161                        comment_line += "###NASA Ames Normal Comments follow###\n"
162
163                        for i in self.na_file_obj.NCOM: 
164                            if i.strip() not in normal_comment_known_strings:
165                                comment_line += ("\n" + i)
166
167                        comment_line += "\n###NASA Ames Normal Comments end###"
168
169                    # Tidy comment line then write to global atts dict
170                    comment_line = comment_line.replace("\n\n", "\n")
171                    #self.cdms_file.comment = comment_line
172                    self.global_attributes["comment"] = comment_line
173
174                elif key == ("ONAME", "ORG"):
175                    # Map the two organisation NA files to the institution field in CDMS (NetCDF)
176                    institution = "%s (ONAME from NASA Ames file); %s (ORG from NASA Ames file)." % \
177                                         (self.na_file_obj.ONAME, self.na_file_obj.ORG)
178                    self.global_attributes["institution"] = institution
179                    #self.cdms_file.institution = "%s (ONAME from NASA Ames file); %s (ORG from NASA Ames file)." % (self.na_file_obj.ONAME, self.na_file_obj.ORG)
180
181                else:
182                    # Any other strange tuple just gets merged into a string
183                        item = (getattr(self.na_file_obj, key[0])) + "\n" + (getattr(self.na_file_obj, key[1]))
184                    # setattr(self.cdms_file, cdms_map.fromNA[key], item)
185                    self.global_attributes[na_to_nc_map[key]] = item
186
187            elif key == "RDATE":
188                # RDATE = Revision date - update this and put in history global attribute
189                date_parts = getattr(self.na_file_obj, "RDATE")
190                date_string = "%.4d-%.2d-%.2d" % tuple(date_parts)
191                hist = date_string + " - NASA Ames File created/revised.\n"
192                time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
193                version = nappy.utils.getVersion()
194                hist = "%s\n%s - Converted to CDMS (NetCDF) format using nappy-%s." % (hist, time_string, version)
195                # self.cdms_file.history = hist
196                print "DIDNT HAVE MAPPING SO PUT HISTORY IN LINE 160 IN na_to_cdms.py"
197                self.global_attributes["history"] = hist
198            else:
199                # Anything else just needs to be stored as a global attribute
200                # setattr(self.cdms_file, cdms_map.fromNA[key], getattr(self, key))
201                self.global_attributes[na_to_nc_map[key]] = getattr(self.na_file_obj, key)
202
203
204    def _convertCdmsVariables(self):
205        """
206        Creates cdms variable list for writing out.
207        """
208        self.cdms_variables = []
209
210        if self.variables == "all":   
211            for var_number in range(self.NV):
212            self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))
213        else:
214            if type(self.variables[0]) == type(1): # They are integers = indices
215                for var_number in self.variables:
216                    self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))   
217            elif type(self.variables[0]) == type("string"):  # Vars are strings
218                for var_name in self.variables:
219                    if var_name in self.VNAME:
220                        var_number = self.VNAME.index(var_name)
221                        self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))
222                    else:
223                        raise Exception("Variable name not known: " + var_name)
224
225
226    def _convertNAToCdmsVariable(self, var_number, attributes={}):
227        """
228        Creates a single cdms variable from the variable number provided in the list.
229        """
230        (var_name, units, miss, scal) = self.na_file_obj.getVariable(var_number)
231        msg = "\nAdding variable: %s" % self.na_file_obj.VNAME[var_number]
232        print msg
233        self.output_message.append(msg)
234        array = Numeric.array(self.na_file_obj.V[var_number])
235        array = array * scal
236        # Set up axes
237        if not hasattr(self, 'cdms_axes'):
238            self._convertCdmsAxes()
239
240        # Set up variable
241        var=cdms.createVariable(array, axes=self.cdms_axes, fill_value=miss, attributes=attributes)
242
243        # Sort units etc
244        if units:   var.units=units
245       
246        # Add the best variable name
247        if len(var_name) < max_id_length:
248            var.id=safe_nc_id.sub("_", var_name).lower()
249        else:
250            var.id="naVariable_%s" % (var_number)
251       
252             # Check if mapping provided for renaming this variable
253        if var_name in self.rename_variables.keys():
254            var_name = self.rename_variables[var_name]
255           
256        var.long_name = var.name = var.title = var_name
257
258        # Add a NASA Ames variable number (for mapping correctly back to NASA Ames)
259        var.nasa_ames_var_number = var_number
260        return var
261
262    def _convertCdmsAuxVariables(self):
263        """
264        Creates a cdms variable from an auxiliary variable
265        """
266        self.cdms_aux_variables = []
267
268        if self.aux_variables == "all":   
269            for avar_number in range(self.NAUXV):
270                self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number))
271        else:
272            if type(self.aux_variables[0]) == type(1): # They are integers = indices
273                for avar_number in self.aux_variables:
274                    self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number))   
275
276            elif type(self.aux_variables[0]) == type("string"): # They are strings
277                for avar_name in self.aux_variables:
278                    if avar_name in self.ANAME:
279                        avar_number = self.ANAME.index(avar_name)
280                        self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number)) 
281            else:
282                raise Exception("Auxiliary variable name not known: " + avar_name)         
283
284    def _convertNAAuxToCdmsVariable(self, avar_number, attributes={}):
285        """
286        Converts an auxiliary variable to a cdms variable.
287        """
288        (var_name, units, miss, scal) = self.na_file_obj(avar_number)
289        array = Numeric.array(self.na_file_obj.A[avar_number])
290        array = array * scal
291
292        msg="\nAdding auxiliary variable: %s" % self.na_file_obj.ANAME[avar_number]
293        print msg
294        self.output_message.append(msg)
295
296        # Set up axes
297        if not hasattr(self, 'cdms_axes'):
298            self._convertCdmsAxes()
299
300        # Set up variable
301        var = cdms.createVariable(array, axes=[self.cdms_axes[0]], fill_value=miss, 
302                                  attributes=attributes)
303
304        # Sort units etc
305        if units:   var.units = units
306        if len(var_name) < max_id_length:
307            var.id = safe_nc_id.sub("_", var_name).lower()
308        else:
309            var.id = "naAuxVariable_%s" % (avar_number)
310
311            # Check if mapping provided for renaming this variable
312        if var_name in self.rename_variables.keys():
313            var_name = self.rename_variables[var_name]
314
315        var.long_name = var.name = var.title = var_name
316
317        # Add a NASA Ames auxiliary variable number (for mapping correctly back to NASA Ames)
318        var.nasa_ames_aux_var_number = avar_number
319        return var       
320
321    def _convertCdmsAxes(self): 
322        """
323        Creates cdms axes from information provided in the NASA Ames dictionary.
324        """
325        if not hasattr(self, 'cdms_axes'):       
326            self.cdms_axes = []
327
328        for ivar_number in range(self.na_file_obj.NIV):
329            self.cdms_axes.append(self._convertNAIndVarToCdmsAxis(ivar_number))
330
331    def _convertNAIndVarToCdmsAxis(self, ivar_number):
332        """
333        Creates a cdms axis from a NASA Ames independent variable.
334        """
335        if self._normalized_X == False:   self.na_file_obj._normalizeIndVars()
336
337        if self.na_file_obj.NIV == 1:
338            array = self.na_file_obj.X
339        else:
340            array = self.na_file_obj.X[ivar_number]
341
342        axis = cdms.createAxis(array)
343        axis.id = axis.name = axis.long_name = self.na_file_obj.XNAME[ivar_number]
344        (var_name, units) = self.getIndependentVariable(ivar_number)
345       
346        # Sort units etc
347        if units:   axis.units = units
348        if len(var_name) < max_id_length:
349            axis.id = safe_nc_id.sub("_", var_name).lower()
350        else:
351            axis.id = "naIndVariable_%s" % (ivar_number)
352
353        if units: axis.units = units
354
355        axis_types = ("longitude", "latitude", "level", "time")
356
357        for axis_type in axis_types:
358            if re.search(axis_type, var_name, re.IGNORECASE):
359                axis.standard_name = axis.id = axis_type
360                # Designate it CF-style if known axis type (e.g. axis.designateTime() etc..)
361                exec "axis.designate%s()" % axis_type.title()
362                print "CHECK AXIS GOT DESIGNATED PROPERLY - RE-FACTORED" * 5
363
364        # Check warning for time units pattern
365        if axis.isTime() and (not hasattr(axis, "units") or not time_units_pattn.match(axis.units)):
366            if self.time_units == None:
367                time_units_input = "I WON'T MATCH"
368
369                while time_units_input != "" and not time_units_pattn.match(time_units_input):
370                    message = time_units_warning_message                           
371                    if self.time_warning == True:
372                        print message
373                        time_units_input = raw_input("Please insert your time unit string here (or leave blank):").strip()
374                    else: 
375                        time_units_input = ""
376                    self.output_message.append(message)
377
378                        self.time_units = time_units_input
379
380                    axis.units = self.time_units
381            axis.long_name = axis.name = "time (%s)" % self.time_units
382
383        if axis.units == None: 
384            if units:
385                axis.units = units     
386            else:
387                axis.units = "Not known"
388
389        return axis
Note: See TracBrowser for help on using the repository browser.