source: nappy/trunk/nappy/nc_interface/na_to_cdms.py @ 3349

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/na_to_cdms.py@3349
Revision 3349, 16.1 KB checked in by astephen, 13 years ago (diff)

Tidied some variable names and boolean types.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6na_to_cdms.py
7=============
8
9Container module for mixin class NAToCdms that is sub-classed
10by NAFile classes.
11
12"""
13
14# Imports from python standard library
15import sys
16import re
17import time
18
19# Import from nappy package
20import utils
21from nappy.na_error import na_error
22import nappy.utils
23
24na_to_nc_map = utils.getConfigDict()["na_to_nc_map"]
25
26# Import external packages (if available)
27if sys.platform.find("win") > -1:
28    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
29try:
30    import cdms, Numeric
31except:
32    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
33cdms.setAutoBounds("off") 
34
35# Define global variables
36safe_nc_id = re.compile("[\/\s\[\(\)\]\=\+\-\?\#\~\@\&\$\%\!\*\{\}\^]+")
37time_units_pattn = re.compile("\w+\s+since\s+\d{4}-\d{1,2}-\d{1,2}\s+\d+:\d+:\d+")
38max_id_length = 40
39special_comment_known_strings = ("###NASA Ames Special Comments follow###", 
40                                     "###NASA Ames Special Comments end###", 
41                                  "Additional Variable Attributes defined in the source file and not translated elsewhere:",
42                                  "Additional Global Attributes defined in the source file and not translated elsewhere:", "\n")
43
44normal_comment_known_strings = ("###NASA Ames Normal Comments follow###", 
45                                "###NASA Ames Normal Comments end###", 
46                                "###Data Section begins on the next line###",
47                                            "Additional Variable Attributes defined in the source file and not translated elsewhere:",
48                                                "Additional Global Attributes defined in the source file and not translated elsewhere:", 
49                                "\n")
50
51time_units_warning_message = """\nWARNING: Could not recognise time units. For true NetCDF compability
52please insert the correct time unit string below in the format:
53   
54    <units> since <YYYY>-<MM>-<DD> <hh>-<mm>-<ss>
55   
56Where:
57    <units> is a known time interval such as years, months, days, etc.
58    <YYYY> is the year, <MM> is the month, <DD> is the day,
59    <hh> is the hour, <mm> is minutes, <ss> is seconds.
60"""
61
62class NADictToCdmsObjects:
63    """
64    Converts a NA File instance to a tuple of CDMS objects.
65    """
66   
67    def __init__(self, na_file_obj, variables="all", aux_variables="all",
68                 global_attributes={"Conventions":"CF-1.0"},
69                 time_units=None, time_warning=True, 
70                 rename_variables={}):
71        """
72        Sets up instance variables.
73        """
74        self.na_file_obj = na_file_obj     
75        self.variables = variables
76        self.aux_variables = aux_variables
77        self.global_attributes = global_attributes
78        self.time_units = time_units
79        self.time_warning = time_warning
80        self.rename_variables = rename_variables
81
82        # Check if we have capability to convert this FFI
83        if self.na_file_obj.ffi in (2110, 2160, 2310): 
84                raise Exception("Cannot convert NASA Ames File Format Index (FFI) " + `self.na_file_obj.ffi` + " to CDMS objects. No mapping implemented yet.")
85
86        self.output_message = []  # for output printing message
87        self.converted = False
88
89    def convert(self):
90        """
91        Reads the NASA Ames file object and converts to CDMS objects.
92        Returns (variable_list, aux_variable_list, global_attribute_dict).
93        All these can be readily written to a CDMS File object.
94        """
95        if self.converted == True:
96            print "Already converted to CDMS objects."
97            return (self.cdms_variables, self.cdms_aux_variables, self.global_attributes)
98
99        self.na_file_obj.readData()
100        #self.cdms_filename = filename
101        #self.cdms_file = cdms.open(self.cdms_filename, 'w')
102       
103        # at file level: write global attributes
104        #for key in global_attributes.keys():
105        #    setattr(self.cdms_file, key, global_attributes[key])
106
107        # Convert global attribute
108        self._mapNACommentsToGlobalAttributes()
109
110        # Convert axes
111        if not hasattr(self, 'cdms_axes'):  self._convertCdmsAxes()
112
113        # Convert main variables
114        if not hasattr(self, 'cdms_variables'):  self._convertCdmsVariables()
115        #for var in self.cdms_variables:
116        #    self.cdms_file.write(var)
117
118        # Then do auxiliary variables
119        if hasattr(self, "NAUXV") and (type(self.NAUXV) == type(1) and self.NAUXV > 0):   # Are there any auxiliary variables?
120            if not hasattr(self, 'cdms_aux_variables'): 
121                self._convertCdmsAuxVariables()
122           
123            #for avar in self.cdms_aux_variables:
124            #    self.cdms_file.write(avar)
125
126        #self.cdms_file.close()
127        #rt = "cdms_file '%s' written successfully." % self.cdms_filename
128        self.converted = True
129        return (self.cdms_variables, self.cdms_aux_variables, self.global_attributes)
130
131
132    def _mapNACommentsToGlobalAttributes(self):
133        """
134        Maps the NASA Ames comments section to global attributes and append them to the
135        self.global_attributes dictionary.
136        """
137       
138        for key in na_to_nc_map.keys():
139
140            if type(key) == type((1,2)):
141
142                if key == ("SCOM", "NCOM"):
143                    # Map special and normal comments into the global comments section
144                    comment_line = ""
145
146                    # Add Special comments first
147                    if self.na_file_obj.NSCOML > 0:
148                        comment_line += "###NASA Ames Special Comments follow###\n"
149
150                        for i in self.na_file_obj.SCOM: 
151                           
152                            if i.strip() not in special_comment_known_strings:
153                                comment_line += ("\n" + i)
154                           
155                        comment_line += "\n###NASA Ames Special Comments end###\n"
156
157                    # Now add normal comments
158                    if self.na_file_obj.NNCOML > 0:
159                        comment_line += "###NASA Ames Normal Comments follow###\n"
160
161                        for i in self.na_file_obj.NCOM: 
162                            if i.strip() not in normal_comment_known_strings:
163                                comment_line += ("\n" + i)
164
165                        comment_line += "\n###NASA Ames Normal Comments end###"
166
167                    # Tidy comment line then write to global atts dict
168                    comment_line = comment_line.replace("\n\n", "\n")
169                    #self.cdms_file.comment = comment_line
170                    self.global_attributes["comment"] = comment_line
171
172                elif key == ("ONAME", "ORG"):
173                    # Map the two organisation NA files to the institution field in CDMS (NetCDF)
174                    institution = "%s (ONAME from NASA Ames file); %s (ORG from NASA Ames file)." % \
175                                         (self.na_file_obj.ONAME, self.na_file_obj.ORG)
176                    self.global_attributes["institution"] = institution
177                    #self.cdms_file.institution = "%s (ONAME from NASA Ames file); %s (ORG from NASA Ames file)." % (self.na_file_obj.ONAME, self.na_file_obj.ORG)
178
179                else:
180                    # Any other strange tuple just gets merged into a string
181                        item = (getattr(self.na_file_obj, key[0])) + "\n" + (getattr(self.na_file_obj, key[1]))
182                    # setattr(self.cdms_file, cdms_map.fromNA[key], item)
183                    self.global_attributes[na_to_nc_map[key]] = item
184
185            elif key == "RDATE":
186                # RDATE = Revision date - update this and put in history global attribute
187                date_parts = getattr(self.na_file_obj, "RDATE")
188                date_string = "%.4d-%.2d-%.2d" % tuple(date_parts)
189                hist = date_string + " - NASA Ames File created/revised.\n"
190                time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
191                version = nappy.utils.getVersion()
192                hist = "%s\n%s - Converted to CDMS (NetCDF) format using nappy-%s." % (hist, time_string, version)
193                # self.cdms_file.history = hist
194                print "DIDNT HAVE MAPPING SO PUT HISTORY IN LINE 160 IN na_to_cdms.py"
195                self.global_attributes["history"] = hist
196            else:
197                # Anything else just needs to be stored as a global attribute
198                # setattr(self.cdms_file, cdms_map.fromNA[key], getattr(self, key))
199                self.global_attributes[na_to_nc_map[key]] = getattr(self.na_file_obj, key)
200
201
202    def _convertCdmsVariables(self):
203        """
204        Creates cdms variable list for writing out.
205        """
206        self.cdms_variables = []
207
208        if self.variables == "all":   
209            for var_number in range(self.NV):
210            self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))
211        else:
212            if type(self.variables[0]) == type(1): # They are integers = indices
213                for var_number in self.variables:
214                    self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))   
215            elif type(self.variables[0]) == type("string"):  # Vars are strings
216                for var_name in self.variables:
217                    if var_name in self.VNAME:
218                        var_number = self.VNAME.index(var_name)
219                        self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))
220                    else:
221                        raise Exception("Variable name not known: " + var_name)
222
223
224    def _convertNAToCdmsVariable(self, var_number, attributes={}):
225        """
226        Creates a single cdms variable from the variable number provided in the list.
227        """
228        (var_name, units, miss, scal) = self.na_file_obj.getVariable(var_number)
229        msg = "\nAdding variable: %s" % self.na_file_obj.VNAME[var_number]
230        print msg
231        self.output_message.append(msg)
232        array = Numeric.array(self.na_file_obj.V[var_number])
233        array = array * scal
234        # Set up axes
235        if not hasattr(self, 'cdms_axes'):
236            self._convertCdmsAxes()
237
238        # Set up variable
239        var=cdms.createVariable(array, axes=self.cdms_axes, fill_value=miss, attributes=attributes)
240
241        # Sort units etc
242        if units:   var.units=units
243       
244        # Add the best variable name
245        if len(var_name) < max_id_length:
246            var.id=safe_nc_id.sub("_", var_name).lower()
247        else:
248            var.id="naVariable_%s" % (var_number)
249       
250             # Check if mapping provided for renaming this variable
251        if var_name in self.rename_variables.keys():
252            var_name = self.rename_variables[var_name]
253           
254        var.long_name = var.name = var.title = var_name
255
256        # Add a NASA Ames variable number (for mapping correctly back to NASA Ames)
257        var.nasa_ames_var_number = var_number
258        return var
259
260    def _convertCdmsAuxVariables(self):
261        """
262        Creates a cdms variable from an auxiliary variable
263        """
264        self.cdms_aux_variables = []
265
266        if self.aux_variables == "all":   
267            for avar_number in range(self.NAUXV):
268                self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number))
269        else:
270            if type(self.aux_variables[0]) == type(1): # They are integers = indices
271                for avar_number in self.aux_variables:
272                    self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number))   
273
274            elif type(self.aux_variables[0]) == type("string"): # They are strings
275                for avar_name in self.aux_variables:
276                    if avar_name in self.ANAME:
277                        avar_number = self.ANAME.index(avar_name)
278                        self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number)) 
279            else:
280                raise Exception("Auxiliary variable name not known: " + avar_name)         
281
282    def _convertNAAuxToCdmsVariable(self, avar_number, attributes={}):
283        """
284        Converts an auxiliary variable to a cdms variable.
285        """
286        (var_name, units, miss, scal) = self.na_file_obj(avar_number)
287        array = Numeric.array(self.na_file_obj.A[avar_number])
288        array = array * scal
289
290        msg="\nAdding auxiliary variable: %s" % self.na_file_obj.ANAME[avar_number]
291        print msg
292        self.output_message.append(msg)
293
294        # Set up axes
295        if not hasattr(self, 'cdms_axes'):
296            self._convertCdmsAxes()
297
298        # Set up variable
299        var = cdms.createVariable(array, axes=[self.cdms_axes[0]], fill_value=miss, 
300                                  attributes=attributes)
301
302        # Sort units etc
303        if units:   var.units = units
304        if len(var_name) < max_id_length:
305            var.id = safe_nc_id.sub("_", var_name).lower()
306        else:
307            var.id = "naAuxVariable_%s" % (avar_number)
308
309            # Check if mapping provided for renaming this variable
310        if var_name in self.rename_variables.keys():
311            var_name = self.rename_variables[var_name]
312
313        var.long_name = var.name = var.title = var_name
314
315        # Add a NASA Ames auxiliary variable number (for mapping correctly back to NASA Ames)
316        var.nasa_ames_aux_var_number = avar_number
317        return var       
318
319    def _convertCdmsAxes(self): 
320        """
321        Creates cdms axes from information provided in the NASA Ames dictionary.
322        """
323        if not hasattr(self, 'cdms_axes'):       
324            self.cdms_axes = []
325
326        for ivar_number in range(self.na_file_obj.NIV):
327            self.cdms_axes.append(self._convertNAIndVarToCdmsAxis(ivar_number))
328
329    def _convertNAIndVarToCdmsAxis(self, ivar_number):
330        """
331        Creates a cdms axis from a NASA Ames independent variable.
332        """
333        if self._normalized_X == False:   self.na_file_obj._normalizeIndVars()
334
335        if self.na_file_obj.NIV == 1:
336            array = self.na_file_obj.X
337        else:
338            array = self.na_file_obj.X[ivar_number]
339
340        axis = cdms.createAxis(array)
341        axis.id = axis.name = axis.long_name = self.na_file_obj.XNAME[ivar_number]
342        (var_name, units) = self.getIndependentVariable(ivar_number)
343       
344        # Sort units etc
345        if units:   axis.units = units
346        if len(var_name) < max_id_length:
347            axis.id = safe_nc_id.sub("_", var_name).lower()
348        else:
349            axis.id = "naIndVariable_%s" % (ivar_number)
350
351        if units: axis.units = units
352
353        axis_types = ("longitude", "latitude", "level", "time")
354
355        for axis_type in axis_types:
356            if re.search(axis_type, var_name, re.IGNORECASE):
357                axis.standard_name = axis.id = axis_type
358                # Designate it CF-style if known axis type (e.g. axis.designateTime() etc..)
359                exec "axis.designate%s()" % axis_type.title()
360                print "CHECK AXIS GOT DESIGNATED PROPERLY - RE-FACTORED" * 5
361
362        # Check warning for time units pattern
363        if axis.isTime() and (not hasattr(axis, "units") or not time_units_pattn.match(axis.units)):
364            if self.time_units == None:
365                time_units_input = "I WON'T MATCH"
366
367                while time_units_input != "" and not time_units_pattn.match(time_units_input):
368                    message = time_units_warning_message                           
369                    if self.time_warning == True:
370                        print message
371                        time_units_input = raw_input("Please insert your time unit string here (or leave blank):").strip()
372                    else: 
373                        time_units_input = ""
374                    self.output_message.append(message)
375
376                        self.time_units = time_units_input
377
378                    axis.units = self.time_units
379            axis.long_name = axis.name = "time (%s)" % self.time_units
380
381        if axis.units == None: 
382            if units:
383                axis.units = units     
384            else:
385                axis.units = "Not known"
386
387        return axis
Note: See TracBrowser for help on using the repository browser.