source: nappy/trunk/nappy/nc_interface/na_to_cdms.py @ 3405

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/na_to_cdms.py@3406
Revision 3405, 15.9 KB checked in by astephen, 12 years ago (diff)

Got some of the code actually working!

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6na_to_cdms.py
7=============
8
9Container module for class NADictToCdmsObjects that is sub-classed
10by NAToNC classes.
11
12"""
13
14# Imports from python standard library
15import sys
16import re
17import time
18
19# Import from nappy package
20from nappy.na_error import na_error
21import nappy.utils
22import nappy.utils.common_utils
23
24na_to_nc_map = nappy.utils.getConfigDict()["na_to_nc_map"]
25print na_to_nc_map
26
27# Import external packages (if available)
28if sys.platform.find("win") > -1:
29    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
30try:
31    import cdms, Numeric
32except:
33    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
34cdms.setAutoBounds("off") 
35
36# Define global variables
37safe_nc_id = re.compile("[\/\s\[\(\)\]\=\+\-\?\#\~\@\&\$\%\!\*\{\}\^]+")
38time_units_pattn = re.compile("\w+\s+since\s+\d{4}-\d{1,2}-\d{1,2}\s+\d+:\d+:\d+")
39max_id_length = 40
40special_comment_known_strings = ("###NASA Ames Special Comments follow###", 
41                                     "###NASA Ames Special Comments end###", 
42                                  "Additional Variable Attributes defined in the source file and not translated elsewhere:",
43                                  "Additional Global Attributes defined in the source file and not translated elsewhere:", "\n")
44
45normal_comment_known_strings = ("###NASA Ames Normal Comments follow###", 
46                                "###NASA Ames Normal Comments end###", 
47                                "###Data Section begins on the next line###",
48                                            "Additional Variable Attributes defined in the source file and not translated elsewhere:",
49                                                "Additional Global Attributes defined in the source file and not translated elsewhere:", 
50                                "\n")
51
52time_units_warning_message = """\nWARNING: Could not recognise time units. For true NetCDF compability
53please insert the correct time unit string below in the format:
54   
55    <units> since <YYYY>-<MM>-<DD> <hh>-<mm>-<ss>
56   
57Where:
58    <units> is a known time interval such as years, months, days, etc.
59    <YYYY> is the year, <MM> is the month, <DD> is the day,
60    <hh> is the hour, <mm> is minutes, <ss> is seconds.
61"""
62
63DEBUG = nappy.utils.getDebug() 
64
65
66class NADictToCdmsObjects:
67    """
68    Converts a NA File instance to a tuple of CDMS objects.
69    """
70   
71    def __init__(self, na_file_obj, variables="all", aux_variables="all",
72                 global_attributes={"Conventions":"CF-1.0"},
73                 time_units=None, time_warning=True, 
74                 rename_variables={}):
75        """
76        Sets up instance variables.
77        """
78        self.na_file_obj = na_file_obj     
79        self.variables = variables
80        self.aux_variables = aux_variables
81        self.global_attributes = global_attributes
82        self.time_units = time_units
83        self.time_warning = time_warning
84        self.rename_variables = rename_variables
85
86        # Check if we have capability to convert this FFI
87        if self.na_file_obj.FFI in (2110, 2160, 2310): 
88                raise Exception("Cannot convert NASA Ames File Format Index (FFI) " + `self.na_file_obj.FFI` + " to CDMS objects. No mapping implemented yet.")
89
90        self.output_message = []  # for output printing message
91        self.converted = False
92
93    def convert(self):
94        """
95        Reads the NASA Ames file object and converts to CDMS objects.
96        Returns (variable_list, aux_variable_list, global_attribute_dict).
97        All these can be readily written to a CDMS File object.
98        """
99        if self.converted == True:
100            print "Already converted to CDMS objects."
101            return (self.cdms_variables, self.cdms_aux_variables, self.global_attributes)
102
103        self.na_file_obj.readData()
104
105        # Convert global attribute
106        self._mapNACommentsToGlobalAttributes()
107
108        # Convert axes
109        if not hasattr(self, 'cdms_axes'):  self._convertCdmsAxes()
110
111        # Convert main variables
112        if not hasattr(self, 'cdms_variables'):  self._convertCdmsVariables()
113
114        # Then do auxiliary variables
115        if hasattr(self.na_file_obj, "NAUXV") and (type(self.na_file_obj.NAUXV) == type(1) and self.na_file_obj.NAUXV > 0):   # Are there any auxiliary variables?
116            if not hasattr(self, 'cdms_aux_variables'): 
117                self._convertCdmsAuxVariables()
118        else:
119            self.cdms_aux_variables = []
120           
121        self.converted = True
122        return (self.cdms_variables, self.cdms_aux_variables, self.global_attributes)
123
124
125    def _mapNACommentsToGlobalAttributes(self):
126        """
127        Maps the NASA Ames comments section to global attributes and append them to the
128        self.global_attributes dictionary.
129        """
130       
131        for key in na_to_nc_map.keys():
132
133            if type(key) == type((1,2)):
134
135                if key == ("SCOM", "NCOM"):
136                    # Map special and normal comments into the global comments section
137                    comment_line = ""
138
139                    # Add Special comments first
140                    if self.na_file_obj.NSCOML > 0:
141                        comment_line += "###NASA Ames Special Comments follow###\n"
142
143                        for i in self.na_file_obj.SCOM: 
144                           
145                            if i.strip() not in special_comment_known_strings:
146                                comment_line += ("\n" + i)
147                           
148                        comment_line += "\n###NASA Ames Special Comments end###\n"
149
150                    # Now add normal comments
151                    if self.na_file_obj.NNCOML > 0:
152                        comment_line += "###NASA Ames Normal Comments follow###\n"
153
154                        for i in self.na_file_obj.NCOM: 
155                            if i.strip() not in normal_comment_known_strings:
156                                comment_line += ("\n" + i)
157
158                        comment_line += "\n###NASA Ames Normal Comments end###"
159
160                    # Tidy comment line then write to global atts dict
161                    comment_line = comment_line.replace("\n\n", "\n")
162                    #self.cdms_file.comment = comment_line
163                    self.global_attributes["comment"] = comment_line
164
165                elif key == ("ONAME", "ORG"):
166                    # Map the two organisation NA files to the institution field in CDMS (NetCDF)
167                    institution = "%s (ONAME from NASA Ames file); %s (ORG from NASA Ames file)." % \
168                                         (self.na_file_obj.ONAME, self.na_file_obj.ORG)
169                    self.global_attributes["institution"] = institution
170                    #self.cdms_file.institution = "%s (ONAME from NASA Ames file); %s (ORG from NASA Ames file)." % (self.na_file_obj.ONAME, self.na_file_obj.ORG)
171
172                else:
173                    # Any other strange tuple just gets merged into a string
174                    item = (getattr(self.na_file_obj, key[0])) + "\n" + (getattr(self.na_file_obj, key[1]))
175                    # setattr(self.cdms_file, cdms_map.fromNA[key], item)
176                    self.global_attributes[na_to_nc_map[key]] = item
177
178            elif key == "RDATE":
179                # RDATE = Revision date - update this and put in history global attribute
180                date_parts = getattr(self.na_file_obj, "RDATE")
181                date_string = "%.4d-%.2d-%.2d" % tuple(date_parts)
182                hist = date_string + " - NASA Ames File created/revised.\n"
183                time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
184                version = nappy.utils.getVersion()
185                hist = "%s\n%s - Converted to CDMS (NetCDF) format using nappy-%s." % (hist, time_string, version)
186                # self.cdms_file.history = hist
187                print "DIDNT HAVE MAPPING SO PUT HISTORY IN LINE 160 IN na_to_cdms.py"
188                self.global_attributes["history"] = hist
189            else:
190                # Anything else just needs to be stored as a global attribute
191                # setattr(self.cdms_file, cdms_map.fromNA[key], getattr(self, key))
192                self.global_attributes[na_to_nc_map[key]] = getattr(self.na_file_obj, key)
193
194
195    def _convertCdmsVariables(self):
196        """
197        Creates cdms variable list for writing out.
198        """
199        self.cdms_variables = []
200
201        if self.variables == None:   
202            for var_number in range(self.na_file_obj.NV):
203                self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))
204        else:
205            if type(self.variables[0]) == type(1): # They are integers = indices
206                for var_number in self.variables:
207                    self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))   
208            elif type(self.variables[0]) == type("string"):  # Vars are strings
209                for var_name in self.variables:
210                    if var_name in self.na_file_obj.VNAME:
211                        var_number = self.na_file_obj.VNAME.index(var_name)
212                        self.cdms_variables.append(self._convertNAToCdmsVariable(var_number))
213                    else:
214                        raise Exception("Variable name not known: " + var_name)
215
216
217    def _convertNAToCdmsVariable(self, var_number, attributes={}):
218        """
219        Creates a single cdms variable from the variable number provided in the list.
220        """
221        (var_name, units, miss, scal) = self.na_file_obj.getVariable(var_number)
222        msg = "\nAdding variable: %s" % self.na_file_obj.VNAME[var_number]
223        print msg
224        self.output_message.append(msg)
225        array = Numeric.array(self.na_file_obj.V[var_number])
226        array = array * scal
227        # Set up axes
228        if not hasattr(self, 'cdms_axes'):
229            self._convertCdmsAxes()
230
231        # Set up variable
232        var=cdms.createVariable(array, axes=self.cdms_axes, fill_value=miss, attributes=attributes)
233
234        # Sort units etc
235        if units:   var.units=units
236       
237        # Add the best variable name
238        if len(var_name) < max_id_length:
239            var.id=safe_nc_id.sub("_", var_name).lower()
240        else:
241            var.id="naVariable_%s" % (var_number)
242       
243             # Check if mapping provided for renaming this variable
244        if var_name in self.rename_variables.keys():
245            var_name = self.rename_variables[var_name]
246           
247        var.long_name = var.name = var.title = var_name
248
249        # Add a NASA Ames variable number (for mapping correctly back to NASA Ames)
250        var.nasa_ames_var_number = var_number
251        return var
252
253    def _convertCdmsAuxVariables(self):
254        """
255        Creates a cdms variable from an auxiliary variable
256        """
257        self.cdms_aux_variables = []
258
259        if self.aux_variables == None:   
260            for avar_number in range(self.na_file_obj.NAUXV):
261                self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number))
262        else:
263            if type(self.aux_variables[0]) == type(1): # They are integers = indices
264                for avar_number in self.aux_variables:
265                    self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number))   
266
267            elif type(self.aux_variables[0]) == type("string"): # They are strings
268                for avar_name in self.aux_variables:
269                    if avar_name in self.na_file_obj.ANAME:
270                        avar_number = self.na_file_obj.ANAME.index(avar_name)
271                        self.cdms_aux_variables.append(self._convertNAAuxToCdmsVariable(avar_number)) 
272            else:
273                raise Exception("Auxiliary variable name not known: " + avar_name)         
274
275    def _convertNAAuxToCdmsVariable(self, avar_number, attributes={}):
276        """
277        Converts an auxiliary variable to a cdms variable.
278        """
279        (var_name, units, miss, scal) = self.na_file_obj.getAuxVariable(avar_number)
280        array = Numeric.array(self.na_file_obj.A[avar_number])
281        array = array * scal
282
283        msg="\nAdding auxiliary variable: %s" % self.na_file_obj.ANAME[avar_number]
284        print msg
285        self.output_message.append(msg)
286
287        # Set up axes
288        if not hasattr(self, 'cdms_axes'):
289            self._convertCdmsAxes()
290
291        # Set up variable
292        var = cdms.createVariable(array, axes=[self.cdms_axes[0]], fill_value=miss, 
293                                  attributes=attributes)
294
295        # Sort units etc
296        if units:   var.units = units
297        if len(var_name) < max_id_length:
298            var.id = safe_nc_id.sub("_", var_name).lower()
299        else:
300            var.id = "naAuxVariable_%s" % (avar_number)
301
302            # Check if mapping provided for renaming this variable
303        if var_name in self.rename_variables.keys():
304            var_name = self.rename_variables[var_name]
305
306        var.long_name = var.name = var.title = var_name
307
308        # Add a NASA Ames auxiliary variable number (for mapping correctly back to NASA Ames)
309        var.nasa_ames_aux_var_number = avar_number
310        return var       
311
312    def _convertCdmsAxes(self): 
313        """
314        Creates cdms axes from information provided in the NASA Ames dictionary.
315        """
316        if not hasattr(self, 'cdms_axes'):       
317            self.cdms_axes = []
318
319        for ivar_number in range(self.na_file_obj.NIV):
320            self.cdms_axes.append(self._convertNAIndVarToCdmsAxis(ivar_number))
321
322
323    def _convertNAIndVarToCdmsAxis(self, ivar_number):
324        """
325        Creates a cdms axis from a NASA Ames independent variable.
326        """
327        if self.na_file_obj._normalized_X == False:   self.na_file_obj._normalizeIndVars()
328
329        if self.na_file_obj.NIV == 1:
330            array = self.na_file_obj.X
331        else:
332            array = self.na_file_obj.X[ivar_number]
333
334        axis = cdms.createAxis(array)
335        axis.id = axis.name = axis.long_name = self.na_file_obj.XNAME[ivar_number]
336        (var_name, units) = self.na_file_obj.getIndependentVariable(ivar_number)
337       
338        # Sort units etc
339        if units:   axis.units = units
340        if len(var_name) < max_id_length:
341            axis.id = safe_nc_id.sub("_", var_name).lower()
342        else:
343            axis.id = "naIndVariable_%s" % (ivar_number)
344
345        if units: axis.units = units
346
347        axis_types = ("longitude", "latitude", "level", "time")
348
349        for axis_type in axis_types:
350            if re.search(axis_type, var_name, re.IGNORECASE):
351                axis.standard_name = axis.id = axis_type
352                # Designate it CF-style if known axis type (e.g. axis.designateTime() etc..)
353                exec "axis.designate%s()" % axis_type.title()
354                print "CHECK AXIS GOT DESIGNATED PROPERLY - RE-FACTORED" * 5
355
356        # Check warning for time units pattern
357        if axis.isTime() and (not hasattr(axis, "units") or not time_units_pattn.match(axis.units)):
358            if self.time_units == None:
359                time_units_input = "I WON'T MATCH"
360
361                while time_units_input != "" and not time_units_pattn.match(time_units_input):
362                    message = time_units_warning_message                           
363                    if self.time_warning == True:
364                        print message
365                        time_units_input = raw_input("Please insert your time unit string here (or leave blank):").strip()
366                    else: 
367                        time_units_input = ""
368
369                self.output_message.append(message)
370                self.time_units = time_units_input
371
372            axis.units = self.time_units
373            axis.long_name = axis.name = "time (%s)" % self.time_units
374
375        if not hasattr(axis, "units") or axis.units == None: 
376            if units:
377                axis.units = units     
378            else:
379                axis.units = "Not known"
380
381        return axis
Note: See TracBrowser for help on using the repository browser.