source: nappy/trunk/nappy/nc_interface/nc_to_na.py @ 4002

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/nc_to_na.py@4002
Revision 4002, 12.7 KB checked in by astephen, 12 years ago (diff)

Minor change

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6nc_to_na.py
7=============
8
9Holds the class NCToNA (sub-classing CDMSToNA) that converts a NetCDF file to
10one or more NASA Ames files.
11
12"""
13
14# Imports from python standard library
15import sys
16
17# Import from nappy package
18import nappy
19from nappy.na_error import na_error
20import nappy.utils
21import nappy.utils.common_utils
22import nappy.nc_interface.cdms_to_na
23import nappy.nc_interface.na_content_collector
24
25# Import external packages (if available)
26if sys.platform.find("win") > -1:
27    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
28
29try:
30    import cdms2 as cdms
31except:
32    try:
33        import cdms
34    except:
35        raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
36
37cdms.setAutoBounds("off") 
38
39# Define global variables
40DEBUG = nappy.utils.getDebug() 
41default_delimiter = nappy.utils.getDefault("default_delimiter")
42default_float_format = nappy.utils.getDefault("default_float_format")
43comment_override_rule = nappy.utils.getDefault("comment_override_rule")
44
45# Define global variables
46permitted_overwrite_metadata = ("DATE",  "RDATE", "ANAME", "MNAME",
47           "ONAME", "ORG", "SNAME", "VNAME", "SCOM", "NCOM")
48items_as_lists = ["DATE", "RDATE", "ANAME", "VNAME"]
49
50
51class NCToNA(nappy.nc_interface.cdms_to_na.CDMSToNA):
52    """
53    Converts a NetCDF file to one or more NASA Ames files.
54    """
55
56    def __init__(self, nc_file, var_ids=None, na_items_to_override={}, 
57            only_return_file_names=False, exclude_vars=[],
58            requested_ffi=None):
59        """
60        Sets up instance variables.
61        Typical usage is:
62        >>>    import nappy.nc_interface.nc_to_na as nc_to_na
63        >>>    c = nc_to_na.NCToNA("old_file.nc")
64        >>>    c.convert()
65        >>>    c.writeNAFiles("new_file.na", delimiter=",")
66
67        OR:
68        >>>    c = nc_to_na.NCToNA("old_file.nc")
69        >>>    file_names = c.constructNAFileNames()
70        """
71        self.nc_file = nc_file
72
73        # Now need to read CDMS file so parent class methods are compatible
74        (cdms_variables, global_attributes) = self._readCDMSFile(var_ids, exclude_vars)
75        nappy.nc_interface.cdms_to_na.CDMSToNA.__init__(self, cdms_variables, global_attributes=global_attributes, 
76                                                        na_items_to_override=na_items_to_override, 
77                                                        only_return_file_names=only_return_file_names,
78                                                        requested_ffi=requested_ffi)
79 
80
81    def _readCDMSFile(self, var_ids=None, exclude_vars=[]):
82        """
83        Reads the file and returns all the CDMS variables in a list as well
84        as the global attributes: (cdms_variable_list, global_atts_list)
85        If var_ids is defined then only get those.
86        """
87        fin = cdms.open(self.nc_file)
88        cdms_variables = []
89
90        # Make sure var_ids is a list
91        if type(var_ids) == type("string"):
92            var_ids = [var_ids]
93
94        for var_id in fin.listvariables():
95            if var_ids == None or var_id in var_ids:
96                if var_ids not in exclude_vars:
97                    cdms_variables.append(fin(var_id))
98
99        globals = fin.attributes.items()
100        return (cdms_variables, globals) 
101
102    def constructNAFileNames(self, na_file=None):
103        """
104        Works out what the file names of the output NA files will be and
105        returns a list of them.
106        """
107        self.convert()
108
109        file_names = []
110        # create file name if not given
111        if na_file == None:
112            base_name = self.nc_file
113            if base_name[-3:] == ".nc":
114                base_name = base_name[:-3]
115            na_file = base_name + ".na"
116
117        file_counter = 1
118        # Now, create some valid file names
119        for this_na_dict in self.na_dict_list:
120            if len(self.na_dict_list) == 1:
121                suffix = ""
122            else:
123                suffix = "_%s" % file_counter
124
125            # Create file name
126            name_parts = na_file.split(".")   
127            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
128            file_names.append(new_name)
129            file_counter += 1
130           
131        return file_names
132
133    def writeNAFiles(self, na_file=None, delimiter=default_delimiter, annotation=False,
134                     float_format=default_float_format, size_limit=None, no_header=False):
135        """
136        Writes the self.na_dict_list content to one or more NASA Ames files.
137        Output file names are based on the self.nc_file name unless specified
138        in the na_file_name argument in which case that provides the main name
139        that is appended to if multiple output file names are required.
140
141        TODO: no_header is NOT implemented.
142        """
143        self.convert() # just in case not already called
144
145        # Gets a list of NA file_names that will be produced.
146        file_names = self.constructNAFileNames(na_file)
147
148        # Set up some counters: file_counter is the expected number of files.
149        #      full_file_counter includes any files that have been split across multiple output NA files
150        #              because size_limit exceeded.
151        file_counter = 1
152        full_file_counter = 1
153
154        # Get any NASA Ames dictionary values that should be overwritten with local values
155        local_attributes = nappy.utils.getLocalAttributesConfigDict()
156        local_na_atts = local_attributes["na_attributes"]
157
158        # define final override list by using defaults then locally provided changes
159        overriders = local_na_atts
160        for (okey, ovalue) in self.na_items_to_override.items():
161            overriders[okey] = ovalue
162
163        # Now loop through writing the outputs
164        for na_dict_and_var_ids in self.na_dict_list:
165            file_name = file_names[file_counter - 1]
166            msg = "\nWriting output NASA Ames file: %s" % file_name
167            if DEBUG: print msg
168            self.output_message.append(msg)
169
170            # Set up current na dict
171            (this_na_dict, vars_to_write) = na_dict_and_var_ids
172
173            # Override content of NASA Ames if they are permitted
174            for key in overriders.keys():
175
176                if key in permitted_overwrite_metadata:   
177                    if key in items_as_lists:
178                        new_item = overriders[key].split()                 
179                    else:
180                        new_item = overriders[key]
181
182                    # Do specific overwrite for comments by inserting lines at start
183                    if key in ("SCOM", "NCOM"):
184
185                        # Use rule defined in config file in terms of where to put new comments
186                        if comment_override_rule == "replace":
187                            comments_list = new_item[:]
188                        elif comment_override_rule == "insert":
189                            comments_list = new_item[:] + this_na_dict.get(key, [])
190                        elif comment_override_rule == "extend": 
191                            comments_list = this_na_dict.get(key, []) + new_item[:]
192
193                        this_na_dict[key] = comments_list
194                        this_na_dict["N%sL" % key] = len(comments_list)
195                        #print "COMMMENTS:", comments_list
196                        #print "Added to comments:", key
197                         
198                    elif not this_na_dict.has_key(key) or new_item != this_na_dict[key]:
199                        this_na_dict[key] = new_item
200                        msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, this_na_dict[key])
201                        if DEBUG: print msg
202                        self.output_message.append(msg)
203       
204            file_list = []
205            # Cope with size limits if specified and FFI is 1001
206            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files.
207            if size_limit is not None and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > size_limit):
208                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter,
209                                                                        float_format=float_format, size_limit=size_limit,
210                                                                        annotation=annotation)
211                file_list.extend(files_written)
212
213            # If not having to split file into multiple outputs (normal condition)
214            else:               
215                x = nappy.openNAFile(file_name, 'w', this_na_dict)
216                x.write(delimiter=delimiter, float_format=float_format, annotation=annotation)
217                x.close()
218                file_list.append(file_name)
219
220            # Report on what has been written
221            msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(vars_to_write[0]))
222            if DEBUG: print msg
223            self.output_message.append(msg)
224       
225            msg = ""
226            aux_var_count = vars_to_write[1]
227            if len(aux_var_count) > 0:
228                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))     
229           
230            singleton_var_count = vars_to_write[2]
231            if len(singleton_var_count) > 0:
232                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count))
233
234            if len(file_list) > 0:
235                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list))
236
237            full_file_counter += len(file_list)
238            file_counter += 1
239
240            if DEBUG: print msg
241            self.output_message.append(msg)
242           
243        full_file_count = full_file_counter - 1
244        if full_file_count == 1:
245            plural = ""
246        else:
247            plural = "s"             
248        msg = "\n%s file%s written." % (full_file_count, plural)
249   
250        if DEBUG: print msg
251        self.output_message.append(msg)
252        self.output_files_written = file_list
253
254        return self.output_message
255
256    def _writeNAFileSubsetsWithinSizeLimit(self, this_na_dict, file_name, delimiter, 
257                      float_format, size_limit, annotation):
258        """
259        If self.size_limit is specified and FFI is 1001 we can chunk the output into
260        different files in a NASA Ames compliant way.
261        Returns list of file names of outputs written.
262        """
263        file_names = []
264        var_list = this_na_dict["V"]
265        array_length = len(var_list[0])
266        nvol_info = divmod(array_length, size_limit)
267        nvol = nvol_info[0]
268
269        # create the number of volumes (files) that need to be written.
270        if nvol_info[1] > 0: nvol = nvol + 1
271
272        start = 0
273        letter_count = 0
274        ivol = 0
275
276        # Loop through until full array length has been written to a set of files.
277        while start < array_length:
278            ivol = ivol + 1
279            end = start + size_limit
280
281            if end > array_length:
282                end = array_length
283
284            current_block = []
285            # Write new V array
286            for v in var_list:
287                current_block.append(v[start:end])
288
289            # Adjust X accordingly in the na dictionary, because independent variable has been reduced in size
290            na_dict_copy = nappy.utils.common_utils.modifyNADictCopy(this_na_dict, current_block, 
291                                                                      start, end, ivol, nvol)
292            # Append a letter to the file name for writing this block to
293            file_name_plus_letter = "%s-%.3d.na" % (file_name[:-3], ivol)
294            file_list.append(file_name_plus_letter)
295
296            # Write data to output file
297            x = nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy)
298            x.write(delimiter=delimiter, float_format=float_format, annotation=annotation)
299            x.close()
300
301            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter)
302            if DEBUG: print msg
303            self.output_message.append(msg)
304            letter_count = letter_count + 1
305            start = end
306
307            file_names.append(file_name_plus_letter) 
308
309        return file_names
310
311
Note: See TracBrowser for help on using the repository browser.