source: nappy/trunk/nappy/nc_interface/nc_to_na.py @ 3615

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/nc_to_na.py@3615
Revision 3615, 10.6 KB checked in by astephen, 12 years ago (diff)

Fixing some bugs and inconsistencies.
Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6nc_to_na.py
7=============
8
9Holds the class NCToNA (sub-classing CDMSToNA) that converts a NetCDF file to
10one or more NASA Ames files.
11
12"""
13
14# Imports from python standard library
15import sys
16
17# Import from nappy package
18import nappy
19from nappy.na_error import na_error
20import nappy.utils
21import nappy.utils.common_utils
22import nappy.nc_interface.cdms_to_na
23import nappy.nc_interface.na_content_collector
24
25# Import external packages (if available)
26if sys.platform.find("win") > -1:
27    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
28try:
29    import cdms, Numeric
30except:
31    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
32
33cdms.setAutoBounds("off") 
34
35# Define global variables
36DEBUG = nappy.utils.getDebug() 
37
38class NCToNA(nappy.nc_interface.cdms_to_na.CDMSToNA):
39    """
40    Converts a NetCDF file to one or more NASA Ames files.
41    """
42
43    def __init__(self, nc_file, var_ids=None, na_items_to_override={}, 
44            only_return_file_names=False, exclude_vars=[],
45            requested_ffi=None):
46        """
47        Sets up instance variables.
48        Typical usage is:
49        >>>    import nappy.nc_interface.nc_to_na as nc_to_na
50        >>>    c = nc_to_na.NCToNA("old_file.nc")
51        >>>    c.convert()
52        >>>    c.writeNAFiles("new_file.na", delimiter=",")
53
54        OR:
55        >>>    c = nc_to_na.NCToNA("old_file.nc")
56        >>>    file_names = c.constructNAFileNames()
57        """
58        self.nc_file = nc_file
59
60        # Now need to read CDMS file so parent class methods are compatible
61        (cdms_variables, global_atts) = self._readCDMSFile(var_ids, exclude_vars)
62        nappy.nc_interface.cdms_to_na.CDMSToNA.__init__(self, cdms_variables, global_atts=global_atts, 
63                                                        na_items_to_override=na_items_to_override, 
64                                                        only_return_file_names=only_return_file_names,
65                                                        requested_ffi=requested_ffi)
66 
67
68    def _readCDMSFile(self, var_ids=None, exclude_vars=[]):
69        """
70        Reads the file and returns all the CDMS variables in a list as well
71        as the global attributes: (cdms_variable_list, global_atts_dict)
72        If var_ids is defined then only get those.
73        """
74        fin = cdms.open(self.nc_file)
75        cdms_variables = []
76
77        # Make sure var_ids is a list
78        if type(var_ids) == type("string"):
79            var_ids = [var_ids]
80
81        for var_id in fin.listvariables():
82            if var_ids == None or var_id in var_ids:
83                if var_ids not in exclude_vars:
84                    cdms_variables.append(fin(var_id))
85
86        globals = fin.attributes
87        return (cdms_variables, globals) 
88
89    def constructNAFileNames(self, na_file=None):
90        """
91        Works out what the file names of the output NA files will be and
92        returns a list of them.
93        """
94        self.convert()
95
96        file_names = []
97        # create file name if not given
98        if na_file == None:
99            base_name = self.nc_file
100            if base_name[-3:] == ".nc":
101                base_name = base_name[:-3]
102            na_file = base_name + ".na"
103
104        file_counter = 1
105        # Now, create some valid file names
106        for this_na_dict in self.na_dict_list:
107            if len(self.na_dict_list) == 1:
108                suffix = ""
109            else:
110                suffix = "_%s" % file_counter
111
112            # Create file name
113            name_parts = na_file.split(".")   
114            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
115            file_names.append(new_name)
116            file_counter += 1
117           
118        return file_names
119
120    def writeNAFiles(self, na_file=None, delimiter="    ", 
121                     float_format="%g", size_limit=None):
122        """
123        Writes the self.na_dict_list content to one or more NASA Ames files.
124        Output file names are based on the self.nc_file name unless specified
125        in the na_file_name argument in which case that provides the main name
126        that is appended to if multiple output file names are required.
127        """
128        self.convert() # just in case not already called
129
130        # Gets a list of NA file_names that will be produced.
131        file_names = self.constructNAFileNames(na_file)
132
133        # Set up some counters: file_counter is the expected number of files.
134        #      full_file_counter includes any files that have been split across multiple output NA files
135        #              because size_limit exceeded.
136        file_counter = 1
137        full_file_counter = 1
138
139        # Now loop through writing the outputs
140        for na_dict_and_var_ids in self.na_dict_list:
141            file_name = file_names[file_counter - 1]
142            msg = "\nWriting output NASA Ames file: %s" % file_name
143            if DEBUG: print msg
144            self.output_message.append(msg)
145
146            # Set up current na dict
147            (this_na_dict, vars_to_write) = na_dict_and_var_ids
148
149            # Override content of NASA Ames if they are permitted
150            for key in self.na_items_to_override.keys():
151                if key in permitted_overwrite_metadata:   
152                    if key in items_as_lists:
153                        new_item = self.na_items_to_override[key].split()                 
154                    else:
155                        new_item = self.na_items_to_override[key]
156                                   
157                    if new_item != this_na_dict[key]:
158                        this_na_dict[key] = new_item
159                        msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, this_na_dict[key])
160                        if DEBUG: print msg
161                        self.output_message.append(msg)
162       
163            file_list = []
164            # Cope with size limits if specified and FFI is 1001
165            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files.
166            if size_limit is not None and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > size_limit):
167                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter,
168                                                                        float_format=float_format, size_limit=size_limit)
169                file_list.extend(files_written)
170
171            # If not having to split file into multiple outputs (normal condition)
172            else:               
173                x = nappy.openNAFile(file_name, 'w', this_na_dict)
174                x.write(delimiter=delimiter, float_format=float_format)
175                x.close()
176                file_list.append(file_name)
177
178            # Report on what has been written
179            msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(vars_to_write[0]))
180            if DEBUG: print msg
181            self.output_message.append(msg)
182       
183            msg = ""
184            aux_var_count = vars_to_write[1]
185            if len(aux_var_count) > 0:
186                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))     
187           
188            singleton_var_count = vars_to_write[2]
189            if len(singleton_var_count) > 0:
190                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count))
191
192            if len(file_list) > 0:
193                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list))
194
195            full_file_counter += len(file_list)
196            file_counter += 1
197
198            if DEBUG: print msg
199            self.output_message.append(msg)
200           
201        full_file_count = full_file_counter - 1
202        if full_file_count == 1:
203            plural = ""
204        else:
205            plural = "s"             
206        msg = "\n%s file%s written." % (full_file_count, plural)
207   
208        if DEBUG: print msg
209        self.output_message.append(msg)
210        return self.output_message
211
212    def _writeNAFileSubsetsWithinSizeLimit(self, this_na_dict, file_name, delimiter, float_format, size_limit):
213        """
214        If self.size_limit is specified and FFI is 1001 we can chunk the output into
215        different files in a NASA Ames compliant way.
216        Returns list of file names of outputs written.
217        """
218        file_names = []
219        var_list = this_na_dict["V"]
220        array_length = len(var_list[0])
221        nvol_info = divmod(array_length, size_limit)
222        nvol = nvol_info[0]
223
224        # create the number of volumes (files) that need to be written.
225        if nvol_info[1] > 0: nvol = nvol + 1
226
227        start = 0
228        letter_count = 0
229        ivol = 0
230
231        # Loop through until full array length has been written to a set of files.
232        while start < array_length:
233            ivol = ivol + 1
234            end = start + size_limit
235
236            if end > array_length:
237                end = array_length
238
239            current_block = []
240            # Write new V array
241            for v in var_list:
242                current_block.append(v[start:end])
243
244            # Adjust X accordingly in the na dictionary, because independent variable has been reduced in size
245            na_dict_copy = nappy.utils.common_utils.modifyNADictCopy(this_na_dict, current_block, 
246                                                                      start, end, ivol, nvol)
247            # Append a letter to the file name for writing this block to
248            file_name_plus_letter = "%s-%.3d.na" % (file_name[:-3], ivol)
249            file_list.append(file_name_plus_letter)
250
251            # Write data to output file
252            x = nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy)
253            x.write(delimiter=delimiter, float_format=float_format)
254            x.close()
255
256            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter)
257            if DEBUG: print msg
258            self.output_message.append(msg)
259            letter_count = letter_count + 1
260            start = end
261
262            file_names.append(file_name_plus_letter) 
263
264        return file_names
Note: See TracBrowser for help on using the repository browser.