source: nappy/trunk/nappy/nc_interface/nc_to_na.py @ 3630

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/nc_to_na.py@3630
Revision 3630, 10.7 KB checked in by astephen, 12 years ago (diff)

Fixed defaults for delimiter and float format so read from nappy.ini config file.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6nc_to_na.py
7=============
8
9Holds the class NCToNA (sub-classing CDMSToNA) that converts a NetCDF file to
10one or more NASA Ames files.
11
12"""
13
14# Imports from python standard library
15import sys
16
17# Import from nappy package
18import nappy
19from nappy.na_error import na_error
20import nappy.utils
21import nappy.utils.common_utils
22import nappy.nc_interface.cdms_to_na
23import nappy.nc_interface.na_content_collector
24
25# Import external packages (if available)
26if sys.platform.find("win") > -1:
27    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
28try:
29    import cdms, Numeric
30except:
31    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
32
33cdms.setAutoBounds("off") 
34
35# Define global variables
36DEBUG = nappy.utils.getDebug() 
37default_delimiter = nappy.utils.getDefault("default_delimiter")
38default_float_format = nappy.utils.getDefault("default_float_format")
39
40
41class NCToNA(nappy.nc_interface.cdms_to_na.CDMSToNA):
42    """
43    Converts a NetCDF file to one or more NASA Ames files.
44    """
45
46    def __init__(self, nc_file, var_ids=None, na_items_to_override={}, 
47            only_return_file_names=False, exclude_vars=[],
48            requested_ffi=None):
49        """
50        Sets up instance variables.
51        Typical usage is:
52        >>>    import nappy.nc_interface.nc_to_na as nc_to_na
53        >>>    c = nc_to_na.NCToNA("old_file.nc")
54        >>>    c.convert()
55        >>>    c.writeNAFiles("new_file.na", delimiter=",")
56
57        OR:
58        >>>    c = nc_to_na.NCToNA("old_file.nc")
59        >>>    file_names = c.constructNAFileNames()
60        """
61        self.nc_file = nc_file
62
63        # Now need to read CDMS file so parent class methods are compatible
64        (cdms_variables, global_atts) = self._readCDMSFile(var_ids, exclude_vars)
65        nappy.nc_interface.cdms_to_na.CDMSToNA.__init__(self, cdms_variables, global_atts=global_atts, 
66                                                        na_items_to_override=na_items_to_override, 
67                                                        only_return_file_names=only_return_file_names,
68                                                        requested_ffi=requested_ffi)
69 
70
71    def _readCDMSFile(self, var_ids=None, exclude_vars=[]):
72        """
73        Reads the file and returns all the CDMS variables in a list as well
74        as the global attributes: (cdms_variable_list, global_atts_dict)
75        If var_ids is defined then only get those.
76        """
77        fin = cdms.open(self.nc_file)
78        cdms_variables = []
79
80        # Make sure var_ids is a list
81        if type(var_ids) == type("string"):
82            var_ids = [var_ids]
83
84        for var_id in fin.listvariables():
85            if var_ids == None or var_id in var_ids:
86                if var_ids not in exclude_vars:
87                    cdms_variables.append(fin(var_id))
88
89        globals = fin.attributes
90        return (cdms_variables, globals) 
91
92    def constructNAFileNames(self, na_file=None):
93        """
94        Works out what the file names of the output NA files will be and
95        returns a list of them.
96        """
97        self.convert()
98
99        file_names = []
100        # create file name if not given
101        if na_file == None:
102            base_name = self.nc_file
103            if base_name[-3:] == ".nc":
104                base_name = base_name[:-3]
105            na_file = base_name + ".na"
106
107        file_counter = 1
108        # Now, create some valid file names
109        for this_na_dict in self.na_dict_list:
110            if len(self.na_dict_list) == 1:
111                suffix = ""
112            else:
113                suffix = "_%s" % file_counter
114
115            # Create file name
116            name_parts = na_file.split(".")   
117            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
118            file_names.append(new_name)
119            file_counter += 1
120           
121        return file_names
122
123    def writeNAFiles(self, na_file=None, delimiter=default_delimiter, 
124                     float_format=default_float_format, size_limit=None):
125        """
126        Writes the self.na_dict_list content to one or more NASA Ames files.
127        Output file names are based on the self.nc_file name unless specified
128        in the na_file_name argument in which case that provides the main name
129        that is appended to if multiple output file names are required.
130        """
131        self.convert() # just in case not already called
132
133        # Gets a list of NA file_names that will be produced.
134        file_names = self.constructNAFileNames(na_file)
135
136        # Set up some counters: file_counter is the expected number of files.
137        #      full_file_counter includes any files that have been split across multiple output NA files
138        #              because size_limit exceeded.
139        file_counter = 1
140        full_file_counter = 1
141
142        # Now loop through writing the outputs
143        for na_dict_and_var_ids in self.na_dict_list:
144            file_name = file_names[file_counter - 1]
145            msg = "\nWriting output NASA Ames file: %s" % file_name
146            if DEBUG: print msg
147            self.output_message.append(msg)
148
149            # Set up current na dict
150            (this_na_dict, vars_to_write) = na_dict_and_var_ids
151
152            # Override content of NASA Ames if they are permitted
153            for key in self.na_items_to_override.keys():
154                if key in permitted_overwrite_metadata:   
155                    if key in items_as_lists:
156                        new_item = self.na_items_to_override[key].split()                 
157                    else:
158                        new_item = self.na_items_to_override[key]
159                                   
160                    if new_item != this_na_dict[key]:
161                        this_na_dict[key] = new_item
162                        msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, this_na_dict[key])
163                        if DEBUG: print msg
164                        self.output_message.append(msg)
165       
166            file_list = []
167            # Cope with size limits if specified and FFI is 1001
168            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files.
169            if size_limit is not None and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > size_limit):
170                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter,
171                                                                        float_format=float_format, size_limit=size_limit)
172                file_list.extend(files_written)
173
174            # If not having to split file into multiple outputs (normal condition)
175            else:               
176                x = nappy.openNAFile(file_name, 'w', this_na_dict)
177                x.write(delimiter=delimiter, float_format=float_format)
178                x.close()
179                file_list.append(file_name)
180
181            # Report on what has been written
182            msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(vars_to_write[0]))
183            if DEBUG: print msg
184            self.output_message.append(msg)
185       
186            msg = ""
187            aux_var_count = vars_to_write[1]
188            if len(aux_var_count) > 0:
189                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))     
190           
191            singleton_var_count = vars_to_write[2]
192            if len(singleton_var_count) > 0:
193                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count))
194
195            if len(file_list) > 0:
196                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list))
197
198            full_file_counter += len(file_list)
199            file_counter += 1
200
201            if DEBUG: print msg
202            self.output_message.append(msg)
203           
204        full_file_count = full_file_counter - 1
205        if full_file_count == 1:
206            plural = ""
207        else:
208            plural = "s"             
209        msg = "\n%s file%s written." % (full_file_count, plural)
210   
211        if DEBUG: print msg
212        self.output_message.append(msg)
213        return self.output_message
214
215    def _writeNAFileSubsetsWithinSizeLimit(self, this_na_dict, file_name, delimiter, float_format, size_limit):
216        """
217        If self.size_limit is specified and FFI is 1001 we can chunk the output into
218        different files in a NASA Ames compliant way.
219        Returns list of file names of outputs written.
220        """
221        file_names = []
222        var_list = this_na_dict["V"]
223        array_length = len(var_list[0])
224        nvol_info = divmod(array_length, size_limit)
225        nvol = nvol_info[0]
226
227        # create the number of volumes (files) that need to be written.
228        if nvol_info[1] > 0: nvol = nvol + 1
229
230        start = 0
231        letter_count = 0
232        ivol = 0
233
234        # Loop through until full array length has been written to a set of files.
235        while start < array_length:
236            ivol = ivol + 1
237            end = start + size_limit
238
239            if end > array_length:
240                end = array_length
241
242            current_block = []
243            # Write new V array
244            for v in var_list:
245                current_block.append(v[start:end])
246
247            # Adjust X accordingly in the na dictionary, because independent variable has been reduced in size
248            na_dict_copy = nappy.utils.common_utils.modifyNADictCopy(this_na_dict, current_block, 
249                                                                      start, end, ivol, nvol)
250            # Append a letter to the file name for writing this block to
251            file_name_plus_letter = "%s-%.3d.na" % (file_name[:-3], ivol)
252            file_list.append(file_name_plus_letter)
253
254            # Write data to output file
255            x = nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy)
256            x.write(delimiter=delimiter, float_format=float_format)
257            x.close()
258
259            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter)
260            if DEBUG: print msg
261            self.output_message.append(msg)
262            letter_count = letter_count + 1
263            start = end
264
265            file_names.append(file_name_plus_letter) 
266
267        return file_names
Note: See TracBrowser for help on using the repository browser.