source: nappy/trunk/nappy/nc_interface/nc_to_na.py @ 3441

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/nc_to_na.py@3441
Revision 3441, 10.3 KB checked in by astephen, 12 years ago (diff)

Fixes to make NC to NA route work.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6nc_to_na.py
7=============
8
9Holds the class NCToNA (sub-classing CDMSToNA) that converts a NetCDF file to
10one or more NASA Ames files.
11
12"""
13
14# Imports from python standard library
15import sys
16
17# Import from nappy package
18import nappy
19from nappy.na_error import na_error
20import nappy.utils
21import nappy.utils.common_utils
22import nappy.nc_interface.cdms_to_na
23import nappy.nc_interface.na_content_collector
24
25# Import external packages (if available)
26if sys.platform.find("win") > -1:
27    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
28try:
29    import cdms, Numeric
30except:
31    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
32
33cdms.setAutoBounds("off") 
34
35# Define global variables
36DEBUG = nappy.utils.getDebug() 
37
38class NCToNA(nappy.nc_interface.cdms_to_na.CDMSToNA):
39    """
40    Converts a NetCDF file to one or more NASA Ames files.
41    """
42
43    def __init__(self, nc_file, var_ids=None, na_items_to_override={}, 
44            only_return_file_names=False, exclude_vars=[]):
45        """
46        Sets up instance variables.
47        Typical usage is:
48        >>>    import nappy.nc_interface.nc_to_na as nc_to_na
49        >>>    c = nc_to_na.NCToNA("old_file.nc")
50        >>>    c.convert()
51        >>>    c.writeNAFiles("new_file.na", delimiter="csv")
52
53        OR:
54        >>>    c = nc_to_na.NCToNA("old_file.nc")
55        >>>    file_names = c.constructNAFileNames()
56        """
57        self.nc_file = nc_file
58
59        # Now need to read CDMS file so parent class methods are compatible
60        (cdms_variables, global_atts) = self._readCDMSFile(var_ids, exclude_vars)
61        nappy.nc_interface.cdms_to_na.CDMSToNA.__init__(self, cdms_variables, global_atts=global_atts, 
62                    na_items_to_override=na_items_to_override, only_return_file_names=only_return_file_names)
63 
64
65    def _readCDMSFile(self, var_ids=None, exclude_vars=[]):
66        """
67        Reads the file and returns all the CDMS variables in a list as well
68        as the global attributes: (cdms_variable_list, global_atts_dict)
69        If var_ids is defined then only get those.
70        """
71        fin = cdms.open(self.nc_file)
72        cdms_variables = []
73
74        # Make sure var_ids is a list
75        if type(var_ids) == type("string"):
76            var_ids = [var_ids]
77
78        for var_id in fin.listvariables():
79            if var_ids == None or var_id in var_ids:
80                if var_ids not in exclude_vars:
81                    cdms_variables.append(fin(var_id))
82
83        globals = fin.attributes
84        return (cdms_variables, globals) 
85
86    def constructNAFileNames(self, na_file=None):
87        """
88        Works out what the file names of the output NA files will be and
89        returns a list of them.
90        """
91        self.convert()
92
93        file_names = []
94        # create file name if not given
95        if na_file == None:
96            base_name = self.nc_file
97            if base_name[-3:] == ".nc":
98                base_name = base_name[:-3]
99            na_file = base_name + ".na"
100
101        file_counter = 1
102        # Now, create some valid file names
103        for this_na_dict in self.na_dict_list:
104            if len(self.na_dict_list) == 1:
105                suffix = ""
106            else:
107                suffix = "_%s" % file_counter
108
109            # Create file name
110            name_parts = na_file.split(".")   
111            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
112            file_names.append(new_name)
113            file_counter += 1
114           
115        return file_names
116
117    def writeNAFiles(self, na_file=None, delimiter="    ", 
118                     float_format="%g", size_limit=None):
119        """
120        Writes the self.na_dict_list content to one or more NASA Ames files.
121        Output file names are based on the self.nc_file name unless specified
122        in the na_file_name argument in which case that provides the main name
123        that is appended to if multiple output file names are required.
124        """
125        self.convert() # just in case not already called
126
127        # Gets a list of NA file_names that will be produced.
128        file_names = self.constructNAFileNames(na_file)
129
130        # Set up some counters: file_counter is the expected number of files.
131        #      full_file_counter includes any files that have been split across multiple output NA files
132        #              because size_limit exceeded.
133        file_counter = 1
134        full_file_counter = 1
135
136        # Now loop through writing the outputs
137        for na_dict_and_var_ids in self.na_dict_list:
138            file_name = file_names[file_counter - 1]
139            msg = "\nWriting output NASA Ames file: %s" % file_name
140            if DEBUG: print msg
141            self.output_message.append(msg)
142
143            # Set up current na dict
144            (this_na_dict, vars_to_write) = na_dict_and_var_ids
145
146            # Override content of NASA Ames if they are permitted
147            for key in self.na_items_to_override.keys():
148                if key in permitted_overwrite_metadata:   
149                    if key in items_as_lists:
150                        new_item = self.na_items_to_override[key].split()                 
151                    else:
152                        new_item = self.na_items_to_override[key]
153                                   
154                    if new_item != this_na_dict[key]:
155                        this_na_dict[key] = new_item
156                        msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, this_na_dict[key])
157                        if DEBUG: print msg
158                        self.output_message.append(msg)
159       
160            file_list = []
161            # Cope with size limits if specified and FFI is 1001
162            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files.
163            if size_limit is not None and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > size_limit):
164                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter,
165                                                                        float_format=float_format, size_limit=size_limit)
166                file_list.extend(files_written)
167
168            # If not having to split file into multiple outputs (normal condition)
169            else:               
170                x = nappy.openNAFile(file_name, 'w', this_na_dict)
171                x.write(delimiter=delimiter, float_format=float_format)
172                x.close()
173                file_list.append(file_name)
174
175            # Report on what has been written
176            msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(vars_to_write[0]))
177            if DEBUG: print msg
178            self.output_message.append(msg)
179       
180            msg = ""
181            aux_var_count = vars_to_write[1]
182            if len(aux_var_count) > 0:
183                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))     
184           
185            singleton_var_count = vars_to_write[2]
186            if len(singleton_var_count) > 0:
187                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count))
188
189            if len(file_list) > 0:
190                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list))
191
192            full_file_counter += len(file_list)
193            file_counter += 1
194
195            if DEBUG: print msg
196            self.output_message.append(msg)
197           
198        full_file_count = full_file_counter - 1
199        if full_file_count == 1:
200            plural = ""
201        else:
202            plural = "s"             
203        msg = "\n%s file%s written." % (full_file_count, plural)
204   
205        if DEBUG: print msg
206        self.output_message.append(msg)
207        return self.output_message
208
209    def _writeNAFileSubsetsWithinSizeLimit(self, this_na_dict, file_name, delimiter, float_format, size_limit):
210        """
211        If self.size_limit is specified and FFI is 1001 we can chunk the output into
212        different files in a NASA Ames compliant way.
213        Returns list of file names of outputs written.
214        """
215        file_names = []
216        var_list = this_na_dict["V"]
217        array_length = len(var_list[0])
218        nvol_info = divmod(array_length, size_limit)
219        nvol = nvol_info[0]
220
221        # create the number of volumes (files) that need to be written.
222        if nvol_info[1] > 0: nvol = nvol + 1
223
224        start = 0
225        letter_count = 0
226        ivol = 0
227
228        # Loop through until full array length has been written to a set of files.
229        while start < array_length:
230            ivol = ivol + 1
231            end = start + size_limit
232
233            if end > array_length:
234                end = array_length
235
236            current_block = []
237            # Write new V array
238            for v in var_list:
239                current_block.append(v[start:end])
240
241            # Adjust X accordingly in the na dictionary, because independent variable has been reduced in size
242            na_dict_copy = nappy.utils.common_utils.modifyNADictCopy(this_na_dict, current_block, 
243                                                                      start, end, ivol, nvol)
244            # Append a letter to the file name for writing this block to
245            file_name_plus_letter = "%s-%.3d.na" % (file_name[:-3], ivol)
246            file_list.append(file_name_plus_letter)
247
248            # Write data to output file
249            x = nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy)
250            x.write(delimiter=delimiter, float_format=float_format)
251            x.close()
252
253            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter)
254            if DEBUG: print msg
255            self.output_message.append(msg)
256            letter_count = letter_count + 1
257            start = end
258
259            file_names.append(file_name_plus_letter) 
260
261        return file_names
Note: See TracBrowser for help on using the repository browser.