source: nappy/trunk/nappy/nc_interface/nc_to_na.py @ 3416

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/nc_to_na.py@3416
Revision 3416, 10.0 KB checked in by astephen, 12 years ago (diff)

Getting NC to NA working but needs nappy_api.py to be changed!

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6nc_to_na.py
7=============
8
9Holds the class NCToNA (sub-classing CDMSToNA) that converts a NetCDF file to
10one or more NASA Ames files.
11
12"""
13
14# Imports from python standard library
15import sys
16
17# Import from nappy package
18import nappy
19from nappy.na_error import na_error
20import nappy.utils
21import nappy.utils.common_utils
22import nappy.nc_interface.cdms_to_na
23import nappy.nc_interface.na_content_collector
24
25# Import external packages (if available)
26if sys.platform.find("win") > -1:
27    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
28try:
29    import cdms, Numeric
30except:
31    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
32
33cdms.setAutoBounds("off") 
34
35# Define global variables
36DEBUG = nappy.utils.getDebug() 
37
38class NCToNA(nappy.nc_interface.cdms_to_na.CDMSToNA):
39    """
40    Converts a NetCDF file to one or more NASA Ames files.
41    """
42
43    def __init__(self, nc_file, var_ids=None, na_items_to_override={}, 
44            only_return_file_names=False, exclude_vars=[]):
45        """
46        Sets up instance variables.
47        Typical usage is:
48        >>>    import nappy.nc_interface.nc_to_na as nc_to_na
49        >>>    c = nc_to_na.NCToNA("old_file.nc")
50        >>>    c.convert()
51        >>>    c.writeNAFiles("new_file.na", delimiter="csv")
52
53        OR:
54        >>>    c = nc_to_na.NCToNA("old_file.nc")
55        >>>    file_names = c.constructNAFileNames()
56        """
57        self.nc_file = nc_file
58
59        # Now need to read CDMS file so parent class methods are compatible
60        (cdms_variables, global_atts) = self._readCDMSFile(self.nc_file, var_ids, exclude_vars)
61        nappy.nc_interface.cdms_to_na.CDMSToNA.__init__(self, cdms_variables, global_atts=global_atts, 
62                    na_items_to_override=na_items_to_override, only_return_file_names=only_return_file_names)
63 
64
65    def _readCDMSFile(self, nc_file, var_ids=None, exclude_vars=[]):
66        """
67        Reads the file and returns all the CDMS variables in a list as well
68        as the global attributes: (cdms_variable_list, global_atts_dict)
69        If var_ids is defined then only get those.
70        """
71        fin = cdms.open(nc_file)
72        cdms_variables = []
73
74        # Make sure var_ids is a list
75        if type(var_ids) == type("string"):
76            var_ids = [var_ids]
77
78        for var_id in fin.listvariables():
79            if var_ids == None or var_id in var_ids:
80                if var_ids not in exclude_vars:
81                    cdms_variables.append(fin(var_id))
82
83        globals = fin.attributes
84        return (cdms_variables, globals) 
85
86    def constructNAFileNames(self, na_file=None):
87        """
88        Works out what the file names of the output NA files will be and
89        returns a list of them.
90        """
91        self.convert()
92        file_names = []
93
94        file_counter = 1
95        for this_na_dict in self.na_dict_list:
96            if len(na_dict_list) == 1:
97                suffix = ""
98            else:
99                suffix = "_%s" % file_counter
100
101            # Create file name
102            name_parts = self.output_file_names[0].split(".")   
103            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
104            file_names.append(new_name)
105            file_counter += 1
106           
107        return file_names
108
109    def writeNAFiles(self, na_file=None, delimiter="    ", 
110                     float_format="%g", size_limit=None):
111        """
112        Writes the self.na_dict_list content to one or more NASA Ames files.
113        Output file names are based on the self.nc_file name unless specified
114        in the na_file_name argument in which case that provides the main name
115        that is appended to if multiple output file names are required.
116        """
117        self.convert() # just in case not already called
118
119        # Gets a list of NA file_names that will be produced.
120        file_names = self.constructNAFileNames(na_file)
121
122        # Set up some counters: file_counter is the expected number of files.
123        #      full_file_counter includes any files that have been split across multiple output NA files
124        #              because size_limit exceeded.
125        file_counter = 1
126        full_file_counter = 1
127
128        # Now loop through writing the outputs
129        for na_dict_and_var_ids in built_na_dicts:
130            file_name = file_names[file_counter - 1]
131            msg = "\nWriting output NASA Ames file: %s" % file_name
132            if DEBUG: print msg
133            self.output_message.append(msg)
134
135            # Set up current na dict
136            this_na_dict = (na_dict_and_var_ids, vars_to_write)
137
138            # Override content of NASA Ames if they are permitted
139            for key in self.na_items_to_override.keys():
140                if key in permitted_overwrite_metadata:   
141                    if key in items_as_lists:
142                        new_item = self.na_items_to_override[key].split()                 
143                    else:
144                        new_item = self.na_items_to_override[key]
145                                   
146                    if new_item != this_na_dict[key]:
147                        this_na_dict[key] = new_item
148                        msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, this_na_dict[key])
149                        if DEBUG: print msg
150                        self.output_message.append(msg)
151       
152            file_list = []
153            # Cope with size limits if specified and FFI is 1001
154            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files.
155            if self.size_limit and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > self.size_limit):
156                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter,
157                                                                        float_format=float_format, size_limit=size_limit)
158                file_list.extend(files_written)
159            # If not having to split file into multiple outputs (normal condition)
160            else:               
161                nappy.openNAFile(file_name, 'w', this_na_dict, delimiter=delimiter, float_format=float_format)
162                file_list.append(file_name)
163
164            # Report on what has been written
165            msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(vars_to_write[0]))
166            if DEBUG: print msg
167            self.output_message.append(msg)
168       
169            msg = ""
170            aux_var_count = this_na_dict[1][1]
171            if len(aux_var_count) > 0:
172                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))     
173           
174            singleton_var_count = this_na_dict[1][2]
175            if len(singleton_var_count) > 0:
176                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count))
177
178            if len(file_list) > 0:
179                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list))
180
181            actual_file_counter += len(file_list)
182            file_counter += 1
183
184            if DEBUG: print msg
185            self.output_message.append(msg)
186           
187        full_file_count = (actual_file_counter - 1)
188        if full_file_count == 1:
189            plural = ""
190        else:
191            plural = "s"             
192        msg = "\n%s file%s written." % (full_file_count, plural)
193   
194        if DEBUG: print msg
195        self.output_message.append(msg)
196        return self.output_message
197
198    def _writeNAFileSubsetsWithinSizeLimit(self, this_na_dict, file_name, delimiter, float_format, size_limit):
199        """
200        If self.size_limit is specified and FFI is 1001 we can chunk the output into
201        different files in a NASA Ames compliant way.
202        Returns list of file names of outputs written.
203        """
204        file_names = []
205        var_list = this_na_dict["V"]
206        array_length = len(var_list[0])
207        nvol_info = divmod(array_length, size_limit)
208        nvol = nvol_info[0]
209
210        # create the number of volumes (files) that need to be written.
211        if nvol_info[1] > 0: nvol = nvol + 1
212
213        start = 0
214        letter_count = 0
215        ivol = 0
216
217        # Loop through until full array length has been written to a set of files.
218        while start < array_length:
219            ivol = ivol + 1
220            end = start + size_limit
221
222            if end > array_length:
223                end = array_length
224
225            current_block = []
226            # Write new V array
227            for v in var_list:
228                current_block.append(v[start:end])
229
230            # Adjust X accordingly in the na dictionary, because independent variable has been reduced in size
231            na_dict_copy = nappy.utils.common_utils.modifyNADictCopy(this_na_dict, current_block, 
232                                                                      start, end, ivol, nvol)
233            # Append a letter to the file name for writing this block to
234            file_name_plus_letter = "%s-%.3d.na" % (file_name[:-3], ivol)
235            file_list.append(file_name_plus_letter)
236
237            # Write data to output file
238            nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy, delimiter=delimiter, 
239                                      float_format=float_format)
240
241            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter)
242            if DEBUG: print msg
243            self.output_message.append(msg)
244            letter_count = letter_count + 1
245            start = end
246
247            file_names.append(file_name_plus_letter) 
248
249        return file_names
Note: See TracBrowser for help on using the repository browser.