1 | """ |
---|
2 | nappy_api.py |
---|
3 | ============ |
---|
4 | |
---|
5 | Top-level API module that allows user to access most of the useful stuff in |
---|
6 | nappy. API examples: |
---|
7 | |
---|
8 | 1. Working with NASA Ames file objects |
---|
9 | 2. Converting between formats (NASA Ames, NetCDF and CSV) |
---|
10 | 3. Comparing NASA Ames files (and/or CSV files) |
---|
11 | 4. General NASA Ames utilities |
---|
12 | |
---|
13 | 1. Working with NASA Ames file objects |
---|
14 | |
---|
15 | # Start python interactive shell |
---|
16 | $ python |
---|
17 | |
---|
18 | # Import the nappy package |
---|
19 | import nappy |
---|
20 | |
---|
21 | # Let's open a NASA Ames file and examine its contents |
---|
22 | f = nappy.openNAFile("data_files/2010.na") |
---|
23 | |
---|
24 | # Get number of header lines |
---|
25 | n_lines = f.getNumHeaderLines() |
---|
26 | |
---|
27 | # Get Organisation from header |
---|
28 | org = f.getOrg() |
---|
29 | # Get the Normal Comments (SCOM) lines. |
---|
30 | norm_comms = f.getNormalComments() |
---|
31 | |
---|
32 | # Get the Special Comments (SCOM) lines. |
---|
33 | spec_comms = f.getSpecialComments() |
---|
34 | |
---|
35 | # Get a list of metadata for all main (non-auxiliary or independent) variables |
---|
36 | var_list = getVariables() |
---|
37 | |
---|
38 | # Get Auxiliary variable metadata for auxiliary variable number 2 |
---|
39 | (variable, units, miss, scale) = f.getAuxVariable(2) |
---|
40 | |
---|
41 | # Get scale factor for primary variable number 3 |
---|
42 | scale_factor = f.getScaleFactor(3) |
---|
43 | |
---|
44 | # Get missing value for primary variable number 1 |
---|
45 | missing = f.getMissingValue(1) |
---|
46 | |
---|
47 | # Let's get the contents dictionary of the whole file |
---|
48 | na_dict = f.getNADict() |
---|
49 | |
---|
50 | # Let's write the na_dict object to a new NASA Ames file |
---|
51 | fout = openNAFile("test_outputs/mytest.na", mode="w", na_dict=na_dict) |
---|
52 | fout.write() |
---|
53 | fout.close() |
---|
54 | |
---|
55 | 2. Converting between formats (NASA Ames, NetCDF and CSV) |
---|
56 | |
---|
57 | # Let's convert a NASA Ames file into a NetCDF file, and add some of our own global attributes |
---|
58 | glob_atts = {"Project": "Really important scientific project involving worms", |
---|
59 | "Errata": "I meant worm holes!"} |
---|
60 | na_file = "data_files/1020.na" |
---|
61 | nc_file = "test_outputs/try_1020.nc" |
---|
62 | nappy.convertNAToNC(na_file, nc_file, global_attributes=glob_atts) |
---|
63 | |
---|
64 | # Let's convert a NASA Ames file to a CSV and add an annotation column to explain the header |
---|
65 | nappy.convertNAToCSV(na_file, annotation=True) |
---|
66 | |
---|
67 | # Let's read a NetCDF and write one (or more) output NASA Ames files, |
---|
68 | # but only including and variables "temp" and "ozone". Also let's write |
---|
69 | # the output using tabs as the delimiters and a float format of "%6.3f". |
---|
70 | nappy.convertNCToNA("data_files/test1.nc", "test_outputs/test1nc.na", |
---|
71 | var_ids=("temp", "ozone"), delimiter="\t", float_format="%6.3f") |
---|
72 | |
---|
73 | # Let's convert a NetCDF file to one (or more) CSV files and don't write the header at all |
---|
74 | nappy.convertNCToCSV("data_files/test1.nc", "test_outputs/test1nc_no_header.csv", |
---|
75 | no_header=True) |
---|
76 | |
---|
77 | # Let's take some in-memory CDMS objects and write them to one, or more, NASA Ames file(s). |
---|
78 | # We need to give it a list of cdms variables and a global attributes dictionary. |
---|
79 | # We also want to instruct nappy to overwrite the content of its |
---|
80 | # MNAME (Mission Name) header line with our specific mission name. |
---|
81 | # Also, tell nappy to write the output to NASA Ames file format index (FFI) 2310 |
---|
82 | # because we know it is compatible. |
---|
83 | nappy.convertCDMSObjectsToNA([cdms_var_1, cdms_var_2], {"Institute": "British Atmospheric Data Centre"}, |
---|
84 | na_file="test_outputs/cdms_to_na.na", |
---|
85 | na_items_to_override={"MNAME": "Atlantic Divergence Mission 2009"}, |
---|
86 | requested_ffi=2310) |
---|
87 | |
---|
88 | # Let's take a list of cdms variables and a global attributes dictionary and write |
---|
89 | # them to a CSV file. |
---|
90 | nappy.convertCDMSObjectsToCSV(cdms_vars, global_atts_dict, csv_file) |
---|
91 | |
---|
92 | # Let's take a NASA Ames dictionary object, and write it to a NetCDF file |
---|
93 | nappy.writeNADictToNC(na_dict, nc_file, mode="w") |
---|
94 | |
---|
95 | # Let's try and write a second na_dict object to the same NetCDF file using mode="a". |
---|
96 | nappy.writeNADictToNC(na_dict_2, nc_file, mode="a") |
---|
97 | |
---|
98 | # Now let's read in a NASA Ames file and convert the contents in-memory into |
---|
99 | # CDMS objects so that we can manipulate them with NetCDF-compatible tools |
---|
100 | (cdms_vars_primary, cdms_vars_aux, global_attributes) = nappy.readCDMSObjectsFromNA(na_file) |
---|
101 | |
---|
102 | # Actually, I only want to get a single variable from that file, so I'll try |
---|
103 | temp_var = getCDMSVariableFromNA(na_file, "temperature") |
---|
104 | |
---|
105 | 3. Comparing NASA Ames files (and/or CSV files) |
---|
106 | |
---|
107 | # I'd like to compare a NASA Ames and CSV file to check they are the same. |
---|
108 | # It will allow for different formatting of numbers as long as the values |
---|
109 | # are the same. Compare both header and body by setting as True (default). |
---|
110 | result = nappy.compareNA(na_file, csv_file, header=True, body=True, |
---|
111 | number_clever=True, delimiter_1=" ", delimiter_2=",") |
---|
112 | |
---|
113 | 4. General NASA Ames utilities |
---|
114 | |
---|
115 | # Get the FFI from a NASA Ames file |
---|
116 | ffi = nappy.readFFI(na_file) |
---|
117 | |
---|
118 | # Given a NASA Ames dictionary (na_dict) get an appropriate FFI. |
---|
119 | ffi = nappy.chooseFFI(na_dict) |
---|
120 | |
---|
121 | |
---|
122 | """ |
---|
123 | |
---|
124 | # Import standard library modules |
---|
125 | |
---|
126 | # Import third-party software |
---|
127 | try: |
---|
128 | import cdms |
---|
129 | except: |
---|
130 | print "WARNING: You cannot use NAPpy's NetCDF conversion tools as your system does not have CDMS installed, or it is not in your sys.path." |
---|
131 | cdms = False |
---|
132 | |
---|
133 | # Import local modules |
---|
134 | import nappy.utils.common_utils |
---|
135 | import nappy.utils.compare_na |
---|
136 | |
---|
137 | # Bring some utils into the API |
---|
138 | compareNA = nappy.utils.compare_na.compareNA |
---|
139 | readFFI = nappy.utils.common_utils.readFFI |
---|
140 | chooseFFI = nappy.utils.common_utils.chooseFFI |
---|
141 | getNAFileClass = nappy.utils.common_utils.getNAFileClass |
---|
142 | __version__ = nappy.utils.common_utils.getVersion() |
---|
143 | default_delimiter = nappy.utils.common_utils.getDefault("default_delimiter") |
---|
144 | default_float_format = nappy.utils.common_utils.getDefault("default_float_format") |
---|
145 | |
---|
146 | |
---|
147 | def openNAFile(filename, mode="r", na_dict=None): |
---|
148 | """ |
---|
149 | Function wrapper around the NASA Ames File classes. Any NASA Ames |
---|
150 | file can be opened through this function and the appropriate read or |
---|
151 | write NASA Ames File class instance is returned. |
---|
152 | """ |
---|
153 | if mode == "r": |
---|
154 | ffi = readFFI(filename) |
---|
155 | return apply(getNAFileClass(ffi), (filename, mode)) |
---|
156 | |
---|
157 | elif mode == "w": |
---|
158 | if na_dict.has_key('FFI') and type(na_dict['FFI']) == type(3): |
---|
159 | ffi = na_dict['FFI'] |
---|
160 | else: |
---|
161 | ffi = chooseFFI(na_dict) |
---|
162 | na_dict['FFI'] = ffi |
---|
163 | print "\nFormat identified as:", ffi |
---|
164 | return apply(getNAFileClass(ffi), (filename,), {"mode":mode, "na_dict":na_dict}) |
---|
165 | else: |
---|
166 | raise Exception("File mode not recognised '" + mode + "'.") |
---|
167 | |
---|
168 | |
---|
169 | def convertNAToNC(na_file, nc_file=None, mode="w", variables=None, aux_variables=None, |
---|
170 | global_attributes={"Conventions":"CF-1.0"}, |
---|
171 | time_units=None, time_warning=True, |
---|
172 | rename_variables={}): |
---|
173 | """ |
---|
174 | Takes a NASA Ames file and converts to a NetCDF file. Options are: |
---|
175 | |
---|
176 | na_file - the input NASA Ames file. |
---|
177 | nc_file - name for the output NetCDF file (default is to replace ".na" from NASA Ames |
---|
178 | file with ".nc"). |
---|
179 | mode - is the file mode, either "w" for write or "a" for append |
---|
180 | variables - is a list of variable names that you wish to be converted. If not set then |
---|
181 | nappy will attempt to convert all files. |
---|
182 | aux_var_list - is a list of auxiliary variables names that you wish to be converted. |
---|
183 | If not set then nappy will use any compatible variables it finds as |
---|
184 | auxiliary variables. |
---|
185 | global_attributes - is a dictionary of global attributes to add to the output file. |
---|
186 | rename_variables - is a dictionary of {old_name: new_name} variable ID pairs that nappy |
---|
187 | should use to rename variables before it writes them to file. |
---|
188 | time_units - is a valid time units string such as "hours since 2003-04-30 10:00:00" to |
---|
189 | use for time units if there is a valid time axis. |
---|
190 | time_warning - suppresses the time units warning for invalid time units if set to False. |
---|
191 | """ |
---|
192 | arg_dict = vars() |
---|
193 | for arg_out in ("nc_file", "mode"): |
---|
194 | del arg_dict[arg_out] |
---|
195 | |
---|
196 | convertor = apply(nappy.nc_convertor.na_to_nc.NAToNC, [], arg_dict) |
---|
197 | convertor.convert() |
---|
198 | if nc_file == None: |
---|
199 | nc_file = nappy.utils.getFileNameWithNewExtension(na_file, "nc") |
---|
200 | convertor.writeNCFile(nc_file, mode) |
---|
201 | print "SHould this return nc file path?" |
---|
202 | return True |
---|
203 | |
---|
204 | |
---|
205 | def convertNAToCSV(na_file, csv_file=None, annotation=False, no_header=False): |
---|
206 | """ |
---|
207 | Reads in a NASA Ames file and writes it out a new CSV file which is identical to the |
---|
208 | input file except that commas are used as the delimiter. Arguments are: |
---|
209 | |
---|
210 | na_file - NASA Ames file path |
---|
211 | csv_file - CSV file path (default is to replace ".na" from NASA Ames file with ".csv"). |
---|
212 | annotation - if set to True write the output file with an additional left-hand column |
---|
213 | describing the contents of each header line. |
---|
214 | no_header - if set to True then only the data blocks are written to file. |
---|
215 | """ |
---|
216 | fin = openNAFile(na_file) |
---|
217 | fin.readData() |
---|
218 | na_dict = fin.getNADict() |
---|
219 | fin.close() |
---|
220 | |
---|
221 | if csv_file == None: |
---|
222 | csv_file = nappy.utils.getFileNameWithNewExtension(nc_file, "csv") |
---|
223 | fout = openNAFile(csv_file, "w", na_dict=na_dict) |
---|
224 | fout.write(delimiter=",", annotation=annotation) |
---|
225 | fout.close() |
---|
226 | return True |
---|
227 | |
---|
228 | |
---|
229 | def convertNCToNA(nc_file, na_file=None, var_ids=None, na_items_to_override={}, |
---|
230 | only_return_file_names=False, exclude_vars=[], |
---|
231 | requested_ffi=None, delimiter=default_delimiter, float_format=default_float_format, |
---|
232 | size_limit=None, annotation=False, no_header=False): |
---|
233 | """ |
---|
234 | Takes a NetCDF file and converts the contents to one or more NASA Ames files. |
---|
235 | Arguments are: |
---|
236 | |
---|
237 | nc_file - is the name of input file (NetCDF). |
---|
238 | na_file - is the name of output file (default is to replace ".nc" from NASA Ames |
---|
239 | file with ".na"). If multiple files produced then this name will be used |
---|
240 | as the base name. |
---|
241 | var_ids - is a list of variables (as ids) to include in the output file(s). |
---|
242 | na_items_to_override - is a dictionary of {key: value} pairs to overwrite in output |
---|
243 | files. Typically the keys are in: |
---|
244 | ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME".) |
---|
245 | only_return_file_names - if set to True then only return a list of file names that |
---|
246 | would be written (i.e. don't convert actual file). |
---|
247 | exclude_vars - is a list of variables (as ids) to exclude in the output file(s). |
---|
248 | requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to. Note |
---|
249 | that there are only limited options available depending on the data |
---|
250 | structures found. |
---|
251 | delimiter - the delimiter you wish to use between data items in the output file such |
---|
252 | as " ", "\t" or ",". |
---|
253 | float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for |
---|
254 | formatting floats when written to file. |
---|
255 | size_limit - if format FFI is 1001 then chop files up into size_limit rows of data. |
---|
256 | annotation - if set to True write the output file with an additional left-hand column |
---|
257 | describing the contents of each header line. |
---|
258 | no_header - if set to True then only the data blocks are written to file. |
---|
259 | """ |
---|
260 | arg_dict = vars() |
---|
261 | for arg_out in ("na_file", "only_return_file_names", "delimiter", "float_format", |
---|
262 | "size_limit", "annotation", "no_header"): |
---|
263 | del arg_dict[arg_out] |
---|
264 | |
---|
265 | if na_file == None: |
---|
266 | na_file = nappy.utils.getFileNameWithNewExtension(nc_file, "na") |
---|
267 | |
---|
268 | convertor = apply(nappy.nc_convertor.nc_to_na.NCToNA, [], arg_dict) |
---|
269 | convertor.convert() |
---|
270 | |
---|
271 | # If user only wants files then only give them that |
---|
272 | if only_return_file_names == True: |
---|
273 | return convertor.constructNAFileNames(na_file) |
---|
274 | else: |
---|
275 | convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, |
---|
276 | size_limit=size_limit, annotation=annotation, no_header=no_header) |
---|
277 | print convertor.output_message |
---|
278 | print "Should NCToNA return a list of the na file names produced?" |
---|
279 | return True |
---|
280 | |
---|
281 | |
---|
282 | def convertNCToCSV(nc_file, csv_file=None, **arg_dict): |
---|
283 | """ |
---|
284 | Reads in a NetCDF file and writes the data out to a CSV file following the |
---|
285 | NASA Ames standard. |
---|
286 | """ |
---|
287 | if csv_file == None: |
---|
288 | csv_file = nappy.utils.getFileNameWithNewExtension(nc_file, "csv") |
---|
289 | arg_dict["na_file"] = csv_file |
---|
290 | arg_dict["delimiter"] = "," |
---|
291 | |
---|
292 | return apply(convertNCToNA, [nc_file], arg_dict) |
---|
293 | |
---|
294 | |
---|
295 | def convertCDMSObjectsToNA(cdms_vars, global_atts_dict, na_file, |
---|
296 | na_items_to_override={}, requested_ffi=None, delimiter=default_delimiter, |
---|
297 | float_format=default_float_format, size_limit=None, annotation=False, no_header=False): |
---|
298 | """ |
---|
299 | Takes a list of cdms variables and a global attributes dictionary and |
---|
300 | writes them to one or more NASA Ames files. Arguments are: |
---|
301 | |
---|
302 | cdms_vars - is a list of CDMS variables |
---|
303 | global_atts_dict - is a dictionary of {key: value} pairs for header |
---|
304 | na_items_to_override - is a dictionary of {key: value} pairs to overwrite in |
---|
305 | output files. Typically the keys are in: |
---|
306 | ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME".) |
---|
307 | requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to. |
---|
308 | Note that there are only limited options available depending on the data |
---|
309 | structures found. |
---|
310 | delimiter - the delimiter you wish to use between data items in the output file |
---|
311 | such as " ", "\t" or ",". |
---|
312 | float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for |
---|
313 | formatting floats when written to file. |
---|
314 | size_limit - if format FFI is 1001 then chop files up into size_limit rows of data. |
---|
315 | annotation - if set to True write the output file with an additional left-hand |
---|
316 | column describing the contents of each header line. |
---|
317 | no_header - if set to True then only the data blocks are written to file. |
---|
318 | """ |
---|
319 | convertor = nappy.nc_interface.cdms_to_na.CDMSToNA(cdms_vars, global_atts=global_atts_dict, |
---|
320 | na_items_to_override=na_items_to_override, requested_ffi=requested_ffi) |
---|
321 | convertor.convert() |
---|
322 | na_files = convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, |
---|
323 | annotation=annotation, no_header=no_header) |
---|
324 | print "SHOULD WE RETURN FILES WRITTEM????" |
---|
325 | return True |
---|
326 | |
---|
327 | |
---|
328 | def convertCDMSObjectsToCSV(cdms_vars, global_atts_dict, csv_file, **arg_dict): |
---|
329 | """ |
---|
330 | Takes a list of cdms variables and a global attributes dictionary and |
---|
331 | writes them to one or more CSV files. |
---|
332 | """ |
---|
333 | arg_dict["delimiter"] = "," |
---|
334 | return apply(convertCDMSObjectsToNA, [cdms_vars, global_atts_dict, csv_file], arg_dict) |
---|
335 | |
---|
336 | |
---|
337 | def writeNADictToNC(na_dict, nc_file, mode="w"): |
---|
338 | """ |
---|
339 | Writes an NASA Ames dictionary object called na_dict to a NetCDF file called nc_file. |
---|
340 | Can set mode="a" or mode="w" to either append to existing nc_file or write new one. |
---|
341 | Note that mode="a" might not always work. |
---|
342 | """ |
---|
343 | # Note, this needs to pretend that the na_dict exists, do this by instantiating NACore and cheating... |
---|
344 | na_file_obj = nappy.na_file.na_core.NACore() |
---|
345 | na_file_obj.setNADict(na_dict) |
---|
346 | # Fake up some required methods |
---|
347 | def fakeCaller():pass |
---|
348 | na_file_obj.readData = fakeCaller |
---|
349 | convertor = nappy.na_to_cdms.NAToCDMS(na_file_obj) |
---|
350 | (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert() |
---|
351 | |
---|
352 | # Now write them out |
---|
353 | fout = cdms.open(nc_file, mode=mode) |
---|
354 | for var in (cdms_primary_vars + cdms_aux_vars): |
---|
355 | fout.write(var) |
---|
356 | |
---|
357 | # Write global attributes |
---|
358 | for (att, value) in global_attributes.items(): |
---|
359 | setattr(fout, att, value) |
---|
360 | |
---|
361 | fout.close() |
---|
362 | print "NetCDF file '%s' written successfully." % file_name |
---|
363 | return True |
---|
364 | |
---|
365 | |
---|
366 | def readCDMSObjectsFromNA(na_file): |
---|
367 | """ |
---|
368 | Reads the NASA Ames file and converts to CDMS objects. |
---|
369 | Returns a tuple containing: |
---|
370 | * a list of primary NASA Ames variables as CDMS variables |
---|
371 | * a list of auxiliary NASA Ames variables as CDMS variables, |
---|
372 | * a dictionary of global attributes |
---|
373 | """ |
---|
374 | cdms_var_list = [] |
---|
375 | global_attributes = {} |
---|
376 | |
---|
377 | # Open the NA file |
---|
378 | na_file_obj = openNAFile(na_file) |
---|
379 | convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj) |
---|
380 | (cdms_vars_primary, cdms_vars_aux, global_attributes) = convertor.convert() |
---|
381 | return (cdms_vars_primary, cdms_vars_aux, global_attributes) |
---|
382 | |
---|
383 | |
---|
384 | def getCDMSVariableFromNA(na_file, var): |
---|
385 | """ |
---|
386 | Returns a CDMS variable object (TransientVariable) identified by the var argument which |
---|
387 | can either be an integer index in the list of variables or the name of the variable. |
---|
388 | The variable is created from the variables found in the NASA Ames file na_file. |
---|
389 | """ |
---|
390 | na_file_obj = openNAFile(na_file) |
---|
391 | convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj, variables=[var]) |
---|
392 | (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert() |
---|
393 | # Must now be a primary var |
---|
394 | return cdms_primary_vars[0] |
---|
395 | |
---|
396 | |
---|