source: nappy/trunk/nappy/nappy_api.py @ 5368

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nappy_api.py@5368
Revision 5368, 17.6 KB checked in by astephen, 11 years ago (diff)

Replace print with logging

Line 
1"""
2nappy_api.py
3============
4
5Top-level API module that allows user to access most of the useful stuff in
6nappy. API examples:
7
8 1. Working with NASA Ames file objects
9 2. Converting between formats (NASA Ames, NetCDF and CSV)
10 3. Comparing NASA Ames files (and/or CSV files)
11 4. General NASA Ames utilities
12
13 1. Working with NASA Ames file objects
14
15# Start python interactive shell
16$ python
17
18# Import the nappy package
19import nappy
20
21# Let's open a NASA Ames file and examine its contents
22f = nappy.openNAFile("data_files/2010.na")
23
24# Get number of header lines
25n_lines = f.getNumHeaderLines()
26
27# Get Organisation from header
28org = f.getOrg()
29# Get the Normal Comments (SCOM) lines.
30norm_comms = f.getNormalComments()
31
32# Get the Special Comments (SCOM) lines.
33spec_comms = f.getSpecialComments()
34
35# Get a list of metadata for all main (non-auxiliary or independent) variables
36var_list = getVariables()
37
38# Get Auxiliary variable metadata for auxiliary variable number 2
39(variable, units, miss, scale) = f.getAuxVariable(2)
40
41# Get scale factor for primary variable number 3
42scale_factor = f.getScaleFactor(3)
43
44# Get missing value for primary variable number 1
45missing = f.getMissingValue(1)
46
47# Let's get the contents dictionary of the whole file
48na_dict = f.getNADict()
49
50# Let's write the na_dict object to a new NASA Ames file
51fout = openNAFile("test_outputs/mytest.na", mode="w", na_dict=na_dict)
52fout.write()
53fout.close()
54
55 2. Converting between formats (NASA Ames, NetCDF and CSV)
56
57# Let's convert a NASA Ames file into a NetCDF file, and add some of our own global attributes
58glob_atts = [("Project", "Really important scientific project involving worms"),
59             ("Errata": "I meant worm holes!")]
60na_file = "data_files/1020.na"
61nc_file = "test_outputs/try_1020.nc"
62nappy.convertNAToNC(na_file, nc_file, global_attributes=glob_atts)
63
64# Let's convert a NASA Ames file to a CSV and add an annotation column to explain the header
65nappy.convertNAToCSV(na_file, annotation=True)
66
67# Let's read a NetCDF and write one (or more) output NASA Ames files,
68# but only including and variables "temp" and "ozone". Also let's write
69# the output using tabs as the delimiters and a float format of "%6.3f".
70nappy.convertNCToNA("data_files/test1.nc", "test_outputs/test1nc.na",
71              var_ids=("temp", "ozone"), delimiter="\t", float_format="%6.3f")
72 
73# Let's convert a NetCDF file to one (or more) CSV files and don't write the header at all
74nappy.convertNCToCSV("data_files/test1.nc", "test_outputs/test1nc_no_header.csv",
75                     no_header=True)
76
77# Let's take some in-memory CDMS objects and write them to one, or more, NASA Ames file(s).
78# We need to give it a list of cdms variables and a global attributes list of tuples/lists.
79# We also want to instruct nappy to overwrite the content of its
80# MNAME (Mission Name) header line with our specific mission name.
81# Also, tell nappy to write the output to NASA Ames file format index (FFI) 2310
82# because we know it is compatible.
83nappy.convertCDMSObjectsToNA([cdms_var_1, cdms_var_2], [("Institute", "British Atmospheric Data Centre")],
84              na_file="test_outputs/cdms_to_na.na",
85              na_items_to_override={"MNAME": "Atlantic Divergence Mission 2009"},
86              requested_ffi=2310)
87
88# Let's take a list of cdms variables and a global attributes list and write
89# them to a CSV file.
90nappy.convertCDMSObjectsToCSV(cdms_vars, global_atttributes, csv_file)
91
92# Let's take a NASA Ames dictionary object, and write it to a NetCDF file
93nappy.writeNADictToNC(na_dict, nc_file, mode="w")
94
95# Let's try and write a second na_dict object to the same NetCDF file using mode="a".
96nappy.writeNADictToNC(na_dict_2, nc_file, mode="a")
97
98# Now let's read in a NASA Ames file and convert the contents in-memory into
99# CDMS objects so that we can manipulate them with NetCDF-compatible tools
100(cdms_vars_primary, cdms_vars_aux, global_attributes) = nappy.readCDMSObjectsFromNA(na_file)
101
102# Actually, I only want to get a single variable from that file, so I'll try
103temp_var = getCDMSVariableFromNA(na_file, "temperature")
104
105 3. Comparing NASA Ames files (and/or CSV files)
106
107# I'd like to compare a NASA Ames and CSV file to check they are the same.
108# It will allow for different formatting of numbers as long as the values
109# are the same. Compare both header and body by setting as True (default).
110result = nappy.compareNA(na_file, csv_file, header=True, body=True,
111            number_clever=True, delimiter_1="    ", delimiter_2=",")
112
113 4. General NASA Ames utilities
114
115# Get the FFI from a NASA Ames file
116ffi = nappy.readFFI(na_file)
117
118# Given a NASA Ames dictionary (na_dict) get an appropriate FFI.
119ffi = nappy.chooseFFI(na_dict)
120
121
122"""
123
124# Import standard library modules
125import logging
126
127# Import third-party software
128try:
129    import cdms2 as cdms
130except:
131    try:
132        import cdms
133    except:
134        log.warn("You cannot use nappy NetCDF conversion tools as your system does not have CDMS installed, or it is not in your sys.path.")
135        cdms = False
136
137# Import local modules
138import nappy.utils.common_utils
139import nappy.utils.compare_na
140
141# Bring some utils into the API
142compareNA = nappy.utils.compare_na.compareNA
143readFFI = nappy.utils.common_utils.readFFI
144chooseFFI = nappy.utils.common_utils.chooseFFI
145getNAFileClass = nappy.utils.common_utils.getNAFileClass
146getFileNameWithNewExtension = nappy.utils.common_utils.getFileNameWithNewExtension
147
148__version__ = nappy.utils.common_utils.getVersion()
149default_delimiter = nappy.utils.common_utils.getDefault("default_delimiter")
150default_float_format = nappy.utils.common_utils.getDefault("default_float_format")
151
152logging.basicConfig(level=logging.INFO)
153log = logging.getLogger(__name__)
154
155
156def openNAFile(filename, mode="r", na_dict=None):
157    """
158    Function wrapper around the NASA Ames File classes. Any NASA Ames
159    file can be opened through this function and the appropriate read or
160    write NASA Ames File class instance is returned.
161    """
162    if mode == "r":
163        ffi = readFFI(filename)
164        return apply(getNAFileClass(ffi), (filename, mode))
165
166    elif mode == "w":
167        if na_dict.has_key('FFI') and type(na_dict['FFI']) == type(3):
168            ffi = na_dict['FFI']
169        else:
170            ffi = chooseFFI(na_dict)
171            na_dict['FFI'] = ffi
172            log.info("\nFormat identified as: %s" % ffi)
173        return apply(getNAFileClass(ffi), (filename,), {"mode":mode, "na_dict":na_dict})
174    else:
175        raise Exception("File mode not recognised '" + mode + "'.")
176
177
178def convertNAToNC(na_file, nc_file=None, mode="w", variables=None, aux_variables=None,
179                 global_attributes=[("Conventions", "CF-1.0")],
180                 time_units=None, time_warning=True,
181                 rename_variables={}):
182    """
183    Takes a NASA Ames file and converts to a NetCDF file. Options are:
184
185    na_file - the input NASA Ames file.
186    nc_file - name for the output NetCDF file (default is to replace ".na" from NASA Ames
187              file with ".nc").
188    mode - is the file mode, either "w" for write or "a" for append
189    variables - is a list of variable names that you wish to be converted. If not set then
190              nappy will attempt to convert all files.
191    aux_var_list - is a list of auxiliary variables names that you wish to be converted.
192              If not set then nappy will use any compatible variables it finds as
193              auxiliary variables.
194    global_attributes - is a list of global attributes to add to the output file.
195    rename_variables - is a dictionary of {old_name: new_name} variable ID pairs that nappy
196              should use to rename variables before it writes them to file. 
197    time_units - is a valid time units string such as "hours since 2003-04-30 10:00:00" to
198              use for time units if there is a valid time axis.
199    time_warning - suppresses the time units warning for invalid time units if set to False.
200    """
201    arg_dict = vars()
202    for arg_out in ("nc_file", "mode"):
203        del arg_dict[arg_out]
204
205    import nappy.nc_interface.na_to_nc
206    convertor = apply(nappy.nc_interface.na_to_nc.NAToNC, [], arg_dict)
207    convertor.convert()
208    if nc_file == None:
209        nc_file = getFileNameWithNewExtension(na_file, "nc")
210    convertor.writeNCFile(nc_file, mode)
211    return nc_file   
212 
213
214def convertNAToCSV(na_file, csv_file=None, annotation=False, no_header=False):
215    """
216    Reads in a NASA Ames file and writes it out a new CSV file which is identical to the
217    input file except that commas are used as the delimiter. Arguments are:
218
219    na_file - NASA Ames file path
220    csv_file - CSV file path (default is to replace ".na" from NASA Ames file with ".csv").
221    annotation - if set to True write the output file with an additional left-hand column
222                 describing the contents of each header line.
223    no_header - if set to True then only the data blocks are written to file.
224    """
225    fin = openNAFile(na_file)
226    fin.readData()
227    na_dict = fin.getNADict()
228    fin.close()
229
230    if csv_file == None:
231        csv_file = getFileNameWithNewExtension(nc_file, "csv")
232    fout = openNAFile(csv_file, "w", na_dict=na_dict)
233    fout.write(delimiter=",", annotation=annotation)
234    fout.close()
235    return True
236
237
238def convertNCToNA(nc_file, na_file=None, var_ids=None, na_items_to_override={},
239            only_return_file_names=False, exclude_vars=[],
240            requested_ffi=None, delimiter=default_delimiter, float_format=default_float_format, 
241            size_limit=None, annotation=False, no_header=False,
242            ):
243    """
244    Takes a NetCDF file and converts the contents to one or more NASA Ames files.
245    Arguments are:
246
247    nc_file - is the name of input file (NetCDF).
248    na_file - is the name of output file (default is to replace ".nc" from NASA Ames
249              file with ".na"). If multiple files produced then this name will be used
250              as the base name.
251    var_ids - is a list of variables (as ids) to include in the output file(s).
252    na_items_to_override - is a dictionary of {key: value} pairs to overwrite in output
253              files. Typically the keys are in: 
254              ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME".)
255    only_return_file_names - if set to True then only return a list of file names that
256              would be written (i.e. don't convert actual file).
257    exclude_vars - is a list of variables (as ids) to exclude in the output file(s).
258    requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to. Note
259              that there are only limited options available depending on the data
260              structures found.
261    delimiter - the delimiter you wish to use between data items in the output file such
262              as "   ", "\t" or ",".
263    float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for
264              formatting floats when written to file.
265    size_limit - if format FFI is 1001 then chop files up into size_limit rows of data.
266    annotation - if set to True write the output file with an additional left-hand column
267              describing the contents of each header line.
268    no_header - if set to True then only the data blocks are written to file.
269    """
270    arg_dict = vars()
271    for arg_out in ("na_file", "only_return_file_names", "delimiter", "float_format", 
272                    "size_limit", "annotation", "no_header"):
273        del arg_dict[arg_out]
274
275    if na_file == None:
276        na_file =  getFileNameWithNewExtension(nc_file, "na")
277
278    import nappy.nc_interface.nc_to_na
279    convertor = apply(nappy.nc_interface.nc_to_na.NCToNA, [], arg_dict)
280    convertor.convert()
281
282    # If user only wants files then only give them that
283    if only_return_file_names == True:
284        return convertor.constructNAFileNames(na_file)
285    else:
286        convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, 
287                               size_limit=size_limit, annotation=annotation, no_header=no_header)
288        log.info(convertor.output_message)
289        output_files_written = convertor.output_files_written
290        log.info(output_files_written)
291        return output_files_written
292
293   
294def convertNCToCSV(nc_file, csv_file=None, **arg_dict):
295    """
296    Reads in a NetCDF file and writes the data out to a CSV file following the
297    NASA Ames standard.
298    """
299    if csv_file == None:
300        csv_file = getFileNameWithNewExtension(nc_file, "csv")
301        arg_dict["na_file"] = csv_file
302        arg_dict["delimiter"] = ","
303 
304    return apply(convertNCToNA, [nc_file], arg_dict)
305   
306
307def convertCDMSObjectsToNA(cdms_vars, global_attributes, na_file, 
308              na_items_to_override={}, requested_ffi=None, delimiter=default_delimiter, 
309              float_format=default_float_format, size_limit=None, annotation=False, no_header=False,
310              ):
311    """
312    Takes a list of cdms variables and a list of global attributes and
313    writes them to one or more NASA Ames files. Arguments are:
314 
315    cdms_vars - is a list of CDMS variables
316    global_attributes - is a list of (key, value) pairs for header
317    na_items_to_override - is a dictionary of {key: value} pairs to overwrite in
318                output files. Typically the keys are in: 
319                ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME", "SCOM", "NCOM".)
320    requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to.
321                Note that there are only limited options available depending on the data
322                structures found.
323    delimiter - the delimiter you wish to use between data items in the output file
324                such as "   ", "\t" or ",".
325    float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for
326                formatting floats when written to file.
327    size_limit - if format FFI is 1001 then chop files up into size_limit rows of data.
328    annotation - if set to True write the output file with an additional left-hand
329                column describing the contents of each header line.
330    no_header - if set to True then only the data blocks are written to file.
331    """
332    import nappy.nc_interface.cdms_objs_to_na_file
333    convertor = nappy.nc_interface.cdms_objs_to_na_file.CDMSObjectsToNAFile(cdms_vars, global_attributes=global_attributes, 
334                        na_items_to_override=na_items_to_override, requested_ffi=requested_ffi,
335                        )
336    convertor.convert()
337    na_files = convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, 
338                                      annotation=annotation, no_header=no_header) 
339    return convertor.output_files_written
340
341
342def convertCDMSObjectsToCSV(cdms_vars, global_attributes, csv_file, **arg_dict):
343    """
344    Takes a list of cdms variables and a global attributes list and
345    writes them to one or more CSV files.
346    """
347    arg_dict["delimiter"] = ","
348    return apply(convertCDMSObjectsToNA, [cdms_vars, global_attributes, csv_file], arg_dict)
349
350
351def writeNADictToNC(na_dict, nc_file, mode="w"):
352    """
353    Writes an NASA Ames dictionary object called na_dict to a NetCDF file called nc_file.
354    Can set mode="a" or mode="w" to either append to existing nc_file or write new one.
355    Note that mode="a" might not always work.
356    """
357    # Note, this needs to pretend that the na_dict exists, do this by instantiating NACore and cheating...
358    import nappy.na_file.na_core
359    na_file_obj = nappy.na_file.na_core.NACore()
360    na_file_obj.setNADict(na_dict)
361    # Fake up some required methods
362    def fakeCaller():pass
363    na_file_obj.readData = fakeCaller
364    import nappy.nc_interface.na_to_cdms
365    convertor = nappy.nc_interface.na_to_cdms.NAToCDMS(na_file_obj)
366    (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert()
367
368    # Now write them out
369    fout = cdms.open(nc_file, mode=mode)
370    for var in (cdms_primary_vars + cdms_aux_vars):
371        fout.write(var)
372
373    # Write global attributes
374    for (att, value) in global_attributes:
375        setattr(fout, att, value)
376       
377    fout.close()
378    log.info("NetCDF file '%s' written successfully." % file_name)
379    return True
380
381
382def readCDMSObjectsFromNA(na_file):
383    """
384    Reads the NASA Ames file and converts to CDMS objects.
385    Returns a tuple containing:
386      * a list of primary NASA Ames variables as CDMS variables
387      * a list of auxiliary NASA Ames variables as CDMS variables,
388      * a list of global attributes
389    """
390    cdms_var_list = []
391    global_attributes = {}
392
393    # Open the NA file
394    na_file_obj = openNAFile(na_file)
395    import nappy.nc_interface.na_to_cdms
396    convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj)
397    (cdms_vars_primary, cdms_vars_aux, global_attributes) = convertor.convert()
398    return (cdms_vars_primary, cdms_vars_aux, global_attributes)
399
400
401def getCDMSVariableFromNA(na_file, var):
402    """
403    Returns a CDMS variable object (TransientVariable) identified by the var argument which
404    can either be an integer index in the list of variables or the name of the variable.
405    The variable is created from the variables found in the NASA Ames file na_file.
406    """
407    na_file_obj = openNAFile(na_file)
408    import nappy.nc_interface.na_to_cdms
409    convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj, variables=[var])
410    (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert()
411    # Must now be a primary var
412    return cdms_primary_vars[0]
413
414
Note: See TracBrowser for help on using the repository browser.