source: nappy/trunk/nappy/nappy_api.py @ 5151

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nappy_api.py@5151
Revision 5151, 17.5 KB checked in by astephen, 11 years ago (diff)

Fixed some import issues.

Line 
1"""
2nappy_api.py
3============
4
5Top-level API module that allows user to access most of the useful stuff in
6nappy. API examples:
7
8 1. Working with NASA Ames file objects
9 2. Converting between formats (NASA Ames, NetCDF and CSV)
10 3. Comparing NASA Ames files (and/or CSV files)
11 4. General NASA Ames utilities
12
13 1. Working with NASA Ames file objects
14
15# Start python interactive shell
16$ python
17
18# Import the nappy package
19import nappy
20
21# Let's open a NASA Ames file and examine its contents
22f = nappy.openNAFile("data_files/2010.na")
23
24# Get number of header lines
25n_lines = f.getNumHeaderLines()
26
27# Get Organisation from header
28org = f.getOrg()
29# Get the Normal Comments (SCOM) lines.
30norm_comms = f.getNormalComments()
31
32# Get the Special Comments (SCOM) lines.
33spec_comms = f.getSpecialComments()
34
35# Get a list of metadata for all main (non-auxiliary or independent) variables
36var_list = getVariables()
37
38# Get Auxiliary variable metadata for auxiliary variable number 2
39(variable, units, miss, scale) = f.getAuxVariable(2)
40
41# Get scale factor for primary variable number 3
42scale_factor = f.getScaleFactor(3)
43
44# Get missing value for primary variable number 1
45missing = f.getMissingValue(1)
46
47# Let's get the contents dictionary of the whole file
48na_dict = f.getNADict()
49
50# Let's write the na_dict object to a new NASA Ames file
51fout = openNAFile("test_outputs/mytest.na", mode="w", na_dict=na_dict)
52fout.write()
53fout.close()
54
55 2. Converting between formats (NASA Ames, NetCDF and CSV)
56
57# Let's convert a NASA Ames file into a NetCDF file, and add some of our own global attributes
58glob_atts = [("Project", "Really important scientific project involving worms"),
59             ("Errata": "I meant worm holes!")]
60na_file = "data_files/1020.na"
61nc_file = "test_outputs/try_1020.nc"
62nappy.convertNAToNC(na_file, nc_file, global_attributes=glob_atts)
63
64# Let's convert a NASA Ames file to a CSV and add an annotation column to explain the header
65nappy.convertNAToCSV(na_file, annotation=True)
66
67# Let's read a NetCDF and write one (or more) output NASA Ames files,
68# but only including and variables "temp" and "ozone". Also let's write
69# the output using tabs as the delimiters and a float format of "%6.3f".
70nappy.convertNCToNA("data_files/test1.nc", "test_outputs/test1nc.na",
71              var_ids=("temp", "ozone"), delimiter="\t", float_format="%6.3f")
72 
73# Let's convert a NetCDF file to one (or more) CSV files and don't write the header at all
74nappy.convertNCToCSV("data_files/test1.nc", "test_outputs/test1nc_no_header.csv",
75                     no_header=True)
76
77# Let's take some in-memory CDMS objects and write them to one, or more, NASA Ames file(s).
78# We need to give it a list of cdms variables and a global attributes list of tuples/lists.
79# We also want to instruct nappy to overwrite the content of its
80# MNAME (Mission Name) header line with our specific mission name.
81# Also, tell nappy to write the output to NASA Ames file format index (FFI) 2310
82# because we know it is compatible.
83nappy.convertCDMSObjectsToNA([cdms_var_1, cdms_var_2], [("Institute", "British Atmospheric Data Centre")],
84              na_file="test_outputs/cdms_to_na.na",
85              na_items_to_override={"MNAME": "Atlantic Divergence Mission 2009"},
86              requested_ffi=2310)
87
88# Let's take a list of cdms variables and a global attributes list and write
89# them to a CSV file.
90nappy.convertCDMSObjectsToCSV(cdms_vars, global_atttributes, csv_file)
91
92# Let's take a NASA Ames dictionary object, and write it to a NetCDF file
93nappy.writeNADictToNC(na_dict, nc_file, mode="w")
94
95# Let's try and write a second na_dict object to the same NetCDF file using mode="a".
96nappy.writeNADictToNC(na_dict_2, nc_file, mode="a")
97
98# Now let's read in a NASA Ames file and convert the contents in-memory into
99# CDMS objects so that we can manipulate them with NetCDF-compatible tools
100(cdms_vars_primary, cdms_vars_aux, global_attributes) = nappy.readCDMSObjectsFromNA(na_file)
101
102# Actually, I only want to get a single variable from that file, so I'll try
103temp_var = getCDMSVariableFromNA(na_file, "temperature")
104
105 3. Comparing NASA Ames files (and/or CSV files)
106
107# I'd like to compare a NASA Ames and CSV file to check they are the same.
108# It will allow for different formatting of numbers as long as the values
109# are the same. Compare both header and body by setting as True (default).
110result = nappy.compareNA(na_file, csv_file, header=True, body=True,
111            number_clever=True, delimiter_1="    ", delimiter_2=",")
112
113 4. General NASA Ames utilities
114
115# Get the FFI from a NASA Ames file
116ffi = nappy.readFFI(na_file)
117
118# Given a NASA Ames dictionary (na_dict) get an appropriate FFI.
119ffi = nappy.chooseFFI(na_dict)
120
121
122"""
123
124# Import standard library modules
125
126# Import third-party software
127try:
128    import cdms2 as cdms
129except:
130    try:
131        import cdms
132    except:
133        print "WARNING: You cannot use NAPpy's NetCDF conversion tools as your system does not have CDMS installed, or it is not in your sys.path."
134        cdms = False
135
136# Import local modules
137import nappy.utils.common_utils
138import nappy.utils.compare_na
139
140# Bring some utils into the API
141compareNA = nappy.utils.compare_na.compareNA
142readFFI = nappy.utils.common_utils.readFFI
143chooseFFI = nappy.utils.common_utils.chooseFFI
144getNAFileClass = nappy.utils.common_utils.getNAFileClass
145getFileNameWithNewExtension = nappy.utils.common_utils.getFileNameWithNewExtension
146
147__version__ = nappy.utils.common_utils.getVersion()
148default_delimiter = nappy.utils.common_utils.getDefault("default_delimiter")
149default_float_format = nappy.utils.common_utils.getDefault("default_float_format")
150
151
152def openNAFile(filename, mode="r", na_dict=None):
153    """
154    Function wrapper around the NASA Ames File classes. Any NASA Ames
155    file can be opened through this function and the appropriate read or
156    write NASA Ames File class instance is returned.
157    """
158    if mode == "r":
159        ffi = readFFI(filename)
160        return apply(getNAFileClass(ffi), (filename, mode))
161
162    elif mode == "w":
163        if na_dict.has_key('FFI') and type(na_dict['FFI']) == type(3):
164            ffi = na_dict['FFI']
165        else:
166            ffi = chooseFFI(na_dict)
167            na_dict['FFI'] = ffi
168            print "\nFormat identified as:", ffi   
169        return apply(getNAFileClass(ffi), (filename,), {"mode":mode, "na_dict":na_dict})
170    else:
171        raise Exception("File mode not recognised '" + mode + "'.")
172
173
174def convertNAToNC(na_file, nc_file=None, mode="w", variables=None, aux_variables=None,
175                 global_attributes=[("Conventions", "CF-1.0")],
176                 time_units=None, time_warning=True,
177                 rename_variables={}):
178    """
179    Takes a NASA Ames file and converts to a NetCDF file. Options are:
180
181    na_file - the input NASA Ames file.
182    nc_file - name for the output NetCDF file (default is to replace ".na" from NASA Ames
183              file with ".nc").
184    mode - is the file mode, either "w" for write or "a" for append
185    variables - is a list of variable names that you wish to be converted. If not set then
186              nappy will attempt to convert all files.
187    aux_var_list - is a list of auxiliary variables names that you wish to be converted.
188              If not set then nappy will use any compatible variables it finds as
189              auxiliary variables.
190    global_attributes - is a list of global attributes to add to the output file.
191    rename_variables - is a dictionary of {old_name: new_name} variable ID pairs that nappy
192              should use to rename variables before it writes them to file. 
193    time_units - is a valid time units string such as "hours since 2003-04-30 10:00:00" to
194              use for time units if there is a valid time axis.
195    time_warning - suppresses the time units warning for invalid time units if set to False.
196    """
197    arg_dict = vars()
198    for arg_out in ("nc_file", "mode"):
199        del arg_dict[arg_out]
200
201    import nappy.nc_interface.na_to_nc
202    convertor = apply(nappy.nc_interface.na_to_nc.NAToNC, [], arg_dict)
203    convertor.convert()
204    if nc_file == None:
205        nc_file = getFileNameWithNewExtension(na_file, "nc")
206    convertor.writeNCFile(nc_file, mode)
207    return nc_file   
208 
209
210def convertNAToCSV(na_file, csv_file=None, annotation=False, no_header=False):
211    """
212    Reads in a NASA Ames file and writes it out a new CSV file which is identical to the
213    input file except that commas are used as the delimiter. Arguments are:
214
215    na_file - NASA Ames file path
216    csv_file - CSV file path (default is to replace ".na" from NASA Ames file with ".csv").
217    annotation - if set to True write the output file with an additional left-hand column
218                 describing the contents of each header line.
219    no_header - if set to True then only the data blocks are written to file.
220    """
221    fin = openNAFile(na_file)
222    fin.readData()
223    na_dict = fin.getNADict()
224    fin.close()
225
226    if csv_file == None:
227        csv_file = getFileNameWithNewExtension(nc_file, "csv")
228    fout = openNAFile(csv_file, "w", na_dict=na_dict)
229    fout.write(delimiter=",", annotation=annotation)
230    fout.close()
231    return True
232
233
234def convertNCToNA(nc_file, na_file=None, var_ids=None, na_items_to_override={},
235            only_return_file_names=False, exclude_vars=[],
236            requested_ffi=None, delimiter=default_delimiter, float_format=default_float_format, 
237            size_limit=None, annotation=False, no_header=False,
238            ):
239    """
240    Takes a NetCDF file and converts the contents to one or more NASA Ames files.
241    Arguments are:
242
243    nc_file - is the name of input file (NetCDF).
244    na_file - is the name of output file (default is to replace ".nc" from NASA Ames
245              file with ".na"). If multiple files produced then this name will be used
246              as the base name.
247    var_ids - is a list of variables (as ids) to include in the output file(s).
248    na_items_to_override - is a dictionary of {key: value} pairs to overwrite in output
249              files. Typically the keys are in: 
250              ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME".)
251    only_return_file_names - if set to True then only return a list of file names that
252              would be written (i.e. don't convert actual file).
253    exclude_vars - is a list of variables (as ids) to exclude in the output file(s).
254    requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to. Note
255              that there are only limited options available depending on the data
256              structures found.
257    delimiter - the delimiter you wish to use between data items in the output file such
258              as "   ", "\t" or ",".
259    float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for
260              formatting floats when written to file.
261    size_limit - if format FFI is 1001 then chop files up into size_limit rows of data.
262    annotation - if set to True write the output file with an additional left-hand column
263              describing the contents of each header line.
264    no_header - if set to True then only the data blocks are written to file.
265    """
266    arg_dict = vars()
267    for arg_out in ("na_file", "only_return_file_names", "delimiter", "float_format", 
268                    "size_limit", "annotation", "no_header"):
269        del arg_dict[arg_out]
270
271    if na_file == None:
272        na_file =  getFileNameWithNewExtension(nc_file, "na")
273
274    import nappy.nc_interface.nc_to_na
275    convertor = apply(nappy.nc_interface.nc_to_na.NCToNA, [], arg_dict)
276    convertor.convert()
277
278    # If user only wants files then only give them that
279    if only_return_file_names == True:
280        return convertor.constructNAFileNames(na_file)
281    else:
282        convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, 
283                               size_limit=size_limit, annotation=annotation, no_header=no_header)
284        print convertor.output_message
285        output_files_written = convertor.output_files_written
286        print output_files_written
287        return output_files_written
288
289   
290def convertNCToCSV(nc_file, csv_file=None, **arg_dict):
291    """
292    Reads in a NetCDF file and writes the data out to a CSV file following the
293    NASA Ames standard.
294    """
295    if csv_file == None:
296        csv_file = getFileNameWithNewExtension(nc_file, "csv")
297        arg_dict["na_file"] = csv_file
298        arg_dict["delimiter"] = ","
299 
300    return apply(convertNCToNA, [nc_file], arg_dict)
301   
302
303def convertCDMSObjectsToNA(cdms_vars, global_attributes, na_file, 
304              na_items_to_override={}, requested_ffi=None, delimiter=default_delimiter, 
305              float_format=default_float_format, size_limit=None, annotation=False, no_header=False,
306              ):
307    """
308    Takes a list of cdms variables and a list of global attributes and
309    writes them to one or more NASA Ames files. Arguments are:
310 
311    cdms_vars - is a list of CDMS variables
312    global_attributes - is a list of (key, value) pairs for header
313    na_items_to_override - is a dictionary of {key: value} pairs to overwrite in
314                output files. Typically the keys are in: 
315                ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME", "SCOM", "NCOM".)
316    requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to.
317                Note that there are only limited options available depending on the data
318                structures found.
319    delimiter - the delimiter you wish to use between data items in the output file
320                such as "   ", "\t" or ",".
321    float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for
322                formatting floats when written to file.
323    size_limit - if format FFI is 1001 then chop files up into size_limit rows of data.
324    annotation - if set to True write the output file with an additional left-hand
325                column describing the contents of each header line.
326    no_header - if set to True then only the data blocks are written to file.
327    """
328    import nappy.nc_interface.cdms_objs_to_na_file
329    convertor = nappy.nc_interface.cdms_objs_to_na_file.CDMSObjectsToNAFile(cdms_vars, global_attributes=global_attributes, 
330                        na_items_to_override=na_items_to_override, requested_ffi=requested_ffi,
331                        )
332    convertor.convert()
333    na_files = convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, 
334                                      annotation=annotation, no_header=no_header) 
335    return convertor.output_files_written
336
337
338def convertCDMSObjectsToCSV(cdms_vars, global_attributes, csv_file, **arg_dict):
339    """
340    Takes a list of cdms variables and a global attributes list and
341    writes them to one or more CSV files.
342    """
343    arg_dict["delimiter"] = ","
344    return apply(convertCDMSObjectsToNA, [cdms_vars, global_attributes, csv_file], arg_dict)
345
346
347def writeNADictToNC(na_dict, nc_file, mode="w"):
348    """
349    Writes an NASA Ames dictionary object called na_dict to a NetCDF file called nc_file.
350    Can set mode="a" or mode="w" to either append to existing nc_file or write new one.
351    Note that mode="a" might not always work.
352    """
353    # Note, this needs to pretend that the na_dict exists, do this by instantiating NACore and cheating...
354    import nappy.na_file.na_core
355    na_file_obj = nappy.na_file.na_core.NACore()
356    na_file_obj.setNADict(na_dict)
357    # Fake up some required methods
358    def fakeCaller():pass
359    na_file_obj.readData = fakeCaller
360    import nappy.nc_interface.na_to_cdms
361    convertor = nappy.nc_interface.na_to_cdms.NAToCDMS(na_file_obj)
362    (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert()
363
364    # Now write them out
365    fout = cdms.open(nc_file, mode=mode)
366    for var in (cdms_primary_vars + cdms_aux_vars):
367        fout.write(var)
368
369    # Write global attributes
370    for (att, value) in global_attributes:
371        setattr(fout, att, value)
372       
373    fout.close()
374    print "NetCDF file '%s' written successfully." % file_name
375    return True
376
377
378def readCDMSObjectsFromNA(na_file):
379    """
380    Reads the NASA Ames file and converts to CDMS objects.
381    Returns a tuple containing:
382      * a list of primary NASA Ames variables as CDMS variables
383      * a list of auxiliary NASA Ames variables as CDMS variables,
384      * a list of global attributes
385    """
386    cdms_var_list = []
387    global_attributes = {}
388
389    # Open the NA file
390    na_file_obj = openNAFile(na_file)
391    import nappy.nc_interface.na_to_cdms
392    convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj)
393    (cdms_vars_primary, cdms_vars_aux, global_attributes) = convertor.convert()
394    return (cdms_vars_primary, cdms_vars_aux, global_attributes)
395
396
397def getCDMSVariableFromNA(na_file, var):
398    """
399    Returns a CDMS variable object (TransientVariable) identified by the var argument which
400    can either be an integer index in the list of variables or the name of the variable.
401    The variable is created from the variables found in the NASA Ames file na_file.
402    """
403    na_file_obj = openNAFile(na_file)
404    import nappy.nc_interface.na_to_cdms
405    convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj, variables=[var])
406    (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert()
407    # Must now be a primary var
408    return cdms_primary_vars[0]
409
410
Note: See TracBrowser for help on using the repository browser.