source: nappy/trunk/nappy/nappy_api.py @ 3630

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nappy_api.py@3630
Revision 3630, 17.0 KB checked in by astephen, 12 years ago (diff)

Fixed defaults for delimiter and float format so read from nappy.ini config file.

Line 
1"""
2nappy_api.py
3============
4
5Top-level API module that allows user to access most of the useful stuff in
6nappy. API examples:
7
8 1. Working with NASA Ames file objects
9 2. Converting between formats (NASA Ames, NetCDF and CSV)
10 3. Comparing NASA Ames files (and/or CSV files)
11 4. General NASA Ames utilities
12
13 1. Working with NASA Ames file objects
14
15# Start python interactive shell
16$ python
17
18# Import the nappy package
19import nappy
20
21# Let's open a NASA Ames file and examine its contents
22f = nappy.openNAFile("data_files/2010.na")
23
24# Get number of header lines
25n_lines = f.getNumHeaderLines()
26
27# Get Organisation from header
28org = f.getOrg()
29# Get the Normal Comments (SCOM) lines.
30norm_comms = f.getNormalComments()
31
32# Get the Special Comments (SCOM) lines.
33spec_comms = f.getSpecialComments()
34
35# Get a list of metadata for all main (non-auxiliary or independent) variables
36var_list = getVariables()
37
38# Get Auxiliary variable metadata for auxiliary variable number 2
39(variable, units, miss, scale) = f.getAuxVariable(2)
40
41# Get scale factor for primary variable number 3
42scale_factor = f.getScaleFactor(3)
43
44# Get missing value for primary variable number 1
45missing = f.getMissingValue(1)
46
47# Let's get the contents dictionary of the whole file
48na_dict = f.getNADict()
49
50# Let's write the na_dict object to a new NASA Ames file
51fout = openNAFile("test_outputs/mytest.na", mode="w", na_dict=na_dict)
52fout.write()
53fout.close()
54
55 2. Converting between formats (NASA Ames, NetCDF and CSV)
56
57# Let's convert a NASA Ames file into a NetCDF file, and add some of our own global attributes
58glob_atts = {"Project": "Really important scientific project involving worms",
59             "Errata": "I meant worm holes!"}
60na_file = "data_files/1020.na"
61nc_file = "test_outputs/try_1020.nc"
62nappy.convertNAToNC(na_file, nc_file, global_attributes=glob_atts)
63
64# Let's convert a NASA Ames file to a CSV and add an annotation column to explain the header
65nappy.convertNAToCSV(na_file, annotation=True)
66
67# Let's read a NetCDF and write one (or more) output NASA Ames files,
68# but only including and variables "temp" and "ozone". Also let's write
69# the output using tabs as the delimiters and a float format of "%6.3f".
70nappy.convertNCToNA("data_files/test1.nc", "test_outputs/test1nc.na",
71              var_ids=("temp", "ozone"), delimiter="\t", float_format="%6.3f")
72 
73# Let's convert a NetCDF file to one (or more) CSV files and don't write the header at all
74nappy.convertNCToCSV("data_files/test1.nc", "test_outputs/test1nc_no_header.csv",
75                     no_header=True)
76
77# Let's take some in-memory CDMS objects and write them to one, or more, NASA Ames file(s).
78# We need to give it a list of cdms variables and a global attributes dictionary.
79# We also want to instruct nappy to overwrite the content of its
80# MNAME (Mission Name) header line with our specific mission name.
81# Also, tell nappy to write the output to NASA Ames file format index (FFI) 2310
82# because we know it is compatible.
83nappy.convertCDMSObjectsToNA([cdms_var_1, cdms_var_2], {"Institute": "British Atmospheric Data Centre"},
84              na_file="test_outputs/cdms_to_na.na",
85              na_items_to_override={"MNAME": "Atlantic Divergence Mission 2009"},
86              requested_ffi=2310)
87
88# Let's take a list of cdms variables and a global attributes dictionary and write
89# them to a CSV file.
90nappy.convertCDMSObjectsToCSV(cdms_vars, global_atts_dict, csv_file)
91
92# Let's take a NASA Ames dictionary object, and write it to a NetCDF file
93nappy.writeNADictToNC(na_dict, nc_file, mode="w")
94
95# Let's try and write a second na_dict object to the same NetCDF file using mode="a".
96nappy.writeNADictToNC(na_dict_2, nc_file, mode="a")
97
98# Now let's read in a NASA Ames file and convert the contents in-memory into
99# CDMS objects so that we can manipulate them with NetCDF-compatible tools
100(cdms_vars_primary, cdms_vars_aux, global_attributes) = nappy.readCDMSObjectsFromNA(na_file)
101
102# Actually, I only want to get a single variable from that file, so I'll try
103temp_var = getCDMSVariableFromNA(na_file, "temperature")
104
105 3. Comparing NASA Ames files (and/or CSV files)
106
107# I'd like to compare a NASA Ames and CSV file to check they are the same.
108# It will allow for different formatting of numbers as long as the values
109# are the same. Compare both header and body by setting as True (default).
110result = nappy.compareNA(na_file, csv_file, header=True, body=True,
111            number_clever=True, delimiter_1="    ", delimiter_2=",")
112
113 4. General NASA Ames utilities
114
115# Get the FFI from a NASA Ames file
116ffi = nappy.readFFI(na_file)
117
118# Given a NASA Ames dictionary (na_dict) get an appropriate FFI.
119ffi = nappy.chooseFFI(na_dict)
120
121
122"""
123
124# Import standard library modules
125
126# Import third-party software
127try:
128    import cdms
129except:
130    print "WARNING: You cannot use NAPpy's NetCDF conversion tools as your system does not have CDMS installed, or it is not in your sys.path."
131    cdms = False
132
133# Import local modules
134import nappy.utils.common_utils
135import nappy.utils.compare_na
136
137# Bring some utils into the API
138compareNA = nappy.utils.compare_na.compareNA
139readFFI = nappy.utils.common_utils.readFFI
140chooseFFI = nappy.utils.common_utils.chooseFFI
141getNAFileClass = nappy.utils.common_utils.getNAFileClass
142__version__ = nappy.utils.common_utils.getVersion()
143default_delimiter = nappy.utils.common_utils.getDefault("default_delimiter")
144default_float_format = nappy.utils.common_utils.getDefault("default_float_format")
145
146
147def openNAFile(filename, mode="r", na_dict=None):
148    """
149    Function wrapper around the NASA Ames File classes. Any NASA Ames
150    file can be opened through this function and the appropriate read or
151    write NASA Ames File class instance is returned.
152    """
153    if mode == "r":
154        ffi = readFFI(filename)
155        return apply(getNAFileClass(ffi), (filename, mode))
156
157    elif mode == "w":
158        if na_dict.has_key('FFI') and type(na_dict['FFI']) == type(3):
159            ffi = na_dict['FFI']
160        else:
161            ffi = chooseFFI(na_dict)
162            na_dict['FFI'] = ffi
163            print "\nFormat identified as:", ffi   
164        return apply(getNAFileClass(ffi), (filename,), {"mode":mode, "na_dict":na_dict})
165    else:
166        raise Exception("File mode not recognised '" + mode + "'.")
167
168
169def convertNAToNC(na_file, nc_file=None, mode="w", variables=None, aux_variables=None,
170                 global_attributes={"Conventions":"CF-1.0"},
171                 time_units=None, time_warning=True,
172                 rename_variables={}):
173    """
174    Takes a NASA Ames file and converts to a NetCDF file. Options are:
175
176    na_file - the input NASA Ames file.
177    nc_file - name for the output NetCDF file (default is to replace ".na" from NASA Ames
178              file with ".nc").
179    mode - is the file mode, either "w" for write or "a" for append
180    variables - is a list of variable names that you wish to be converted. If not set then
181              nappy will attempt to convert all files.
182    aux_var_list - is a list of auxiliary variables names that you wish to be converted.
183              If not set then nappy will use any compatible variables it finds as
184              auxiliary variables.
185    global_attributes - is a dictionary of global attributes to add to the output file.
186    rename_variables - is a dictionary of {old_name: new_name} variable ID pairs that nappy
187              should use to rename variables before it writes them to file. 
188    time_units - is a valid time units string such as "hours since 2003-04-30 10:00:00" to
189              use for time units if there is a valid time axis.
190    time_warning - suppresses the time units warning for invalid time units if set to False.
191    """
192    arg_dict = vars()
193    for arg_out in ("nc_file", "mode"):
194        del arg_dict[arg_out]
195
196    convertor = apply(nappy.nc_convertor.na_to_nc.NAToNC, [], arg_dict)
197    convertor.convert()
198    if nc_file == None:
199        nc_file = nappy.utils.getFileNameWithNewExtension(na_file, "nc")
200    convertor.writeNCFile(nc_file, mode)
201    print "SHould this return nc file path?"
202    return True   
203 
204
205def convertNAToCSV(na_file, csv_file=None, annotation=False, no_header=False):
206    """
207    Reads in a NASA Ames file and writes it out a new CSV file which is identical to the
208    input file except that commas are used as the delimiter. Arguments are:
209
210    na_file - NASA Ames file path
211    csv_file - CSV file path (default is to replace ".na" from NASA Ames file with ".csv").
212    annotation - if set to True write the output file with an additional left-hand column
213                 describing the contents of each header line.
214    no_header - if set to True then only the data blocks are written to file.
215    """
216    fin = openNAFile(na_file)
217    fin.readData()
218    na_dict = fin.getNADict()
219    fin.close()
220
221    if csv_file == None:
222        csv_file = nappy.utils.getFileNameWithNewExtension(nc_file, "csv")
223    fout = openNAFile(csv_file, "w", na_dict=na_dict)
224    fout.write(delimiter=",", annotation=annotation)
225    fout.close()
226    return True
227
228
229def convertNCToNA(nc_file, na_file=None, var_ids=None, na_items_to_override={},
230            only_return_file_names=False, exclude_vars=[],
231            requested_ffi=None, delimiter=default_delimiter, float_format=default_float_format, 
232            size_limit=None, annotation=False, no_header=False):
233    """
234    Takes a NetCDF file and converts the contents to one or more NASA Ames files.
235    Arguments are:
236
237    nc_file - is the name of input file (NetCDF).
238    na_file - is the name of output file (default is to replace ".nc" from NASA Ames
239              file with ".na"). If multiple files produced then this name will be used
240              as the base name.
241    var_ids - is a list of variables (as ids) to include in the output file(s).
242    na_items_to_override - is a dictionary of {key: value} pairs to overwrite in output
243              files. Typically the keys are in: 
244              ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME".)
245    only_return_file_names - if set to True then only return a list of file names that
246              would be written (i.e. don't convert actual file).
247    exclude_vars - is a list of variables (as ids) to exclude in the output file(s).
248    requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to. Note
249              that there are only limited options available depending on the data
250              structures found.
251    delimiter - the delimiter you wish to use between data items in the output file such
252              as "   ", "\t" or ",".
253    float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for
254              formatting floats when written to file.
255    size_limit - if format FFI is 1001 then chop files up into size_limit rows of data.
256    annotation - if set to True write the output file with an additional left-hand column
257              describing the contents of each header line.
258    no_header - if set to True then only the data blocks are written to file.
259    """
260    arg_dict = vars()
261    for arg_out in ("na_file", "only_return_file_names", "delimiter", "float_format", 
262                    "size_limit", "annotation", "no_header"):
263        del arg_dict[arg_out]
264
265    if na_file == None:
266        na_file =  nappy.utils.getFileNameWithNewExtension(nc_file, "na")
267
268    convertor = apply(nappy.nc_convertor.nc_to_na.NCToNA, [], arg_dict)
269    convertor.convert()
270
271    # If user only wants files then only give them that
272    if only_return_file_names == True:
273        return convertor.constructNAFileNames(na_file)
274    else:
275        convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, 
276                               size_limit=size_limit, annotation=annotation, no_header=no_header)
277        print convertor.output_message
278        print "Should NCToNA return a list of the na file names produced?"
279        return True   
280
281   
282def convertNCToCSV(nc_file, csv_file=None, **arg_dict):
283    """
284    Reads in a NetCDF file and writes the data out to a CSV file following the
285    NASA Ames standard.
286    """
287    if csv_file == None:
288        csv_file = nappy.utils.getFileNameWithNewExtension(nc_file, "csv")
289        arg_dict["na_file"] = csv_file
290        arg_dict["delimiter"] = ","
291 
292    return apply(convertNCToNA, [nc_file], arg_dict)
293   
294
295def convertCDMSObjectsToNA(cdms_vars, global_atts_dict, na_file, 
296              na_items_to_override={}, requested_ffi=None, delimiter=default_delimiter, 
297              float_format=default_float_format, size_limit=None, annotation=False, no_header=False):
298    """
299    Takes a list of cdms variables and a global attributes dictionary and
300    writes them to one or more NASA Ames files. Arguments are:
301 
302    cdms_vars - is a list of CDMS variables
303    global_atts_dict - is a dictionary of {key: value} pairs for header
304    na_items_to_override - is a dictionary of {key: value} pairs to overwrite in
305                output files. Typically the keys are in: 
306                ("DATE", "RDATE", "ANAME", "MNAME","ONAME", "ORG", "SNAME", "VNAME".)
307    requested_ffi - is the NASA Ames File Format Index (FFI) you wish to write to.
308                Note that there are only limited options available depending on the data
309                structures found.
310    delimiter - the delimiter you wish to use between data items in the output file
311                such as "   ", "\t" or ",".
312    float_format - a python formatting string such as "%s", "%g" or "%5.2f" used for
313                formatting floats when written to file.
314    size_limit - if format FFI is 1001 then chop files up into size_limit rows of data.
315    annotation - if set to True write the output file with an additional left-hand
316                column describing the contents of each header line.
317    no_header - if set to True then only the data blocks are written to file.
318    """
319    convertor = nappy.nc_interface.cdms_to_na.CDMSToNA(cdms_vars, global_atts=global_atts_dict, 
320                       na_items_to_override=na_items_to_override, requested_ffi=requested_ffi)
321    convertor.convert()
322    na_files = convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format, 
323                                      annotation=annotation, no_header=no_header) 
324    print "SHOULD WE RETURN FILES WRITTEM????"
325    return True
326
327
328def convertCDMSObjectsToCSV(cdms_vars, global_atts_dict, csv_file, **arg_dict):
329    """
330    Takes a list of cdms variables and a global attributes dictionary and
331    writes them to one or more CSV files.
332    """
333    arg_dict["delimiter"] = ","
334    return apply(convertCDMSObjectsToNA, [cdms_vars, global_atts_dict, csv_file], arg_dict)
335
336
337def writeNADictToNC(na_dict, nc_file, mode="w"):
338    """
339    Writes an NASA Ames dictionary object called na_dict to a NetCDF file called nc_file.
340    Can set mode="a" or mode="w" to either append to existing nc_file or write new one.
341    Note that mode="a" might not always work.
342    """
343    # Note, this needs to pretend that the na_dict exists, do this by instantiating NACore and cheating...
344    na_file_obj = nappy.na_file.na_core.NACore()
345    na_file_obj.setNADict(na_dict)
346    # Fake up some required methods
347    def fakeCaller():pass
348    na_file_obj.readData = fakeCaller
349    convertor = nappy.na_to_cdms.NAToCDMS(na_file_obj)
350    (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert()
351
352    # Now write them out
353    fout = cdms.open(nc_file, mode=mode)
354    for var in (cdms_primary_vars + cdms_aux_vars):
355        fout.write(var)
356
357    # Write global attributes
358    for (att, value) in global_attributes.items():
359        setattr(fout, att, value)
360       
361    fout.close()
362    print "NetCDF file '%s' written successfully." % file_name
363    return True
364
365
366def readCDMSObjectsFromNA(na_file):
367    """
368    Reads the NASA Ames file and converts to CDMS objects.
369    Returns a tuple containing:
370      * a list of primary NASA Ames variables as CDMS variables
371      * a list of auxiliary NASA Ames variables as CDMS variables,
372      * a dictionary of global attributes
373    """
374    cdms_var_list = []
375    global_attributes = {}
376
377    # Open the NA file
378    na_file_obj = openNAFile(na_file)
379    convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj)
380    (cdms_vars_primary, cdms_vars_aux, global_attributes) = convertor.convert()
381    return (cdms_vars_primary, cdms_vars_aux, global_attributes)
382
383
384def getCDMSVariableFromNA(na_file, var):
385    """
386    Returns a CDMS variable object (TransientVariable) identified by the var argument which
387    can either be an integer index in the list of variables or the name of the variable.
388    The variable is created from the variables found in the NASA Ames file na_file.
389    """
390    na_file_obj = openNAFile(na_file)
391    convertor = nappy.nc_interface.na_to_cdms.NADictToCdmsObjects(na_file_obj, variables=[var])
392    (cdms_primary_vars, cdms_aux_vars, global_attributes) = convertor.convert()
393    # Must now be a primary var
394    return cdms_primary_vars[0]
395
396
Note: See TracBrowser for help on using the repository browser.