source: nappy/trunk/nappy/nc_interface/cdms_to_na.py @ 3349

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/cdms_to_na.py@3349
Revision 3349, 7.7 KB checked in by astephen, 12 years ago (diff)

Tidied some variable names and boolean types.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6cdms_to_na.py
7=============
8
9Holds the class CDMSToNA that converts a set of CDMS variables and global attributes.
10
11"""
12
13# Imports from python standard library
14import sys
15import os
16import time
17import string
18import re
19
20# Import from nappy package
21from nappy.na_error import na_error
22import nappy.utils
23import nappy.utils.common_utils
24import nappy.cdms_utils.var_utils
25import nappy.na_file.na_core
26
27nc_to_na_map = utils.getConfigDict()["nc_to_na_map"]
28
29# Import external packages (if available)
30if sys.platform.find("win") > -1:
31    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
32try:
33    import cdms, Numeric
34except:
35    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
36
37cdms.setAutoBounds("off") 
38
39cdms2na - 200 lines of code to do main conversion, needs to be split out into other stuff.
40 * getVariableCollections(f and varlist) --> (ordered_vars, other_vars)
41 * buildNADicts()
42 * writeToOutputFiles()
43
44
45
46def cdms2na(ncfile, na_file_names, na_vars={}, variables=None, only_return_file_names="no", 
47            ffi="automatic", spacer="    ", float_format="%g", size_limit=None):
48    """
49    Main conversion function that calls the appropriate classes and functions
50    to write a NASA Ames file.
51    """
52    if type(na_file_names) == type("string"): 
53        na_file_names = [na_file_names]
54   
55    # Get which NASA Ames internal variables are allowed to be overwritten in the output files (i.e. by user inputs)
56    allowed_overwrite_metadata = ("DATE",  "RDATE", "ANAME", "MNAME",
57           "ONAME", "ORG", "SNAME", "VNAME")
58    array_args = ["DATE", "RDATE", "ANAME", "VNAME"]
59    output_message = []
60    msg = "Reading data from: %s\n" % infilename
61    print msg
62    output_message.append(msg)
63    cdms_file = cdms.open(infilename)
64    globals = cdms_file.attributes
65   
66    vars = []
67    if not variables:
68        variables = cdms_file.listvariables()
69
70    for variable in variables:
71        var_obj = cdms_file(variable)
72
73        # Deal with singleton variables
74        if not hasattr(var_obj, "rank"):
75                varMetadata = cdms_file[variable].attributes
76                varValue = var_obj
77                var_obj = cdms.createVariable(Numeric.array(var_obj), id=nappy.cdms_utils.var_utils.getBestName(varMetadata).replace(" ", "_"), attributes=varMetadata)
78                var_obj.value = var_obj._data[0]                 
79        vars.append(var_obj)
80       
81    # Re-order variables if they have the attribute 'nasa_ames_var_number'
82    ordered_vars = [None] * 1000
83    other_vars = []
84    for var in vars:
85        varMetadata = cdms_file[var]
86        if hasattr(varMetadata, "nasa_ames_var_number"):
87            num = varMetadata.nasa_ames_var_number
88            ordered_vars[num] = var
89        else:
90            other_vars.append(var)
91   
92    vars = []
93    for var in ordered_vars:
94        if var != None:
95            vars.append(var)
96           
97    vars = vars + other_vars
98   
99    builder = NAContentCollector(vars, globals, rule=rule, cdms_file=cdms_file)
100    built_na_dicts = [[builder.na_dict, builder.var_ids]]
101
102    if builder.var_ids == None:
103        msg = "\nNo files created after variables parsed."
104        print msg
105        output_message.append(msg)
106        return output_message
107
108    while len(builder.varBin) > 0:
109        builder = NAContentCollector(builder.varBin, globals, rule=rule, cdms_file=cdms_file)
110        output_message = output_message + builder.output_message
111        if builder.var_ids != None:  built_na_dicts.append([builder.na_dict, builder.var_ids])
112
113    # Return only file_names if only want to know them now.
114    ncount = 1
115    file_names = []
116    if only_return_file_names == "yes": 
117        for i in built_na_dicts:
118            if len(built_na_dicts) == 1:
119                suffix = ""
120            else:
121                suffix = "_%s" % ncount
122            name_parts = output_file_names[0].split(".")   
123            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
124            file_names.append(new_name)
125        ncount = ncount + 1
126           
127        return file_names
128               
129    msg = "\n%s files to write" % len(built_na_dicts)
130    print msg
131    output_message.append(msg)
132
133    count = 1
134    ncount = 1
135    for i in built_na_dicts:
136        if len(output_file_names) == 1:
137            if len(built_na_dicts) == 1:
138                suffix = ""
139            else:
140                suffix = "_%s" % ncount
141            name_parts = output_file_names[0].split(".")   
142            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1]
143        else:
144            new_name = output_file_names[count - 1]
145 
146        msg = "\nWriting output NASA Ames file: %s" % new_name
147        print msg
148        output_message.append(msg)
149       
150        builtNADict = i[0]
151        for key in na_vars.keys():
152            if key in allowed_overwrite_metadata:
153           
154                if key in array_args:
155                    newItem = na_vars[key].split()                 
156                else:
157                    newItem = na_vars[key]
158                                   
159                if newItem != builtNADict[key]:
160                    builtNADict[key] = newItem
161                    msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, builtNADict[key])
162                    print msg
163                    output_message.append(msg)
164       
165        fileList = []
166        # Cope with size limits if specified and FFI is 1001
167        if size_limit and (builtNADict["FFI"] == 1001 and len(builtNADict["V"][0]) > size_limit):
168            varList = builtNADict["V"]
169            arrayLength = len(varList[0])
170            nvolInfo = divmod(arrayLength, size_limit)
171            nvol = nvolInfo[0]
172            if nvolInfo[1] > 0: nvol = nvol + 1
173            start = 0
174            letterCount = 0
175            ivol = 0
176            while start < arrayLength:
177                ivol = ivol + 1
178                end = start + size_limit
179                if end > arrayLength:
180                    end = arrayLength
181                currentBlock = []
182                # Write new V array
183                for v in varList:
184                    currentBlock.append(v[start:end])
185
186                # Adjust X accordingly
187                NADictCopy = nappy.utils.common_utils.modifyNADictCopy(builtNADict, currentBlock, start, end, ivol, nvol)
188               
189                # Write data to output file
190                new_namePlusLetter = "%s-%.3d.na" % (new_name[:-3], ivol)
191                fileList.append(new_namePlusLetter)
192                general.openNAFile(new_namePlusLetter, 'w', NADictCopy, spacer=spacer, float_format=float_format)
193                msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, new_namePlusLetter)
194                print msg
195                output_message.append(msg)
196                letterCount = letterCount + 1
197                start = end
198
199
200        else:           
201            general.openNAFile(new_name, 'w', builtNADict, spacer=spacer, float_format=float_format)
202
203        msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(i[1][0]))
204        print msg
205        output_message.append(msg)
206       
207        if len(i[1][1]) > 0:
208            msg = "\nWrote the following auxiliary variables:"
209            msg = msg + "\n\t" + ("\n\t".join(i[1][1])) 
210           
211        if len(i[1][2]) > 0:
212            msg = "\nWrote the following Singleton variables:"
213            msg = msg + "\n\t" + ("\n\t".join(i[1][2]))
214
215        if len(fileList) > 0:
216            msg = msg + ("\n\nNASA Ames files written successfully: \n%s" % "\n".join(fileList))
217            count = count + len(fileList)
218        else:
219            msg = msg + "\n\nNASA Ames file written successfully: %s" % new_name
220            count = count + 1
221        ncount = ncount + 1
222
223        print msg
224        output_message.append(msg)
225           
226    if (count - 1) == 1:
227        plural = ""
228    else:
229        plural = "s"         
230    msg = "\n%s file%s written." % ((count - 1), plural)
231    print msg
232    output_message.append(msg)
233    return output_message
234
Note: See TracBrowser for help on using the repository browser.