source: nappy/trunk/nappy/nc_interface/cdms_to_na.py @ 3347

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/cdms_to_na.py@3347
Revision 3347, 8.0 KB checked in by astephen, 13 years ago (diff)

Splitting up cdms to na stuff into more modules to make more readable.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6cdms_to_na.py
7=============
8
9Holds the class CDMSToNA that converts a set of CDMS variables and global attributes.
10
11"""
12
13# Imports from python standard library
14import sys
15import os
16import time
17import string
18import re
19
20# Import from nappy package
21from nappy.na_error import na_error
22import nappy.utils
23import nappy.utils.common_utils
24import nappy.cdms_utils.var_utils
25import nappy.na_file.na_core
26
27nc_to_na_map = utils.getConfigDict()["nc_to_na_map"]
28
29# Import external packages (if available)
30if sys.platform.find("win") > -1:
31    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
32try:
33    import cdms, Numeric
34except:
35    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
36
37cdms.setAutoBounds("off") 
38
39cdms2na - 200 lines of code to do main conversion, needs to be split out into other stuff.
40 * getVariableCollections(f and varlist) --> (ordered_vars, other_vars)
41 * buildNADicts()
42 * writeToOutputFiles()
43
44
45
46def cdms2na(ncfile, na_file_names, naVars={}, variables=None, nFilesOnly="no", 
47            ffi="automatic", spacer="    ", floatFormat="%g", size_limit=None):
48    """
49    Main conversion function that calls the appropriate classes and functions
50    to write a NASA Ames file.
51    """
52    #print infilename, outfilenames, nFilesOnly, naVars, variables
53    if type(na_file_names) == type("string"): 
54        na_file_names = [na_file_names]
55   
56    # Get which NASA Ames internal variables are allowed to be overwritten in the output files (i.e. by user inputs)
57    allowed_overwrite_metadata = ("DATE",  "RDATE", "ANAME", "MNAME",
58           "ONAME", "ORG", "SNAME", "VNAME")
59    array_args = ["DATE", "RDATE", "ANAME", "VNAME"]
60    output_message = []
61    msg = "Reading data from: %s\n" % infilename
62    print msg
63    output_message.append(msg)
64    cdms_file = cdms.open(infilename)
65    globals = cdms_file.attributes
66   
67    vars = []
68    if not variables:
69        variables = cdms_file.listvariables()
70        #for var in cdms_file.listvariables():
71            #vars.append(cdms_file(var))   
72           
73    for variable in variables:
74        varObj = cdms_file(variable)
75        # Deal with singleton variables
76        if not hasattr(varObj, "rank"):
77                varMetadata = cdms_file[variable].attributes
78                varValue = varObj
79                #print varMetadata, varValue, varMetadata.keys(), varMetadata._obj_.id
80                varObj = cdms.createVariable(Numeric.array(varObj), id=nappy.cdms_utils.var_utils.getBestName(varMetadata).replace(" ", "_"), attributes=varMetadata)
81                #print varObj, dir(varObj); sys.exit()
82                varObj.value = varObj._data[0]
83                #varObj.rank = 0
84               
85        #print varObj, varObj.attributes                         
86        vars.append(varObj)
87       
88    # Re-order variables if they have the attribute 'nasa_ames_var_number'
89    ordered_vars = [None] * 1000
90    other_vars = []
91    for var in vars:
92        varMetadata = cdms_file[var]
93        if hasattr(varMetadata, "nasa_ames_var_number"):
94            num = varMetadata.nasa_ames_var_number
95            ordered_vars[num] = var
96        else:
97            other_vars.append(var)
98   
99    vars = []
100    for var in ordered_vars:
101        if var != None:
102            vars.append(var)
103           
104    vars = vars + other_vars
105   
106    builder = NAContentCollector(vars, globals, rule=rule, cdms_file=cdms_file)
107    #print builder.naDict["X"]
108    builtNADicts = [[builder.naDict, builder.varIDs]]
109    if builder.varIDs == None:
110        msg = "\nNo files created after variables parsed."
111        print msg
112        output_message.append(msg)
113        return output_message
114
115    while len(builder.varBin) > 0:
116        builder = NAContentCollector(builder.varBin, globals, rule=rule, cdms_file=cdms_file)
117        output_message = output_message + builder.output_message
118        if builder.varIDs != None:  builtNADicts.append([builder.naDict, builder.varIDs])
119
120    # Return only filenames if only want to know them now.
121    ncount = 1
122    fileNames = []
123    if nFilesOnly == "yes": 
124        for i in builtNADicts:
125            if len(builtNADicts) == 1:
126                suffix = ""
127            else:
128                suffix = "_%s" % ncount
129            nameparts = outfilenames[0].split(".")   
130            newname = (".".join(nameparts[:-1])) + suffix + "." + nameparts[-1]
131            fileNames.append(newname)
132        ncount = ncount + 1
133           
134        return fileNames
135               
136    msg = "\n%s files to write" % len(builtNADicts)
137    print msg
138    output_message.append(msg)
139
140    count = 1
141    ncount = 1
142    for i in builtNADicts:
143        if len(outfilenames) == 1:
144            if len(builtNADicts) == 1:
145                suffix = ""
146            else:
147                suffix = "_%s" % ncount
148            nameparts = outfilenames[0].split(".")   
149            newname = (".".join(nameparts[:-1])) + suffix + "." + nameparts[-1]
150        else:
151            newname = outfilenames[count - 1]
152 
153        msg = "\nWriting output NASA Ames file: %s" % newname
154        print msg
155        output_message.append(msg)
156       
157        builtNADict = i[0]
158        for key in naVars.keys():
159            if key in allowed_overwrite_metadata:
160           
161                if key in array_args:
162                    newItem = naVars[key].split()                 
163                else:
164                    newItem = naVars[key]
165                                   
166                if newItem != builtNADict[key]:
167                    builtNADict[key] = newItem
168                    msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, builtNADict[key])
169                    print msg
170                    output_message.append(msg)
171       
172        fileList = []
173        # Cope with size limits if specified and FFI is 1001
174        if size_limit and (builtNADict["FFI"] == 1001 and len(builtNADict["V"][0]) > size_limit):
175            varList = builtNADict["V"]
176            arrayLength = len(varList[0])
177            nvolInfo = divmod(arrayLength, size_limit)
178            nvol = nvolInfo[0]
179            if nvolInfo[1] > 0: nvol = nvol + 1
180            start = 0
181            letterCount = 0
182            ivol = 0
183            while start < arrayLength:
184                ivol = ivol + 1
185                end = start + size_limit
186                if end > arrayLength:
187                    end = arrayLength
188                currentBlock = []
189                # Write new V array
190                for v in varList:
191                    currentBlock.append(v[start:end])
192
193                # Adjust X accordingly
194                NADictCopy = nappy.utils.common_utils.modifyNADictCopy(builtNADict, currentBlock, start, end, ivol, nvol)
195               
196                # Write data to output file
197                newnamePlusLetter = "%s-%.3d.na" % (newname[:-3], ivol)
198                fileList.append(newnamePlusLetter)
199                general.openNAFile(newnamePlusLetter, 'w', NADictCopy, spacer=spacer, floatFormat=floatFormat)
200                msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, newnamePlusLetter)
201                print msg
202                output_message.append(msg)
203                letterCount = letterCount + 1
204                start = end
205
206
207        else:           
208            general.openNAFile(newname, 'w', builtNADict, spacer=spacer, floatFormat=floatFormat)
209
210        msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(i[1][0]))
211        print msg
212        output_message.append(msg)
213       
214        if len(i[1][1]) > 0:
215            msg = "\nWrote the following auxiliary variables:"
216            msg = msg + "\n\t" + ("\n\t".join(i[1][1])) 
217           
218        if len(i[1][2]) > 0:
219            msg = "\nWrote the following Singleton variables:"
220            msg = msg + "\n\t" + ("\n\t".join(i[1][2]))
221
222        if len(fileList) > 0:
223            msg = msg + ("\n\nNASA Ames files written successfully: \n%s" % "\n".join(fileList))
224            count = count + len(fileList)
225        else:
226            msg = msg + "\n\nNASA Ames file written successfully: %s" % newname
227            count = count + 1
228        ncount = ncount + 1
229
230        print msg
231        output_message.append(msg)
232           
233    if (count - 1) == 1:
234        plural = ""
235    else:
236        plural = "s"         
237    msg = "\n%s file%s written." % ((count - 1), plural)
238    print msg
239    output_message.append(msg)
240    return output_message
241
Note: See TracBrowser for help on using the repository browser.