source: nappy/trunk/nappy/nc_interface/cdms_to_na.py @ 3469

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/cdms_to_na.py@3469
Revision 3469, 6.8 KB checked in by astephen, 12 years ago (diff)

Some temporary scripts to test functionality before formalising into unit tests.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6cdms_to_na.py
7=============
8
9Holds the class CDMSToNA that converts a set of CDMS variables and global attributes.
10
11"""
12
13# Imports from python standard library
14import sys
15
16# Import from nappy package
17import nappy
18from nappy.na_error import na_error
19import nappy.utils
20import nappy.utils.common_utils
21import nappy.cdms_utils.var_utils
22import nappy.na_file.na_core
23import nappy.nc_interface.na_content_collector
24
25# Import external packages (if available)
26if sys.platform.find("win") > -1:
27    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
28try:
29    import cdms, Numeric
30except:
31    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
32
33cdms.setAutoBounds("off") 
34
35# Define global variables
36permitted_overwrite_metadata = ("DATE",  "RDATE", "ANAME", "MNAME",
37           "ONAME", "ORG", "SNAME", "VNAME")
38items_as_lists = ["DATE", "RDATE", "ANAME", "VNAME"]
39var_limit = 5000 # surely never going to get this many vars in a file!
40
41DEBUG = nappy.utils.getDebug() 
42
43class CDMSToNA:
44    """
45    Converts CDMS objects to NASA Ames file dictionaries.
46    """
47
48    def __init__(self, cdms_variables, global_atts={}, na_items_to_override={}, 
49                 only_return_file_names=False, requested_ffi=None):
50        """
51        Sets up instance variables.     
52        """
53        self.cdms_variables = cdms_variables
54        self.global_atts = global_atts
55        self.na_items_to_override = na_items_to_override
56        self.only_return_file_names = only_return_file_names
57        self.requested_ffi = requested_ffi
58
59        self.converted = False
60        self.output_message = []
61   
62    def convert(self):
63        """
64        Reads the CDMS objects and convert to a set of dictionaries that
65        provide the structure for a NA File object.
66        Returns [(na_dict, var_ids), (na_dict, var_ids), ....]
67        All these na_dict dictionaries can be readily written to a NA File object.
68
69        Note that NASA Ames is not as flexible as NetCDF so you cannot just send any
70        set of variables to write to a NASA Ames file. Essentially there is one
71        multi-dimensional structure and all variables must be defined against it.
72
73        Otherwise variables must be auxiliary variables within that structure (i.e. only
74        defined once per the least changing dimension.
75        """
76        if self.converted == True:
77            print "Already converted to NA dictionary objects."
78            return self.na_dict_list
79       
80        msg = "Reading data from: %s\n" % self.nc_file
81        if DEBUG: print msg
82        self.output_message.append(msg)
83
84        # Convert any singleton variables to CDMS variables
85        variables = self._convertSingletonVars(self.cdms_variables)
86
87        # Re-order variables if they have the attribute "nasa_ames_var_number" which means they came from a NASA Ames file originally
88        variables = self._reorderVars(variables)
89
90        # Make first call to collector class that creates NA dict from CDMS variables and global atts dicts
91        collector = nappy.nc_interface.na_content_collector.NAContentCollector(variables, 
92                                        self.global_atts, requested_ffi=self.requested_ffi)
93        collector.collectNAContent()
94
95        # Return if no files returned
96        if collector.found_na == False:
97            msg = "\nNo files created after variables parsed."
98            if DEBUG: print msg
99            self.output_message.append(msg)
100            return
101
102        # NOTE: collector has attributes: na_dict, var_ids, unused_vars
103
104        # Set up a list to collect multiple calls to content collector
105        na_dict_list = []
106        na_dict_list.append((collector.na_dict, collector.var_ids))
107
108        # If there are variables that were not captured (i.e. unused) by NAContentCollector then loop through these
109        # in attempt to convert all to a set of na_dicts
110        while len(collector.unused_vars) > 0:
111            collector = nappy.nc_interface.na_content_collector.NAContentCollector(collector.unused_vars, 
112                                       self.global_atts, requested_ffi=self.requested_ffi)
113            collector.collectNAContent()           
114            self.output_message += collector.output_message
115
116            # Append to list if more variables were captured
117            if collector.found_na == True: 
118                na_dict_list.append((collector.na_dict, collector.var_ids))
119
120        self.na_dict_list = na_dict_list
121        self.converted = True
122        return self.na_dict_list
123
124    def _convertSingletonVars(self, variables):
125        """
126        Loops through variables to convert singleton variables (i.e. Masked Arrays/Numeric Arrays)
127        to proper CDMS variables. Then code won't break when asking for rank attribute later.
128        Returns a list of CDMS variable objects
129        """
130        vars = []
131
132        for variable in variables:
133            var_obj = variable
134
135            # If singleton variable then convert into proper CDMS variables so code doesn't break later
136            if not hasattr(var_obj, "rank"):
137                var_metadata = var_obj.attributes       
138                var_value = var_obj
139                var_obj = cdms.createVariable(Numeric.array(var_obj), 
140                id=nappy.cdms_utils.var_utils.getBestName(var_metadata).replace(" ", "_"), 
141                                   attributes=var_metadata)
142                var_obj.value = var_obj._data[0]                 
143               
144            vars.append(var_obj)
145
146        return vars
147
148    def _reorderVars(self, variables):
149        """
150        Returns a reordered list of variables. Any that have the attribute
151        "nasa_ames_var_number" get ordered first in the list (according to numbering).
152        """
153        # Set up a long list (longer than number of vars)
154        if len(variables) > var_limit:
155            raise Exception("Can only handle converting less than " + `var_limit` + " variables in any batch.")
156
157        # Collect up those that are ordered and unordered
158        ordered_vars = [None] * var_limit
159        unordered_vars = []
160        for var in variables:
161            var_metadata = var.attributes
162            if hasattr(var_metadata, "nasa_ames_var_number"):
163                num = var_metadata.nasa_ames_var_number
164                ordered_vars[num] = var
165            else:
166                unordered_vars.append(var)
167   
168        vars = []
169        # Clear any None values in ordered_vars and place in final vars list
170        for var in ordered_vars + unordered_vars:
171            if var != None: vars.append(var)
172           
173        return vars
174
Note: See TracBrowser for help on using the repository browser.