source: nappy/trunk/nappy/nc_interface/na_content_collector.py @ 3441

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/nappy/trunk/nappy/nc_interface/na_content_collector.py@3441
Revision 3441, 25.0 KB checked in by astephen, 12 years ago (diff)

Fixes to make NC to NA route work.

Line 
1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ).
2#   This software may be distributed under the terms of the
3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt
4
5"""
6cdms_to_na.py
7=============
8
9Holds the class CDMSToNA that converts a set of CDMS variables and global attributes.
10
11"""
12
13# Imports from python standard library
14import sys
15import time
16import re
17
18# Import from nappy package
19from nappy.na_error import na_error
20import nappy.utils
21import nappy.cdms_utils.axis_utils
22import nappy.cdms_utils.var_utils
23import nappy.utils.common_utils
24import nappy.na_file.na_core
25
26config_dict = nappy.utils.getConfigDict()
27nc_to_na_map = config_dict["nc_to_na_map"]
28version = nappy.utils.getVersion()
29
30# Import external packages (if available)
31if sys.platform.find("win") > -1:
32    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.")
33try:
34    import cdms, Numeric
35except:
36    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.")
37
38cdms.setAutoBounds("off") 
39
40DEBUG = nappy.utils.getDebug() 
41
42class NAContentCollector(nappy.na_file.na_core.NACore):
43    """
44    Class to build a NASA Ames File object from a set of
45    CDMS variables and global attributes (optional).
46    """
47   
48    def __init__(self, variables, global_attributes={}):
49        """
50        Sets up instance variables and calls appropriate methods to
51        generate sections of NASA Ames file object.
52
53        Input arguments are:
54          * variables - list/tuple of actual CDMS variables
55          * global_attributes - dictionary of user-defined globals to include.
56
57        Typical usage:
58        >>> x = NAContentCollector(["temp", "precip"])
59        >>> x.collectNAContent()
60        >>> if x.found_na == True:
61        ...     print x.na_dict, x.var_ids, x.unused_vars
62        """
63        self.output_message = []
64        self.na_dict = {}
65        self.vars = variables
66
67        # Note that self.var_ids will be a list containing:
68        #    [ordered_vars,  auxiliary_vars,   rank_zero_vars]
69        self.var_ids = None
70        self.globals = global_attributes       
71        self.rank_zero_vars = []
72        self.rank_zero_var_ids = []
73
74        # Create a flag to check if anything found
75        self.found_na = False
76
77
78    def collectNAContent(self):
79        """
80        Collect NASA Ames content. Save the contents to the following instance
81        attributes:
82         * self.na_dict
83         * self.var_ids
84         * self.unused_vars
85        """
86        (self.ordered_vars, aux_vars) = self._analyseVariables()
87     
88        if self.ordered_vars == []:
89            print "WARNING: No NASA Ames content created."
90            self.unused_vars = []
91        else:
92            self.var_ids = [[var.id for var in self.ordered_vars],
93                            [var.id for var in aux_vars], 
94                            self.rank_zero_var_ids]
95            self.na_dict["NLHEAD"] = "-999"
96            self._defineNAVars(self.ordered_vars)
97            self._defineNAaux_vars(aux_vars)
98            self._defineNAGlobals()
99            self._defineNAComments()
100            self._defineGeneralHeader()
101            # Quick fudge to cope with 1001 issue
102            if self.na_dict["FFI"] == 1001: 
103                self.na_dict["X"] = self.na_dict["X"][0]
104
105            self.found_na = True
106
107
108    def _analyseVariables(self):
109        """
110        Method to examine the content of CDMS variables to return
111        a tuple of two lists containing variables and auxiliary variables
112        for the NASA Ames file object.
113        Variables not compatible with the first file are put in self.unused_vars
114        """
115        self.unused_vars = []
116
117        highest_rank = -1
118        best_var = None
119        count = 0
120
121        # Need to get highest ranked variable (most dimensions) so that we can work out FFI
122        for var in self.vars:
123            msg = "Analysing: %s" % var.id
124            print msg
125            self.output_message.append(msg)
126            count = count + 1
127            # get rank
128            rank = var.rank()
129
130            # Deal with singleton variables
131            if rank == 0: 
132                self.rank_zero_vars.append(var)
133                self.rank_zero_var_ids.append(var.id)
134                continue
135           
136            # Update highest if highest found or if equals highest with bigger size
137            if rank > highest_rank or (rank == highest_rank and var.size() > best_var.size()):
138                highest_rank = rank
139                best_var = var
140                best_var_index = count
141
142        # If all are zero ranked variables or no vars identified/found then we cannot write any to NASA Ames and return ([], [])
143        if len(self.rank_zero_vars) == len(self.vars) or best_var is None: 
144            return ([], [])
145
146        # Now start to sort the variables into main and auxiliary
147        vars_for_na = [best_var]
148        aux_vars_for_na = []
149        shape = best_var.shape
150        number_of_dims = len(shape)
151        self.na_dict["NIV"] = number_of_dims
152
153        # Get the axes for the main variable being used
154        best_var_axes = best_var.getAxisList()
155       
156        # Get other variable info
157        rest_of_the_vars = self.vars[:best_var_index - 1] + self.vars[best_var_index:]
158
159        for var in rest_of_the_vars:
160
161            if var.id in self.rank_zero_var_ids: continue
162
163            # What to do with variables that have different number of dimensions or different shape
164            if len(var.shape) != number_of_dims or var.shape != shape: 
165                # Could it be an auxiliary variable?
166                if len(var.shape) != 1: 
167                    self.unused_vars.append(var)
168                    continue
169
170                first_axis = var.getAxis(0)
171
172                if nappy.cdms_utils.axis_utils.areAxesIdentical(best_var_axes[0], first_axis) == False: 
173                    self.unused_vars.append(var)
174                    continue
175
176                # I think it is an auxiliary variable
177                aux_vars_for_na.append(var) 
178                # Also put it in unused var bin because auxiliary vars might be useful later on in there own right
179                print "NOTE: Auxiliary variables are recorded in file but also placed in unused category just in case they should be re-used in other files."
180                self.unused_vars.append(var)
181            else:
182                this_var_axes = var.getAxisList()
183
184                # Loop through dimensions
185                for i in range(number_of_dims):           
186                    if nappy.cdms_utils.axis_utils.areAxesIdentical(best_var_axes[i], this_var_axes[i]) == False:
187                        self.unused_vars.append(var)
188                        continue
189
190                # OK, I think the current variable is compatible to write with the best variable along with a NASA Ames file
191                vars_for_na.append(var)
192               
193        # Send vars_for_na AND aux_vars_for_na to a method to check if they have previously been mapped
194        # from NASA Ames. In which case we'll write them back in the order they were initially read from the input file.
195        (vars_for_na, aux_vars_for_na) = self._reorderVarsIfPreviouslyNA(vars_for_na, aux_vars_for_na)
196
197        # Get the FFI
198        self.na_dict["FFI"] = self._decideFileFormatIndex(number_of_dims, aux_vars_for_na)
199        return (vars_for_na, aux_vars_for_na)
200
201
202    def _reorderVarsIfPreviouslyNA(self, vars_for_na, aux_vars_for_na):
203        """
204        Re-order if they previously came from NASA Ames files (i.e. including the
205        attribute 'nasa_ames_var_number'). Return re-ordered or unchanged pair of
206        (vars_for_na, aux_vars_for_na).
207        """
208        # THIS SHOULD REALLY BE DONE IN A LOOP
209        # First do the main variables
210        ordered_vars = [None] * 1000 # Make a long list to put vars in
211        # Create a list of other variables to collect up any that are not labelled as nasa ames variables
212        other_vars = []
213        for var in vars_for_na:
214            if hasattr(var, "nasa_ames_var_number"):
215                ordered_vars[var.nasa_ames_var_number[0]] = var
216            else:
217                other_vars.append(var)
218
219        # Remake vars_for_na now in new order and clean out any that are "None"
220        vars_for_na = []
221        for var in ordered_vars:
222            if var != None: 
223                vars_for_na.append(var)
224
225        vars_for_na = vars_for_na + other_vars
226
227        # Now re-order the Auxiliary variables if they previously came from NASA
228        ordered_aux_vars = [None] * 1000
229        other_aux_vars = []
230
231        for var in aux_vars_for_na:
232            if hasattr(var, "nasa_ames_aux_var_number"):
233                ordered_aux_vars[var.nasa_ames_aux_var_number[0]] = var
234            else:
235                other_aux_vars.append(var)
236
237        # Remake aux_vars_for_na now in order
238        aux_vars_for_na = []
239        for var in ordered_aux_vars:
240            if var != None: 
241                aux_vars_for_na.append(var)
242
243        aux_vars_for_na = aux_vars_for_na + other_aux_vars     
244        return (vars_for_na, aux_vars_for_na)
245
246
247    def _decideFileFormatIndex(self, number_of_dims, aux_vars_for_na):
248        """
249        Based on the number of dimensions and the NASA Ames dictionary return
250        the File Format Index.
251        """
252        if number_of_dims in (2,3,4):
253            ffi = 10 + (number_of_dims * 1000)
254        elif number_of_dims > 4:
255            raise Exception("Cannot write variables defined against greater than 4 axes in NASA Ames format.")
256        else:
257            if len(aux_vars_for_na) > 0 or (self.na_dict.has_key("NAUXV") and self.na_dict["NAUXV"] > 0):
258                ffi = 1010
259            else:
260                ffi = 1001
261        return ffi
262
263
264    def _defineNAVars(self, vars):
265        """
266        Method to define NASA Ames file object variables and their
267        associated metadata.
268        """
269        self.na_dict["NV"] = len(vars)
270        self.na_dict["VNAME"] = []
271        self.na_dict["VMISS"] = []
272        self.na_dict["VSCAL"] = []
273        self.na_dict["V"] = []
274
275        for var in vars:
276            name = nappy.cdms_utils.var_utils.getBestName(var)
277            self.na_dict["VNAME"].append(name)
278            miss = nappy.cdms_utils.var_utils.getMissingValue(var)
279            if type(miss) not in (type(1.2), type(1), type(1L)): 
280                miss = miss[0]
281            self.na_dict["VMISS"].append(miss)
282            self.na_dict["VSCAL"].append(1)
283            # Populate the variable list with the array
284            self.na_dict["V"].append(var._data)
285
286            if not self.na_dict.has_key("X"):
287                self.na_dict["NXDEF"] = []
288                self.na_dict["NX"] = []
289
290                # Create independent variable information
291                self.ax0 = var.getAxis(0)
292                self.na_dict["X"] = [self.ax0[:].tolist()]
293                self.na_dict["XNAME"] = [nappy.cdms_utils.var_utils.getBestName(self.ax0)]
294                if len(self.ax0) == 1:
295                    self.na_dict["DX"] = [0]
296                else:
297                    incr = self.ax0[1] - self.ax0[0]
298                    # Set default increment as gap between first two
299                    self.na_dict["DX"] = [incr]
300                    # Now overwrite it as zero if non-uniform interval in axis
301                    for i in range(1, len(self.ax0)):
302                        if (self.ax0[i] - self.ax0[i - 1]) != incr:
303                            self.na_dict["DX"] = [0]
304                            break
305
306                # Now add the rest of the axes to the self.na_dict objects
307                for axis in var.getAxisList()[1:]:
308                    self._appendAxisDefinition(axis)
309
310
311    def _defineNAaux_vars(self, aux_vars):
312        """
313        Method to define NASA Ames file object auxiliary variables and their
314        associated metadata.
315        """
316        self.na_dict["NAUXV"] = len(aux_vars)
317        self.na_dict["ANAME"] = []
318        self.na_dict["AMISS"] = []
319        self.na_dict["ASCAL"] = []
320        self.na_dict["A"] = []
321
322        for var in aux_vars:
323            name = nappy.cdms_utils.var_utils.getBestName(var)
324            self.na_dict["ANAME"].append(name)
325            miss = nappy.cdms_utils.var_utils.getMissingValue(var)
326            if type(miss) != type(1.1):  miss = miss[0]
327            self.na_dict["AMISS"].append(miss)
328            self.na_dict["ASCAL"].append(1)
329            # Populate the variable list with the array
330            self.na_dict["A"].append(var._data)
331
332    def _appendAxisDefinition(self, axis):
333        """
334        Method to create the appropriate NASA Ames file object
335        items associated with an axis (independent variable in
336        NASA Ames). It appends to the various self.na_dict containers.
337        """
338        length = len(axis)
339
340        self.na_dict["NX"].append(length)
341        self.na_dict["XNAME"].append(nappy.cdms_utils.var_utils.getBestName(axis))
342        # If only one item in axis values
343        if length < 2:
344            self.na_dict["DX"].append(0)
345            self.na_dict["NXDEF"].append(length)
346            self.na_dict["X"].append(axis[:].tolist())       
347            return
348   
349        incr = axis[1] - axis[0]
350        for i in range(1, length):
351            if (axis[i] - axis[i - 1]) != incr:
352                self.na_dict["DX"].append(0)
353                self.na_dict["NXDEF"].append(length)
354                self.na_dict["X"].append(axis.tolist())
355                break
356        else: # If did not break out of the loop
357            max_length = length
358            if length > 3: 
359                max_length = 3
360            self.na_dict["DX"].append(incr)
361            self.na_dict["NXDEF"].append(max_length)
362            self.na_dict["X"].append(axis[:max_length])
363
364    def _defineNAGlobals(self):
365        """
366        Maps CDMS (NetCDF) global attributes into NASA Ames Header fields.
367        """
368        # Check if we should add to it with locally set rules
369        local_attributes = nappy.utils.getConfigDict()["local_attributes"]
370        for att, value in local_attributes.items():
371            if not nc_to_na_map.has_key(att):
372                nc_to_na_map[key] = value
373
374        self.extra_comments = [[],[],[]]  # Normal comments, special comments, other comments
375        convention_or_reference_comments = []
376
377        for key in self.globals.keys():
378            if key != "first_valid_date_of_data" and type(self.globals[key]) \
379                                       not in (type("s"), type(1.1), type(1)):
380                continue
381
382            # Loop through keys of header/comment items to map
383            if key in nc_to_na_map.keys():
384                if key == "history":
385                    time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
386                    history = "History:\t%s - Converted to NASA Ames format using nappy-%s.\n\t%s" % \
387                                                 (time_string, version, self.globals[key])
388                    history = history.split("\n") 
389                    self.history = []
390                    for h in history:
391                        if h[:8] != "History:" and h[:1] != "\t": 
392                            h = "\t" + h
393                        self.history.append(h) 
394                   
395                elif key == "institution":
396                    # If fields came from NA then extract appropriate fields.
397                    match = re.match(r"(.*)\s+\(ONAME from NASA Ames file\);\s+(.*)\s+\(ORG from NASA Ames file\)\.", 
398                             self.globals[key])
399                    if match:
400                        self.na_dict["ONAME"] = match.groups()[0]
401                        self.na_dict["ORG"] = match.groups()[1]
402                    else:
403                        self.na_dict["ONAME"] = self.globals[key]
404                        self.na_dict["ORG"] = self.globals[key]           
405                    # NOTE: should probably do the following search and replace on all string lines
406                    self.na_dict["ONAME"] = self.na_dict["ONAME"].replace("\n", "  ")
407                    self.na_dict["ORG"] = self.na_dict["ORG"].replace("\n", "  ")
408                                   
409                elif key == "comment":
410                    # Need to work out if they are actually comments from NASA Ames in the first place
411                    comment_lines = self.globals[key].split("\n")
412                    normal_comments = []
413                    normal_comm_flag = None
414                    special_comments = []
415                    special_comm_flag = None
416
417                    for line in comment_lines:
418                        if line.find("###NASA Ames Special Comments follow###") > -1:
419                            special_comm_flag = 1
420                        elif line.find("###NASA Ames Special Comments end###") > -1:
421                            special_comm_flag = None
422                        elif line.find("###NASA Ames Normal Comments follow###") > -1:
423                            normal_comm_flag = 1
424                        elif line.find("###NASA Ames Normal Comments end###") > -1:
425                            normal_comm_flag = None     
426                        elif special_comm_flag == 1:
427                            special_comments.append(line)
428                        elif normal_comm_flag == 1:
429                            normal_comments.append(line)
430                        elif line.find("###Data Section begins on the next line###") > -1:
431                            pass
432                        else:
433                            normal_comments.append(line)           
434                   
435                    self.extra_comments = [special_comments, normal_comments, []]                   
436                                   
437                elif key == "first_valid_date_of_data":
438                    self.na_dict["DATE"] = self.globals[key]
439               
440                elif key in ("Conventions", "references"):
441                    #convention_or_reference_comments.append("%s:   %s" % (key, self.globals[key]))
442                    self.extra_comments[2].append("%s:   %s" % (key, self.globals[key]))
443                else:
444                    self.na_dict[nc_to_na_map[key]] = self.globals[key]
445            else:
446                self.extra_comments[2].append("%s:   %s" % (key, self.globals[key]))
447        return
448
449
450    def _defineNAComments(self, normal_comments=[], special_comments=[]):
451        """
452        Defines the Special and Normal comments sections in the NASA Ames file
453        object - including information gathered from the defineNAGlobals method.
454        """
455       
456        if hasattr(self, "ncom"):  normal_comments = self.ncom + normal_comments
457
458        NCOM = []
459        for ncom in normal_comments:
460            NCOM.append(ncom)
461
462        if len(NCOM) > 0:   NCOM.append("")
463       
464        if len(self.extra_comments[2]) > 0:
465            for excom in self.extra_comments[2]:
466                NCOM.append(excom)
467       
468        if len(self.extra_comments[1]) > 0: 
469            NCOM.append("Additional Global Attributes defined in the source file and not translated elsewhere:")
470            for excom in self.extra_comments[1]:
471                NCOM.append(excom)
472
473        if hasattr(self, "history"):
474            for h in self.history:
475                NCOM.append(h)
476       
477        if len(NCOM) > 0:
478            NCOM.insert(0, "###NASA Ames Normal Comments follow###")
479            NCOM.append("")
480            NCOM.append("###NASA Ames Normal Comments end###")
481            NCOM.append("###Data Section begins on the next line###")
482
483        spec_comm_flag = None
484        SCOM = []
485        special_comments = self.extra_comments[0]
486        if len(special_comments) > 0: 
487            SCOM = ["###NASA Ames Special Comments follow###"]
488
489            spec_comm_flag = 1
490        for scom in special_comments:
491            SCOM.append(scom)
492
493        used_var_atts = ("id",  "missing_value", "fill_value", "units", 
494                   "nasa_ames_var_number", "nasa_ames_aux_var_number")
495        var_comm_flag = None
496
497        # Create a string for the Special comments to hold rank-zero vars
498        rank_zero_vars_string = []
499
500        for var in self.rank_zero_vars:
501            rank_zero_vars_string.append("\tVariable %s: %s" % (var.id, nappy.cdms_utils.var_utils.getBestName(var)))
502
503            for att in var.attributes.keys():
504                value = var.attributes[att]
505
506                if type(value) in (type("s"), type(1.0), type(1)):
507
508                    rank_zero_vars_string.append("\t\t%s = %s" % (att, var.attributes[att]))
509
510        if len(rank_zero_vars_string) > 0:
511            rank_zero_vars_string.insert(0, "###Singleton Variables defined in the source file follow###")
512            rank_zero_vars_string.append("###Singleton Variables defined in the source file end###")
513
514        for var in self.ordered_vars:
515            varflag = "unused"
516            name = nappy.cdms_utils.var_utils.getBestName(var)
517
518            for scom,value in var.attributes.items():
519                if type(value) in (type([]), type(Numeric.array([0]))) and len(value) == 1:
520                    value = value[0]
521
522                if type(value) in (type("s"), type(1.1), type(1)) and scom not in used_var_atts:
523                    if varflag == "unused":
524                        if var_comm_flag == None:
525                            var_comm_flag = 1
526
527                    if spec_comm_flag == None:
528                        SCOM = ["###NASA Ames Special Comments follow###"] + rank_zero_vars_string
529                        SCOM.append("Additional Variable Attributes defined in the source file and not translated elsewhere:")
530                        SCOM.append("###Variable attributes from source (NetCDF) file follow###")
531                        varflag = "using" 
532
533                    SCOM.append("\tVariable %s: %s" % (var.id, name))
534                    SCOM.append("\t\t%s = %s" % (scom, value))
535
536        if var_comm_flag == 1: 
537            SCOM.append("###Variable attributes from source (NetCDF) file end###")
538        if spec_comm_flag == 1:
539            SCOM.append("###NASA Ames Special Comments end###")
540
541        # Strip out empty lines (or returns)
542        NCOM_cleaned = []
543        SCOM_cleaned = []
544
545        for c in NCOM:
546            if c.strip() not in ("", " ", "  "):
547                # Replace new lines within one attribute with a newline and tab so easier to read
548                lines = c.split("\n")
549                for line in lines:
550                    if line != lines[0]: 
551                        line = "\t" + line
552                    NCOM_cleaned.append(line)
553                       
554        for c in SCOM:
555            if c.strip() not in ("", " ", "  "):               
556                        # Replace new lines within one attribute with a newline and tab so easier to read
557                lines = c.split("\n")
558                for line in lines:
559                    if line != lines[0]: 
560                        line = "\t" + line
561                    SCOM_cleaned.append(line)
562                   
563        self.na_dict["NCOM"] = NCOM_cleaned
564        self.na_dict["NNCOML"] = len(self.na_dict["NCOM"])
565        self.na_dict["SCOM"] = SCOM_cleaned
566        self.na_dict["NSCOML"] = len(self.na_dict["SCOM"])
567        return
568
569
570    def _defineGeneralHeader(self, header_items={}):
571        """
572        Defines known header items and overwrites any with header_items
573        key/value pairs.
574        """
575        # Check if DATE field previously known in NASA Ames file
576        time_now = time.strftime("%Y %m %d", time.localtime(time.time())).split()
577        if not self.na_dict.has_key("RDATE"):
578            self.na_dict["RDATE"] = time_now
579       
580        if self.ax0.isTime():
581            # Get first date in list
582            try:
583                (unit, start_date) = re.match("(\w+)\s+?since\s+?(\d+-\d+-\d+)", self.ax0.units).groups()           
584                comptime = cdtime.s2c(start_date)
585                first_day = comptime.add(self.na_dict["X"][0][0], getattr(cdtime, unit.capitalize()))
586                self.na_dict["DATE"] = str(first_day).split(" ")[0].replace("-", " ").split()
587            except:
588                msg = "Nappy Warning: Could not get the first date in the file. You will need to manually edit the output file."
589                print msg
590                self.output_message.append(msg)
591                self.na_dict["DATE"] = ("DATE", "NOT", "KNOWN")
592        else: 
593            if not self.na_dict.has_key("DATE"):
594                msg = "Nappy Warning: Could not get the first date in the file. You will need to manually edit the output file."
595                print msg
596                self.output_message.append(msg)
597                self.na_dict["DATE"] = ("DATE", "NOT", "KNOWN")
598            else:
599                pass # i.e. use existing DATE
600
601        self.na_dict["IVOL"] = 1
602        self.na_dict["NVOL"] = 1
603        for key in header_items.keys():
604             self.na_dict[key] = header_items[key]
Note: See TracBrowser for help on using the repository browser.