Changeset 3392 for nappy


Ignore:
Timestamp:
12/02/08 18:42:18 (12 years ago)
Author:
astephen
Message:
 
File:
1 edited

Legend:

Unmodified
Added
Removed
  • nappy/trunk/nappy/nc_interface/nc_to_na.py

    r3333 r3392  
     1#   Copyright (C) 2004 CCLRC & NERC( Natural Environment Research Council ). 
     2#   This software may be distributed under the terms of the 
     3#   Q Public License, version 1.0 or later. http://ndg.nerc.ac.uk/public_docs/QPublic_license.txt 
     4 
     5""" 
     6nc_to_na.py 
     7============= 
     8 
     9Holds the class NCToNA (sub-classing CDMSToNA) that converts a NetCDF file to 
     10one or more NASA Ames files. 
     11 
     12""" 
     13 
     14# Imports from python standard library 
     15import sys 
     16 
     17# Import from nappy package 
     18import nappy 
     19from nappy.na_error import na_error 
     20import nappy.utils 
     21import nappy.utils.common_utils 
     22import nappy.cdms_utils.var_utils 
     23import nappy.na_file.na_core 
     24import nappy.nc_interface.na_content_collector 
     25 
     26# Import external packages (if available) 
     27if sys.platform.find("win") > -1: 
     28    raise na_error.NAPlatformError("Windows does not support CDMS. CDMS is required to convert to CDMS objects and NetCDF.") 
     29try: 
     30    import cdms, Numeric 
     31except: 
     32    raise Exception("Could not import third-party software. Nappy requires the CDMS and Numeric packages to be installed to convert to CDMS and NetCDF.") 
     33 
     34cdms.setAutoBounds("off")  
     35 
     36# Define global variables 
     37DEBUG = nappy.utils.DEBUG  
     38 
     39class NCToNA(nappy.nc_interface.cdms_to_na.CDMSToNA): 
     40    """ 
     41    Converts a NetCDF file to one or more NASA Ames files. 
     42    """ 
     43 
     44    def __init__(self, nc_file, var_ids=None, na_items_to_override={},  
     45            only_return_file_names=False, exclude_vars=[]): 
     46        """ 
     47        Sets up instance variables. 
     48        Typical usage is: 
     49        >>>    import nappy.nc_interface.nc_to_na as nc_to_na 
     50        >>>    c = nc_to_na.NCToNA("old_file.nc")  
     51        >>>    c.convert() 
     52        >>>    c.writeNAFiles("new_file.na", delimiter="csv")  
     53 
     54        OR: 
     55        >>>    c = nc_to_na.NCToNA("old_file.nc")  
     56        >>>    file_names = c.constructNAFileNames()  
     57        """ 
     58        self.nc_file = nc_file 
     59 
     60        # Now need to read CDMS file so parent class methods are compatible 
     61        (cdms_variables, global_atts) = self._readCDMSFile(self.nc_file, var_ids, exclude_vars) 
     62        nappy.nc_interface.cdms_to_na.CDMSToNA(self, cdms_variables, global_atts=global_atts,  
     63                    na_items_to_override=na_items_to_override, only_return_file_names=only_return_file_names) 
     64  
     65 
     66    def _readCDMSFile(self, nc_file, var_ids=None, exclude_vars=[]): 
     67        """ 
     68        Reads the file and returns all the CDMS variables in a list as well 
     69        as the global attributes: (cdms_variable_list, global_atts_dict) 
     70        If var_ids is defined then only get those. 
     71        """ 
     72        fin = cdms.open(nc_file) 
     73        cdms_variables = [] 
     74 
     75        # Make sure var_ids is a list 
     76        if type(var_ids) == type("string"): 
     77            var_ids = [var_ids] 
     78 
     79        for var_id in f.listvariables(): 
     80            if var_ids == None or var_id in var_ids: 
     81                if var_ids not in exclude_vars: 
     82                    cdms_variables.append(fin(var_id)) 
     83 
     84        globals = cdms_file.attributes 
     85        return (cdms_variables, globals)  
     86 
     87    def constructNAFileNames(self, na_file=None): 
     88        """ 
     89        Works out what the file names of the output NA files will be and  
     90        returns a list of them. 
     91        """ 
     92        self.convert() 
     93        file_names = [] 
     94 
     95        file_counter = 1 
     96        for this_na_dict in self.na_dict_list: 
     97            if len(na_dict_list) == 1: 
     98                    suffix = "" 
     99                else: 
     100                    suffix = "_%s" % file_counter 
     101 
     102            # Create file name 
     103            name_parts = self.output_file_names[0].split(".")     
     104            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1] 
     105            file_names.append(new_name) 
     106            file_counter += 1 
     107             
     108        return file_names 
     109 
     110    def writeNAFiles(self, na_file=None, delimiter="    ",  
     111                     float_format="%g", size_limit=None): 
     112        """ 
     113        Writes the self.na_dict_list content to one or more NASA Ames files. 
     114        Output file names are based on the self.nc_file name unless specified 
     115        in the na_file_name argument in which case that provides the main name 
     116        that is appended to if multiple output file names are required. 
     117        """ 
     118        self.convert() # just in case not already called 
     119 
     120        # Gets a list of NA file_names that will be produced. 
     121        file_names = self.constructNAFileNames(na_file) 
     122 
     123        # Set up some counters: file_counter is the expected number of files. 
     124        #      full_file_counter includes any files that have been split across multiple output NA files 
     125        #              because size_limit exceeded. 
     126        file_counter = 1 
     127        full_file_counter = 1 
     128 
     129        # Now loop through writing the outputs 
     130        for na_dict_and_var_ids in built_na_dicts: 
     131            file_name = file_names[file_counter - 1] 
     132            msg = "\nWriting output NASA Ames file: %s" % file_name 
     133            if DEBUG: print msg 
     134            self.output_message.append(msg) 
     135 
     136            # Set up current na dict 
     137            this_na_dict = (na_dict_and_var_ids, vars_to_write) 
     138 
     139            # Override content of NASA Ames if they are permitted 
     140            for key in self.na_items_to_override.keys(): 
     141                if key in permitted_overwrite_metadata:     
     142                    if key in items_as_lists: 
     143                        new_item = self.na_items_to_override[key].split()                   
     144                    else: 
     145                        new_item = self.na_items_to_override[key] 
     146                                     
     147                    if new_item != this_na_dict[key]: 
     148                        this_na_dict[key] = new_item 
     149                        msg = "Metadata overwritten in output file: '%s' is now '%s'" % (key, this_na_dict[key]) 
     150                        if DEBUG: print msg 
     151                        self.output_message.append(msg) 
     152         
     153            file_list = [] 
     154            # Cope with size limits if specified and FFI is 1001 
     155            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files. 
     156            if self.size_limit and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > self.size_limit): 
     157                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter, 
     158                                                                        float_format=float_format, size_limit=size_limit) 
     159                file_list.extend(files_written) 
     160            # If not having to split file into multiple outputs (normal condition) 
     161            else:                
     162                nappy.openNAFile(file_name, 'w', this_na_dict, delimiter=delimiter, float_format=float_format) 
     163                file_list.append(file_name) 
     164 
     165            # Report on what has been written 
     166            msg = "\nWrote the following variables:" + "\n\t" + ("\n\t".join(vars_to_write[0])) 
     167            if DEBUG: print msg 
     168            self.output_message.append(msg) 
     169         
     170            msg = "" 
     171            aux_var_count = this_na_dict[1][1] 
     172            if len(aux_var_count) > 0: 
     173                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))       
     174             
     175            singleton_var_count = this_na_dict[1][2] 
     176            if len(singleton_var_count) > 0: 
     177                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count)) 
     178 
     179            if len(file_list) > 0: 
     180                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list)) 
     181 
     182            actual_file_counter += len(file_list) 
     183            file_counter += 1 
     184 
     185                if DEBUG: print msg 
     186                self.output_message.append(msg) 
     187             
     188        full_file_count = (actual_file_counter - 1) 
     189        if full_file_count == 1: 
     190            plural = "" 
     191        else: 
     192            plural = "s"               
     193        msg = "\n%s file%s written." % (full_file_count, plural) 
     194     
     195        if DEBUG: print msg 
     196        self.output_message.append(msg) 
     197        return self.output_message 
     198 
     199    def _writeNAFileSubsetsWithinSizeLimit(self, this_na_dict, file_name, delimiter, float_format, size_limit): 
     200        """ 
     201        If self.size_limit is specified and FFI is 1001 we can chunk the output into  
     202        different files in a NASA Ames compliant way.  
     203        Returns list of file names of outputs written. 
     204        """ 
     205        file_names = [] 
     206        var_list = this_na_dict["V"] 
     207        array_length = len(var_list[0]) 
     208        nvol_info = divmod(array_length, size_limit) 
     209        nvol = nvol_info[0] 
     210 
     211        # create the number of volumes (files) that need to be written. 
     212        if nvol_info[1] > 0: nvol = nvol + 1 
     213 
     214        start = 0 
     215        letter_count = 0 
     216        ivol = 0 
     217 
     218        # Loop through until full array length has been written to a set of files. 
     219        while start < array_length: 
     220            ivol = ivol + 1 
     221            end = start + size_limit 
     222 
     223            if end > array_length: 
     224                end = array_length 
     225 
     226            current_block = [] 
     227            # Write new V array 
     228            for v in var_list: 
     229                current_block.append(v[start:end]) 
     230 
     231            # Adjust X accordingly in the na dictionary, because independent variable has been reduced in size 
     232            na_dict_copy = nappy.utils.common_utils.modifyNADictCopy(this_na_dict, current_block,  
     233                                                                      start, end, ivol, nvol) 
     234            # Append a letter to the file name for writing this block to 
     235            file_name_plus_letter = "%s-%.3d.na" % (file_name[:-3], ivol) 
     236            file_list.append(file_name_plus_letter) 
     237 
     238            # Write data to output file 
     239            nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy, delimiter=delimiter,  
     240                                      float_format=float_format) 
     241 
     242            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter) 
     243            if DEBUG: print msg 
     244            self.output_message.append(msg) 
     245            letter_count = letter_count + 1 
     246            start = end 
     247 
     248            file_names.append(file_name_plus_letter)  
     249 
     250        return file_names 
Note: See TracChangeset for help on using the changeset viewer.