Changeset 3441


Ignore:
Timestamp:
15/02/08 18:46:46 (12 years ago)
Author:
astephen
Message:

Fixes to make NC to NA route work.

Location:
nappy/trunk/nappy
Files:
5 edited

Legend:

Unmodified
Added
Removed
  • nappy/trunk/nappy/cdms_utils/var_utils.py

    r3380 r3441  
    1616 
    1717# Imports from python standard library 
     18import re 
    1819 
    1920# Import external packages 
  • nappy/trunk/nappy/nc_interface/cdms_to_na.py

    r3416 r3441  
    8888        # Make first call to collector class that creates NA dict from CDMS variables and global atts dicts 
    8989        collector = nappy.nc_interface.na_content_collector.NAContentCollector(variables, self.global_atts) 
    90         collected_dict = collector.collectNAContent() 
    91         # NOTE: collected_dict has attributes: na_dict, var_ids, unused_vars 
     90        collector.collectNAContent() 
    9291 
    9392        # Return if no files returned 
    94         if collected_dict.var_ids == None: 
     93        if collector.found_na == False: 
    9594            msg = "\nNo files created after variables parsed." 
    9695            if DEBUG: print msg 
    9796            self.output_message.append(msg) 
    98             return  
     97            return 
     98 
     99        # NOTE: collector has attributes: na_dict, var_ids, unused_vars 
    99100 
    100101        # Set up a list to collect multiple calls to content collector 
    101102        na_dict_list = [] 
    102         na_dict_list.append((collected_dict.na_dict, collected_dict.var_ids)) 
     103        na_dict_list.append((collector.na_dict, collector.var_ids)) 
    103104 
    104105        # If there are variables that were not captured (i.e. unused) by NAContentCollector then loop through these 
     
    106107        while len(collector.unused_vars) > 0: 
    107108            collector = nappy.nc_interface.na_content_collector.NAContentCollector(collector.unused_vars, self.global_atts) 
    108             collected_dict = collector.collectNAContent()            
     109            collector.collectNAContent()            
    109110            self.output_message += collector.output_message 
     111 
    110112            # Append to list if more variables were captured 
    111             if collector.var_ids != None:   
    112                 na_dict_list.append((collected_dict.na_dict, collected_dict.var_ids)) 
     113            if collector.found_na == True:   
     114                na_dict_list.append((collector.na_dict, collector.var_ids)) 
    113115 
    114116        self.na_dict_list = na_dict_list 
  • nappy/trunk/nappy/nc_interface/na_content_collector.py

    r3423 r3441  
    2424import nappy.na_file.na_core 
    2525 
    26 nc_to_na_map = nappy.utils.getConfigDict()["nc_to_na_map"] 
     26config_dict = nappy.utils.getConfigDict() 
     27nc_to_na_map = config_dict["nc_to_na_map"] 
     28version = nappy.utils.getVersion() 
    2729 
    2830# Import external packages (if available) 
     
    5658        >>> x = NAContentCollector(["temp", "precip"]) 
    5759        >>> x.collectNAContent() 
    58         >>> print x.na_dict, x.var_ids, x.unused_vars  
     60        >>> if x.found_na == True: 
     61        ...     print x.na_dict, x.var_ids, x.unused_vars  
    5962        """ 
    6063        self.output_message = [] 
     
    6972        self.rank_zero_var_ids = [] 
    7073 
     74        # Create a flag to check if anything found 
     75        self.found_na = False 
     76 
    7177 
    7278    def collectNAContent(self): 
     
    7985        """ 
    8086        (self.ordered_vars, aux_vars) = self._analyseVariables() 
    81         if self.ordered_vars == None: 
     87      
     88        if self.ordered_vars == []: 
     89            print "WARNING: No NASA Ames content created." 
    8290            self.unused_vars = [] 
    8391        else: 
     
    95103                self.na_dict["X"] = self.na_dict["X"][0] 
    96104 
     105            self.found_na = True 
     106 
     107 
    97108    def _analyseVariables(self): 
    98109        """ 
     
    122133                self.rank_zero_var_ids.append(var.id) 
    123134                continue 
    124              
     135            
    125136            # Update highest if highest found or if equals highest with bigger size 
    126137            if rank > highest_rank or (rank == highest_rank and var.size() > best_var.size()): 
     
    128139                best_var = var 
    129140                best_var_index = count 
    130                 print "WARNING WARNING WARNING....CHANGED CODE HERE from len(var.flat) to var.size() which should be quicker!!!" 
    131  
    132         # If all are zero ranked variables or no vars identified/found then we cannot write any to NASA Ames and return (None, None) 
    133         if len(self.rank_zero_vars) == len(self.vars) or best_var == None:  
    134             print "No usable variables found."  
    135             return (None, None) 
     141 
     142        # If all are zero ranked variables or no vars identified/found then we cannot write any to NASA Ames and return ([], []) 
     143        if len(self.rank_zero_vars) == len(self.vars) or best_var is None:  
     144            return ([], []) 
    136145 
    137146        # Now start to sort the variables into main and auxiliary  
     
    168177                aux_vars_for_na.append(var)  
    169178                # Also put it in unused var bin because auxiliary vars might be useful later on in there own right 
     179                print "NOTE: Auxiliary variables are recorded in file but also placed in unused category just in case they should be re-used in other files." 
    170180                self.unused_vars.append(var) 
    171181            else: 
     
    190200 
    191201 
    192     def _reorderVarsIfPreviouslyNA(vars_for_na, aux_vars_for_na): 
     202    def _reorderVarsIfPreviouslyNA(self, vars_for_na, aux_vars_for_na): 
    193203        """ 
    194204        Re-order if they previously came from NASA Ames files (i.e. including the  
     
    327337        """ 
    328338        length = len(axis) 
     339 
    329340        self.na_dict["NX"].append(length) 
    330341        self.na_dict["XNAME"].append(nappy.cdms_utils.var_utils.getBestName(axis)) 
     
    343354                self.na_dict["X"].append(axis.tolist()) 
    344355                break 
    345             else: # If did not break out of the loop 
    346                 max_length = length 
    347                 if length > 3: max_length = 3 
    348  
    349                 self.na_dict["DX"].append(incr) 
    350                 self.na_dict["NXDEF"].append(max_length) 
    351                 self.na_dict["X"].append(axis[:max_length]) 
    352  
     356        else: # If did not break out of the loop 
     357            max_length = length 
     358            if length > 3:  
     359                max_length = 3 
     360            self.na_dict["DX"].append(incr) 
     361            self.na_dict["NXDEF"].append(max_length) 
     362            self.na_dict["X"].append(axis[:max_length]) 
    353363 
    354364    def _defineNAGlobals(self): 
     
    375385                    time_string = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time())) 
    376386                    history = "History:\t%s - Converted to NASA Ames format using nappy-%s.\n\t%s" % \ 
    377                                                  (time_string, version.version, self.globals[key]) 
     387                                                 (time_string, version, self.globals[key]) 
    378388                    history = history.split("\n")  
    379389                    self.history = [] 
     
    487497        # Create a string for the Special comments to hold rank-zero vars 
    488498        rank_zero_vars_string = [] 
     499 
    489500        for var in self.rank_zero_vars: 
    490501            rank_zero_vars_string.append("\tVariable %s: %s" % (var.id, nappy.cdms_utils.var_utils.getBestName(var))) 
    491502 
    492         for att in var.attributes.keys(): 
    493             value = var.attributes[att] 
    494             if type(value) in (type("s"), type(1.0), type(1)): 
    495                 rank_zero_vars_string.append("\t\t%s = %s" % (att, var.attributes[att])) 
    496          
     503            for att in var.attributes.keys(): 
     504                value = var.attributes[att] 
     505 
     506                if type(value) in (type("s"), type(1.0), type(1)): 
     507 
     508                    rank_zero_vars_string.append("\t\t%s = %s" % (att, var.attributes[att])) 
     509 
    497510        if len(rank_zero_vars_string) > 0: 
    498511            rank_zero_vars_string.insert(0, "###Singleton Variables defined in the source file follow###") 
  • nappy/trunk/nappy/nc_interface/nc_to_na.py

    r3416 r3441  
    5858 
    5959        # Now need to read CDMS file so parent class methods are compatible 
    60         (cdms_variables, global_atts) = self._readCDMSFile(self.nc_file, var_ids, exclude_vars) 
     60        (cdms_variables, global_atts) = self._readCDMSFile(var_ids, exclude_vars) 
    6161        nappy.nc_interface.cdms_to_na.CDMSToNA.__init__(self, cdms_variables, global_atts=global_atts,  
    6262                    na_items_to_override=na_items_to_override, only_return_file_names=only_return_file_names) 
    6363  
    6464 
    65     def _readCDMSFile(self, nc_file, var_ids=None, exclude_vars=[]): 
     65    def _readCDMSFile(self, var_ids=None, exclude_vars=[]): 
    6666        """ 
    6767        Reads the file and returns all the CDMS variables in a list as well 
     
    6969        If var_ids is defined then only get those. 
    7070        """ 
    71         fin = cdms.open(nc_file) 
     71        fin = cdms.open(self.nc_file) 
    7272        cdms_variables = [] 
    7373 
     
    9090        """ 
    9191        self.convert() 
     92 
    9293        file_names = [] 
     94        # create file name if not given 
     95        if na_file == None: 
     96            base_name = self.nc_file 
     97            if base_name[-3:] == ".nc": 
     98                base_name = base_name[:-3] 
     99            na_file = base_name + ".na" 
    93100 
    94101        file_counter = 1 
     102        # Now, create some valid file names 
    95103        for this_na_dict in self.na_dict_list: 
    96             if len(na_dict_list) == 1: 
     104            if len(self.na_dict_list) == 1: 
    97105                suffix = "" 
    98106            else: 
     
    100108 
    101109            # Create file name 
    102             name_parts = self.output_file_names[0].split(".")     
     110            name_parts = na_file.split(".")     
    103111            new_name = (".".join(name_parts[:-1])) + suffix + "." + name_parts[-1] 
    104112            file_names.append(new_name) 
     
    127135 
    128136        # Now loop through writing the outputs 
    129         for na_dict_and_var_ids in built_na_dicts: 
     137        for na_dict_and_var_ids in self.na_dict_list: 
    130138            file_name = file_names[file_counter - 1] 
    131139            msg = "\nWriting output NASA Ames file: %s" % file_name 
     
    134142 
    135143            # Set up current na dict 
    136             this_na_dict = (na_dict_and_var_ids, vars_to_write) 
     144            (this_na_dict, vars_to_write) = na_dict_and_var_ids 
    137145 
    138146            # Override content of NASA Ames if they are permitted 
     
    153161            # Cope with size limits if specified and FFI is 1001 
    154162            # Seems to be writing different chunks of a too long array to different na_dicts to then write to separate files. 
    155             if self.size_limit and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > self.size_limit): 
     163            if size_limit is not None and (this_na_dict["FFI"] == 1001 and len(this_na_dict["V"][0]) > size_limit): 
    156164                files_written = self._writeNAFileSubsetsWithinSizeLimit(this_na_dict, file_name, delimiter=delimiter, 
    157165                                                                        float_format=float_format, size_limit=size_limit) 
    158166                file_list.extend(files_written) 
     167 
    159168            # If not having to split file into multiple outputs (normal condition) 
    160169            else:                
    161                 nappy.openNAFile(file_name, 'w', this_na_dict, delimiter=delimiter, float_format=float_format) 
     170                x = nappy.openNAFile(file_name, 'w', this_na_dict) 
     171                x.write(delimiter=delimiter, float_format=float_format) 
     172                x.close() 
    162173                file_list.append(file_name) 
    163174 
     
    168179         
    169180            msg = "" 
    170             aux_var_count = this_na_dict[1][1] 
     181            aux_var_count = vars_to_write[1] 
    171182            if len(aux_var_count) > 0: 
    172183                msg = "\nWrote the following auxiliary variables:" + "\n\t" + ("\n\t".join(aux_var_count))       
    173184             
    174             singleton_var_count = this_na_dict[1][2] 
     185            singleton_var_count = vars_to_write[2] 
    175186            if len(singleton_var_count) > 0: 
    176187                msg = "\nWrote the following Singleton variables:" + "\n\t" + ("\n\t".join(singleton_var_count)) 
     
    179190                msg = msg + ("\n\nNASA Ames file(s) written successfully: \n%s" % "\n".join(file_list)) 
    180191 
    181             actual_file_counter += len(file_list) 
     192            full_file_counter += len(file_list) 
    182193            file_counter += 1 
    183194 
     
    185196            self.output_message.append(msg) 
    186197             
    187         full_file_count = (actual_file_counter - 1) 
     198        full_file_count = full_file_counter - 1 
    188199        if full_file_count == 1: 
    189200            plural = "" 
     
    236247 
    237248            # Write data to output file 
    238             nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy, delimiter=delimiter,  
    239                                       float_format=float_format) 
     249            x = nappy.openNAFile(file_name_plus_letter, 'w', na_dict_copy) 
     250            x.write(delimiter=delimiter, float_format=float_format) 
     251            x.close() 
    240252 
    241253            msg = "\nOutput files split on size limit: %s\nFilename used: %s" % (size_limit, file_name_plus_letter) 
  • nappy/trunk/nappy/nc_interface/quick_tests_nc_to_na.py

    r3416 r3441  
    1616 
    1717    print "Writing:", outfile 
    18     print dir(x) 
    1918    x.writeNAFiles(outfile) 
Note: See TracChangeset for help on using the changeset viewer.