Changeset 4693 for nappy/trunk


Ignore:
Timestamp:
19/12/08 23:00:18 (11 years ago)
Author:
astephen
Message:

Updated nappy code to allow us to update header info better.

Location:
nappy/trunk
Files:
8 edited

Legend:

Unmodified
Added
Removed
  • nappy/trunk/nappy.ini

    r3988 r4693  
    44default_delimiter = __space____space____space____space__ 
    55default_float_format = %g 
    6 comment_override_rule = replace 
     6comment_override_rule = extend 
     7add_column_headers = True 
    78annotations_file = annotations.ini 
    89local_attributes_file = local_attributes.ini 
     10 
     11[header_partitions] 
     12sc_start = "=== Special Comments follow ===" 
     13sc_end = "=== Special Comments end ===" 
     14nc_start = "=== Normal Comments follow ===" 
     15nc_end = "=== Normal Comments end ===" 
     16data_next = "=== Data Section begins on the next line ===" 
     17sing_start = "=== Singleton Variables defined in the source file follow ===" 
     18sing_end = "=== Singleton Variables defined in the source file end ===" 
     19ncatts_start = "=== Variable attributes from source (NetCDF) file follow ===" 
     20ncatts_end = "=== Variable attributes from source (NetCDF) file end ===" 
     21addl_globals = "Additional Global Attributes defined in the source file and not translated elsewhere:" 
     22addl_vatts = "Additional Variable Attributes defined in the source file and not translated elsewhere:" 
    923 
    1024[na_to_nc_map] 
  • nappy/trunk/nappy/cdms_utils/axis_utils.py

    r3987 r4693  
    4040 
    4141    # Check units 
    42     if ax1.units != ax2.units: return False 
     42    print ax1.id, ax2.id 
     43    if hasattr(ax1, "units"): 
     44        if ax1.units != ax2.units: return False 
    4345     
    4446    # Do different comparisons depending on 'is_subset' argument 
  • nappy/trunk/nappy/nappy_api.py

    r3991 r4693  
    232232            only_return_file_names=False, exclude_vars=[], 
    233233            requested_ffi=None, delimiter=default_delimiter, float_format=default_float_format,  
    234             size_limit=None, annotation=False, no_header=False): 
     234            size_limit=None, annotation=False, no_header=False, 
     235            ): 
    235236    """ 
    236237    Takes a NetCDF file and converts the contents to one or more NASA Ames files.  
     
    298299def convertCDMSObjectsToNA(cdms_vars, global_attributes, na_file,  
    299300              na_items_to_override={}, requested_ffi=None, delimiter=default_delimiter,  
    300               float_format=default_float_format, size_limit=None, annotation=False, no_header=False): 
     301              float_format=default_float_format, size_limit=None, annotation=False, no_header=False, 
     302              ): 
    301303    """ 
    302304    Takes a list of cdms variables and a list of global attributes and 
     
    322324    import nappy.nc_interface.cdms_objs_to_na_file 
    323325    convertor = nappy.nc_interface.cdms_objs_to_na_file.CDMSObjectsToNAFile(cdms_vars, global_attributes=global_attributes,  
    324                         na_items_to_override=na_items_to_override, requested_ffi=requested_ffi) 
     326                        na_items_to_override=na_items_to_override, requested_ffi=requested_ffi, 
     327                        ) 
    325328    convertor.convert() 
    326329    na_files = convertor.writeNAFiles(na_file, delimiter=delimiter, float_format=float_format,  
  • nappy/trunk/nappy/nc_interface/cdms_objs_to_na_file.py

    r3991 r4693  
    4848    def __init__(self, cdms_variables, global_attributes=[], na_items_to_override={}, 
    4949                 only_return_file_names=False, 
    50                  requested_ffi=None): 
     50                 requested_ffi=None, 
     51                 ): 
    5152        """ 
    5253        Sets up instance variables. 
  • nappy/trunk/nappy/nc_interface/cdms_to_na.py

    r3991 r4693  
    4949 
    5050    def __init__(self, cdms_variables, global_attributes=[], na_items_to_override={},  
    51                  only_return_file_names=False, requested_ffi=None): 
     51                 only_return_file_names=False, requested_ffi=None, 
     52                 ): 
    5253        """ 
    5354        Sets up instance variables.       
     
    8889        # Make first call to collector class that creates NA dict from CDMS variables and global atts list  
    8990        collector = nappy.nc_interface.na_content_collector.NAContentCollector(variables,  
    90                                         self.global_attributes, requested_ffi=self.requested_ffi) 
     91                                        self.global_attributes, requested_ffi=self.requested_ffi, 
     92                                        ) 
    9193        collector.collectNAContent() 
    9294 
     
    108110        while len(collector.unused_vars) > 0: 
    109111            collector = nappy.nc_interface.na_content_collector.NAContentCollector(collector.unused_vars,  
    110                                        self.global_attributes, requested_ffi=self.requested_ffi) 
     112                                        self.global_attributes, requested_ffi=self.requested_ffi, 
     113                                        ) 
    111114            collector.collectNAContent()            
    112115            self.output_message += collector.output_message 
  • nappy/trunk/nappy/nc_interface/na_content_collector.py

    r3991 r4693  
    2626config_dict = nappy.utils.getConfigDict() 
    2727nc_to_na_map = config_dict["nc_to_na_map"] 
     28header_partitions = config_dict["header_partitions"] 
     29hp = header_partitions 
     30 
    2831version = nappy.utils.getVersion() 
    2932 
     
    7578        self.globals = dict(global_attributes)   
    7679        self.requested_ffi = requested_ffi 
     80 
    7781        self.rank_zero_vars = [] 
    7882        self.rank_zero_var_ids = [] 
     
    536540 
    537541                    for line in comment_lines: 
    538                         if line.find("###NASA Ames Special Comments follow###") > -1: 
     542                        if line.find(hp["sc_start"]) > -1: 
    539543                            special_comm_flag = 1 
    540                         elif line.find("###NASA Ames Special Comments end###") > -1: 
     544                        elif line.find(hp["sc_end"]) > -1: 
    541545                            special_comm_flag = None 
    542                         elif line.find("###NASA Ames Normal Comments follow###") > -1: 
     546                        elif line.find(hp["nc_start"]) > -1: 
    543547                            normal_comm_flag = 1 
    544                         elif line.find("###NASA Ames Normal Comments end###") > -1: 
     548                        elif line.find(hp["nc_end"]) > -1: 
    545549                            normal_comm_flag = None      
    546550                        elif special_comm_flag == 1: 
     
    548552                        elif normal_comm_flag == 1: 
    549553                            normal_comments.append(line) 
    550                         elif line.find("###Data Section begins on the next line###") > -1: 
     554                        elif line.find(hp["data_next"]) > -1: 
    551555                            pass 
    552556                        else: 
     
    572576        Defines the Special and Normal comments sections in the NASA Ames file  
    573577        object - including information gathered from the defineNAGlobals method. 
    574         """ 
    575          
     578 
     579        Starts with values provided for normal_comments and special_comments. 
     580        """ 
    576581        if hasattr(self, "ncom"):  normal_comments = self.ncom + normal_comments 
    577582 
     
    581586 
    582587        if len(NCOM) > 0:   NCOM.append("") 
    583          
     588 
     589        # Use third item in self.extra_comments and adds to NCOM         
    584590        if len(self.extra_comments[2]) > 0: 
    585591            for excom in self.extra_comments[2]: 
     
    587593         
    588594        if len(self.extra_comments[1]) > 0:   
    589             NCOM.append("Additional Global Attributes defined in the source file and not translated elsewhere:") 
     595            NCOM.append(hp["addl_globals"]) 
    590596            for excom in self.extra_comments[1]: 
    591597                NCOM.append(excom) 
     
    594600            for h in self.history: 
    595601                NCOM.append(h) 
    596          
     602        
     603        # When NCOM has been defined then surround it in some extras  
    597604        if len(NCOM) > 0: 
    598             NCOM.insert(0, "###NASA Ames Normal Comments follow###") 
     605            NCOM.insert(0, hp["nc_start"])  
    599606            NCOM.append("") 
    600             NCOM.append("###NASA Ames Normal Comments end###") 
    601             NCOM.append("###Data Section begins on the next line###") 
     607            NCOM.append(hp["nc_end"]) 
     608            NCOM.append(hp["data_next"]) 
    602609 
    603610        spec_comm_flag = None 
     611        # Start with special_comments added in 
    604612        SCOM = [] 
    605         special_comments = self.extra_comments[0] 
     613 
     614        # Uses first item in self.extra_comments to start SCOM 
     615        special_comments = special_comments + self.extra_comments[0] 
     616 
    606617        if len(special_comments) > 0:  
    607             SCOM = ["###NASA Ames Special Comments follow###"] 
    608  
    609             spec_comm_flag = 1 
     618            SCOM = [hp["sc_start"]] 
     619            spec_comm_flag = 1 
     620 
    610621        for scom in special_comments: 
    611622            SCOM.append(scom) 
     
    629640 
    630641        if len(rank_zero_vars_string) > 0: 
    631             rank_zero_vars_string.insert(0, "###Singleton Variables defined in the source file follow###") 
    632             rank_zero_vars_string.append("###Singleton Variables defined in the source file end###") 
     642            rank_zero_vars_string.insert(0, hp["sing_start"]) 
     643            rank_zero_vars_string.append(hp["sing_end"]) 
    633644 
    634645        for var in self.ordered_vars: 
    635646            varflag = "unused" 
     647            var_name_written = False 
     648 
    636649            name = nappy.cdms_utils.var_utils.getBestName(var) 
    637650 
     
    646659 
    647660                    if spec_comm_flag == None: 
    648                         SCOM = ["###NASA Ames Special Comments follow###"] + rank_zero_vars_string 
    649                         SCOM.append("Additional Variable Attributes defined in the source file and not translated elsewhere:") 
    650                         SCOM.append("###Variable attributes from source (NetCDF) file follow###") 
     661                        SCOM = [hp["sc_start"]] + rank_zero_vars_string 
     662                        SCOM.append(hp["addl_vatts"]) 
     663                        SCOM.append(hp["ncatts_start"]) 
    651664                        varflag = "using"  
    652  
    653                     SCOM.append("\tVariable %s: %s" % (var.id, name)) 
     665                        spec_comm_flag = 1 
     666 
     667                    if var_name_written == False: 
     668                        SCOM.append("\tVariable %s: %s" % (var.id, name)) 
     669                        var_name_written = True 
     670 
    654671                    SCOM.append("\t\t%s = %s" % (scom, value)) 
    655672 
    656673        if var_comm_flag == 1:   
    657             SCOM.append("###Variable attributes from source (NetCDF) file end###") 
     674            SCOM.append(hp["ncatts_end"]) 
    658675        if spec_comm_flag == 1: 
    659             SCOM.append("###NASA Ames Special Comments end###") 
     676            SCOM.append(hp["sc_end"]) 
    660677 
    661678        # Strip out empty lines (or returns) 
     
    670687                    if line != lines[0]:  
    671688                        line = "\t" + line 
     689 
    672690                    NCOM_cleaned.append(line) 
    673                          
     691                 
    674692        for c in SCOM: 
    675693            if c.strip() not in ("", " ", "  "):                 
    676                         # Replace new lines within one attribute with a newline and tab so easier to read 
     694                # Replace new lines within one attribute with a newline and tab so easier to read 
    677695                lines = c.split("\n") 
    678696                for line in lines: 
    679697                    if line != lines[0]:  
    680698                        line = "\t" + line 
     699 
    681700                    SCOM_cleaned.append(line) 
    682                      
     701 
    683702        self.na_dict["NCOM"] = NCOM_cleaned 
    684703        self.na_dict["NNCOML"] = len(self.na_dict["NCOM"]) 
     
    726745        for key in header_items.keys(): 
    727746             self.na_dict[key] = header_items[key] 
     747 
  • nappy/trunk/nappy/nc_interface/na_to_cdms.py

    r3991 r4693  
    2222import nappy.utils.common_utils 
    2323 
    24 na_to_nc_map = nappy.utils.getConfigDict()["na_to_nc_map"] 
     24config_dict = nappy.utils.getConfigDict() 
     25na_to_nc_map = config_dict["na_to_nc_map"] 
     26header_partitions = config_dict["header_partitions"] 
     27hp = header_partitions 
    2528 
    2629# Import external packages (if available) 
     
    4346time_units_pattn = re.compile("\w+\s+since\s+\d{4}-\d{1,2}-\d{1,2}\s+\d+:\d+:\d+") 
    4447max_id_length = 40 
    45 special_comment_known_strings = ("###NASA Ames Special Comments follow###",  
    46                                      "###NASA Ames Special Comments end###",   
    47                                   "Additional Variable Attributes defined in the source file and not translated elsewhere:", 
    48                                   "Additional Global Attributes defined in the source file and not translated elsewhere:", "\n") 
    49  
    50 normal_comment_known_strings = ("###NASA Ames Normal Comments follow###",  
    51                                 "###NASA Ames Normal Comments end###",  
    52                                 "###Data Section begins on the next line###", 
    53                                             "Additional Variable Attributes defined in the source file and not translated elsewhere:", 
    54                                                 "Additional Global Attributes defined in the source file and not translated elsewhere:",  
    55                                 "\n") 
     48special_comment_known_strings = (hp["sc_start"], hp["sc_end"], hp["addl_vatts"], 
     49                                  hp["addl_globals"], "\n") 
     50 
     51normal_comment_known_strings = (hp["nc_start"], hp["nc_end"], hp["data_next"],  
     52                                  hp["addl_vatts"], hp["addl_globals"], "\n") 
    5653 
    5754time_units_warning_message = """\nWARNING: Could not recognise time units. For true NetCDF compability 
     
    7572     
    7673    def __init__(self, na_file_obj, variables="all", aux_variables="all", 
    77                  global_attributes=[("Conventions":"CF-1.0")], 
     74                 global_attributes=[("Conventions", "CF-1.0")], 
    7875                 time_units=None, time_warning=True,  
    7976                 rename_variables={}): 
     
    145142                    # Add Special comments first 
    146143                    if self.na_file_obj.NSCOML > 0: 
    147                         comment_line += "###NASA Ames Special Comments follow###\n" 
     144                        comment_line += (hp["sc_start"] + "\n") 
    148145 
    149146                        for i in self.na_file_obj.SCOM:  
     
    152149                                comment_line += ("\n" + i) 
    153150                             
    154                         comment_line += "\n###NASA Ames Special Comments end###\n" 
     151                        comment_line += ("\n" + hp["sc_end"] + "\n") 
    155152 
    156153                    # Now add normal comments 
    157154                    if self.na_file_obj.NNCOML > 0: 
    158                         comment_line += "###NASA Ames Normal Comments follow###\n" 
     155                        comment_line += (hp["nc_start"] + "\n") 
    159156 
    160157                        for i in self.na_file_obj.NCOM:  
     
    162159                                comment_line += ("\n" + i) 
    163160 
    164                         comment_line += "\n###NASA Ames Normal Comments end###" 
     161                        comment_line += ("\n" + hp["nc_end"]) 
    165162 
    166163                    # Tidy comment line then write to global atts dict 
  • nappy/trunk/nappy/nc_interface/nc_to_na.py

    r4002 r4693  
    4242default_float_format = nappy.utils.getDefault("default_float_format") 
    4343comment_override_rule = nappy.utils.getDefault("comment_override_rule") 
     44add_column_headers = bool(nappy.utils.getDefault("add_column_headers")) 
    4445 
    4546# Define global variables 
     
    5657    def __init__(self, nc_file, var_ids=None, na_items_to_override={},  
    5758            only_return_file_names=False, exclude_vars=[], 
    58             requested_ffi=None): 
     59            requested_ffi=None, 
     60            ): 
    5961        """ 
    6062        Sets up instance variables. 
     
    201203                        if DEBUG: print msg 
    202204                        self.output_message.append(msg) 
     205 
     206            # For certain FFIs create final Normal comments as a list of column headers before data section  
     207            if add_column_headers == True: 
     208                self._updateWithColumnHeaders(this_na_dict, delimiter) 
    203209         
    204210            file_list = [] 
     
    310316 
    311317 
     318    def _updateWithColumnHeaders(self, na_dict, delimiter): 
     319        """ 
     320        Updates the NCOM and NCOML parts of the na_dict so that  
     321        the last normal comments line is in fact a set of headers  
     322        for the data section. E.g.: 
     323 
     324           UTs     Spd  Direc 
     325           30446.9 305  2592 
     326           20447.9 204  2596 
     327 
     328        The 'delimiter' argument is used to separate out the arguments. 
     329 
     330        This option is only compatible with a limited range of FFIs and 
     331        only works if there are no Auxiliary variables defined. 
     332        """ 
     333        ffi = na_dict["FFI"] 
     334        compatible_ffis = (1001, 1010, 2110) 
     335 
     336        if ffi not in compatible_ffis or na_dict["NAUXV"] > 0: 
     337            print "WARNING: Column Headers are not written for FFIs other than: " + str(compatible_ffis) 
     338            return 
     339 
     340        if ffi in (1001, 2110): 
     341            col_names = [na_dict["XNAME"][0]] 
     342        elif ffi in (1010,): 
     343            col_names = [] 
     344 
     345        col_names.extend(na_dict["VNAME"]) 
     346        col_names_line = ",".join(col_names) 
     347        na_dict["NCOM"].append(col_names_line)  
     348        na_dict["NNCOML"] = len(na_dict["NCOM"]) 
     349        return   
Note: See TracChangeset for help on using the changeset viewer.