source: TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/ConfigFileParsers.py @ 4134

"""
Generic parsers to use when reading in configuration data
- methods available to deal with both XML and INI (flat text key/val) formats
"""
__author__ = "C Byrom - Tessella"
__date__ = "20/05/08"
__copyright__ = "(C) 2008 STFC & NERC"
__license__ = \
"""This software may be distributed under the terms of the Q Public 
License, version 1.0 or later."""
__contact__ = "P.J.Kershaw@rl.ac.uk"
__revision__ = '$Id$'

from ConfigParser import SafeConfigParser
from ndg.security.common.wssecurity import WSSecurityConfig
# For parsing of properties file
try: # python 2.5
    from xml.etree import cElementTree as ElementTree
except ImportError:
    # if you've installed it yourself it comes this way
    import cElementTree as ElementTree

import logging, os
log = logging.getLogger(__name__)

# lambda function to expand out any environment variables in properties read in
expandEnvVars = lambda x: isinstance(x, basestring) and os.path.expandvars(x).strip() or x

# since they are used across services, define the valid keys for the WS-Security section
# here for easy access
__wsSecurityValidKeys = {
        'certFile':               None,
        'keyFile':                None,
        'keyPwd':                 None,
        'clntCertFile':           None,
        'caCertFileList':         [],
        'wssRefInclNS':           [],
        'wssSignedInfoInclNS':    [],
        'reqBinSecTokValType':    'X509v3',
        'applySignatureConfirmation': True,
        'useSignatureHandler':    True
        }

class CaseSensitiveConfigParser(SafeConfigParser):
    '''
    Subclass the SafeConfigParser - to preserve the original string case of the
    cfg section names - NB, the RawConfigParser default is to lowercase these by default
    '''
    
    def optionxform(self, optionstr):
        return optionstr

def readAndValidateProperties(propFilePath, validKeys={}):
    """
    Determine the type of properties file and load the contents appropriately.
    If a dict of valid keys is also specified, check the loaded properties against these.
    @param propFilePath: file path to properties file - either in xml or ini format
    @type propFilePath: string
    @keywords validKeys: a dictionary of valid values to be read from the file
    - if values are encountered that are not in this list, an exception will be thrown
    - if all info should be read, this keyword should be left to its default value
    - NB, this dict will also ensure list data is read in correctely
    @type validKeys: dict
    @raise ValueError: if a key is read in from the file that is not included in the
    specified validKeys dict
    """
    log.debug("Reading properties from %s" %propFilePath)
    properties = {}
    if propFilePath.lower().endswith('xml'):
        log.debug("File has 'xml' suffix - treating as standard XML formatted properties file")
        properties = readXMLPropertyFile(propFilePath, validKeys)
    else:
        properties = readINIPropertyFile(propFilePath, validKeys)

    # if validKeys set, check that all loaded property values are featured in 
    # this list
    if validKeys:
        validateProperties(properties, validKeys)
    
        # lastly set any default values from the validKeys dict for vals not read in
        # from the property file
        _setDefaultValues(properties, validKeys)
    
    # lastly, expand out any environment variables set in the properties file
    properties = _expandEnvironmentVariables(properties)
    log.info('Properties loaded')
    return properties


def readProperties(propFilePath, validKeys={}):
    """
    Determine the type of properties file and load the contents appropriately.
    @param propFilePath: file path to properties file - either in xml or ini format
    @type propFilePath: string
    """
    log.debug("Reading properties from %s" %propFilePath)
    properties = {}
    if propFilePath.lower().endswith('xml'):
        log.debug("File has 'xml' suffix - treating as standard XML formatted properties file")
        properties = readXMLPropertyFile(propFilePath, validKeys)
    else:
        properties = readINIPropertyFile(propFilePath, validKeys)
    
    # lastly, expand out any environment variables set in the properties file
    properties = _expandEnvironmentVariables(properties)
    log.info('Properties loaded')
    return properties
        

def readINIPropertyFile(propFilePath, validKeys):
    """
    Read 'ini' type property file - i.e. a flat text file with key/value
    data separated into sections

    @param propFilePath: file path to properties file - either in xml or ini format
    @type propFilePath: string
    @param validKeys: a dictionary of valid values to be read from the file
    - if values are encountered that are not in this list, an exception will be thrown
    - if all info should be read, set this param to 'None'
    @type validKeys: dict
    @return: dict with the loaded properties in
    @raise ValueError: if a key is read in from the file that is not included in the
    specified validKeys dict
    """
    log.debug("File is not marked as XML - treating as flat, 'ini' format file")
    
    if not os.path.isfile(propFilePath):
        raise ValueError("Error parsing properties file \"%s\": No such file" %propFilePath)

    cfg = CaseSensitiveConfigParser()
    cfg.read(propFilePath)
    properties = {}
    
    # NB, add 'DEFAULT' section since this isn't returned by the 'sections()'
    sections = cfg.sections()
    sections.append('DEFAULT')
    
    # parse data from the specified sections of the config file
    for section in sections:
        if section == 'DEFAULT':
            properties.update(_parseConfig(cfg, validKeys, section=section))
        else:
            keys = validKeys
            if section == 'WS-Security':
                keys = __wsSecurityValidKeys #WSSecurityConfig.defParam
            properties[section] = _parseConfig(cfg, keys, section=section)

    log.debug("Finished reading from INI properties file")
    return properties
    

def _parseConfig(cfg, validKeys, section='DEFAULT'):
    '''
    Extract parameters from cfg config object
    @param cfg: config object
    @type cfg: CaseSensitiveConfigParser
    @param validKeys: a dictionary of valid values to be read from the file - used to check the
    type of the input parameter to ensure (lists) are handled correctly
    @type validKeys: dict
    @keyword section: section of config file to parse from
    @type section: string
    @return: dict with the loaded properties in
    '''
    log.debug("Parsing section: %s" %section)
    keys = {}
    properties = {}
    if section == 'DEFAULT':
        keys = cfg.defaults().keys()
    else:
        keys = cfg.options(section)
        # NB, we need to be careful here - since this will return the section keywords
        # AND the 'DEFAULT' section entries - so use the difference between the two
        keys = filter(lambda x:x not in cfg.defaults().keys(), keys)

    for key in keys:
        val = cfg.get(section, key)
        if val:
            
            # expand out any env vars
            val = expandEnvVars(val)
            
            # if the tag contains an integer, convert this appropriately
            if val.isdigit():
                val = int(val)
            
            # if the value is in the validKeys dict, ensure it is read in as the correct type
            if key in validKeys:
                if isinstance(validKeys[key], list):
                    # Treat as a list of space separated elements
                    val = val.split()
        else:
            # NB, the XML parser will return empty vals as None, so ensure consistency here
            val = None

        # check if key already exists; if so, append to list
        if properties.has_key(key):
            properties[key] = __listify(properties[key]).extend(val)
        else:
            properties[key] = val

    log.debug("Finished parsing section")
    return properties


def readXMLPropertyFile(propFilePath, validKeys, rootElem=None):
    """
    Read property file - assuming the standard XML schema

    @param propFilePath: file path to properties file - either in xml or ini format
    @type propFilePath: string
    @param validKeys: a dictionary of valid values to be read from the file - used to check the
    type of the input parameter to ensure (lists) are handled correctly
    @keyword rootElem: a particular element of an ElementTree can be passed in to
    use as the root element; NB, if this is set, it will take precedence over any
    propFilePath specified
    @type rootElem: ElementTree.Element
    @return: dict with the loaded properties in
    """
    if rootElem is None:
        try:
            tree = ElementTree.parse(propFilePath)
            
        except IOError, ioErr:
            raise ValueError, "Error parsing properties file \"%s\": %s" % \
                                (ioErr.filename, ioErr.strerror)
    
        rootElem = tree.getroot()
        if rootElem is None:
            raise ValueError, \
                "Parsing properties file \"%s\": root element is not defined" % \
                propFilePath

    properties = {}
    # Copy properties from file into a dictionary
    try:
        for elem in rootElem:
            key = elem.tag
            val = elem.text

            # expand out any env vars
            val = expandEnvVars(val)

            # if the tag contains an integer, convert this appropriately
            if val and val.isdigit():
                val = int(val)
            
            # check for lists - don't recurse into these else the key names
            # will end up being wrong
            if key in validKeys and isinstance(validKeys[key], list):
                # handle lists of elements
                if len(elem) == 0:
                    if elem.text is not None:
                        # Treat as a list of space separated elements
                        val = val.split()
                else:
                    # Parse from a list of sub-elements
                    val = [expandEnvVars(subElem.text.strip()) for subElem in elem]
            
            # otherwise check for subelements; if these exist, recurse and store properties
            # in an inner dictionary
            elif len(elem) > 0:
                keys = validKeys
                if key == 'WS-Security':
                    keys = __wsSecurityValidKeys #WSSecurityConfig.defParam
                val = readXMLPropertyFile(propFilePath, keys, rootElem=elem)

            # check if key already exists; if so, append to list
            if properties.has_key(key):
                properties[key] = __listify(properties[key]).extend(val)
            else:
                properties[key] = val
            #print "XML: Key: %s, Val: %s" %(key, properties[key])
            
    except Exception, e:
        raise ValueError, \
            "Error parsing tag \"%s\" in properties file \"%s\": %s" % \
            (elem.tag, propFilePath, e)

    log.debug("Finished reading from XML properties file")
    return properties


def __listify(val):
    '''
    Checks if val is a list; if so return as is, if not return as list
    @param val: object to turn into a list
    @return: val as a list (if it is not already)
    '''
    if isinstance(val, list):
        return val
    return [val]


def validateProperties(properties, validKeys):
    '''
    Check the contents of the properties dict to ensure it doesn't contain
    any keys not featured in the validKeys dict; if it does, throw an exception
    @param properties: dictionary storing loaded properties
    @type properties: dict
    @param validKeys: a dictionary of valid values
    @type validKeys: dict
    @raise ValueError: if a key is read in from the file that is not included in the
    specified validKeys dict
    '''
    log.debug("Checking for invalid properties")
    invalidKeys = []
    for key in properties:
        # NB, this is a standard property used across most services - so check
        # using the properties listed here
        if key == 'WS-Security':
            validateProperties(properties[key], __wsSecurityValidKeys)
        else:
            if key not in validKeys:
                invalidKeys.append(key)

    if invalidKeys != []:
        errorMessage = "The following properties file " + \
            "elements are invalid: " + ', '.join(invalidKeys)
        print errorMessage
        log.error(errorMessage)
        raise ValueError, errorMessage


def _expandEnvironmentVariables(properties):
    '''
    Iterate through the values in a dict and expand out environment variables
    specified in any non password option entries
    @param properties: dict of properties to expand
    @type properties: dict
    @return: dict with expanded values
    '''
    log.debug("Expanding out environment variables in properties dictionary")
    for (key, val) in properties.items():
        # only check strings or lists of strings
        if (isinstance(val, list)):
            newList = []
            for item in val:
                newList.append(_expandEnvironmentVariable(key, item))
            
            properties[key] = newList
            
        elif (isinstance(val, str)):
            properties[key] = _expandEnvironmentVariable(key, val)
            
    log.debug("Finished expanding environment variables")
    return properties


def _expandEnvironmentVariable(key, val):
    '''
    Expand out a val, if it contains environment variables and
    is not password related
    @param key: key name for the value
    @type key: str
    @param val: value to expand env vars out in
    @type val: str
    @return: val - with any environment variables expanded out
    '''
    if key.lower().find('pwd') == -1 and key.lower().find('password') == -1:
        val = os.path.expandvars(val)
    return val

    
def _setDefaultValues(properties, validKeys):
    '''
    Check the contents of the properties dict to ensure it contains all the
    keys featured in the validKeys dict; if any of these are missing or have no value
    set for them, set up default values for these in the properties dict
    @param properties: dictionary storing loaded properties
    @type properties: dict
    @param validKeys: a dictionary of valid values
    @type validKeys: dict
    @return properties: updated dict with default values for any missing values
    '''
    log.debug("Checking for any unset keys")
    for key in validKeys:
        if key not in properties or not properties[key]:
            log.debug("Found missing/unset property - setting default values: %s = %s" \
                      %(key, validKeys[key]))
            properties[key] = validKeys[key]
    log.debug("Finished checking for unset keys")