source: TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/ConfigFileParsers.py @ 4134

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/ConfigFileParsers.py@4134
Revision 4134, 14.5 KB checked in by cbyrom, 11 years ago (diff)

Extend config parser utilities - to cleanly separate the input and
validation - and setting of default values - of properties. Also,
allow separate property sections to be stored in child dictionaries +
properly traverse elementtree docs to retrieve all data + fix a few
issues along the way.

Line 
1"""
2Generic parsers to use when reading in configuration data
3- methods available to deal with both XML and INI (flat text key/val) formats
4"""
5__author__ = "C Byrom - Tessella"
6__date__ = "20/05/08"
7__copyright__ = "(C) 2008 STFC & NERC"
8__license__ = \
9"""This software may be distributed under the terms of the Q Public
10License, version 1.0 or later."""
11__contact__ = "P.J.Kershaw@rl.ac.uk"
12__revision__ = '$Id$'
13
14from ConfigParser import SafeConfigParser
15from ndg.security.common.wssecurity import WSSecurityConfig
16# For parsing of properties file
17try: # python 2.5
18    from xml.etree import cElementTree as ElementTree
19except ImportError:
20    # if you've installed it yourself it comes this way
21    import cElementTree as ElementTree
22
23import logging, os
24log = logging.getLogger(__name__)
25
26# lambda function to expand out any environment variables in properties read in
27expandEnvVars = lambda x: isinstance(x, basestring) and os.path.expandvars(x).strip() or x
28
29# since they are used across services, define the valid keys for the WS-Security section
30# here for easy access
31__wsSecurityValidKeys = {
32        'certFile':               None,
33        'keyFile':                None,
34        'keyPwd':                 None,
35        'clntCertFile':           None,
36        'caCertFileList':         [],
37        'wssRefInclNS':           [],
38        'wssSignedInfoInclNS':    [],
39        'reqBinSecTokValType':    'X509v3',
40        'applySignatureConfirmation': True,
41        'useSignatureHandler':    True
42        }
43
44class CaseSensitiveConfigParser(SafeConfigParser):
45    '''
46    Subclass the SafeConfigParser - to preserve the original string case of the
47    cfg section names - NB, the RawConfigParser default is to lowercase these by default
48    '''
49   
50    def optionxform(self, optionstr):
51        return optionstr
52
53def readAndValidateProperties(propFilePath, validKeys={}):
54    """
55    Determine the type of properties file and load the contents appropriately.
56    If a dict of valid keys is also specified, check the loaded properties against these.
57    @param propFilePath: file path to properties file - either in xml or ini format
58    @type propFilePath: string
59    @keywords validKeys: a dictionary of valid values to be read from the file
60    - if values are encountered that are not in this list, an exception will be thrown
61    - if all info should be read, this keyword should be left to its default value
62    - NB, this dict will also ensure list data is read in correctely
63    @type validKeys: dict
64    @raise ValueError: if a key is read in from the file that is not included in the
65    specified validKeys dict
66    """
67    log.debug("Reading properties from %s" %propFilePath)
68    properties = {}
69    if propFilePath.lower().endswith('xml'):
70        log.debug("File has 'xml' suffix - treating as standard XML formatted properties file")
71        properties = readXMLPropertyFile(propFilePath, validKeys)
72    else:
73        properties = readINIPropertyFile(propFilePath, validKeys)
74
75    # if validKeys set, check that all loaded property values are featured in
76    # this list
77    if validKeys:
78        validateProperties(properties, validKeys)
79   
80        # lastly set any default values from the validKeys dict for vals not read in
81        # from the property file
82        _setDefaultValues(properties, validKeys)
83   
84    # lastly, expand out any environment variables set in the properties file
85    properties = _expandEnvironmentVariables(properties)
86    log.info('Properties loaded')
87    return properties
88
89
90def readProperties(propFilePath, validKeys={}):
91    """
92    Determine the type of properties file and load the contents appropriately.
93    @param propFilePath: file path to properties file - either in xml or ini format
94    @type propFilePath: string
95    """
96    log.debug("Reading properties from %s" %propFilePath)
97    properties = {}
98    if propFilePath.lower().endswith('xml'):
99        log.debug("File has 'xml' suffix - treating as standard XML formatted properties file")
100        properties = readXMLPropertyFile(propFilePath, validKeys)
101    else:
102        properties = readINIPropertyFile(propFilePath, validKeys)
103   
104    # lastly, expand out any environment variables set in the properties file
105    properties = _expandEnvironmentVariables(properties)
106    log.info('Properties loaded')
107    return properties
108       
109
110def readINIPropertyFile(propFilePath, validKeys):
111    """
112    Read 'ini' type property file - i.e. a flat text file with key/value
113    data separated into sections
114
115    @param propFilePath: file path to properties file - either in xml or ini format
116    @type propFilePath: string
117    @param validKeys: a dictionary of valid values to be read from the file
118    - if values are encountered that are not in this list, an exception will be thrown
119    - if all info should be read, set this param to 'None'
120    @type validKeys: dict
121    @return: dict with the loaded properties in
122    @raise ValueError: if a key is read in from the file that is not included in the
123    specified validKeys dict
124    """
125    log.debug("File is not marked as XML - treating as flat, 'ini' format file")
126   
127    if not os.path.isfile(propFilePath):
128        raise ValueError("Error parsing properties file \"%s\": No such file" %propFilePath)
129
130    cfg = CaseSensitiveConfigParser()
131    cfg.read(propFilePath)
132    properties = {}
133   
134    # NB, add 'DEFAULT' section since this isn't returned by the 'sections()'
135    sections = cfg.sections()
136    sections.append('DEFAULT')
137   
138    # parse data from the specified sections of the config file
139    for section in sections:
140        if section == 'DEFAULT':
141            properties.update(_parseConfig(cfg, validKeys, section=section))
142        else:
143            keys = validKeys
144            if section == 'WS-Security':
145                keys = __wsSecurityValidKeys #WSSecurityConfig.defParam
146            properties[section] = _parseConfig(cfg, keys, section=section)
147
148    log.debug("Finished reading from INI properties file")
149    return properties
150   
151
152def _parseConfig(cfg, validKeys, section='DEFAULT'):
153    '''
154    Extract parameters from cfg config object
155    @param cfg: config object
156    @type cfg: CaseSensitiveConfigParser
157    @param validKeys: a dictionary of valid values to be read from the file - used to check the
158    type of the input parameter to ensure (lists) are handled correctly
159    @type validKeys: dict
160    @keyword section: section of config file to parse from
161    @type section: string
162    @return: dict with the loaded properties in
163    '''
164    log.debug("Parsing section: %s" %section)
165    keys = {}
166    properties = {}
167    if section == 'DEFAULT':
168        keys = cfg.defaults().keys()
169    else:
170        keys = cfg.options(section)
171        # NB, we need to be careful here - since this will return the section keywords
172        # AND the 'DEFAULT' section entries - so use the difference between the two
173        keys = filter(lambda x:x not in cfg.defaults().keys(), keys)
174
175    for key in keys:
176        val = cfg.get(section, key)
177        if val:
178           
179            # expand out any env vars
180            val = expandEnvVars(val)
181           
182            # if the tag contains an integer, convert this appropriately
183            if val.isdigit():
184                val = int(val)
185           
186            # if the value is in the validKeys dict, ensure it is read in as the correct type
187            if key in validKeys:
188                if isinstance(validKeys[key], list):
189                    # Treat as a list of space separated elements
190                    val = val.split()
191        else:
192            # NB, the XML parser will return empty vals as None, so ensure consistency here
193            val = None
194
195        # check if key already exists; if so, append to list
196        if properties.has_key(key):
197            properties[key] = __listify(properties[key]).extend(val)
198        else:
199            properties[key] = val
200
201    log.debug("Finished parsing section")
202    return properties
203
204
205def readXMLPropertyFile(propFilePath, validKeys, rootElem=None):
206    """
207    Read property file - assuming the standard XML schema
208
209    @param propFilePath: file path to properties file - either in xml or ini format
210    @type propFilePath: string
211    @param validKeys: a dictionary of valid values to be read from the file - used to check the
212    type of the input parameter to ensure (lists) are handled correctly
213    @keyword rootElem: a particular element of an ElementTree can be passed in to
214    use as the root element; NB, if this is set, it will take precedence over any
215    propFilePath specified
216    @type rootElem: ElementTree.Element
217    @return: dict with the loaded properties in
218    """
219    if rootElem is None:
220        try:
221            tree = ElementTree.parse(propFilePath)
222           
223        except IOError, ioErr:
224            raise ValueError, "Error parsing properties file \"%s\": %s" % \
225                                (ioErr.filename, ioErr.strerror)
226   
227        rootElem = tree.getroot()
228        if rootElem is None:
229            raise ValueError, \
230                "Parsing properties file \"%s\": root element is not defined" % \
231                propFilePath
232
233    properties = {}
234    # Copy properties from file into a dictionary
235    try:
236        for elem in rootElem:
237            key = elem.tag
238            val = elem.text
239
240            # expand out any env vars
241            val = expandEnvVars(val)
242
243            # if the tag contains an integer, convert this appropriately
244            if val and val.isdigit():
245                val = int(val)
246           
247            # check for lists - don't recurse into these else the key names
248            # will end up being wrong
249            if key in validKeys and isinstance(validKeys[key], list):
250                # handle lists of elements
251                if len(elem) == 0:
252                    if elem.text is not None:
253                        # Treat as a list of space separated elements
254                        val = val.split()
255                else:
256                    # Parse from a list of sub-elements
257                    val = [expandEnvVars(subElem.text.strip()) for subElem in elem]
258           
259            # otherwise check for subelements; if these exist, recurse and store properties
260            # in an inner dictionary
261            elif len(elem) > 0:
262                keys = validKeys
263                if key == 'WS-Security':
264                    keys = __wsSecurityValidKeys #WSSecurityConfig.defParam
265                val = readXMLPropertyFile(propFilePath, keys, rootElem=elem)
266
267            # check if key already exists; if so, append to list
268            if properties.has_key(key):
269                properties[key] = __listify(properties[key]).extend(val)
270            else:
271                properties[key] = val
272            #print "XML: Key: %s, Val: %s" %(key, properties[key])
273           
274    except Exception, e:
275        raise ValueError, \
276            "Error parsing tag \"%s\" in properties file \"%s\": %s" % \
277            (elem.tag, propFilePath, e)
278
279    log.debug("Finished reading from XML properties file")
280    return properties
281
282
283def __listify(val):
284    '''
285    Checks if val is a list; if so return as is, if not return as list
286    @param val: object to turn into a list
287    @return: val as a list (if it is not already)
288    '''
289    if isinstance(val, list):
290        return val
291    return [val]
292
293
294def validateProperties(properties, validKeys):
295    '''
296    Check the contents of the properties dict to ensure it doesn't contain
297    any keys not featured in the validKeys dict; if it does, throw an exception
298    @param properties: dictionary storing loaded properties
299    @type properties: dict
300    @param validKeys: a dictionary of valid values
301    @type validKeys: dict
302    @raise ValueError: if a key is read in from the file that is not included in the
303    specified validKeys dict
304    '''
305    log.debug("Checking for invalid properties")
306    invalidKeys = []
307    for key in properties:
308        # NB, this is a standard property used across most services - so check
309        # using the properties listed here
310        if key == 'WS-Security':
311            validateProperties(properties[key], __wsSecurityValidKeys)
312        else:
313            if key not in validKeys:
314                invalidKeys.append(key)
315
316    if invalidKeys != []:
317        errorMessage = "The following properties file " + \
318            "elements are invalid: " + ', '.join(invalidKeys)
319        print errorMessage
320        log.error(errorMessage)
321        raise ValueError, errorMessage
322
323
324def _expandEnvironmentVariables(properties):
325    '''
326    Iterate through the values in a dict and expand out environment variables
327    specified in any non password option entries
328    @param properties: dict of properties to expand
329    @type properties: dict
330    @return: dict with expanded values
331    '''
332    log.debug("Expanding out environment variables in properties dictionary")
333    for (key, val) in properties.items():
334        # only check strings or lists of strings
335        if (isinstance(val, list)):
336            newList = []
337            for item in val:
338                newList.append(_expandEnvironmentVariable(key, item))
339           
340            properties[key] = newList
341           
342        elif (isinstance(val, str)):
343            properties[key] = _expandEnvironmentVariable(key, val)
344           
345    log.debug("Finished expanding environment variables")
346    return properties
347
348
349def _expandEnvironmentVariable(key, val):
350    '''
351    Expand out a val, if it contains environment variables and
352    is not password related
353    @param key: key name for the value
354    @type key: str
355    @param val: value to expand env vars out in
356    @type val: str
357    @return: val - with any environment variables expanded out
358    '''
359    if key.lower().find('pwd') == -1 and key.lower().find('password') == -1:
360        val = os.path.expandvars(val)
361    return val
362
363   
364def _setDefaultValues(properties, validKeys):
365    '''
366    Check the contents of the properties dict to ensure it contains all the
367    keys featured in the validKeys dict; if any of these are missing or have no value
368    set for them, set up default values for these in the properties dict
369    @param properties: dictionary storing loaded properties
370    @type properties: dict
371    @param validKeys: a dictionary of valid values
372    @type validKeys: dict
373    @return properties: updated dict with default values for any missing values
374    '''
375    log.debug("Checking for any unset keys")
376    for key in validKeys:
377        if key not in properties or not properties[key]:
378            log.debug("Found missing/unset property - setting default values: %s = %s" \
379                      %(key, validKeys[key]))
380            properties[key] = validKeys[key]
381    log.debug("Finished checking for unset keys")
382
Note: See TracBrowser for help on using the repository browser.