source: TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/configfileparsers.py @ 4569

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/configfileparsers.py@4569
Revision 4569, 22.3 KB checked in by pjkersha, 12 years ago (diff)

Renamed classfactory and configfileparsers modules.

Line 
1"""
2Generic parsers to use when reading in configuration data
3- methods available to deal with both XML and INI (flat text key/val) formats
4"""
5__author__ = "C Byrom - Tessella"
6__date__ = "20/05/08"
7__copyright__ = "(C) 2008 STFC & NERC"
8__license__ = \
9"""This software may be distributed under the terms of the Q Public
10License, version 1.0 or later."""
11__contact__ = "Philip.Kershaw@stfc.ac.uk"
12__revision__ = '$Id$'
13
14from ConfigParser import SafeConfigParser, InterpolationMissingOptionError, \
15    NoOptionError
16from ndg.security.common.wssecurity import WSSecurityConfig
17# For parsing of properties file
18try: # python 2.5
19    from xml.etree import cElementTree as ElementTree
20except ImportError:
21    # if you've installed it yourself it comes this way
22    import cElementTree as ElementTree
23
24import logging, os
25log = logging.getLogger(__name__)
26
27# lambda function to expand out any environment variables in properties read in
28expandEnvVars = lambda x: isinstance(x, basestring) and \
29                    os.path.expandvars(x).strip() or x
30
31
32class CaseSensitiveConfigParser(SafeConfigParser):
33    '''
34    Subclass the SafeConfigParser - to preserve the original string case of the
35    cfg section names - NB, the RawConfigParser default is to lowercase these
36    by default
37    '''
38   
39    def optionxform(self, optionstr):
40        return optionstr
41       
42class ConfigFileParseError(Exception):
43    """Raise for errors in configuration file formatting"""
44
45def readAndValidateProperties(propFilePath, validKeys={}, **iniPropertyFileKw):
46    """
47    Determine the type of properties file and load the contents appropriately.
48    If a dict of valid keys is also specified, check the loaded properties
49    against these.
50   
51    @param propFilePath: file path to properties file - either in xml or ini
52    format
53    @type propFilePath: string
54    @keywords validKeys: a dictionary of valid values to be read from the file
55    - if values are encountered that are not in this list, an exception will be
56    thrown
57    - if all info should be read, this keyword should be left to its default
58    value
59    - NB, this dict will also ensure list data is read in correctly
60    @type validKeys: dict
61    @raise ValueError: if a key is read in from the file that is not included
62    in the specified validKeys dict
63    """
64    log.debug("Reading properties from %s" % propFilePath)
65    properties = {}
66    if propFilePath.lower().endswith('.xml'):
67        log.debug("File has 'xml' suffix - treating as standard XML formatted "
68                  "properties file")
69        log.warning("Current version of code for properties handling with "
70                    "XML is untested - may be deprecated")
71        properties = readXMLPropertyFile(propFilePath, validKeys)
72        # if validKeys set, check that all loaded property values are featured
73        # in this list
74        if validKeys:
75            validateProperties(properties, validKeys)
76       
77            # lastly set any default values from the validKeys dict for vals
78            # not read in from the property file
79            _setDefaultValues(properties, validKeys)
80    else:
81        properties = readINIPropertyFile(propFilePath, validKeys,
82                                         **iniPropertyFileKw)
83       
84        # Ugly hack to allow for sections and option prefixes in the validation
85        # and setting of defaults
86        if validKeys:
87            sections = iniPropertyFileKw.get('sections')
88            prefix = iniPropertyFileKw.get('prefix')
89            if sections is not None:
90                for section in sections:
91                    if section == 'DEFAULT':
92                        propBranch = properties
93                    else:
94                        propBranch = properties[section]
95                       
96                    validateProperties(propBranch, validKeys)
97                    _setDefaultValues(propBranch, validKeys)
98                   
99            else:
100                validateProperties(properties, validKeys)
101                _setDefaultValues(properties, validKeys)
102
103   
104    # lastly, expand out any environment variables set in the properties file
105    properties = _expandEnvironmentVariables(properties)
106    log.info('Properties loaded')
107    return properties
108
109
110def readProperties(propFilePath, validKeys={}, **iniPropertyFileKw):
111    """
112    Determine the type of properties file and load the contents appropriately.
113    @param propFilePath: file path to properties file - either in xml or ini
114    format
115    @type propFilePath: string
116    """
117    log.debug("Reading properties from %s" %propFilePath)
118    properties = {}
119    if propFilePath.lower().endswith('.xml'):
120        log.debug("File has 'xml' suffix - treating as standard XML formatted "
121                  "properties file")
122        log.warning("Current version of code for properties handling with "
123                    "XML is untested - may be deprecated")
124        properties = readXMLPropertyFile(propFilePath, validKeys)
125    else:
126        properties = readINIPropertyFile(propFilePath, validKeys,
127                                         **iniPropertyFileKw)
128   
129    # lastly, expand out any environment variables set in the properties file
130    properties = _expandEnvironmentVariables(properties)
131    log.info('Properties loaded')
132    return properties
133       
134
135class INIPropertyFile(object):
136    '''INI Property file reading class
137   
138    __call__ method enables a standalone read function'''
139       
140    def read(self, propFilePath, validKeys, cfg=None, sections=None,
141             wsseSection='WS-Security', prefix=''):
142        """
143        Read 'ini' type property file - i.e. a flat text file with key/value
144        data separated into sections
145   
146        @param propFilePath: file path to properties file - either in xml or
147        ini format
148        @type propFilePath: string
149        @param validKeys: a dictionary of valid values to be read from the file
150        - if values are encountered that are not in this list, an exception
151        will be thrown
152        - if all info should be read, set this param to 'None'
153        @type validKeys: dict
154        @type sections: basestring
155        @param sections: sections to be read from - defaults to all sections in the
156        file
157        @type wsseSection: basestring
158        @param wsseSection: section to read WS-Security settings from as
159        specified by WSSecurityConfig class.  WS-Security section doesn't need
160        to be present and can be ignored.
161        @rtype: dict
162        @return: dict with the loaded properties in
163        @raise ValueError: if a key is read in from the file that is not
164        included in the specified validKeys dict
165        """
166        log.debug("File is not marked as XML - treating as flat 'ini' format "
167                  "file")
168       
169        # Keep a record of property file path setting
170        self.propFilePath = propFilePath
171           
172        if cfg is None:
173            self.cfg = CaseSensitiveConfigParser()
174            self.cfg.read(propFilePath)
175            if not os.path.isfile(propFilePath):
176                raise IOError('Error parsing properties file "%s": No such '
177                              'file' % propFilePath)
178        else:
179            self.cfg = cfg
180               
181        properties = {}
182       
183        if sections is None:
184            # NB, add 'DEFAULT' section since this isn't returned by the
185            # 'sections()'
186            sections = self.cfg.sections()
187            sections.append('DEFAULT')
188       
189        # parse data from the specified sections of the config file
190        for section in sections:
191            if section == 'DEFAULT':
192                properties.update(_parseConfig(self.cfg, 
193                                               validKeys, 
194                                               section=section,
195                                               prefix=prefix))
196            else:
197                if section == wsseSection:
198                    keys = WSSecurityConfig.propertyDefaults
199                else:
200                    keys = validKeys
201                   
202                properties[section] = _parseConfig(self.cfg, 
203                                                   keys, 
204                                                   section=section,
205                                                   prefix=prefix)
206   
207        log.debug("Finished reading from INI properties file")
208        return properties
209   
210    # Enables use of this class like a function see below ...
211    __call__ = read
212   
213   
214# Enable read INI of file as a one shot call
215readINIPropertyFile = INIPropertyFile()   
216
217
218class INIPropertyFileWithValidation(INIPropertyFile):
219    '''Extension of INI Property file reading class to make a callable that
220    validates as well as reads in the properties.  Also see
221    readAndValidateINIPropertyFile in this module'''
222   
223    def readAndValidate(self, propFilePath, validKeys, **kw):
224        prop = super(INIPropertyFileWithValidation,self).__call__(propFilePath,
225                                                                  validKeys, 
226                                                                  **kw)
227       
228        # Pass wsseSection but respect validateProperties default value
229        wsseSection = kw.get('wssSection')
230        if wsseSection is not None:
231            validatePropKw = {'wsseSection': wsseSection}
232        else:
233            validatePropKw = {}
234           
235        validateProperties(prop, validKeys, **validatePropKw)
236        return prop
237   
238    __call__ = readAndValidate
239   
240# Enable read and validation of INI file as a one shot call
241readAndValidateINIPropertyFile = INIPropertyFileWithValidation()
242
243
244def _parseConfig(cfg, validKeys, section='DEFAULT', prefix=''):
245    '''
246    Extract parameters from cfg config object
247    @param cfg: config object
248    @type cfg: CaseSensitiveConfigParser
249    @param validKeys: a dictionary of valid values to be read from the file -
250    used to check the type of the input parameter to ensure (lists) are handled
251    correctly
252    @type validKeys: dict
253    @keyword section: section of config file to parse from
254    @type section: string
255    @return: dict with the loaded properties in
256    '''
257    log.debug("Parsing section: %s" % section)
258
259    propRoot = {}
260    propThisBranch = propRoot
261   
262    if section == 'DEFAULT':
263        keys = cfg.defaults().keys()
264    else:
265        keys = cfg.options(section)
266        # NB, we need to be careful here - since this will return the section
267        # keywords AND the 'DEFAULT' section entries - so use the difference
268        # between the two
269        keys = filter(lambda x:x not in cfg.defaults().keys(), keys)
270
271    for key in keys:
272        try:
273            val = cfg.get(section, key)
274        except InterpolationMissingOptionError, e:
275            log.warning('Ignoring property "%s": %s' % (key, e))
276            continue
277       
278        # Allow for prefixes - 1st a prefix global to all parameters
279#        keyLevels = key.split('.')
280#        if prefix:
281#            if keyLevels[0] == prefix:
282#                keyLevels = keyLevels[1:]
283#                if keyLevels == []:
284#                    raise ConfigFileParseError('Expecting "%s.<option>"; got '
285#                                               '"%s"' % ((prefix,)*2))
286#            else:
287#                continue           
288        if prefix:
289            if key.startswith(prefix):
290                keyLevels = key.replace(prefix+'.', '', 1).split('.') 
291                if keyLevels == []:
292                    raise ConfigFileParseError('Expecting "%s.<option>"; got '
293                                               '"%s"' % ((prefix,)*2))
294            else:
295                continue
296        else:
297            keyLevels = key.split('.')
298                       
299        # 2nd - prefixes to denote sections
300        if len(keyLevels) > 1:
301               
302            # Nb. This allows only one level of nesting - subsequent levels if
303            # present are represented by a concatenation of the levels joined
304            # by underscores
305            subSectionKey = keyLevels[0]
306            subKey = '_'.join(keyLevels[1:])
307            if subSectionKey in validKeys and \
308               isinstance(validKeys[subSectionKey], dict):
309                val = _parseVal(cfg, section, key, validKeys[subSectionKey],
310                                subKey=subKey)
311                if subSectionKey in propThisBranch:
312                    propThisBranch[subSectionKey][subKey] = val
313                else:
314                    propThisBranch[subSectionKey] = {subKey: val}
315        else: 
316            # No sub-section present           
317            subKey = keyLevels[0]
318            val = _parseVal(cfg, section, key, validKeys, subKey=subKey)
319           
320            # check if key already exists; if so, append to list
321            if propThisBranch.has_key(subKey):
322                propThisBranch[subKey] = __listify(
323                                            propThisBranch[subKey]).extend(val)
324            else:
325                propThisBranch[subKey] = val
326
327    log.debug("Finished parsing section")
328    return propRoot
329
330def _parseVal(cfg, section, option, validKeys, subKey=None):
331    '''Convert option to correct type trying each parser config routine in
332    turn.  Convert to a list if validKeys dict item indicates so
333   
334    @type cfg: ndg.security.common.utils.ConfigFileParsers.CaseSensitiveConfigParser
335    @param cfg: config file object
336    @type section: basestring
337    @param section: section in config file to read from
338    @type key: basestring
339    @param key: section option to read
340    @type validKeys: dict
341    @param validKeys: key look-up - if item is set to list type then the option
342    value in the config file will be split into a list.'''
343   
344    if subKey:
345        key = subKey
346    else:
347        key = option
348         
349    conversionFuncs = (cfg.getint, cfg.getfloat, cfg.getboolean, cfg.get)
350    for conversionFunc in conversionFuncs:
351        try:
352            val = conversionFunc(section, option)
353            if val == '':
354                # NB, the XML parser will return empty vals as None, so ensure
355                # consistency here
356                val = None
357               
358            elif isinstance(val, basestring):
359                # expand out any env vars
360                val = expandEnvVars(val)
361               
362                # ensure it is read in as the correct type
363                if key in validKeys and isinstance(validKeys[key], list):
364                    # Treat as a list of space separated string type elements
365                    # Nb. lists only cater for string type elements
366                    val = val.split()
367             
368            return val
369        except ValueError:
370            continue
371        except Exception, e:
372            log.error('Error parsing option "%s" in section "%s": %s' %
373                      (section, key, e))
374            raise
375
376    raise ValueError('Error parsing option "%s" in section "%s"'%(section,key))
377
378         
379def readXMLPropertyFile(propFilePath, validKeys, rootElem=None):
380    """
381    Read property file - assuming the standard XML schema
382
383    @param propFilePath: file path to properties file - either in xml or ini
384    format
385    @type propFilePath: string
386    @param validKeys: a dictionary of valid values to be read from the file -
387    used to check the type of the input parameter to ensure (lists) are handled
388    correctly
389    @keyword rootElem: a particular element of an ElementTree can be passed in
390    to use as the root element; NB, if this is set, it will take precedence
391    over any propFilePath specified
392    @type rootElem: ElementTree.Element
393    @return: dict with the loaded properties in
394    """
395    if rootElem is None:
396        try:
397            tree = ElementTree.parse(propFilePath)
398           
399        except IOError, ioErr:
400            raise ValueError("Error parsing properties file \"%s\": %s" % 
401                             (ioErr.filename, ioErr.strerror))
402   
403        rootElem = tree.getroot()
404        if rootElem is None:
405            raise ValueError('Parsing properties file "%s": root element is '
406                             'not defined' % propFilePath)
407
408    properties = {}
409    # Copy properties from file into a dictionary
410    try:
411        for elem in rootElem:
412            key = elem.tag
413            val = elem.text
414
415            # expand out any env vars
416            val = expandEnvVars(val)
417
418            # if the tag contains an integer, convert this appropriately
419            if val and val.isdigit():
420                val = int(val)
421           
422            # check for lists - don't recurse into these else the key names
423            # will end up being wrong
424            if key in validKeys and isinstance(validKeys[key], list):
425                # handle lists of elements
426                if len(elem) == 0:
427                    if elem.text is not None:
428                        # Treat as a list of space separated elements
429                        val = val.split()
430                else:
431                    # Parse from a list of sub-elements
432                    val = [expandEnvVars(subElem.text.strip()) \
433                           for subElem in elem]
434           
435            # otherwise check for subelements; if these exist, recurse and
436            # store properties in an inner dictionary
437            elif len(elem) > 0:
438                keys = validKeys
439                if key == 'WS-Security':
440                    keys = WSSecurityConfig.propertyDefaults
441                val = readXMLPropertyFile(propFilePath, keys, rootElem=elem)
442
443            # check if key already exists; if so, append to list
444            if properties.has_key(key):
445                properties[key] = __listify(properties[key]).extend(val)
446            else:
447                properties[key] = val
448           
449    except Exception, e:
450        raise ValueError('Error parsing tag "%s" in properties file "%s": %s' %
451                         (elem.tag, propFilePath, e))
452
453    log.debug("Finished reading from XML properties file")
454    return properties
455
456
457def __listify(val):
458    '''
459    Checks if val is a list; if so return as is, if not return as list
460   
461    @type val: list
462    @param val: object to turn into a list
463    @rtype: list
464    @return: val as a list (if it is not already)
465    '''
466    if isinstance(val, list):
467        return val
468    return [val]
469
470
471def validateProperties(properties, validKeys, wsseSection='WS-Security'):
472    '''
473    Check the contents of the properties dict to ensure it doesn't contain
474    any keys not featured in the validKeys dict; if it does, throw an exception
475    @param properties: dictionary storing loaded properties
476    @type properties: dict
477    @param validKeys: a dictionary of valid values
478    @type validKeys: dict
479    @raise ValueError: if a key is read in from the file that is not included
480    in the specified validKeys dict
481    '''
482    log.debug("Checking for invalid properties")
483    invalidKeys = []
484    for key in validKeys:
485        # NB, this is a standard property used across most services - so check
486        # using the properties listed here
487        if key == wsseSection:
488            validateProperties(properties[key], WSSecurityConfig.propertyDefaults)
489           
490        elif validKeys[key] and isinstance(validKeys[key], dict):
491            validateProperties(properties.get(key, {}), validKeys[key])
492               
493        elif key not in properties and nonDefaultProperty(validKeys[key]):
494            invalidKeys += [key]
495
496    if invalidKeys != []:
497        errorMessage = "The following properties file " + \
498            "elements are missing and must be set: " + ', '.join(invalidKeys)
499        log.error(errorMessage)
500        raise ValueError(errorMessage)
501
502nonDefaultProperty = lambda prop:prop==NotImplemented or prop==[NotImplemented]
503
504def _expandEnvironmentVariables(properties):
505    '''
506    Iterate through the values in a dict and expand out environment variables
507    specified in any non password option entries
508    @param properties: dict of properties to expand
509    @type properties: dict
510    @return: dict with expanded values
511    '''
512    log.debug("Expanding out environment variables in properties dictionary")
513    for key, val in properties.items():
514        # only check strings or lists of strings
515        if isinstance(val, list):           
516            properties[key] = [_expandEnvironmentVariable(key, item) \
517                               for item in val]
518           
519        elif isinstance(val, str):
520            properties[key] = _expandEnvironmentVariable(key, val)
521           
522    log.debug("Finished expanding environment variables")
523    return properties
524
525
526def _expandEnvironmentVariable(key, val):
527    '''
528    Expand out a val, if it contains environment variables and
529    is not password related
530    @param key: key name for the value
531    @type key: str
532    @param val: value to expand env vars out in
533    @type val: str
534    @rtype: basestring
535    @return: val - with any environment variables expanded out
536    '''
537    if key.lower().find('pwd') == -1 and key.lower().find('password') == -1:
538        val = os.path.expandvars(val)
539    return val
540
541   
542def _setDefaultValues(properties, validKeys, sectionKey=''):
543    '''
544    Check the contents of the properties dict to ensure it contains all the
545    keys featured in the validKeys dict; if any of these are missing or have
546    no value set for them, set up default values for these in the properties
547    dict
548    @param properties: dictionary storing loaded properties
549    @type properties: dict
550    @param validKeys: a dictionary of valid values
551    @type validKeys: dict
552    @rtype: dict
553    @return properties: updated dict with default values for any missing values
554    '''
555   
556   
557    if sectionKey:
558        sectionKeyDot = sectionKey+'.'
559        log.debug("Checking for any unset keys for %s sub-section"%sectionKey)
560    else:
561        sectionKeyDot = ''
562        log.debug("Checking for any unset keys")
563       
564    for key in validKeys:
565        if key not in properties or not properties[key]:
566            if validKeys[key] == NotImplemented:
567                errorMessage = 'Missing property "%s" must be set.' % key
568                log.error(errorMessage)
569                raise ValueError(errorMessage)
570           
571            log.warning("Found missing/unset property - setting default "
572                        "values: %s%s=%s" % (sectionKeyDot,key,validKeys[key]))
573            properties[key] = validKeys[key]
574           
575        elif isinstance(properties[key], dict):
576            _setDefaultValues(properties[key], validKeys[key], sectionKey=key)
577       
578    log.debug("Finished checking for unset keys")
579
Note: See TracBrowser for help on using the repository browser.