source: TI12-security/trunk/NDG_XACML/ndg/xacml/utils/configfileparsers.py @ 6746

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI12-security/trunk/NDG_XACML/ndg/xacml/utils/configfileparsers.py@6746
Revision 6746, 22.3 KB checked in by pjkersha, 10 years ago (diff)

Added more !ETree readers for the different types.

Line 
1"""
2Generic parsers to use when reading in configuration data
3- methods available to deal with both XML and INI (flat text key/val) formats
4"""
5__author__ = "C Byrom - Tessella"
6__date__ = "20/05/08"
7__copyright__ = "(C) 2009 Science and Technology Facilities Council"
8__license__ = "BSD - see LICENSE file in top-level directory"
9__contact__ = "Philip.Kershaw@stfc.ac.uk"
10__revision__ = '$Id: $'
11
12from ConfigParser import SafeConfigParser, InterpolationMissingOptionError, \
13    NoOptionError
14
15# For parsing of properties file
16try: # python 2.5
17    from xml.etree import cElementTree as ElementTree
18except ImportError:
19    # if you've installed it yourself it comes this way
20    import cElementTree as ElementTree
21
22import logging, os
23log = logging.getLogger(__name__)
24
25# lambda function to expand out any environment variables in properties read in
26expandEnvVars = lambda x: isinstance(x, basestring) and \
27                    os.path.expandvars(x).strip() or x
28
29
30class CaseSensitiveConfigParser(SafeConfigParser):
31    '''
32    Subclass the SafeConfigParser - to preserve the original string case of the
33    cfg section names - NB, the RawConfigParser default is to lowercase these
34    by default
35    '''
36   
37    def optionxform(self, optionstr):
38        return optionstr
39       
40class ConfigFileParseError(Exception):
41    """Raise for errors in configuration file formatting"""
42
43def readAndValidateProperties(propFilePath, validKeys={}, **iniPropertyFileKw):
44    """
45    Determine the type of properties file and load the contents appropriately.
46    If a dict of valid keys is also specified, check the loaded properties
47    against these.
48   
49    @param propFilePath: file path to properties file - either in xml or ini
50    format
51    @type propFilePath: string
52    @keywords validKeys: a dictionary of valid values to be read from the file
53    - if values are encountered that are not in this list, an exception will be
54    thrown
55    - if all info should be read, this keyword should be left to its default
56    value
57    - NB, this dict will also ensure list data is read in correctly
58    @type validKeys: dict
59    @raise ValueError: if a key is read in from the file that is not included
60    in the specified validKeys dict
61    """
62    log.debug("Reading properties from %s" % propFilePath)
63    properties = {}
64    if propFilePath.lower().endswith('.xml'):
65        log.debug("File has 'xml' suffix - treating as standard XML formatted "
66                  "properties file")
67        log.warning("Current version of code for properties handling with "
68                    "XML is untested - may be deprecated")
69        properties = readXMLPropertyFile(propFilePath, validKeys)
70        # if validKeys set, check that all loaded property values are featured
71        # in this list
72        if validKeys:
73            validateProperties(properties, validKeys)
74       
75            # lastly set any default values from the validKeys dict for vals
76            # not read in from the property file
77            _setDefaultValues(properties, validKeys)
78    else:
79        properties = readINIPropertyFile(propFilePath, validKeys,
80                                         **iniPropertyFileKw)
81       
82        # Ugly hack to allow for sections and option prefixes in the validation
83        # and setting of defaults
84        if validKeys:
85            sections = iniPropertyFileKw.get('sections')
86            prefix = iniPropertyFileKw.get('prefix')
87            if sections is not None:
88                for section in sections:
89                    if section == 'DEFAULT':
90                        propBranch = properties
91                    else:
92                        propBranch = properties[section]
93                       
94                    validateProperties(propBranch, validKeys)
95                    _setDefaultValues(propBranch, validKeys)
96                   
97            else:
98                validateProperties(properties, validKeys)
99                _setDefaultValues(properties, validKeys)
100
101   
102    # lastly, expand out any environment variables set in the properties file
103    properties = _expandEnvironmentVariables(properties)
104    log.info('Properties loaded')
105    return properties
106
107
108def readProperties(propFilePath, validKeys={}, **iniPropertyFileKw):
109    """
110    Determine the type of properties file and load the contents appropriately.
111    @param propFilePath: file path to properties file - either in xml or ini
112    format
113    @type propFilePath: string
114    """
115    log.debug("Reading properties from %s" %propFilePath)
116    properties = {}
117    if propFilePath.lower().endswith('.xml'):
118        log.debug("File has 'xml' suffix - treating as standard XML formatted "
119                  "properties file")
120        log.warning("Current version of code for properties handling with "
121                    "XML is untested - may be deprecated")
122        properties = readXMLPropertyFile(propFilePath, validKeys)
123    else:
124        properties = readINIPropertyFile(propFilePath, validKeys,
125                                         **iniPropertyFileKw)
126   
127    # lastly, expand out any environment variables set in the properties file
128    properties = _expandEnvironmentVariables(properties)
129    log.info('Properties loaded')
130    return properties
131       
132
133class INIPropertyFile(object):
134    '''INI Property file reading class
135   
136    __call__ method enables a standalone read function'''
137   
138    defaultOptionNames = ('here',)
139   
140    def read(self, 
141             propFilePath, 
142             validKeys, 
143             cfg=None, 
144             sections=None,
145             defaultItems={}, 
146             prefix=''):
147        """
148        Read 'ini' type property file - i.e. a flat text file with key/value
149        data separated into sections
150   
151        @param propFilePath: file path to properties file - either in xml or
152        ini format
153        @type propFilePath: string
154        @param validKeys: a dictionary of valid values to be read from the file
155        - if values are encountered that are not in this list, an exception
156        will be thrown
157        - if all info should be read, set this param to 'None'
158        @type validKeys: dict
159        @type sections: basestring
160        @param sections: sections to be read from - defaults to all sections in the
161        file
162        @type defaultItems: dict
163        @param defaultItems: add items via this input dictionary as well as by
164        retrieval from config file itself.  This only comes into effect if
165        cfg was not set and a new config object is created locally.
166        @rtype: dict
167        @return: dict with the loaded properties in
168        @raise ValueError: if a key is read in from the file that is not
169        included in the specified validKeys dict
170        """
171        log.debug("File is not marked as XML - treating as flat 'ini' format "
172                  "file")
173       
174        # Keep a record of property file path setting
175        self.propFilePath = propFilePath
176           
177        if cfg is None:
178            # Add default item for file location to enable convenient
179            # substitutions within the file
180            defaultItems['here'] = os.path.dirname(propFilePath)
181           
182            self.cfg = CaseSensitiveConfigParser(defaults=defaultItems)
183            self.cfg.read(propFilePath)
184            if not os.path.isfile(propFilePath):
185                raise IOError('Error parsing properties file "%s": No such '
186                              'file' % propFilePath)
187        else:
188            self.cfg = cfg
189               
190        properties = {}
191       
192        if sections is None:
193            # NB, add 'DEFAULT' section since this isn't returned by the
194            # 'sections()'
195            sections = self.cfg.sections()
196            sections.append('DEFAULT')
197       
198        # parse data from the specified sections of the config file
199        for section in sections:
200            if section == 'DEFAULT':
201                properties.update(_parseConfig(self.cfg, 
202                                               validKeys, 
203                                               section=section,
204                                               prefix=prefix))
205            else:                   
206                properties[section] = _parseConfig(self.cfg, 
207                                                   validKeys, 
208                                                   section=section,
209                                                   prefix=prefix)
210   
211               
212        # Get rid of 'here' default item to avoid interfering with later
213        # processing
214        for opt in INIPropertyFile.defaultOptionNames:
215            properties.pop(opt, None)
216       
217        log.debug("Finished reading from INI properties file")
218        return properties
219   
220    # Enables use of this class like a function see below ...
221    __call__ = read
222   
223   
224# Enable read INI of file as a one shot call
225readINIPropertyFile = INIPropertyFile()   
226
227
228class INIPropertyFileWithValidation(INIPropertyFile):
229    '''Extension of INI Property file reading class to make a callable that
230    validates as well as reads in the properties.  Also see
231    readAndValidateINIPropertyFile in this module'''
232   
233    def readAndValidate(self, propFilePath, validKeys, **kw):
234        prop = super(INIPropertyFileWithValidation,self).__call__(propFilePath,
235                                                                  validKeys, 
236                                                                  **kw)
237       
238        # Pass wsseSection but respect validateProperties default value
239        wsseSection = kw.get('wssSection')
240        if wsseSection is not None:
241            validatePropKw = {'wsseSection': wsseSection}
242        else:
243            validatePropKw = {}
244           
245        validateProperties(prop, validKeys, **validatePropKw)
246        return prop
247   
248    __call__ = readAndValidate
249   
250# Enable read and validation of INI file as a one shot call
251readAndValidateINIPropertyFile = INIPropertyFileWithValidation()
252
253
254def _parseConfig(cfg, validKeys, section='DEFAULT', prefix=''):
255    '''
256    Extract parameters from cfg config object
257    @param cfg: config object
258    @type cfg: CaseSensitiveConfigParser
259    @param validKeys: a dictionary of valid values to be read from the file -
260    used to check the type of the input parameter to ensure (lists) are handled
261    correctly
262    @type validKeys: dict
263    @keyword section: section of config file to parse from
264    @type section: string
265    @return: dict with the loaded properties in
266    '''
267    log.debug("Parsing section: %s" % section)
268
269    propRoot = {}
270    propThisBranch = propRoot
271   
272    if section == 'DEFAULT':
273        keys = cfg.defaults().keys()
274    else:
275        keys = cfg.options(section)
276        # NB, we need to be careful here - since this will return the section
277        # keywords AND the 'DEFAULT' section entries - so use the difference
278        # between the two
279        keys = filter(lambda x:x not in cfg.defaults().keys(), keys)
280
281    for key in keys:
282        try:
283            val = cfg.get(section, key)
284        except InterpolationMissingOptionError, e:
285            log.warning('Ignoring property "%s": %s' % (key, e))
286            continue
287       
288        # Allow for prefixes - 1st a prefix global to all parameters
289#        keyLevels = key.split('.')
290#        if prefix:
291#            if keyLevels[0] == prefix:
292#                keyLevels = keyLevels[1:]
293#                if keyLevels == []:
294#                    raise ConfigFileParseError('Expecting "%s.<option>"; got '
295#                                               '"%s"' % ((prefix,)*2))
296#            else:
297#                continue           
298        if prefix:
299            if key.startswith(prefix):
300                keyLevels = key.replace(prefix+'.', '', 1).split('.') 
301                if keyLevels == []:
302                    raise ConfigFileParseError('Expecting "%s.<option>"; got '
303                                               '"%s"' % ((prefix,)*2))
304            else:
305                continue
306        else:
307            keyLevels = key.split('.')
308                       
309        # 2nd - prefixes to denote sections
310        if len(keyLevels) > 1:
311               
312            # Nb. This allows only one level of nesting - subsequent levels if
313            # present are represented by a concatenation of the levels joined
314            # by underscores
315            subSectionKey = keyLevels[0]
316            subKey = '_'.join(keyLevels[1:])
317            if subSectionKey in validKeys and \
318               isinstance(validKeys[subSectionKey], dict):
319                val = _parseVal(cfg, section, key, validKeys[subSectionKey],
320                                subKey=subKey)
321                if subSectionKey in propThisBranch:
322                    propThisBranch[subSectionKey][subKey] = val
323                else:
324                    propThisBranch[subSectionKey] = {subKey: val}
325        else: 
326            # No sub-section present           
327            subKey = keyLevels[0]
328            val = _parseVal(cfg, section, key, validKeys, subKey=subKey)
329           
330            # check if key already exists; if so, append to list
331            if propThisBranch.has_key(subKey):
332                propThisBranch[subKey] = __listify(
333                                            propThisBranch[subKey]).extend(val)
334            else:
335                propThisBranch[subKey] = val
336
337    log.debug("Finished parsing section")
338    return propRoot
339
340def _parseVal(cfg, section, option, validKeys, subKey=None):
341    '''Convert option to correct type trying each parser config routine in
342    turn.  Convert to a list if validKeys dict item indicates so
343   
344    @type cfg: ndg.xacml.utils.configfileparsers.CaseSensitiveConfigParser
345    @param cfg: config file object
346    @type section: basestring
347    @param section: section in config file to read from
348    @type key: basestring
349    @param key: section option to read
350    @type validKeys: dict
351    @param validKeys: key look-up - if item is set to list type then the option
352    value in the config file will be split into a list.'''
353   
354    if subKey:
355        key = subKey
356    else:
357        key = option
358         
359    conversionFuncs = (cfg.getint, cfg.getfloat, cfg.getboolean, cfg.get)
360    for conversionFunc in conversionFuncs:
361        try:
362            val = conversionFunc(section, option)
363            if val == '':
364                # NB, the XML parser will return empty vals as None, so ensure
365                # consistency here
366                val = None
367               
368            elif isinstance(val, basestring):
369                # expand out any env vars
370                val = expandEnvVars(val)
371               
372                # ensure it is read in as the correct type
373                if key in validKeys and isinstance(validKeys[key], list):
374                    # Treat as a list of space separated string type elements
375                    # Nb. lists only cater for string type elements
376                    val = val.split()
377             
378            return val
379        except ValueError:
380            continue
381        except Exception, e:
382            log.error('Error parsing option "%s" in section "%s": %s' %
383                      (section, key, e))
384            raise
385
386    raise ValueError('Error parsing option "%s" in section "%s"'%(section,key))
387
388         
389def readXMLPropertyFile(propFilePath, validKeys, rootElem=None):
390    """
391    Read property file - assuming the standard XML schema
392
393    @param propFilePath: file path to properties file - either in xml or ini
394    format
395    @type propFilePath: string
396    @param validKeys: a dictionary of valid values to be read from the file -
397    used to check the type of the input parameter to ensure (lists) are handled
398    correctly
399    @keyword rootElem: a particular element of an ElementTree can be passed in
400    to use as the root element; NB, if this is set, it will take precedence
401    over any propFilePath specified
402    @type rootElem: ElementTree.Element
403    @return: dict with the loaded properties in
404    """
405    if rootElem is None:
406        try:
407            tree = ElementTree.parse(propFilePath)
408           
409        except IOError, ioErr:
410            raise ValueError("Error parsing properties file \"%s\": %s" % 
411                             (ioErr.filename, ioErr.strerror))
412   
413        rootElem = tree.getroot()
414        if rootElem is None:
415            raise ValueError('Parsing properties file "%s": root element is '
416                             'not defined' % propFilePath)
417
418    properties = {}
419    # Copy properties from file into a dictionary
420    try:
421        for elem in rootElem:
422            key = elem.tag
423            val = elem.text
424
425            # expand out any env vars
426            val = expandEnvVars(val)
427
428            # if the tag contains an integer, convert this appropriately
429            if val and val.isdigit():
430                val = int(val)
431           
432            # check for lists - don't recurse into these else the key names
433            # will end up being wrong
434            if key in validKeys and isinstance(validKeys[key], list):
435                # handle lists of elements
436                if len(elem) == 0:
437                    if elem.text is not None:
438                        # Treat as a list of space separated elements
439                        val = val.split()
440                else:
441                    # Parse from a list of sub-elements
442                    val = [expandEnvVars(subElem.text.strip()) \
443                           for subElem in elem]
444           
445            # otherwise check for subelements; if these exist, recurse and
446            # store properties in an inner dictionary
447            elif len(elem) > 0:
448                val = readXMLPropertyFile(propFilePath,validKeys,rootElem=elem)
449
450            # check if key already exists; if so, append to list
451            if properties.has_key(key):
452                properties[key] = __listify(properties[key]).extend(val)
453            else:
454                properties[key] = val
455           
456    except Exception, e:
457        raise ValueError('Error parsing tag "%s" in properties file "%s": %s' %
458                         (elem.tag, propFilePath, e))
459
460    log.debug("Finished reading from XML properties file")
461    return properties
462
463
464def __listify(val):
465    '''
466    Checks if val is a list; if so return as is, if not return as list
467   
468    @type val: list
469    @param val: object to turn into a list
470    @rtype: list
471    @return: val as a list (if it is not already)
472    '''
473    if isinstance(val, list):
474        return val
475    return [val]
476
477
478def validateProperties(properties, validKeys):
479    '''
480    Check the contents of the properties dict to ensure it doesn't contain
481    any keys not featured in the validKeys dict; if it does, throw an exception
482    @param properties: dictionary storing loaded properties
483    @type properties: dict
484    @param validKeys: a dictionary of valid values
485    @type validKeys: dict
486    @raise ValueError: if a key is read in from the file that is not included
487    in the specified validKeys dict
488    '''
489    log.debug("Checking for invalid properties")
490    invalidKeys = []
491    for key in validKeys:
492        # NB, this is a standard property used across most services - so check
493        # using the properties listed here
494        if validKeys[key] and isinstance(validKeys[key], dict):
495            validateProperties(properties.get(key, {}), validKeys[key])
496               
497        elif key not in properties and nonDefaultProperty(validKeys[key]):
498            invalidKeys += [key]
499
500    if invalidKeys != []:
501        errorMessage = "The following properties file " + \
502            "elements are missing and must be set: " + ', '.join(invalidKeys)
503        log.error(errorMessage)
504        raise ValueError(errorMessage)
505
506nonDefaultProperty = lambda prop:prop==NotImplemented or prop==[NotImplemented]
507
508def _expandEnvironmentVariables(properties):
509    '''
510    Iterate through the values in a dict and expand out environment variables
511    specified in any non password option entries
512    @param properties: dict of properties to expand
513    @type properties: dict
514    @return: dict with expanded values
515    '''
516    log.debug("Expanding out environment variables in properties dictionary")
517    for key, val in properties.items():
518        # only check strings or lists of strings
519        if isinstance(val, list):           
520            properties[key] = [_expandEnvironmentVariable(key, item) \
521                               for item in val]
522           
523        elif isinstance(val, str):
524            properties[key] = _expandEnvironmentVariable(key, val)
525           
526    log.debug("Finished expanding environment variables")
527    return properties
528
529
530def _expandEnvironmentVariable(key, val):
531    '''
532    Expand out a val, if it contains environment variables and
533    is not password related
534    @param key: key name for the value
535    @type key: str
536    @param val: value to expand env vars out in
537    @type val: str
538    @rtype: basestring
539    @return: val - with any environment variables expanded out
540    '''
541    if key.lower().find('pwd') == -1 and key.lower().find('password') == -1:
542        val = os.path.expandvars(val)
543    return val
544
545   
546def _setDefaultValues(properties, validKeys, sectionKey=''):
547    '''
548    Check the contents of the properties dict to ensure it contains all the
549    keys featured in the validKeys dict; if any of these are missing or have
550    no value set for them, set up default values for these in the properties
551    dict
552    @param properties: dictionary storing loaded properties
553    @type properties: dict
554    @param validKeys: a dictionary of valid values
555    @type validKeys: dict
556    @rtype: dict
557    @return properties: updated dict with default values for any missing values
558    '''
559   
560   
561    if sectionKey:
562        sectionKeyDot = sectionKey+'.'
563        log.debug("Checking for any unset keys for %s sub-section"%sectionKey)
564    else:
565        sectionKeyDot = ''
566        log.debug("Checking for any unset keys")
567       
568    for key in validKeys:
569        if key not in properties or not properties[key]:
570            if validKeys[key] == NotImplemented:
571                errorMessage = 'Missing property "%s" must be set.' % key
572                log.error(errorMessage)
573                raise ValueError(errorMessage)
574           
575            log.warning("Found missing/unset property - setting default "
576                        "values: %s%s=%s" % (sectionKeyDot,key,validKeys[key]))
577            properties[key] = validKeys[key]
578           
579        elif isinstance(properties[key], dict):
580            _setDefaultValues(properties[key], validKeys[key], sectionKey=key)
581       
582    log.debug("Finished checking for unset keys")
583
Note: See TracBrowser for help on using the repository browser.