source: TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/ConfigFileParsers.py @ 4131

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/TI12-security/trunk/python/ndg.security.common/ndg/security/common/utils/ConfigFileParsers.py@4131
Revision 4131, 11.1 KB checked in by cbyrom, 11 years ago (diff)

Create new utility package with class, ConfigFileParsers? - a utility
class with parsers for XML and INI style config files. This takes
a filename, together with an optional dictionary of valid keys (to
check for invalid config inputs) + optional section list (to restrict
parsing of INI files to particular sections) and returns a
dictionary of read in properties. NB, if valid keys are specified
and not featured in the prop file, default values are set up in the
returned property dict.
Implemented use of the ConfigFileParsers? in the AttAuthority? service.
Added new testsuite (together with noseTests class to drive tests) to
exercise the new parsers in the context of the AttAuthority? section +
added test config files.

Line 
1"""
2Generic parsers to use when reading in configuration data
3- methods available to deal with both XML and INI (flat text key/val) formats
4"""
5__author__ = "C Byrom - Tessella"
6__date__ = "20/05/08"
7__copyright__ = "(C) 2008 STFC & NERC"
8__license__ = \
9"""This software may be distributed under the terms of the Q Public
10License, version 1.0 or later."""
11__contact__ = "P.J.Kershaw@rl.ac.uk"
12__revision__ = '$Id$'
13
14from ConfigParser import SafeConfigParser
15
16# For parsing of properties file
17try: # python 2.5
18    from xml.etree import cElementTree as ElementTree
19except ImportError:
20    # if you've installed it yourself it comes this way
21    import cElementTree as ElementTree
22
23import logging, os
24log = logging.getLogger(__name__)
25
26
27class CaseSensitiveConfigParser(SafeConfigParser):
28    '''
29    Subclass the SafeConfigParser - to preserve the original string case of the
30    cfg section names - NB, the RawConfigParser default is to lowercase these by default
31    '''
32   
33    def optionxform(self, optionstr):
34        return optionstr
35
36
37def readProperties(propFilePath, validKeys={}, sections=[]):
38    """
39    Determine the type of properties file and load the contents appropriately.
40    @param propFilePath: file path to properties file - either in xml or ini format
41    @type propFilePath: string
42    @keywords validKeys: a dictionary of valid values to be read from the file
43    - if values are encountered that are not in this list, an exception will be thrown
44    - if all info should be read, this keyword should be left to its default value
45    @type validKeys: dict
46    @keyword sections: a list of sections to include data from - if reading from an 'ini' type file
47    @type sections: list
48    @return: dict with the loaded properties in
49    @raise ValueError: if a key is read in from the file that is not included in the
50    specified validKeys dict
51    """
52    log.debug("Reading properties from %s" %propFilePath)
53    properties = {}
54    if propFilePath.lower().endswith('xml'):
55        properties = readXMLPropertyFile(propFilePath, validKeys)
56    else:
57        properties = readINIPropertyFile(propFilePath, validKeys, sections)
58
59    # if validKeys set, check that all loaded property values are featured in
60    # this list
61    if validKeys:
62        _checkForInvalidElements(properties, validKeys)
63   
64        # lastly set any default values from the validKeys dict for vals not read in
65        # from the property file
66        _setDefaultValues(properties, validKeys)
67   
68    # lastly, expand out any environment variables set in the properties file
69    properties = _expandEnvironmentVariables(properties)
70    log.info('Properties loaded')
71    return properties
72
73
74def _expandEnvironmentVariables(properties):
75    '''
76    Iterate through the values in a dict and expand out environment variables
77    specified in any non password option entries
78    @param properties: dict of properties to expand
79    @type properties: dict
80    @return: dict with expanded values
81    '''
82    log.debug("Expanding out environment variables in properties dictionary")
83    for (key, val) in properties.items():
84        # only check strings or lists of strings
85        if (isinstance(val, list)):
86            newList = []
87            for item in val:
88                newList.append(_expandEnvironmentVariable(key, item))
89           
90            properties[key] = newList
91           
92        elif (isinstance(val, str)):
93            properties[key] = _expandEnvironmentVariable(key, val)
94           
95    log.debug("Finished expanding environment variables")
96    return properties
97
98
99def _expandEnvironmentVariable(key, val):
100    '''
101    Expand out a val, if it contains environment variables and
102    is not password related
103    @param key: key name for the value
104    @type key: str
105    @param val: value to expand env vars out in
106    @type val: str
107    @return: val - with any environment variables expanded out
108    '''
109    if key.lower().find('pwd') == -1 and key.lower().find('password') == -1:
110        val = os.path.expandvars(val)
111    return val
112       
113
114def readINIPropertyFile(propFilePath, validKeys, sections=[]):
115    """
116    Read 'ini' type property file - i.e. a flat text file with key/value
117    data separated into sections
118
119    @param propFilePath: file path to properties file - either in xml or ini format
120    @type propFilePath: string
121    @param validKeys: a dictionary of valid values to be read from the file
122    - if values are encountered that are not in this list, an exception will be thrown
123    - if all info should be read, set this param to 'None'
124    @type validKeys: dict
125    @keyword sections: a list of sections to include data from
126    @type sections: list
127    @return: dict with the loaded properties in
128    @raise ValueError: if a key is read in from the file that is not included in the
129    specified validKeys dict
130    """
131    log.debug("File is not marked as XML - treating as flat, 'ini' format file")
132   
133    if not os.path.isfile(propFilePath):
134        raise ValueError("Error parsing properties file \"%s\": No such file" %propFilePath)
135
136    cfg = CaseSensitiveConfigParser()
137    cfg.read(propFilePath)
138    properties = {}
139   
140    # if no sections are set, use all sections - including the default one
141    # - NB, this isn't returned by the 'sections()' method so needs to be added explicitly
142    if not sections:
143        sections = cfg.sections()
144        sections.append('DEFAULT')
145   
146    # parse data from the specified sections of the config file
147    if len(sections):
148        for section in sections:
149            properties = _parseConfig(cfg, properties, validKeys, section=section)
150
151    log.debug("Finished reading from INI properties file")
152    return properties
153
154
155def _checkForInvalidElements(properties, validKeys):
156    '''
157    Check the contents of the properties dict to ensure it doesn't contain
158    any keys not featured in the validKeys dict; if it does, throw an exception
159    @param properties: dictionary storing loaded properties
160    @type properties: dict
161    @param validKeys: a dictionary of valid values
162    @type validKeys: dict
163    @raise ValueError: if a key is read in from the file that is not included in the
164    specified validKeys dict
165    '''
166    log.debug("Checking for invalid properties")
167    invalidKeys = []
168    for key in properties:
169        if key not in validKeys:
170            invalidKeys.append(key)
171
172    if invalidKeys != []:
173        errorMessage = "The following properties file " + \
174            "elements are invalid: " + ', '.join(invalidKeys)
175        log.error(errorMessage)
176        raise ValueError, errorMessage
177
178   
179def _setDefaultValues(properties, validKeys):
180    '''
181    Check the contents of the properties dict to ensure it contains all the
182    keys featured in the validKeys dict; if any of these are missing or have no value
183    set for them, set up default values for these in the properties dict
184    @param properties: dictionary storing loaded properties
185    @type properties: dict
186    @param validKeys: a dictionary of valid values
187    @type validKeys: dict
188    @return properties: updated dict with default values for any missing values
189    '''
190    log.debug("Checking for any unset keys")
191    for key in validKeys:
192        if key not in properties or not properties[key]:
193            log.debug("Found missing/unset property - setting default values: %s = %s" \
194                      %(key, validKeys[key]))
195            properties[key] = validKeys[key]
196    log.debug("Finished checking for unset keys")
197   
198
199def _parseConfig(cfg, properties, validKeys, section='DEFAULT'):
200    '''
201    Extract parameters from cfg config object
202    @param cfg: config object
203    @type cfg: CaseSensitiveConfigParser
204    @param properties: dictionary to store properties in
205    @type properties: dict
206    @param validKeys: a dictionary of valid values to be read from the file
207    - if values are encountered that are not in this list, an exception will be thrown
208    - if all info should be read, set this param to 'None'
209    @type validKeys: dict
210    @keyword section: section of config file to parse from
211    @type section: string
212    @return: dict with the loaded properties in
213    '''
214    log.debug("Parsing section: %s" %section)
215    keys = {}
216    if section == 'DEFAULT':
217        keys = cfg.defaults().keys()
218    else:
219        keys = cfg.options(section)
220       
221    for key in keys:
222        val = cfg.get(section, key)
223       
224        # NB, the XML parser will return empty vals as None, so ensure consistency here
225        if not val:
226            properties[key] = None
227            continue
228       
229        # if the value is in the validKeys dict, ensure it is read in as the correct type
230        if key in validKeys:
231            if isinstance(validKeys[key], list):
232                # Treat as a list of space separated elements
233                properties[key] = val.split()
234            else:
235                properties[key] = val
236        else:
237            properties[key] = val
238                   
239    log.debug("Finished parsing section")
240    return properties
241
242
243def readXMLPropertyFile(propFilePath, validKeys):
244    """
245    Read property file - assuming the standard XML schema
246
247    @param propFilePath: file path to properties file - either in xml or ini format
248    @type propFilePath: string
249    @param validKeys: a dictionary of valid values to be read from the file
250    - if values are encountered that are not in this list, an exception will be thrown
251    - if all info should be read, set this param to 'None'
252    @type validKeys: dict
253    @return: dict with the loaded properties in
254    """
255    log.debug("File has 'xml' suffix - treating as standard XML formatted properties file")
256    try:
257        tree = ElementTree.parse(propFilePath)
258       
259    except IOError, ioErr:
260        raise ValueError, "Error parsing properties file \"%s\": %s" % \
261                            (ioErr.filename, ioErr.strerror)
262
263    rootElem = tree.getroot()
264    if rootElem is None:
265        raise ValueError, \
266            "Parsing properties file \"%s\": root element is not defined" % \
267            propFilePath
268
269    # Copy properties from file into a dictionary
270    properties = {}
271    try:
272        for elem in rootElem:
273            # if the value is in the validKeys dict, ensure it is read in as the correct type
274            if elem.tag in validKeys:
275                if isinstance(validKeys[elem.tag], list):
276                    if len(elem) == 0:
277                        if elem.text is not None:
278                            # Treat as a list of space separated elements
279                            properties[elem.tag] = elem.text.split()
280                        else:
281                            properties[elem.tag] = []
282                    else:
283                        # Parse from a list of sub-elements
284                        properties[elem.tag] = \
285                            [subElem.text.strip() for subElem in elem]
286                else:
287                    properties[elem.tag] = elem.text
288            else:
289                properties[elem.tag] = elem.text
290           
291    except Exception, e:
292        raise ValueError, \
293            "Error parsing tag \"%s\" in properties file \"%s\": %s" % \
294            (elem.tag, propFilePath, e)
295
296    log.debug("Finished reading from XML properties file")
297    return properties
Note: See TracBrowser for help on using the repository browser.