source: exist/trunk/python/elementtree-1.3a6-20070220-badc/elementtree/HTMLTreeBuilder.py @ 3578

Subversion URL: http://proj.badc.rl.ac.uk/svn/ndg/exist/trunk/python/elementtree-1.3a6-20070220-badc/elementtree/HTMLTreeBuilder.py@3578
Revision 3578, 7.9 KB checked in by pjkersha, 11 years ago (diff)

Latest releases from Fredrik Lundh. 10 March release has exclusive C14N support with namespace prefixes.

Line 
1#
2# ElementTree
3# $Id: HTMLTreeBuilder.py 3375 2008-02-13 08:05:08Z fredrik $
4#
5# a simple tree builder, for HTML input
6#
7# history:
8# 2002-04-06 fl   created
9# 2002-04-07 fl   ignore IMG and HR end tags
10# 2002-04-07 fl   added support for 1.5.2 and later
11# 2003-04-13 fl   added HTMLTreeBuilder alias
12# 2004-12-02 fl   don't feed non-ASCII charrefs/entities as 8-bit strings
13# 2004-12-05 fl   don't feed non-ASCII CDATA as 8-bit strings
14#
15# Copyright (c) 1999-2004 by Fredrik Lundh.  All rights reserved.
16#
17# fredrik@pythonware.com
18# http://www.pythonware.com
19#
20# --------------------------------------------------------------------
21# The ElementTree toolkit is
22#
23# Copyright (c) 1999-2008 by Fredrik Lundh
24#
25# By obtaining, using, and/or copying this software and/or its
26# associated documentation, you agree that you have read, understood,
27# and will comply with the following terms and conditions:
28#
29# Permission to use, copy, modify, and distribute this software and
30# its associated documentation for any purpose and without fee is
31# hereby granted, provided that the above copyright notice appears in
32# all copies, and that both that copyright notice and this permission
33# notice appear in supporting documentation, and that the name of
34# Secret Labs AB or the author not be used in advertising or publicity
35# pertaining to distribution of the software without specific, written
36# prior permission.
37#
38# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD
39# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT-
40# ABILITY AND FITNESS.  IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR
41# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY
42# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS,
43# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS
44# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
45# OF THIS SOFTWARE.
46# --------------------------------------------------------------------
47
48##
49# Tools to build element trees from HTML files.
50##
51
52import htmlentitydefs
53import re, string, sys
54import mimetools, StringIO
55
56import ElementTree
57
58AUTOCLOSE = "p", "li", "tr", "th", "td", "head", "body"
59IGNOREEND = "img", "hr", "meta", "link", "br"
60
61if sys.version[:3] == "1.5":
62    is_not_ascii = re.compile(r"[\x80-\xff]").search # 1.5.2
63else:
64    is_not_ascii = re.compile(eval(r'u"[\u0080-\uffff]"')).search
65
66try:
67    from HTMLParser import HTMLParser
68except ImportError:
69    from sgmllib import SGMLParser
70    # hack to use sgmllib's SGMLParser to emulate 2.2's HTMLParser
71    class HTMLParser(SGMLParser):
72        # the following only works as long as this class doesn't
73        # provide any do, start, or end handlers
74        def unknown_starttag(self, tag, attrs):
75            self.handle_starttag(tag, attrs)
76        def unknown_endtag(self, tag):
77            self.handle_endtag(tag)
78
79##
80# ElementTree builder for HTML source code.  This builder converts an
81# HTML document or fragment to an ElementTree.
82# <p>
83# The parser is relatively picky, and requires balanced tags for most
84# elements.  However, elements belonging to the following group are
85# automatically closed: P, LI, TR, TH, and TD.  In addition, the
86# parser automatically inserts end tags immediately after the start
87# tag, and ignores any end tags for the following group: IMG, HR,
88# META, and LINK.
89#
90# @keyparam builder Optional builder object.  If omitted, the parser
91#     uses the standard <b>elementtree</b> builder.
92# @keyparam encoding Optional character encoding, if known.  If omitted,
93#     the parser looks for META tags inside the document.  If no tags
94#     are found, the parser defaults to ISO-8859-1.  Note that if your
95#     document uses a non-ASCII compatible encoding, you must decode
96#     the document before parsing.
97#
98# @see elementtree.ElementTree
99
100class HTMLTreeBuilder(HTMLParser):
101
102    # FIXME: shouldn't this class be named Parser, not Builder?
103
104    def __init__(self, builder=None, encoding=None):
105        self.__stack = []
106        if builder is None:
107            builder = ElementTree.TreeBuilder()
108        self.__builder = builder
109        self.encoding = encoding or "iso-8859-1"
110        HTMLParser.__init__(self)
111
112    ##
113    # Flushes parser buffers, and return the root element.
114    #
115    # @return An Element instance.
116
117    def close(self):
118        HTMLParser.close(self)
119        return self.__builder.close()
120
121    ##
122    # (Internal) Handles start tags.
123
124    def handle_starttag(self, tag, attrs):
125        if tag == "meta":
126            # look for encoding directives
127            http_equiv = content = None
128            for k, v in attrs:
129                if k == "http-equiv":
130                    http_equiv = string.lower(v)
131                elif k == "content":
132                    content = v
133            if http_equiv == "content-type" and content:
134                # use mimetools to parse the http header
135                header = mimetools.Message(
136                    StringIO.StringIO("%s: %s\n\n" % (http_equiv, content))
137                    )
138                encoding = header.getparam("charset")
139                if encoding:
140                    self.encoding = encoding
141        if tag in AUTOCLOSE:
142            if self.__stack and self.__stack[-1] == tag:
143                self.handle_endtag(tag)
144        self.__stack.append(tag)
145        attrib = {}
146        if attrs:
147            for k, v in attrs:
148                attrib[string.lower(k)] = v
149        self.__builder.start(tag, attrib)
150        if tag in IGNOREEND:
151            self.__stack.pop()
152            self.__builder.end(tag)
153
154    ##
155    # (Internal) Handles end tags.
156
157    def handle_endtag(self, tag):
158        if tag in IGNOREEND:
159            return
160        lasttag = self.__stack.pop()
161        if tag != lasttag and lasttag in AUTOCLOSE:
162            self.handle_endtag(lasttag)
163        self.__builder.end(tag)
164
165    ##
166    # (Internal) Handles character references.
167
168    def handle_charref(self, char):
169        if char[:1] == "x":
170            char = int(char[1:], 16)
171        else:
172            char = int(char)
173        if 0 <= char < 128:
174            self.__builder.data(chr(char))
175        else:
176            self.__builder.data(unichr(char))
177
178    ##
179    # (Internal) Handles entity references.
180
181    def handle_entityref(self, name):
182        entity = htmlentitydefs.entitydefs.get(name)
183        if entity:
184            if len(entity) == 1:
185                entity = ord(entity)
186            else:
187                entity = int(entity[2:-1])
188            if 0 <= entity < 128:
189                self.__builder.data(chr(entity))
190            else:
191                self.__builder.data(unichr(entity))
192        else:
193            self.unknown_entityref(name)
194
195    ##
196    # (Internal) Handles character data.
197
198    def handle_data(self, data):
199        if isinstance(data, type('')) and is_not_ascii(data):
200            # convert to unicode, but only if necessary
201            data = unicode(data, self.encoding, "ignore")
202        self.__builder.data(data)
203
204    ##
205    # (Hook) Handles unknown entity references.  The default action
206    # is to ignore unknown entities.
207
208    def unknown_entityref(self, name):
209        pass # ignore by default; override if necessary
210
211##
212# An alias for the <b>HTMLTreeBuilder</b> class.
213
214TreeBuilder = HTMLTreeBuilder
215
216##
217# Parse an HTML document or document fragment.
218#
219# @param source A filename or file object containing HTML data.
220# @param encoding Optional character encoding, if known.  If omitted,
221#     the parser looks for META tags inside the document.  If no tags
222#     are found, the parser defaults to ISO-8859-1.
223# @return An ElementTree instance
224
225def parse(source, encoding=None):
226    return ElementTree.parse(source, HTMLTreeBuilder(encoding=encoding))
227
228if __name__ == "__main__":
229    import sys
230    ElementTree.dump(parse(open(sys.argv[1])))
Note: See TracBrowser for help on using the repository browser.