monkee.ch/SickGear
1
0
Fork 0
mirror of https://github.com/SickGear/SickGear.git synced 2024-12-18 08:43:37 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'feature/UpdateBSoup' into dev

Browse source
This commit is contained in:
JackDandy 2023-02-09 14:39:08 +00:00
commit c24d19beda
11 changed files with 819 additions and 296 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
CHANGES.md
View file

@ -1,6 +1,7 @@
### 3.27.0 (202x-xx-xx xx:xx:00 UTC) ### 3.27.0 (202x-xx-xx xx:xx:00 UTC)
* Update attr 20.3.0 (f3762ba) to 22.2.0 (a9960de) * Update attr 20.3.0 (f3762ba) to 22.2.0 (a9960de)
* Update Beautiful Soup 4.9.3 (r593) to 4.11.1 (r642)
* Update cachecontrol 0.12.6 (167a605) to 0.12.11 (c05ef9e) * Update cachecontrol 0.12.6 (167a605) to 0.12.11 (c05ef9e)
* Add filelock 3.9.0 (ce3e891) * Add filelock 3.9.0 (ce3e891)
* Remove lockfile no longer used by cachecontrol * Remove lockfile no longer used by cachecontrol

133
lib/bs4/__init__.py
View file

@ -7,7 +7,7 @@ Beautiful Soup uses a pluggable XML or HTML parser to parse a
provides methods and Pythonic idioms that make it easy to navigate, provides methods and Pythonic idioms that make it easy to navigate,
search, and modify the parse tree. search, and modify the parse tree.
Beautiful Soup works with Python 2.7 and up. It works better if lxml Beautiful Soup works with Python 3.5 and up. It works better if lxml
and/or html5lib is installed. and/or html5lib is installed.
For more than you ever wanted to know about Beautiful Soup, see the For more than you ever wanted to know about Beautiful Soup, see the
@ -15,8 +15,8 @@ documentation: http://www.crummy.com/software/BeautifulSoup/bs4/doc/
""" """
__author__ = "Leonard Richardson (leonardr@segfault.org)" __author__ = "Leonard Richardson (leonardr@segfault.org)"
__version__ = "4.9.3" __version__ = "4.11.1"
__copyright__ = "Copyright (c) 2004-2020 Leonard Richardson" __copyright__ = "Copyright (c) 2004-2022 Leonard Richardson"
# Use of this source code is governed by the MIT license. # Use of this source code is governed by the MIT license.
__license__ = "MIT" __license__ = "MIT"
@ -29,7 +29,16 @@ import sys
import traceback import traceback
import warnings import warnings
from .builder import builder_registry, ParserRejectedMarkup # The very first thing we do is give a useful error if someone is
# running this code under Python 2.
if sys.version_info.major < 3:
raise ImportError('You are trying to use a Python 3-specific version of Beautiful Soup under Python 2. This will not work. The final version of Beautiful Soup to support Python 2 was 4.9.3.')
from .builder import (
builder_registry,
ParserRejectedMarkup,
XMLParsedAsHTMLWarning,
)
from .dammit import UnicodeDammit from .dammit import UnicodeDammit
from .element import ( from .element import (
CData, CData,
@ -49,10 +58,6 @@ from .element import (
TemplateString, TemplateString,
) )
# The very first thing we do is give a useful error if someone is
# running this code under Python 3 without converting it.
'You are trying to run the Python 2 version of Beautiful Soup under Python 3. This will not work.'!='You need to convert the code, either by installing it (`python setup.py install`) or by running 2to3 (`2to3 -w bs4`).'
# Define some custom warnings. # Define some custom warnings.
class GuessedAtParserWarning(UserWarning): class GuessedAtParserWarning(UserWarning):
"""The warning issued when BeautifulSoup has to guess what parser to """The warning issued when BeautifulSoup has to guess what parser to
@ -65,7 +70,7 @@ class MarkupResemblesLocatorWarning(UserWarning):
on disk. on disk.
""" """
class BeautifulSoup(Tag): class BeautifulSoup(Tag):
"""A data structure representing a parsed HTML or XML document. """A data structure representing a parsed HTML or XML document.
@ -205,10 +210,10 @@ class BeautifulSoup(Tag):
if old_name in kwargs: if old_name in kwargs:
warnings.warn( warnings.warn(
'The "%s" argument to the BeautifulSoup constructor ' 'The "%s" argument to the BeautifulSoup constructor '
'has been renamed to "%s."' % (old_name, new_name)) 'has been renamed to "%s."' % (old_name, new_name),
value = kwargs[old_name] DeprecationWarning
del kwargs[old_name] )
return value return kwargs.pop(old_name)
return None return None
parse_only = parse_only or deprecated_argument( parse_only = parse_only or deprecated_argument(
@ -303,39 +308,18 @@ class BeautifulSoup(Tag):
self._namespaces = dict() self._namespaces = dict()
self.parse_only = parse_only self.parse_only = parse_only
self.builder.initialize_soup(self)
if hasattr(markup, 'read'): # It's a file-type object. if hasattr(markup, 'read'): # It's a file-type object.
markup = markup.read() markup = markup.read()
elif len(markup) <= 256 and ( elif len(markup) <= 256 and (
(isinstance(markup, bytes) and not b'<' in markup) (isinstance(markup, bytes) and not b'<' in markup)
or (isinstance(markup, str) and not '<' in markup) or (isinstance(markup, str) and not '<' in markup)
): ):
# Print out warnings for a couple beginner problems # Issue warnings for a couple beginner problems
# involving passing non-markup to Beautiful Soup. # involving passing non-markup to Beautiful Soup.
# Beautiful Soup will still parse the input as markup, # Beautiful Soup will still parse the input as markup,
# just in case that's what the user really wants. # since that is sometimes the intended behavior.
if (isinstance(markup, str) if not self._markup_is_url(markup):
and not os.path.supports_unicode_filenames): self._markup_resembles_filename(markup)
possible_filename = markup.encode("utf8")
else:
possible_filename = markup
is_file = False
try:
is_file = os.path.exists(possible_filename)
except Exception as e:
# This is almost certainly a problem involving
# characters not valid in filenames on this
# system. Just let it go.
pass
if is_file:
warnings.warn(
'"%s" looks like a filename, not markup. You should'
' probably open this file and pass the filehandle into'
' Beautiful Soup.' % self._decode_markup(markup),
MarkupResemblesLocatorWarning
)
self._check_markup_is_url(markup)
rejections = [] rejections = []
success = False success = False
@ -344,6 +328,7 @@ class BeautifulSoup(Tag):
self.builder.prepare_markup( self.builder.prepare_markup(
markup, from_encoding, exclude_encodings=exclude_encodings)): markup, from_encoding, exclude_encodings=exclude_encodings)):
self.reset() self.reset()
self.builder.initialize_soup(self)
try: try:
self._feed() self._feed()
success = True success = True
@ -379,10 +364,10 @@ class BeautifulSoup(Tag):
def __getstate__(self): def __getstate__(self):
# Frequently a tree builder can't be pickled. # Frequently a tree builder can't be pickled.
d = dict(self.__dict__) d = dict(self.__dict__)
if 'builder' in d and not self.builder.picklable: if 'builder' in d and d['builder'] is not None and not self.builder.picklable:
d['builder'] = None d['builder'] = None
return d return d
@classmethod @classmethod
def _decode_markup(cls, markup): def _decode_markup(cls, markup):
"""Ensure `markup` is bytes so it's safe to send into warnings.warn. """Ensure `markup` is bytes so it's safe to send into warnings.warn.
@ -397,11 +382,13 @@ class BeautifulSoup(Tag):
return decoded return decoded
@classmethod @classmethod
def _check_markup_is_url(cls, markup): def _markup_is_url(cls, markup):
"""Error-handling method to raise a warning if incoming markup looks """Error-handling method to raise a warning if incoming markup looks
like a URL. like a URL.
:param markup: A string. :param markup: A string.
:return: Whether or not the markup resembles a URL
closely enough to justify a warning.
""" """
if isinstance(markup, bytes): if isinstance(markup, bytes):
space = b' ' space = b' '
@ -410,20 +397,50 @@ class BeautifulSoup(Tag):
space = ' ' space = ' '
cant_start_with = ("http:", "https:") cant_start_with = ("http:", "https:")
else: else:
return return False
if any(markup.startswith(prefix) for prefix in cant_start_with): if any(markup.startswith(prefix) for prefix in cant_start_with):
if not space in markup: if not space in markup:
warnings.warn( warnings.warn(
'"%s" looks like a URL. Beautiful Soup is not an' 'The input looks more like a URL than markup. You may want to use'
' HTTP client. You should probably use an HTTP client like' ' an HTTP client like requests to get the document behind'
' requests to get the document behind the URL, and feed' ' the URL, and feed that document to Beautiful Soup.',
' that document to Beautiful Soup.' % cls._decode_markup(
markup
),
MarkupResemblesLocatorWarning MarkupResemblesLocatorWarning
) )
return True
return False
@classmethod
def _markup_resembles_filename(cls, markup):
"""Error-handling method to raise a warning if incoming markup
resembles a filename.
:param markup: A bytestring or string.
:return: Whether or not the markup resembles a filename
closely enough to justify a warning.
"""
path_characters = '/\\'
extensions = ['.html', '.htm', '.xml', '.xhtml', '.txt']
if isinstance(markup, bytes):
path_characters = path_characters.encode("utf8")
extensions = [x.encode('utf8') for x in extensions]
filelike = False
if any(x in markup for x in path_characters):
filelike = True
else:
lower = markup.lower()
if any(lower.endswith(ext) for ext in extensions):
filelike = True
if filelike:
warnings.warn(
'The input looks more like a filename than markup. You may'
' want to open this file and pass the filehandle into'
' Beautiful Soup.',
MarkupResemblesLocatorWarning
)
return True
return False
def _feed(self): def _feed(self):
"""Internal method that parses previously set markup, creating a large """Internal method that parses previously set markup, creating a large
number of Tag and NavigableString objects. number of Tag and NavigableString objects.
@ -485,7 +502,7 @@ class BeautifulSoup(Tag):
# On top of that, we may be inside a tag that needs a special # On top of that, we may be inside a tag that needs a special
# container class. # container class.
if self.string_container_stack: if self.string_container_stack and container is NavigableString:
container = self.builder.string_containers.get( container = self.builder.string_containers.get(
self.string_container_stack[-1].name, container self.string_container_stack[-1].name, container
) )
@ -541,9 +558,7 @@ class BeautifulSoup(Tag):
def endData(self, containerClass=None): def endData(self, containerClass=None):
"""Method called by the TreeBuilder when the end of a data segment """Method called by the TreeBuilder when the end of a data segment
occurs. occurs.
""" """
containerClass = self.string_container(containerClass)
if self.current_data: if self.current_data:
current_data = ''.join(self.current_data) current_data = ''.join(self.current_data)
# If whitespace is not preserved, and this string contains # If whitespace is not preserved, and this string contains
@ -570,6 +585,7 @@ class BeautifulSoup(Tag):
not self.parse_only.search(current_data)): not self.parse_only.search(current_data)):
return return
containerClass = self.string_container(containerClass)
o = containerClass(current_data) o = containerClass(current_data)
self.object_was_parsed(o) self.object_was_parsed(o)
@ -676,7 +692,7 @@ class BeautifulSoup(Tag):
return most_recently_popped return most_recently_popped
def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None, def handle_starttag(self, name, namespace, nsprefix, attrs, sourceline=None,
sourcepos=None): sourcepos=None, namespaces=None):
"""Called by the tree builder when a new tag is encountered. """Called by the tree builder when a new tag is encountered.
:param name: Name of the tag. :param name: Name of the tag.
@ -686,6 +702,8 @@ class BeautifulSoup(Tag):
source document. source document.
:param sourcepos: The character position within `sourceline` where this :param sourcepos: The character position within `sourceline` where this
tag was found. tag was found.
:param namespaces: A dictionary of all namespace prefix mappings
currently in scope in the document.
If this method returns None, the tag was rejected by an active If this method returns None, the tag was rejected by an active
SoupStrainer. You should proceed as if the tag had not occurred SoupStrainer. You should proceed as if the tag had not occurred
@ -703,7 +721,8 @@ class BeautifulSoup(Tag):
tag = self.element_classes.get(Tag, Tag)( tag = self.element_classes.get(Tag, Tag)(
self, self.builder, name, namespace, nsprefix, attrs, self, self.builder, name, namespace, nsprefix, attrs,
self.currentTag, self._most_recent_element, self.currentTag, self._most_recent_element,
sourceline=sourceline, sourcepos=sourcepos sourceline=sourceline, sourcepos=sourcepos,
namespaces=namespaces
) )
if tag is None: if tag is None:
return tag return tag
@ -722,7 +741,7 @@ class BeautifulSoup(Tag):
#print("End tag: " + name) #print("End tag: " + name)
self.endData() self.endData()
self._popToTag(name, nsprefix) self._popToTag(name, nsprefix)
def handle_data(self, data): def handle_data(self, data):
"""Called by the tree builder when a chunk of textual data is encountered.""" """Called by the tree builder when a chunk of textual data is encountered."""
self.current_data.append(data) self.current_data.append(data)
@ -769,7 +788,9 @@ class BeautifulStoneSoup(BeautifulSoup):
kwargs['features'] = 'xml' kwargs['features'] = 'xml'
warnings.warn( warnings.warn(
'The BeautifulStoneSoup class is deprecated. Instead of using ' 'The BeautifulStoneSoup class is deprecated. Instead of using '
'it, pass features="xml" into the BeautifulSoup constructor.') 'it, pass features="xml" into the BeautifulSoup constructor.',
DeprecationWarning
)
super(BeautifulStoneSoup, self).__init__(*args, **kwargs) super(BeautifulStoneSoup, self).__init__(*args, **kwargs)

128
lib/bs4/builder/__init__.py
View file

@ -3,10 +3,14 @@ __license__ = "MIT"
from collections import defaultdict from collections import defaultdict
import itertools import itertools
import re
import warnings
import sys import sys
from ..element import ( from ..element import (
CharsetMetaAttributeValue, CharsetMetaAttributeValue,
ContentMetaAttributeValue, ContentMetaAttributeValue,
RubyParenthesisString,
RubyTextString,
Stylesheet, Stylesheet,
Script, Script,
TemplateString, TemplateString,
@ -28,6 +32,12 @@ XML = 'xml'
HTML = 'html' HTML = 'html'
HTML_5 = 'html5' HTML_5 = 'html5'
class XMLParsedAsHTMLWarning(UserWarning):
"""The warning issued when an HTML parser is used to parse
XML that is not XHTML.
"""
MESSAGE = """It looks like you're parsing an XML document using an HTML parser. If this really is an HTML document (maybe it's XHTML?), you can ignore or filter this warning. If it's XML, you should know that using an XML parser will be more reliable. To parse this document as XML, make sure you have the lxml package installed, and pass the keyword argument `features="xml"` into the BeautifulSoup constructor."""
class TreeBuilderRegistry(object): class TreeBuilderRegistry(object):
"""A way of looking up TreeBuilder subclasses by their name or by desired """A way of looking up TreeBuilder subclasses by their name or by desired
@ -112,7 +122,7 @@ class TreeBuilder(object):
# A value for these tag/attribute combinations is a space- or # A value for these tag/attribute combinations is a space- or
# comma-separated list of CDATA, rather than a single CDATA. # comma-separated list of CDATA, rather than a single CDATA.
DEFAULT_CDATA_LIST_ATTRIBUTES = {} DEFAULT_CDATA_LIST_ATTRIBUTES = defaultdict(list)
# Whitespace should be preserved inside these tags. # Whitespace should be preserved inside these tags.
DEFAULT_PRESERVE_WHITESPACE_TAGS = set() DEFAULT_PRESERVE_WHITESPACE_TAGS = set()
@ -234,7 +244,8 @@ class TreeBuilder(object):
:param markup: Some markup -- probably a bytestring. :param markup: Some markup -- probably a bytestring.
:param user_specified_encoding: The user asked to try this encoding. :param user_specified_encoding: The user asked to try this encoding.
:param document_declared_encoding: The markup itself claims to be :param document_declared_encoding: The markup itself claims to be
in this encoding. in this encoding. NOTE: This argument is not used by the
calling code and can probably be removed.
:param exclude_encodings: The user asked _not_ to try any of :param exclude_encodings: The user asked _not_ to try any of
these encodings. these encodings.
@ -318,7 +329,7 @@ class TreeBuilder(object):
values = value values = value
attrs[attr] = values attrs[attr] = values
return attrs return attrs
class SAXTreeBuilder(TreeBuilder): class SAXTreeBuilder(TreeBuilder):
"""A Beautiful Soup treebuilder that listens for SAX events. """A Beautiful Soup treebuilder that listens for SAX events.
@ -389,17 +400,25 @@ class HTMLTreeBuilder(TreeBuilder):
# you need to use it. # you need to use it.
block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"]) block_elements = set(["address", "article", "aside", "blockquote", "canvas", "dd", "div", "dl", "dt", "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "header", "hr", "li", "main", "nav", "noscript", "ol", "output", "p", "pre", "section", "table", "tfoot", "ul", "video"])
# The HTML standard defines an unusual content model for these tags. # These HTML tags need special treatment so they can be
# We represent this by using a string class other than NavigableString # represented by a string class other than NavigableString.
# inside these tags.
# #
# I made this list by going through the HTML spec # For some of these tags, it's because the HTML standard defines
# an unusual content model for them. I made this list by going
# through the HTML spec
# (https://html.spec.whatwg.org/#metadata-content) and looking for # (https://html.spec.whatwg.org/#metadata-content) and looking for
# "metadata content" elements that can contain strings. # "metadata content" elements that can contain strings.
# #
# The Ruby tags (<rt> and <rp>) are here despite being normal
# "phrasing content" tags, because the content they contain is
# qualitatively different from other text in the document, and it
# can be useful to be able to distinguish it.
#
# TODO: Arguably <noscript> could go here but it seems # TODO: Arguably <noscript> could go here but it seems
# qualitatively different from the other tags. # qualitatively different from the other tags.
DEFAULT_STRING_CONTAINERS = { DEFAULT_STRING_CONTAINERS = {
'rt' : RubyTextString,
'rp' : RubyParenthesisString,
'style': Stylesheet, 'style': Stylesheet,
'script': Script, 'script': Script,
'template': TemplateString, 'template': TemplateString,
@ -430,7 +449,7 @@ class HTMLTreeBuilder(TreeBuilder):
} }
DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea']) DEFAULT_PRESERVE_WHITESPACE_TAGS = set(['pre', 'textarea'])
def set_up_substitutions(self, tag): def set_up_substitutions(self, tag):
"""Replace the declared encoding in a <meta> tag with a placeholder, """Replace the declared encoding in a <meta> tag with a placeholder,
to be substituted when the tag is output to a string. to be substituted when the tag is output to a string.
@ -474,6 +493,99 @@ class HTMLTreeBuilder(TreeBuilder):
return (meta_encoding is not None) return (meta_encoding is not None)
class DetectsXMLParsedAsHTML(object):
"""A mixin class for any class (a TreeBuilder, or some class used by a
TreeBuilder) that's in a position to detect whether an XML
document is being incorrectly parsed as HTML, and issue an
appropriate warning.
This requires being able to observe an incoming processing
instruction that might be an XML declaration, and also able to
observe tags as they're opened. If you can't do that for a given
TreeBuilder, there's a less reliable implementation based on
examining the raw markup.
"""
# Regular expression for seeing if markup has an <html> tag.
LOOKS_LIKE_HTML = re.compile("<[^ +]html", re.I)
LOOKS_LIKE_HTML_B = re.compile(b"<[^ +]html", re.I)
XML_PREFIX = '<?xml'
XML_PREFIX_B = b'<?xml'
@classmethod
def warn_if_markup_looks_like_xml(cls, markup):
"""Perform a check on some markup to see if it looks like XML
that's not XHTML. If so, issue a warning.
This is much less reliable than doing the check while parsing,
but some of the tree builders can't do that.
:return: True if the markup looks like non-XHTML XML, False
otherwise.
"""
if isinstance(markup, bytes):
prefix = cls.XML_PREFIX_B
looks_like_html = cls.LOOKS_LIKE_HTML_B
else:
prefix = cls.XML_PREFIX
looks_like_html = cls.LOOKS_LIKE_HTML
if (markup is not None
and markup.startswith(prefix)
and not looks_like_html.search(markup[:500])
):
cls._warn()
return True
return False
@classmethod
def _warn(cls):
"""Issue a warning about XML being parsed as HTML."""
warnings.warn(
XMLParsedAsHTMLWarning.MESSAGE, XMLParsedAsHTMLWarning
)
def _initialize_xml_detector(self):
"""Call this method before parsing a document."""
self._first_processing_instruction = None
self._root_tag = None
def _document_might_be_xml(self, processing_instruction):
"""Call this method when encountering an XML declaration, or a
"processing instruction" that might be an XML declaration.
"""
if (self._first_processing_instruction is not None
or self._root_tag is not None):
# The document has already started. Don't bother checking
# anymore.
return
self._first_processing_instruction = processing_instruction
# We won't know until we encounter the first tag whether or
# not this is actually a problem.
def _root_tag_encountered(self, name):
"""Call this when you encounter the document's root tag.
This is where we actually check whether an XML document is
being incorrectly parsed as HTML, and issue the warning.
"""
if self._root_tag is not None:
# This method was incorrectly called multiple times. Do
# nothing.
return
self._root_tag = name
if (name != 'html' and self._first_processing_instruction is not None
and self._first_processing_instruction.lower().startswith('xml ')):
# We encountered an XML declaration and then a tag other
# than 'html'. This is a reliable indicator that a
# non-XHTML document is being parsed as XML.
self._warn()
def register_treebuilders_from(module): def register_treebuilders_from(module):
"""Copy TreeBuilders from the given module into this module.""" """Copy TreeBuilders from the given module into this module."""
this_module = sys.modules[__name__] this_module = sys.modules[__name__]

12
lib/bs4/builder/_html5lib.py
View file

@ -8,6 +8,7 @@ __all__ = [
import warnings import warnings
import re import re
from ..builder import ( from ..builder import (
DetectsXMLParsedAsHTML,
PERMISSIVE, PERMISSIVE,
HTML, HTML,
HTML_5, HTML_5,
@ -70,6 +71,11 @@ class HTML5TreeBuilder(HTMLTreeBuilder):
# UnicodeDammit. # UnicodeDammit.
if exclude_encodings: if exclude_encodings:
warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.") warnings.warn("You provided a value for exclude_encoding, but the html5lib tree builder doesn't support exclude_encoding.")
# html5lib only parses HTML, so if it's given XML that's worth
# noting.
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
yield (markup, None, None, False) yield (markup, None, None, False)
# These methods are defined by Beautiful Soup. # These methods are defined by Beautiful Soup.
@ -242,10 +248,10 @@ class AttrList(object):
def __setitem__(self, name, value): def __setitem__(self, name, value):
# If this attribute is a multi-valued attribute for this element, # If this attribute is a multi-valued attribute for this element,
# turn its value into a list. # turn its value into a list.
list_attr = self.element.cdata_list_attributes list_attr = self.element.cdata_list_attributes or {}
if (name in list_attr['*'] if (name in list_attr.get('*', [])
or (self.element.name in list_attr or (self.element.name in list_attr
and name in list_attr[self.element.name])): and name in list_attr.get(self.element.name, []))):
# A node that is being cloned may have already undergone # A node that is being cloned may have already undergone
# this procedure. # this procedure.
if not isinstance(value, list): if not isinstance(value, list):

32
lib/bs4/builder/_htmlparser.py
View file

@ -44,6 +44,7 @@ from ..element import (
from ..dammit import EntitySubstitution, UnicodeDammit from ..dammit import EntitySubstitution, UnicodeDammit
from ..builder import ( from ..builder import (
DetectsXMLParsedAsHTML,
HTML, HTML,
HTMLTreeBuilder, HTMLTreeBuilder,
STRICT, STRICT,
@ -52,7 +53,7 @@ from ..builder import (
HTMLPARSER = 'html.parser' HTMLPARSER = 'html.parser'
class BeautifulSoupHTMLParser(HTMLParser): class BeautifulSoupHTMLParser(HTMLParser, DetectsXMLParsedAsHTML):
"""A subclass of the Python standard library's HTMLParser class, which """A subclass of the Python standard library's HTMLParser class, which
listens for HTMLParser events and translates them into calls listens for HTMLParser events and translates them into calls
to Beautiful Soup's tree construction API. to Beautiful Soup's tree construction API.
@ -88,6 +89,8 @@ class BeautifulSoupHTMLParser(HTMLParser):
# will ignore, assuming they ever show up. # will ignore, assuming they ever show up.
self.already_closed_empty_element = [] self.already_closed_empty_element = []
self._initialize_xml_detector()
def error(self, msg): def error(self, msg):
"""In Python 3, HTMLParser subclasses must implement error(), although """In Python 3, HTMLParser subclasses must implement error(), although
this requirement doesn't appear to be documented. this requirement doesn't appear to be documented.
@ -167,6 +170,9 @@ class BeautifulSoupHTMLParser(HTMLParser):
# But we might encounter an explicit closing tag for this tag # But we might encounter an explicit closing tag for this tag
# later on. If so, we want to ignore it. # later on. If so, we want to ignore it.
self.already_closed_empty_element.append(name) self.already_closed_empty_element.append(name)
if self._root_tag is None:
self._root_tag_encountered(name)
def handle_endtag(self, name, check_already_closed=True): def handle_endtag(self, name, check_already_closed=True):
"""Handle a closing tag, e.g. '</tag>' """Handle a closing tag, e.g. '</tag>'
@ -185,7 +191,7 @@ class BeautifulSoupHTMLParser(HTMLParser):
self.already_closed_empty_element.remove(name) self.already_closed_empty_element.remove(name)
else: else:
self.soup.handle_endtag(name) self.soup.handle_endtag(name)
def handle_data(self, data): def handle_data(self, data):
"""Handle some textual data that shows up between tags.""" """Handle some textual data that shows up between tags."""
self.soup.handle_data(data) self.soup.handle_data(data)
@ -231,7 +237,7 @@ class BeautifulSoupHTMLParser(HTMLParser):
def handle_entityref(self, name): def handle_entityref(self, name):
"""Handle a named entity reference by converting it to the """Handle a named entity reference by converting it to the
corresponding Unicode character and treating it as textual corresponding Unicode character(s) and treating it as textual
data. data.
:param name: Name of the entity reference. :param name: Name of the entity reference.
@ -288,6 +294,7 @@ class BeautifulSoupHTMLParser(HTMLParser):
""" """
self.soup.endData() self.soup.endData()
self.soup.handle_data(data) self.soup.handle_data(data)
self._document_might_be_xml(data)
self.soup.endData(ProcessingInstruction) self.soup.endData(ProcessingInstruction)
@ -359,9 +366,24 @@ class HTMLParserTreeBuilder(HTMLTreeBuilder):
return return
# Ask UnicodeDammit to sniff the most likely encoding. # Ask UnicodeDammit to sniff the most likely encoding.
# This was provided by the end-user; treat it as a known
# definite encoding per the algorithm laid out in the HTML5
# spec. (See the EncodingDetector class for details.)
known_definite_encodings = [user_specified_encoding]
# This was found in the document; treat it as a slightly lower-priority
# user encoding.
user_encodings = [document_declared_encoding]
try_encodings = [user_specified_encoding, document_declared_encoding] try_encodings = [user_specified_encoding, document_declared_encoding]
dammit = UnicodeDammit(markup, try_encodings, is_html=True, dammit = UnicodeDammit(
exclude_encodings=exclude_encodings) markup,
known_definite_encodings=known_definite_encodings,
user_encodings=user_encodings,
is_html=True,
exclude_encodings=exclude_encodings
)
yield (dammit.markup, dammit.original_encoding, yield (dammit.markup, dammit.original_encoding,
dammit.declared_html_encoding, dammit.declared_html_encoding,
dammit.contains_replacement_characters) dammit.contains_replacement_characters)

70
lib/bs4/builder/_lxml.py
View file

@ -22,6 +22,7 @@ from ..element import (
XMLProcessingInstruction, XMLProcessingInstruction,
) )
from ..builder import ( from ..builder import (
DetectsXMLParsedAsHTML,
FAST, FAST,
HTML, HTML,
HTMLTreeBuilder, HTMLTreeBuilder,
@ -79,15 +80,24 @@ class LXMLTreeBuilderForXML(TreeBuilder):
This might be useful later on when creating CSS selectors. This might be useful later on when creating CSS selectors.
This will track (almost) all namespaces, even ones that were
only in scope for part of the document. If two namespaces have
the same prefix, only the first one encountered will be
tracked. Un-prefixed namespaces are not tracked.
:param mapping: A dictionary mapping namespace prefixes to URIs. :param mapping: A dictionary mapping namespace prefixes to URIs.
""" """
for key, value in list(mapping.items()): for key, value in list(mapping.items()):
# This is 'if key' and not 'if key is not None' because we
# don't track un-prefixed namespaces. Soupselect will
# treat an un-prefixed namespace as the default, which
# causes confusion in some cases.
if key and key not in self.soup._namespaces: if key and key not in self.soup._namespaces:
# Let the BeautifulSoup object know about a new namespace. # Let the BeautifulSoup object know about a new namespace.
# If there are multiple namespaces defined with the same # If there are multiple namespaces defined with the same
# prefix, the first one in the document takes precedence. # prefix, the first one in the document takes precedence.
self.soup._namespaces[key] = value self.soup._namespaces[key] = value
def default_parser(self, encoding): def default_parser(self, encoding):
"""Find the default parser for the given encoding. """Find the default parser for the given encoding.
@ -125,6 +135,7 @@ class LXMLTreeBuilderForXML(TreeBuilder):
self.empty_element_tags = set(empty_element_tags) self.empty_element_tags = set(empty_element_tags)
self.soup = None self.soup = None
self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED] self.nsmaps = [self.DEFAULT_NSMAPS_INVERTED]
self.active_namespace_prefixes = [dict(self.DEFAULT_NSMAPS)]
super(LXMLTreeBuilderForXML, self).__init__(**kwargs) super(LXMLTreeBuilderForXML, self).__init__(**kwargs)
def _getNsTag(self, tag): def _getNsTag(self, tag):
@ -166,12 +177,21 @@ class LXMLTreeBuilderForXML(TreeBuilder):
is_html = not self.is_xml is_html = not self.is_xml
if is_html: if is_html:
self.processing_instruction_class = ProcessingInstruction self.processing_instruction_class = ProcessingInstruction
# We're in HTML mode, so if we're given XML, that's worth
# noting.
DetectsXMLParsedAsHTML.warn_if_markup_looks_like_xml(markup)
else: else:
self.processing_instruction_class = XMLProcessingInstruction self.processing_instruction_class = XMLProcessingInstruction
if isinstance(markup, str): if isinstance(markup, str):
# We were given Unicode. Maybe lxml can parse Unicode on # We were given Unicode. Maybe lxml can parse Unicode on
# this system? # this system?
# TODO: This is a workaround for
# https://bugs.launchpad.net/lxml/+bug/1948551.
# We can remove it once the upstream issue is fixed.
if len(markup) > 0 and markup[0] == u'\N{BYTE ORDER MARK}':
markup = markup[1:]
yield markup, None, document_declared_encoding, False yield markup, None, document_declared_encoding, False
if isinstance(markup, str): if isinstance(markup, str):
@ -180,9 +200,19 @@ class LXMLTreeBuilderForXML(TreeBuilder):
yield (markup.encode("utf8"), "utf8", yield (markup.encode("utf8"), "utf8",
document_declared_encoding, False) document_declared_encoding, False)
try_encodings = [user_specified_encoding, document_declared_encoding] # This was provided by the end-user; treat it as a known
# definite encoding per the algorithm laid out in the HTML5
# spec. (See the EncodingDetector class for details.)
known_definite_encodings = [user_specified_encoding]
# This was found in the document; treat it as a slightly lower-priority
# user encoding.
user_encodings = [document_declared_encoding]
detector = EncodingDetector( detector = EncodingDetector(
markup, try_encodings, is_html, exclude_encodings) markup, known_definite_encodings=known_definite_encodings,
user_encodings=user_encodings, is_html=is_html,
exclude_encodings=exclude_encodings
)
for encoding in detector.encodings: for encoding in detector.encodings:
yield (detector.markup, encoding, document_declared_encoding, False) yield (detector.markup, encoding, document_declared_encoding, False)
@ -230,6 +260,20 @@ class LXMLTreeBuilderForXML(TreeBuilder):
# mappings. # mappings.
self.nsmaps.append(_invert(nsmap)) self.nsmaps.append(_invert(nsmap))
# The currently active namespace prefixes have
# changed. Calculate the new mapping so it can be stored
# with all Tag objects created while these prefixes are in
# scope.
current_mapping = dict(self.active_namespace_prefixes[-1])
current_mapping.update(nsmap)
# We should not track un-prefixed namespaces as we can only hold one
# and it will be recognized as the default namespace by soupsieve,
# which may be confusing in some situations.
if '' in current_mapping:
del current_mapping['']
self.active_namespace_prefixes.append(current_mapping)
# Also treat the namespace mapping as a set of attributes on the # Also treat the namespace mapping as a set of attributes on the
# tag, so we can recreate it later. # tag, so we can recreate it later.
attrs = attrs.copy() attrs = attrs.copy()
@ -254,8 +298,11 @@ class LXMLTreeBuilderForXML(TreeBuilder):
namespace, name = self._getNsTag(name) namespace, name = self._getNsTag(name)
nsprefix = self._prefix_for_namespace(namespace) nsprefix = self._prefix_for_namespace(namespace)
self.soup.handle_starttag(name, namespace, nsprefix, attrs) self.soup.handle_starttag(
name, namespace, nsprefix, attrs,
namespaces=self.active_namespace_prefixes[-1]
)
def _prefix_for_namespace(self, namespace): def _prefix_for_namespace(self, namespace):
"""Find the currently active prefix for the given namespace.""" """Find the currently active prefix for the given namespace."""
if namespace is None: if namespace is None:
@ -279,13 +326,20 @@ class LXMLTreeBuilderForXML(TreeBuilder):
if len(self.nsmaps) > 1: if len(self.nsmaps) > 1:
# This tag, or one of its parents, introduced a namespace # This tag, or one of its parents, introduced a namespace
# mapping, so pop it off the stack. # mapping, so pop it off the stack.
self.nsmaps.pop() out_of_scope_nsmap = self.nsmaps.pop()
if out_of_scope_nsmap is not None:
# This tag introduced a namespace mapping which is no
# longer in scope. Recalculate the currently active
# namespace prefixes.
self.active_namespace_prefixes.pop()
def pi(self, target, data): def pi(self, target, data):
self.soup.endData() self.soup.endData()
self.soup.handle_data(target + ' ' + data) data = target + ' ' + data
self.soup.handle_data(data)
self.soup.endData(self.processing_instruction_class) self.soup.endData(self.processing_instruction_class)
def data(self, content): def data(self, content):
self.soup.handle_data(content) self.soup.handle_data(content)

4
lib/bs4/check_block.py
View file

@ -1,4 +0,0 @@
import requests
data = requests.get("https://www.crummy.com/").content
from . import _s
data = [x for x in _s(data).block_text()]

300
lib/bs4/dammit.py
View file

@ -9,47 +9,44 @@ XML or HTML to reflect a new encoding; that's the tree builder's job.
# Use of this source code is governed by the MIT license. # Use of this source code is governed by the MIT license.
__license__ = "MIT" __license__ = "MIT"
import codecs
from html.entities import codepoint2name from html.entities import codepoint2name
from collections import defaultdict
import codecs
import re import re
import logging import logging
import string import string
# Import a library to autodetect character encodings. # Import a library to autodetect character encodings. We'll support
chardet_type = None # any of a number of libraries that all support the same API:
#
# * cchardet
# * chardet
# * charset-normalizer
chardet_module = None
try: try:
# First try the fast C implementation.
# PyPI package: cchardet # PyPI package: cchardet
import cchardet import cchardet as chardet_module
except ImportError:
try:
# Debian package: python-chardet
# PyPI package: chardet
import chardet as chardet_module
except ImportError:
try:
# PyPI package: charset-normalizer
import charset_normalizer as chardet_module
except ImportError:
# No chardet available.
chardet_module = None
if chardet_module:
def chardet_dammit(s): def chardet_dammit(s):
if isinstance(s, str): if isinstance(s, str):
return None return None
return cchardet.detect(s)['encoding'] return chardet_module.detect(s)['encoding']
except ImportError: else:
try: def chardet_dammit(s):
# Fall back to the pure Python implementation return None
# Debian package: python-chardet
# PyPI package: chardet
import chardet
def chardet_dammit(s):
if isinstance(s, str):
return None
return chardet.detect(s)['encoding']
#import chardet.constants
#chardet.constants._debug = 1
except ImportError:
# No chardet available.
def chardet_dammit(s):
return None
# Available from http://cjkpython.i18n.org/.
#
# TODO: This doesn't work anymore and the closest thing, iconv_codecs,
# is GPL-licensed. Check whether this is still necessary.
try:
import iconv_codec
except ImportError:
pass
# Build bytestring and Unicode versions of regular expressions for finding # Build bytestring and Unicode versions of regular expressions for finding
# a declared encoding inside an XML or HTML document. # a declared encoding inside an XML or HTML document.
@ -65,34 +62,129 @@ encoding_res[str] = {
'xml' : re.compile(xml_encoding, re.I) 'xml' : re.compile(xml_encoding, re.I)
} }
from html.entities import html5
class EntitySubstitution(object): class EntitySubstitution(object):
"""The ability to substitute XML or HTML entities for certain characters.""" """The ability to substitute XML or HTML entities for certain characters."""
def _populate_class_variables(): def _populate_class_variables():
lookup = {} """Initialize variables used by this class to manage the plethora of
reverse_lookup = {} HTML5 named entities.
characters_for_re = []
# &apos is an XHTML entity and an HTML 5, but not an HTML 4 This function returns a 3-tuple containing two dictionaries
# entity. We don't want to use it, but we want to recognize it on the way in. and a regular expression:
#
# TODO: Ideally we would be able to recognize all HTML 5 named unicode_to_name - A mapping of Unicode strings like "⦨" to
# entities, but that's a little tricky. entity names like "angmsdaa". When a single Unicode string has
extra = [(39, 'apos')] multiple entity names, we try to choose the most commonly-used
for codepoint, name in list(codepoint2name.items()) + extra: name.
name_to_unicode: A mapping of entity names like "angmsdaa" to
Unicode strings like "⦨".
named_entity_re: A regular expression matching (almost) any
Unicode string that corresponds to an HTML5 named entity.
"""
unicode_to_name = {}
name_to_unicode = {}
short_entities = set()
long_entities_by_first_character = defaultdict(set)
for name_with_semicolon, character in sorted(html5.items()):
# "It is intentional, for legacy compatibility, that many
# code points have multiple character reference names. For
# example, some appear both with and without the trailing
# semicolon, or with different capitalizations."
# - https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references
#
# The parsers are in charge of handling (or not) character
# references with no trailing semicolon, so we remove the
# semicolon whenever it appears.
if name_with_semicolon.endswith(';'):
name = name_with_semicolon[:-1]
else:
name = name_with_semicolon
# When parsing HTML, we want to recognize any known named
# entity and convert it to a sequence of Unicode
# characters.
if name not in name_to_unicode:
name_to_unicode[name] = character
# When _generating_ HTML, we want to recognize special
# character sequences that _could_ be converted to named
# entities.
unicode_to_name[character] = name
# We also need to build a regular expression that lets us
# _find_ those characters in output strings so we can
# replace them.
#
# This is tricky, for two reasons.
if (len(character) == 1 and ord(character) < 128
and character not in '<>&'):
# First, it would be annoying to turn single ASCII
# characters like | into named entities like
# &verbar;. The exceptions are <>&, which we _must_
# turn into named entities to produce valid HTML.
continue
if len(character) > 1 and all(ord(x) < 128 for x in character):
# We also do not want to turn _combinations_ of ASCII
# characters like 'fj' into named entities like '&fjlig;',
# though that's more debateable.
continue
# Second, some named entities have a Unicode value that's
# a subset of the Unicode value for some _other_ named
# entity. As an example, \u2267' is &GreaterFullEqual;,
# but '\u2267\u0338' is &NotGreaterFullEqual;. Our regular
# expression needs to match the first two characters of
# "\u2267\u0338foo", but only the first character of
# "\u2267foo".
#
# In this step, we build two sets of characters that
# _eventually_ need to go into the regular expression. But
# we won't know exactly what the regular expression needs
# to look like until we've gone through the entire list of
# named entities.
if len(character) == 1:
short_entities.add(character)
else:
long_entities_by_first_character[character[0]].add(character)
# Now that we've been through the entire list of entities, we
# can create a regular expression that matches any of them.
particles = set()
for short in short_entities:
long_versions = long_entities_by_first_character[short]
if not long_versions:
particles.add(short)
else:
ignore = "".join([x[1] for x in long_versions])
# This finds, e.g. \u2267 but only if it is _not_
# followed by \u0338.
particles.add("%s(?![%s])" % (short, ignore))
for long_entities in list(long_entities_by_first_character.values()):
for long_entity in long_entities:
particles.add(long_entity)
re_definition = "(%s)" % "|".join(particles)
# If an entity shows up in both html5 and codepoint2name, it's
# likely that HTML5 gives it several different names, such as
# 'rsquo' and 'rsquor'. When converting Unicode characters to
# named entities, the codepoint2name name should take
# precedence where possible, since that's the more easily
# recognizable one.
for codepoint, name in list(codepoint2name.items()):
character = chr(codepoint) character = chr(codepoint)
if codepoint not in (34, 39): unicode_to_name[character] = name
# There's no point in turning the quotation mark into
# &quot; or the single quote into &apos;, unless it return unicode_to_name, name_to_unicode, re.compile(re_definition)
# happens within an attribute value, which is handled
# elsewhere.
characters_for_re.append(character)
lookup[character] = name
# But we do want to recognize those entities on the way in and
# convert them to Unicode characters.
reverse_lookup[name] = character
re_definition = "[%s]" % "".join(characters_for_re)
return lookup, reverse_lookup, re.compile(re_definition)
(CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER, (CHARACTER_TO_HTML_ENTITY, HTML_ENTITY_TO_CHARACTER,
CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables() CHARACTER_TO_HTML_ENTITY_RE) = _populate_class_variables()
@ -113,14 +205,14 @@ class EntitySubstitution(object):
@classmethod @classmethod
def _substitute_html_entity(cls, matchobj): def _substitute_html_entity(cls, matchobj):
"""Used with a regular expression to substitute the """Used with a regular expression to substitute the
appropriate HTML entity for a special character.""" appropriate HTML entity for a special character string."""
entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0)) entity = cls.CHARACTER_TO_HTML_ENTITY.get(matchobj.group(0))
return "&%s;" % entity return "&%s;" % entity
@classmethod @classmethod
def _substitute_xml_entity(cls, matchobj): def _substitute_xml_entity(cls, matchobj):
"""Used with a regular expression to substitute the """Used with a regular expression to substitute the
appropriate XML entity for a special character.""" appropriate XML entity for a special character string."""
entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)] entity = cls.CHARACTER_TO_XML_ENTITY[matchobj.group(0)]
return "&%s;" % entity return "&%s;" % entity
@ -228,32 +320,65 @@ class EncodingDetector:
Order of precedence: Order of precedence:
1. Encodings you specifically tell EncodingDetector to try first 1. Encodings you specifically tell EncodingDetector to try first
(the override_encodings argument to the constructor). (the known_definite_encodings argument to the constructor).
2. An encoding declared within the bytestring itself, either in an 2. An encoding determined by sniffing the document's byte-order mark.
3. Encodings you specifically tell EncodingDetector to try if
byte-order mark sniffing fails (the user_encodings argument to the
constructor).
4. An encoding declared within the bytestring itself, either in an
XML declaration (if the bytestring is to be interpreted as an XML XML declaration (if the bytestring is to be interpreted as an XML
document), or in a <meta> tag (if the bytestring is to be document), or in a <meta> tag (if the bytestring is to be
interpreted as an HTML document.) interpreted as an HTML document.)
3. An encoding detected through textual analysis by chardet, 5. An encoding detected through textual analysis by chardet,
cchardet, or a similar external library. cchardet, or a similar external library.
4. UTF-8. 4. UTF-8.
5. Windows-1252. 5. Windows-1252.
""" """
def __init__(self, markup, override_encodings=None, is_html=False, def __init__(self, markup, known_definite_encodings=None,
exclude_encodings=None): is_html=False, exclude_encodings=None,
user_encodings=None, override_encodings=None):
"""Constructor. """Constructor.
:param markup: Some markup in an unknown encoding. :param markup: Some markup in an unknown encoding.
:param override_encodings: These encodings will be tried first.
:param is_html: If True, this markup is considered to be HTML. Otherwise :param known_definite_encodings: When determining the encoding
it's assumed to be XML. of `markup`, these encodings will be tried first, in
:param exclude_encodings: These encodings will not be tried, even order. In HTML terms, this corresponds to the "known
if they otherwise would be. definite encoding" step defined here:
https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
:param user_encodings: These encodings will be tried after the
`known_definite_encodings` have been tried and failed, and
after an attempt to sniff the encoding by looking at a
byte order mark has failed. In HTML terms, this
corresponds to the step "user has explicitly instructed
the user agent to override the document's character
encoding", defined here:
https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
:param override_encodings: A deprecated alias for
known_definite_encodings. Any encodings here will be tried
immediately after the encodings in
known_definite_encodings.
:param is_html: If True, this markup is considered to be
HTML. Otherwise it's assumed to be XML.
:param exclude_encodings: These encodings will not be tried,
even if they otherwise would be.
""" """
self.override_encodings = override_encodings or [] self.known_definite_encodings = list(known_definite_encodings or [])
if override_encodings:
self.known_definite_encodings += override_encodings
self.user_encodings = user_encodings or []
exclude_encodings = exclude_encodings or [] exclude_encodings = exclude_encodings or []
self.exclude_encodings = set([x.lower() for x in exclude_encodings]) self.exclude_encodings = set([x.lower() for x in exclude_encodings])
self.chardet_encoding = None self.chardet_encoding = None
@ -286,7 +411,9 @@ class EncodingDetector:
:yield: A sequence of strings. :yield: A sequence of strings.
""" """
tried = set() tried = set()
for e in self.override_encodings:
# First, try the known definite encodings
for e in self.known_definite_encodings:
if self._usable(e, tried): if self._usable(e, tried):
yield e yield e
@ -295,6 +422,12 @@ class EncodingDetector:
if self._usable(self.sniffed_encoding, tried): if self._usable(self.sniffed_encoding, tried):
yield self.sniffed_encoding yield self.sniffed_encoding
# Sniffing the byte-order mark did nothing; try the user
# encodings.
for e in self.user_encodings:
if self._usable(e, tried):
yield e
# Look within the document for an XML or HTML encoding # Look within the document for an XML or HTML encoding
# declaration. # declaration.
if self.declared_encoding is None: if self.declared_encoding is None:
@ -405,13 +538,33 @@ class UnicodeDammit:
"iso-8859-2", "iso-8859-2",
] ]
def __init__(self, markup, override_encodings=[], def __init__(self, markup, known_definite_encodings=[],
smart_quotes_to=None, is_html=False, exclude_encodings=[]): smart_quotes_to=None, is_html=False, exclude_encodings=[],
user_encodings=None, override_encodings=None
):
"""Constructor. """Constructor.
:param markup: A bytestring representing markup in an unknown encoding. :param markup: A bytestring representing markup in an unknown encoding.
:param override_encodings: These encodings will be tried first,
before any sniffing code is run. :param known_definite_encodings: When determining the encoding
of `markup`, these encodings will be tried first, in
order. In HTML terms, this corresponds to the "known
definite encoding" step defined here:
https://html.spec.whatwg.org/multipage/parsing.html#parsing-with-a-known-character-encoding
:param user_encodings: These encodings will be tried after the
`known_definite_encodings` have been tried and failed, and
after an attempt to sniff the encoding by looking at a
byte order mark has failed. In HTML terms, this
corresponds to the step "user has explicitly instructed
the user agent to override the document's character
encoding", defined here:
https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
:param override_encodings: A deprecated alias for
known_definite_encodings. Any encodings here will be tried
immediately after the encodings in
known_definite_encodings.
:param smart_quotes_to: By default, Microsoft smart quotes will, like all other characters, be converted :param smart_quotes_to: By default, Microsoft smart quotes will, like all other characters, be converted
to Unicode characters. Setting this to 'ascii' will convert them to ASCII quotes instead. to Unicode characters. Setting this to 'ascii' will convert them to ASCII quotes instead.
@ -421,6 +574,7 @@ class UnicodeDammit:
it's assumed to be XML. it's assumed to be XML.
:param exclude_encodings: These encodings will not be considered, even :param exclude_encodings: These encodings will not be considered, even
if the sniffing code thinks they might make sense. if the sniffing code thinks they might make sense.
""" """
self.smart_quotes_to = smart_quotes_to self.smart_quotes_to = smart_quotes_to
self.tried_encodings = [] self.tried_encodings = []
@ -428,7 +582,9 @@ class UnicodeDammit:
self.is_html = is_html self.is_html = is_html
self.log = logging.getLogger(__name__) self.log = logging.getLogger(__name__)
self.detector = EncodingDetector( self.detector = EncodingDetector(
markup, override_encodings, is_html, exclude_encodings) markup, known_definite_encodings, is_html, exclude_encodings,
user_encodings, override_encodings
)
# Short-circuit if the data is in Unicode to begin with. # Short-circuit if the data is in Unicode to begin with.
if isinstance(markup, str) or markup == '': if isinstance(markup, str) or markup == '':

10
lib/bs4/diagnose.py
View file

@ -4,7 +4,7 @@
__license__ = "MIT" __license__ = "MIT"
import cProfile import cProfile
from io import StringIO from io import BytesIO
from html.parser import HTMLParser from html.parser import HTMLParser
import bs4 import bs4
from . import BeautifulSoup, __version__ from . import BeautifulSoup, __version__
@ -103,7 +103,13 @@ def lxml_trace(data, html=True, **kwargs):
if False, lxml's XML parser will be used. if False, lxml's XML parser will be used.
""" """
from lxml import etree from lxml import etree
for event, element in etree.iterparse(StringIO(data), html=html, **kwargs): recover = kwargs.pop('recover', True)
if isinstance(data, str):
data = data.encode("utf8")
reader = BytesIO(data)
for event, element in etree.iterparse(
reader, html=html, recover=recover, **kwargs
):
print(("%s, %4s, %s" % (event, element.tag, element.text))) print(("%s, %4s, %s" % (event, element.tag, element.text)))
class AnnouncingParser(HTMLParser): class AnnouncingParser(HTMLParser):

382
lib/bs4/element.py
View file

@ -23,7 +23,6 @@ from .formatter import (
) )
DEFAULT_OUTPUT_ENCODING = "utf-8" DEFAULT_OUTPUT_ENCODING = "utf-8"
PY3K = (sys.version_info[0] > 2)
nonwhitespace_re = re.compile(r"\S+") nonwhitespace_re = re.compile(r"\S+")
@ -83,9 +82,9 @@ class NamespacedAttribute(str):
# per https://www.w3.org/TR/xml-names/#defaulting # per https://www.w3.org/TR/xml-names/#defaulting
name = None name = None
if name is None: if not name:
obj = str.__new__(cls, prefix) obj = str.__new__(cls, prefix)
elif prefix is None: elif not prefix:
# Not really namespaced. # Not really namespaced.
obj = str.__new__(cls, name) obj = str.__new__(cls, name)
else: else:
@ -255,25 +254,67 @@ class PageElement(object):
nextSibling = _alias("next_sibling") # BS3 nextSibling = _alias("next_sibling") # BS3
previousSibling = _alias("previous_sibling") # BS3 previousSibling = _alias("previous_sibling") # BS3
def replace_with(self, replace_with): default = object()
"""Replace this PageElement with another one, keeping the rest of the def _all_strings(self, strip=False, types=default):
tree the same. """Yield all strings of certain classes, possibly stripping them.
:param replace_with: A PageElement. This is implemented differently in Tag and NavigableString.
"""
raise NotImplementedError()
@property
def stripped_strings(self):
"""Yield all strings in this PageElement, stripping them first.
:yield: A sequence of stripped strings.
"""
for string in self._all_strings(True):
yield string
def get_text(self, separator="", strip=False,
types=default):
"""Get all child strings of this PageElement, concatenated using the
given separator.
:param separator: Strings will be concatenated using this separator.
:param strip: If True, strings will be stripped before being
concatenated.
:param types: A tuple of NavigableString subclasses. Any
strings of a subclass not found in this list will be
ignored. Although there are exceptions, the default
behavior in most cases is to consider only NavigableString
and CData objects. That means no comments, processing
instructions, etc.
:return: A string.
"""
return separator.join([s for s in self._all_strings(
strip, types=types)])
getText = get_text
text = property(get_text)
def replace_with(self, *args):
"""Replace this PageElement with one or more PageElements, keeping the
rest of the tree the same.
:param args: One or more PageElements.
:return: `self`, no longer part of the tree. :return: `self`, no longer part of the tree.
""" """
if self.parent is None: if self.parent is None:
raise ValueError( raise ValueError(
"Cannot replace one element with another when the " "Cannot replace one element with another when the "
"element to be replaced is not part of a tree.") "element to be replaced is not part of a tree.")
if replace_with is self: if len(args) == 1 and args[0] is self:
return return
if replace_with is self.parent: if any(x is self.parent for x in args):
raise ValueError("Cannot replace a Tag with its parent.") raise ValueError("Cannot replace a Tag with its parent.")
old_parent = self.parent old_parent = self.parent
my_index = self.parent.index(self) my_index = self.parent.index(self)
self.extract(_self_index=my_index) self.extract(_self_index=my_index)
old_parent.insert(my_index, replace_with) for idx, replace_with in enumerate(args, start=my_index):
old_parent.insert(idx, replace_with)
return self return self
replaceWith = replace_with # BS3 replaceWith = replace_with # BS3
@ -513,7 +554,7 @@ class PageElement(object):
parent.insert(index+1+offset, successor) parent.insert(index+1+offset, successor)
offset += 1 offset += 1
def find_next(self, name=None, attrs={}, text=None, **kwargs): def find_next(self, name=None, attrs={}, string=None, **kwargs):
"""Find the first PageElement that matches the given criteria and """Find the first PageElement that matches the given criteria and
appears later in the document than this PageElement. appears later in the document than this PageElement.
@ -522,15 +563,15 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A PageElement. :return: A PageElement.
:rtype: bs4.element.Tag | bs4.element.NavigableString :rtype: bs4.element.Tag | bs4.element.NavigableString
""" """
return self._find_one(self.find_all_next, name, attrs, text, **kwargs) return self._find_one(self.find_all_next, name, attrs, string, **kwargs)
findNext = find_next # BS3 findNext = find_next # BS3
def find_all_next(self, name=None, attrs={}, text=None, limit=None, def find_all_next(self, name=None, attrs={}, string=None, limit=None,
**kwargs): **kwargs):
"""Find all PageElements that match the given criteria and appear """Find all PageElements that match the given criteria and appear
later in the document than this PageElement. later in the document than this PageElement.
@ -540,16 +581,16 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:param limit: Stop looking after finding this many results. :param limit: Stop looking after finding this many results.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A ResultSet containing PageElements. :return: A ResultSet containing PageElements.
""" """
return self._find_all(name, attrs, text, limit, self.next_elements, return self._find_all(name, attrs, string, limit, self.next_elements,
**kwargs) **kwargs)
findAllNext = find_all_next # BS3 findAllNext = find_all_next # BS3
def find_next_sibling(self, name=None, attrs={}, text=None, **kwargs): def find_next_sibling(self, name=None, attrs={}, string=None, **kwargs):
"""Find the closest sibling to this PageElement that matches the """Find the closest sibling to this PageElement that matches the
given criteria and appears later in the document. given criteria and appears later in the document.
@ -558,16 +599,16 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A PageElement. :return: A PageElement.
:rtype: bs4.element.Tag | bs4.element.NavigableString :rtype: bs4.element.Tag | bs4.element.NavigableString
""" """
return self._find_one(self.find_next_siblings, name, attrs, text, return self._find_one(self.find_next_siblings, name, attrs, string,
**kwargs) **kwargs)
findNextSibling = find_next_sibling # BS3 findNextSibling = find_next_sibling # BS3
def find_next_siblings(self, name=None, attrs={}, text=None, limit=None, def find_next_siblings(self, name=None, attrs={}, string=None, limit=None,
**kwargs): **kwargs):
"""Find all siblings of this PageElement that match the given criteria """Find all siblings of this PageElement that match the given criteria
and appear later in the document. and appear later in the document.
@ -577,18 +618,18 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:param limit: Stop looking after finding this many results. :param limit: Stop looking after finding this many results.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A ResultSet of PageElements. :return: A ResultSet of PageElements.
:rtype: bs4.element.ResultSet :rtype: bs4.element.ResultSet
""" """
return self._find_all(name, attrs, text, limit, return self._find_all(name, attrs, string, limit,
self.next_siblings, **kwargs) self.next_siblings, **kwargs)
findNextSiblings = find_next_siblings # BS3 findNextSiblings = find_next_siblings # BS3
fetchNextSiblings = find_next_siblings # BS2 fetchNextSiblings = find_next_siblings # BS2
def find_previous(self, name=None, attrs={}, text=None, **kwargs): def find_previous(self, name=None, attrs={}, string=None, **kwargs):
"""Look backwards in the document from this PageElement and find the """Look backwards in the document from this PageElement and find the
first PageElement that matches the given criteria. first PageElement that matches the given criteria.
@ -597,16 +638,16 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A PageElement. :return: A PageElement.
:rtype: bs4.element.Tag | bs4.element.NavigableString :rtype: bs4.element.Tag | bs4.element.NavigableString
""" """
return self._find_one( return self._find_one(
self.find_all_previous, name, attrs, text, **kwargs) self.find_all_previous, name, attrs, string, **kwargs)
findPrevious = find_previous # BS3 findPrevious = find_previous # BS3
def find_all_previous(self, name=None, attrs={}, text=None, limit=None, def find_all_previous(self, name=None, attrs={}, string=None, limit=None,
**kwargs): **kwargs):
"""Look backwards in the document from this PageElement and find all """Look backwards in the document from this PageElement and find all
PageElements that match the given criteria. PageElements that match the given criteria.
@ -616,18 +657,18 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:param limit: Stop looking after finding this many results. :param limit: Stop looking after finding this many results.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A ResultSet of PageElements. :return: A ResultSet of PageElements.
:rtype: bs4.element.ResultSet :rtype: bs4.element.ResultSet
""" """
return self._find_all(name, attrs, text, limit, self.previous_elements, return self._find_all(name, attrs, string, limit, self.previous_elements,
**kwargs) **kwargs)
findAllPrevious = find_all_previous # BS3 findAllPrevious = find_all_previous # BS3
fetchPrevious = find_all_previous # BS2 fetchPrevious = find_all_previous # BS2
def find_previous_sibling(self, name=None, attrs={}, text=None, **kwargs): def find_previous_sibling(self, name=None, attrs={}, string=None, **kwargs):
"""Returns the closest sibling to this PageElement that matches the """Returns the closest sibling to this PageElement that matches the
given criteria and appears earlier in the document. given criteria and appears earlier in the document.
@ -636,16 +677,16 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A PageElement. :return: A PageElement.
:rtype: bs4.element.Tag | bs4.element.NavigableString :rtype: bs4.element.Tag | bs4.element.NavigableString
""" """
return self._find_one(self.find_previous_siblings, name, attrs, text, return self._find_one(self.find_previous_siblings, name, attrs, string,
**kwargs) **kwargs)
findPreviousSibling = find_previous_sibling # BS3 findPreviousSibling = find_previous_sibling # BS3
def find_previous_siblings(self, name=None, attrs={}, text=None, def find_previous_siblings(self, name=None, attrs={}, string=None,
limit=None, **kwargs): limit=None, **kwargs):
"""Returns all siblings to this PageElement that match the """Returns all siblings to this PageElement that match the
given criteria and appear earlier in the document. given criteria and appear earlier in the document.
@ -655,13 +696,13 @@ class PageElement(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:param limit: Stop looking after finding this many results. :param limit: Stop looking after finding this many results.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
:return: A ResultSet of PageElements. :return: A ResultSet of PageElements.
:rtype: bs4.element.ResultSet :rtype: bs4.element.ResultSet
""" """
return self._find_all(name, attrs, text, limit, return self._find_all(name, attrs, string, limit,
self.previous_siblings, **kwargs) self.previous_siblings, **kwargs)
findPreviousSiblings = find_previous_siblings # BS3 findPreviousSiblings = find_previous_siblings # BS3
fetchPreviousSiblings = find_previous_siblings # BS2 fetchPreviousSiblings = find_previous_siblings # BS2
@ -728,26 +769,29 @@ class PageElement(object):
#These methods do the real heavy lifting. #These methods do the real heavy lifting.
def _find_one(self, method, name, attrs, text, **kwargs): def _find_one(self, method, name, attrs, string, **kwargs):
r = None r = None
l = method(name, attrs, text, 1, **kwargs) l = method(name, attrs, string, 1, **kwargs)
if l: if l:
r = l[0] r = l[0]
return r return r
def _find_all(self, name, attrs, text, limit, generator, **kwargs): def _find_all(self, name, attrs, string, limit, generator, **kwargs):
"Iterates over a generator looking for things that match." "Iterates over a generator looking for things that match."
if text is None and 'string' in kwargs: if string is None and 'text' in kwargs:
text = kwargs['string'] string = kwargs.pop('text')
del kwargs['string'] warnings.warn(
"The 'text' argument to find()-type methods is deprecated. Use 'string' instead.",
DeprecationWarning
)
if isinstance(name, SoupStrainer): if isinstance(name, SoupStrainer):
strainer = name strainer = name
else: else:
strainer = SoupStrainer(name, attrs, text, **kwargs) strainer = SoupStrainer(name, attrs, string, **kwargs)
if text is None and not limit and not attrs and not kwargs: if string is None and not limit and not attrs and not kwargs:
if name is True or name is None: if name is True or name is None:
# Optimization to find all tags. # Optimization to find all tags.
result = (element for element in generator result = (element for element in generator
@ -945,7 +989,54 @@ class NavigableString(str, PageElement):
"""Prevent NavigableString.name from ever being set.""" """Prevent NavigableString.name from ever being set."""
raise AttributeError("A NavigableString cannot be given a name.") raise AttributeError("A NavigableString cannot be given a name.")
def _all_strings(self, strip=False, types=PageElement.default):
"""Yield all strings of certain classes, possibly stripping them.
This makes it easy for NavigableString to implement methods
like get_text() as conveniences, creating a consistent
text-extraction API across all PageElements.
:param strip: If True, all strings will be stripped before being
yielded.
:param types: A tuple of NavigableString subclasses. If this
NavigableString isn't one of those subclasses, the
sequence will be empty. By default, the subclasses
considered are NavigableString and CData objects. That
means no comments, processing instructions, etc.
:yield: A sequence that either contains this string, or is empty.
"""
if types is self.default:
# This is kept in Tag because it's full of subclasses of
# this class, which aren't defined until later in the file.
types = Tag.DEFAULT_INTERESTING_STRING_TYPES
# Do nothing if the caller is looking for specific types of
# string, and we're of a different type.
#
# We check specific types instead of using isinstance(self,
# types) because all of these classes subclass
# NavigableString. Anyone who's using this feature probably
# wants generic NavigableStrings but not other stuff.
my_type = type(self)
if types is not None:
if isinstance(types, type):
# Looking for a single type.
if my_type is not types:
return
elif my_type not in types:
# Looking for one of a list of types.
return
value = self
if strip:
value = value.strip()
if len(value) > 0:
yield value
strings = property(_all_strings)
class PreformattedString(NavigableString): class PreformattedString(NavigableString):
"""A NavigableString not subject to the normal formatting rules. """A NavigableString not subject to the normal formatting rules.
@ -1057,6 +1148,27 @@ class TemplateString(NavigableString):
pass pass
class RubyTextString(NavigableString):
"""A NavigableString representing the contents of the <rt> HTML
element.
https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rt-element
Can be used to distinguish such strings from the strings they're
annotating.
"""
pass
class RubyParenthesisString(NavigableString):
"""A NavigableString representing the contents of the <rp> HTML
element.
https://dev.w3.org/html5/spec-LC/text-level-semantics.html#the-rp-element
"""
pass
class Tag(PageElement): class Tag(PageElement):
"""Represents an HTML or XML tag that is part of a parse tree, along """Represents an HTML or XML tag that is part of a parse tree, along
with its attributes and contents. with its attributes and contents.
@ -1069,7 +1181,9 @@ class Tag(PageElement):
prefix=None, attrs=None, parent=None, previous=None, prefix=None, attrs=None, parent=None, previous=None,
is_xml=None, sourceline=None, sourcepos=None, is_xml=None, sourceline=None, sourcepos=None,
can_be_empty_element=None, cdata_list_attributes=None, can_be_empty_element=None, cdata_list_attributes=None,
preserve_whitespace_tags=None preserve_whitespace_tags=None,
interesting_string_types=None,
namespaces=None
): ):
"""Basic constructor. """Basic constructor.
@ -1095,6 +1209,16 @@ class Tag(PageElement):
be treated as CDATA if they ever show up on this tag. be treated as CDATA if they ever show up on this tag.
:param preserve_whitespace_tags: A list of tag names whose contents :param preserve_whitespace_tags: A list of tag names whose contents
should have their whitespace preserved. should have their whitespace preserved.
:param interesting_string_types: This is a NavigableString
subclass or a tuple of them. When iterating over this
Tag's strings in methods like Tag.strings or Tag.get_text,
these are the types of strings that are interesting enough
to be considered. The default is to consider
NavigableString and CData the only interesting string
subtypes.
:param namespaces: A dictionary mapping currently active
namespace prefixes to URIs. This can be used later to
construct CSS selectors.
""" """
if parser is None: if parser is None:
self.parser_class = None self.parser_class = None
@ -1106,6 +1230,7 @@ class Tag(PageElement):
raise ValueError("No value provided for new tag's name.") raise ValueError("No value provided for new tag's name.")
self.name = name self.name = name
self.namespace = namespace self.namespace = namespace
self._namespaces = namespaces or {}
self.prefix = prefix self.prefix = prefix
if ((not builder or builder.store_line_numbers) if ((not builder or builder.store_line_numbers)
and (sourceline is not None or sourcepos is not None)): and (sourceline is not None or sourcepos is not None)):
@ -1140,6 +1265,7 @@ class Tag(PageElement):
self.can_be_empty_element = can_be_empty_element self.can_be_empty_element = can_be_empty_element
self.cdata_list_attributes = cdata_list_attributes self.cdata_list_attributes = cdata_list_attributes
self.preserve_whitespace_tags = preserve_whitespace_tags self.preserve_whitespace_tags = preserve_whitespace_tags
self.interesting_string_types = interesting_string_types
else: else:
# Set up any substitutions for this tag, such as the charset in a META tag. # Set up any substitutions for this tag, such as the charset in a META tag.
builder.set_up_substitutions(self) builder.set_up_substitutions(self)
@ -1160,6 +1286,13 @@ class Tag(PageElement):
# Keep track of the names that might cause this tag to be treated as a # Keep track of the names that might cause this tag to be treated as a
# whitespace-preserved tag. # whitespace-preserved tag.
self.preserve_whitespace_tags = builder.preserve_whitespace_tags self.preserve_whitespace_tags = builder.preserve_whitespace_tags
if self.name in builder.string_containers:
# This sort of tag uses a special string container
# subclass for most of its strings. When we ask the
self.interesting_string_types = builder.string_containers[self.name]
else:
self.interesting_string_types = self.DEFAULT_INTERESTING_STRING_TYPES
parserClass = _alias("parser_class") # BS3 parserClass = _alias("parser_class") # BS3
@ -1180,7 +1313,7 @@ class Tag(PageElement):
for child in self.contents: for child in self.contents:
clone.append(child.__copy__()) clone.append(child.__copy__())
return clone return clone
@property @property
def is_empty_element(self): def is_empty_element(self):
"""Is this tag an empty-element tag? (aka a self-closing tag) """Is this tag an empty-element tag? (aka a self-closing tag)
@ -1226,65 +1359,45 @@ class Tag(PageElement):
self.clear() self.clear()
self.append(string.__class__(string)) self.append(string.__class__(string))
def _all_strings(self, strip=False, types=(NavigableString, CData)): DEFAULT_INTERESTING_STRING_TYPES = (NavigableString, CData)
def _all_strings(self, strip=False, types=PageElement.default):
"""Yield all strings of certain classes, possibly stripping them. """Yield all strings of certain classes, possibly stripping them.
:param strip: If True, all strings will be stripped before being :param strip: If True, all strings will be stripped before being
yielded. yielded.
:types: A tuple of NavigableString subclasses. Any strings of :param types: A tuple of NavigableString subclasses. Any strings of
a subclass not found in this list will be ignored. By a subclass not found in this list will be ignored. By
default, this means only NavigableString and CData objects default, the subclasses considered are the ones found in
will be considered. So no comments, processing instructions, self.interesting_string_types. If that's not specified,
etc. only NavigableString and CData objects will be
considered. That means no comments, processing
instructions, etc.
:yield: A sequence of strings. :yield: A sequence of strings.
""" """
if types is self.default:
types = self.interesting_string_types
for descendant in self.descendants: for descendant in self.descendants:
if ( if (types is None and not isinstance(descendant, NavigableString)):
(types is None and not isinstance(descendant, NavigableString)) continue
or descendant_type = type(descendant)
(types is not None and type(descendant) not in types)): if isinstance(types, type):
if descendant_type is not types:
# We're not interested in strings of this type.
continue
elif types is not None and descendant_type not in types:
# We're not interested in strings of this type.
continue continue
if strip: if strip:
descendant = descendant.strip() descendant = descendant.strip()
if len(descendant) == 0: if len(descendant) == 0:
continue continue
yield descendant yield descendant
strings = property(_all_strings) strings = property(_all_strings)
@property
def stripped_strings(self):
"""Yield all strings in the document, stripping them first.
:yield: A sequence of stripped strings.
"""
for string in self._all_strings(True):
yield string
def get_text(self, separator="", strip=False,
types=(NavigableString, CData)):
"""Get all child strings, concatenated using the given separator.
:param separator: Strings will be concatenated using this separator.
:param strip: If True, strings will be stripped before being
concatenated.
:types: A tuple of NavigableString subclasses. Any strings of
a subclass not found in this list will be ignored. By
default, this means only NavigableString and CData objects
will be considered. So no comments, processing instructions,
stylesheets, etc.
:return: A string.
"""
return separator.join([s for s in self._all_strings(
strip, types=types)])
getText = get_text
text = property(get_text)
def decompose(self): def decompose(self):
"""Recursively destroys this PageElement and its children. """Recursively destroys this PageElement and its children.
@ -1444,7 +1557,8 @@ class Tag(PageElement):
warnings.warn( warnings.warn(
'.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag")' % dict( '.%(name)sTag is deprecated, use .find("%(name)s") instead. If you really were looking for a tag called %(name)sTag, use .find("%(name)sTag")' % dict(
name=tag_name name=tag_name
) ),
DeprecationWarning
) )
return self.find(tag_name) return self.find(tag_name)
# We special case contents to avoid recursion. # We special case contents to avoid recursion.
@ -1478,36 +1592,19 @@ class Tag(PageElement):
def __repr__(self, encoding="unicode-escape"): def __repr__(self, encoding="unicode-escape"):
"""Renders this PageElement as a string. """Renders this PageElement as a string.
:param encoding: The encoding to use (Python 2 only). :param encoding: The encoding to use (Python 2 only).
:return: Under Python 2, a bytestring; under Python 3, TODO: This is now ignored and a warning should be issued
a Unicode string. if a value is provided.
:return: A (Unicode) string.
""" """
if PY3K: # "The return value must be a string object", i.e. Unicode
# "The return value must be a string object", i.e. Unicode return self.decode()
return self.decode()
else:
# "The return value must be a string object", i.e. a bytestring.
# By convention, the return value of __repr__ should also be
# an ASCII string.
return self.encode(encoding)
def __unicode__(self): def __unicode__(self):
"""Renders this PageElement as a Unicode string.""" """Renders this PageElement as a Unicode string."""
return self.decode() return self.decode()
def __str__(self): __str__ = __repr__ = __unicode__
"""Renders this PageElement as a generic string.
:return: Under Python 2, a UTF-8 bytestring; under Python 3,
a Unicode string.
"""
if PY3K:
return self.decode()
else:
return self.encode()
if PY3K:
__str__ = __repr__ = __unicode__
def encode(self, encoding=DEFAULT_OUTPUT_ENCODING, def encode(self, encoding=DEFAULT_OUTPUT_ENCODING,
indent_level=None, formatter="minimal", indent_level=None, formatter="minimal",
@ -1517,8 +1614,10 @@ class Tag(PageElement):
:param encoding: The destination encoding. :param encoding: The destination encoding.
:param indent_level: Each line of the rendering will be :param indent_level: Each line of the rendering will be
indented this many spaces. Used internally in indented this many levels. (The formatter decides what a
recursive calls while pretty-printing. 'level' means in terms of spaces or other characters
output.) Used internally in recursive calls while
pretty-printing.
:param formatter: A Formatter object, or a string naming one of :param formatter: A Formatter object, or a string naming one of
the standard formatters. the standard formatters.
:param errors: An error handling strategy such as :param errors: An error handling strategy such as
@ -1594,7 +1693,7 @@ class Tag(PageElement):
space = '' space = ''
indent_space = '' indent_space = ''
if indent_level is not None: if indent_level is not None:
indent_space = (' ' * (indent_level - 1)) indent_space = (formatter.indent * (indent_level - 1))
if pretty_print: if pretty_print:
space = indent_space space = indent_space
indent_contents = indent_level + 1 indent_contents = indent_level + 1
@ -1669,8 +1768,10 @@ class Tag(PageElement):
"""Renders the contents of this tag as a Unicode string. """Renders the contents of this tag as a Unicode string.
:param indent_level: Each line of the rendering will be :param indent_level: Each line of the rendering will be
indented this many spaces. Used internally in indented this many levels. (The formatter decides what a
recursive calls while pretty-printing. 'level' means in terms of spaces or other characters
output.) Used internally in recursive calls while
pretty-printing.
:param eventual_encoding: The tag is destined to be :param eventual_encoding: The tag is destined to be
encoded into this encoding. decode_contents() is _not_ encoded into this encoding. decode_contents() is _not_
@ -1681,6 +1782,7 @@ class Tag(PageElement):
:param formatter: A Formatter object, or a string naming one of :param formatter: A Formatter object, or a string naming one of
the standard Formatters. the standard Formatters.
""" """
# First off, turn a string formatter into a Formatter object. This # First off, turn a string formatter into a Formatter object. This
# will stop the lookup from happening over and over again. # will stop the lookup from happening over and over again.
@ -1703,7 +1805,7 @@ class Tag(PageElement):
text = text.strip() text = text.strip()
if text: if text:
if pretty_print and not preserve_whitespace: if pretty_print and not preserve_whitespace:
s.append(" " * (indent_level - 1)) s.append(formatter.indent * (indent_level - 1))
s.append(text) s.append(text)
if pretty_print and not preserve_whitespace: if pretty_print and not preserve_whitespace:
s.append("\n") s.append("\n")
@ -1715,8 +1817,10 @@ class Tag(PageElement):
"""Renders the contents of this PageElement as a bytestring. """Renders the contents of this PageElement as a bytestring.
:param indent_level: Each line of the rendering will be :param indent_level: Each line of the rendering will be
indented this many spaces. Used internally in indented this many levels. (The formatter decides what a
recursive calls while pretty-printing. 'level' means in terms of spaces or other characters
output.) Used internally in recursive calls while
pretty-printing.
:param eventual_encoding: The bytestring will be in this encoding. :param eventual_encoding: The bytestring will be in this encoding.
@ -1739,7 +1843,7 @@ class Tag(PageElement):
#Soup methods #Soup methods
def find(self, name=None, attrs={}, recursive=True, text=None, def find(self, name=None, attrs={}, recursive=True, string=None,
**kwargs): **kwargs):
"""Look in the children of this PageElement and find the first """Look in the children of this PageElement and find the first
PageElement that matches the given criteria. PageElement that matches the given criteria.
@ -1758,13 +1862,13 @@ class Tag(PageElement):
:rtype: bs4.element.Tag | bs4.element.NavigableString :rtype: bs4.element.Tag | bs4.element.NavigableString
""" """
r = None r = None
l = self.find_all(name, attrs, recursive, text, 1, **kwargs) l = self.find_all(name, attrs, recursive, string, 1, **kwargs)
if l: if l:
r = l[0] r = l[0]
return r return r
findChild = find #BS2 findChild = find #BS2
def find_all(self, name=None, attrs={}, recursive=True, text=None, def find_all(self, name=None, attrs={}, recursive=True, string=None,
limit=None, **kwargs): limit=None, **kwargs):
"""Look in the children of this PageElement and find all """Look in the children of this PageElement and find all
PageElements that match the given criteria. PageElements that match the given criteria.
@ -1785,7 +1889,7 @@ class Tag(PageElement):
generator = self.descendants generator = self.descendants
if not recursive: if not recursive:
generator = self.children generator = self.children
return self._find_all(name, attrs, text, limit, generator, **kwargs) return self._find_all(name, attrs, string, limit, generator, **kwargs)
findAll = find_all # BS3 findAll = find_all # BS3
findChildren = find_all # BS2 findChildren = find_all # BS2
@ -1887,8 +1991,10 @@ class Tag(PageElement):
has_key() is gone in Python 3, anyway. has_key() is gone in Python 3, anyway.
""" """
warnings.warn('has_key is deprecated. Use has_attr("%s") instead.' % ( warnings.warn(
key)) 'has_key is deprecated. Use has_attr(key) instead.',
DeprecationWarning
)
return self.has_attr(key) return self.has_attr(key)
# Next, a couple classes to represent queries and their results. # Next, a couple classes to represent queries and their results.
@ -1902,7 +2008,7 @@ class SoupStrainer(object):
document. document.
""" """
def __init__(self, name=None, attrs={}, text=None, **kwargs): def __init__(self, name=None, attrs={}, string=None, **kwargs):
"""Constructor. """Constructor.
The SoupStrainer constructor takes the same arguments passed The SoupStrainer constructor takes the same arguments passed
@ -1911,9 +2017,16 @@ class SoupStrainer(object):
:param name: A filter on tag name. :param name: A filter on tag name.
:param attrs: A dictionary of filters on attribute values. :param attrs: A dictionary of filters on attribute values.
:param text: A filter for a NavigableString with specific text. :param string: A filter for a NavigableString with specific text.
:kwargs: A dictionary of filters on attribute values. :kwargs: A dictionary of filters on attribute values.
""" """
if string is None and 'text' in kwargs:
string = kwargs.pop('text')
warnings.warn(
"The 'text' argument to the SoupStrainer constructor is deprecated. Use 'string' instead.",
DeprecationWarning
)
self.name = self._normalize_search_value(name) self.name = self._normalize_search_value(name)
if not isinstance(attrs, dict): if not isinstance(attrs, dict):
# Treat a non-dict value for attrs as a search for the 'class' # Treat a non-dict value for attrs as a search for the 'class'
@ -1938,7 +2051,10 @@ class SoupStrainer(object):
normalized_attrs[key] = self._normalize_search_value(value) normalized_attrs[key] = self._normalize_search_value(value)
self.attrs = normalized_attrs self.attrs = normalized_attrs
self.text = self._normalize_search_value(text) self.string = self._normalize_search_value(string)
# DEPRECATED but just in case someone is checking this.
self.text = self.string
def _normalize_search_value(self, value): def _normalize_search_value(self, value):
# Leave it alone if it's a Unicode string, a callable, a # Leave it alone if it's a Unicode string, a callable, a
@ -1972,8 +2088,8 @@ class SoupStrainer(object):
def __str__(self): def __str__(self):
"""A human-readable representation of this SoupStrainer.""" """A human-readable representation of this SoupStrainer."""
if self.text: if self.string:
return self.text return self.string
else: else:
return "%s|%s" % (self.name, self.attrs) return "%s|%s" % (self.name, self.attrs)
@ -2033,7 +2149,7 @@ class SoupStrainer(object):
found = markup found = markup
else: else:
found = markup_name found = markup_name
if found and self.text and not self._matches(found.string, self.text): if found and self.string and not self._matches(found.string, self.string):
found = None found = None
return found return found
@ -2061,12 +2177,12 @@ class SoupStrainer(object):
# If it's a Tag, make sure its name or attributes match. # If it's a Tag, make sure its name or attributes match.
# Don't bother with Tags if we're searching for text. # Don't bother with Tags if we're searching for text.
elif isinstance(markup, Tag): elif isinstance(markup, Tag):
if not self.text or self.name or self.attrs: if not self.string or self.name or self.attrs:
found = self.search_tag(markup) found = self.search_tag(markup)
# If it's text, make sure the text matches. # If it's text, make sure the text matches.
elif isinstance(markup, NavigableString) or \ elif isinstance(markup, NavigableString) or \
isinstance(markup, str): isinstance(markup, str):
if not self.name and not self.attrs and self._matches(markup, self.text): if not self.name and not self.attrs and self._matches(markup, self.string):
found = markup found = markup
else: else:
raise Exception( raise Exception(

43
lib/bs4/formatter.py
View file

@ -14,7 +14,8 @@ class Formatter(EntitySubstitution):
For HTML documents: For HTML documents:
* 'html' - HTML entity substitution for generic HTML documents. (default) * 'html' - HTML entity substitution for generic HTML documents. (default)
* 'html5' - HTML entity substitution for HTML5 documents. * 'html5' - HTML entity substitution for HTML5 documents, as
well as some optimizations in the way tags are rendered.
* 'minimal' - Only make the substitutions necessary to guarantee * 'minimal' - Only make the substitutions necessary to guarantee
valid HTML. valid HTML.
* None - Do not perform any substitution. This will be faster * None - Do not perform any substitution. This will be faster
@ -48,6 +49,7 @@ class Formatter(EntitySubstitution):
def __init__( def __init__(
self, language=None, entity_substitution=None, self, language=None, entity_substitution=None,
void_element_close_prefix='/', cdata_containing_tags=None, void_element_close_prefix='/', cdata_containing_tags=None,
empty_attributes_are_booleans=False, indent=1,
): ):
"""Constructor. """Constructor.
@ -64,6 +66,18 @@ class Formatter(EntitySubstitution):
as containing CDATA in this dialect. For example, in HTML, as containing CDATA in this dialect. For example, in HTML,
<script> and <style> tags are defined as containing CDATA, <script> and <style> tags are defined as containing CDATA,
and their contents should not be formatted. and their contents should not be formatted.
:param blank_attributes_are_booleans: Render attributes whose value
is the empty string as HTML-style boolean attributes.
(Attributes whose value is None are always rendered this way.)
:param indent: If indent is a non-negative integer or string,
then the contents of elements will be indented
appropriately when pretty-printing. An indent level of 0,
negative, or "" will only insert newlines. Using a
positive integer indent indents that many spaces per
level. If indent is a string (such as "\t"), that string
is used to indent each level. The default behavior to
indent one space per level.
""" """
self.language = language self.language = language
self.entity_substitution = entity_substitution self.entity_substitution = entity_substitution
@ -71,7 +85,19 @@ class Formatter(EntitySubstitution):
self.cdata_containing_tags = self._default( self.cdata_containing_tags = self._default(
language, cdata_containing_tags, 'cdata_containing_tags' language, cdata_containing_tags, 'cdata_containing_tags'
) )
self.empty_attributes_are_booleans=empty_attributes_are_booleans
if indent is None:
indent = 0
if isinstance(indent, int):
if indent < 0:
indent = 0
indent = ' ' * indent
elif isinstance(indent, str):
indent = indent
else:
indent = ' '
self.indent = indent
def substitute(self, ns): def substitute(self, ns):
"""Process a string that needs to undergo entity substitution. """Process a string that needs to undergo entity substitution.
This may be a string encountered in an attribute value or as This may be a string encountered in an attribute value or as
@ -107,11 +133,17 @@ class Formatter(EntitySubstitution):
By default, attributes are sorted alphabetically. This makes By default, attributes are sorted alphabetically. This makes
behavior consistent between Python 2 and Python 3, and preserves behavior consistent between Python 2 and Python 3, and preserves
backwards compatibility with older versions of Beautiful Soup. backwards compatibility with older versions of Beautiful Soup.
If `empty_boolean_attributes` is True, then attributes whose
values are set to the empty string will be treated as boolean
attributes.
""" """
if tag.attrs is None: if tag.attrs is None:
return [] return []
return sorted(tag.attrs.items()) return sorted(
(k, (None if self.empty_attributes_are_booleans and v == '' else v))
for k, v in list(tag.attrs.items())
)
class HTMLFormatter(Formatter): class HTMLFormatter(Formatter):
"""A generic Formatter for HTML.""" """A generic Formatter for HTML."""
@ -133,7 +165,8 @@ HTMLFormatter.REGISTRY['html'] = HTMLFormatter(
) )
HTMLFormatter.REGISTRY["html5"] = HTMLFormatter( HTMLFormatter.REGISTRY["html5"] = HTMLFormatter(
entity_substitution=EntitySubstitution.substitute_html, entity_substitution=EntitySubstitution.substitute_html,
void_element_close_prefix = None void_element_close_prefix=None,
empty_attributes_are_booleans=True,
) )
HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter( HTMLFormatter.REGISTRY["minimal"] = HTMLFormatter(
entity_substitution=EntitySubstitution.substitute_xml entity_substitution=EntitySubstitution.substitute_xml