"""Tests of the bs4.element.PageElement class"""
import copy
import pickle
import pytest
import sys
from bs4 import BeautifulSoup
from bs4.element import (
Comment,
ResultSet,
SoupStrainer,
)
from . import (
SoupTest,
)
class TestEncoding(SoupTest):
"""Test the ability to encode objects into strings."""
def test_unicode_string_can_be_encoded(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
assert soup.b.string.encode("utf-8") == "\N{SNOWMAN}".encode("utf-8")
def test_tag_containing_unicode_string_can_be_encoded(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
assert soup.b.encode("utf-8") == html.encode("utf-8")
def test_encoding_substitutes_unrecognized_characters_by_default(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
assert soup.b.encode("ascii") == b"☃"
def test_encoding_can_be_made_strict(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
with pytest.raises(UnicodeEncodeError):
soup.encode("ascii", errors="strict")
def test_decode_contents(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
assert "\N{SNOWMAN}" == soup.b.decode_contents()
def test_encode_contents(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
assert "\N{SNOWMAN}".encode("utf8") == soup.b.encode_contents(
encoding="utf8"
)
def test_encode_deeply_nested_document(self):
# This test verifies that encoding a string doesn't involve
# any recursive function calls. If it did, this test would
# overflow the Python interpreter stack.
limit = sys.getrecursionlimit() + 1
markup = "" * limit
soup = self.soup(markup)
encoded = soup.encode()
assert limit == encoded.count(b"")
def test_deprecated_renderContents(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
soup.renderContents()
assert "\N{SNOWMAN}".encode("utf8") == soup.b.renderContents()
def test_repr(self):
html = "\N{SNOWMAN}"
soup = self.soup(html)
assert html == repr(soup)
class TestFormatters(SoupTest):
"""Test the formatting feature, used by methods like decode() and
prettify(), and the formatters themselves.
"""
def test_default_formatter_is_minimal(self):
markup = "<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
soup = self.soup(markup)
decoded = soup.decode(formatter="minimal")
# The < is converted back into < but the e-with-acute is left alone.
assert decoded == self.document_for(
"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
)
def test_formatter_html(self):
markup = "
<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
soup = self.soup(markup)
decoded = soup.decode(formatter="html")
assert decoded == self.document_for(
"
<<Sacré bleu!>>"
)
def test_formatter_html5(self):
markup = "
<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
soup = self.soup(markup)
decoded = soup.decode(formatter="html5")
assert decoded == self.document_for(
"
<<Sacré bleu!>>"
)
def test_formatter_minimal(self):
markup = "<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
soup = self.soup(markup)
decoded = soup.decode(formatter="minimal")
# The < is converted back into < but the e-with-acute is left alone.
assert decoded == self.document_for(
"<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
)
def test_formatter_null(self):
markup = "<<Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!>>"
soup = self.soup(markup)
decoded = soup.decode(formatter=None)
# Neither the angle brackets nor the e-with-acute are converted.
# This is not valid HTML, but it's what the user wanted.
assert decoded == self.document_for(
"<> "
)
def test_formatter_custom(self):
markup = "<foo>bar
"
soup = self.soup(markup)
decoded = soup.decode(formatter = lambda x: x.upper())
# Instead of normal entity conversion code, the custom
# callable is called on every string.
assert decoded == self.document_for("
")
def test_formatter_is_run_on_attribute_values(self):
markup = 'e'
soup = self.soup(markup)
a = soup.a
expect_minimal = 'e'
assert expect_minimal == a.decode()
assert expect_minimal == a.decode(formatter="minimal")
expect_html = 'e'
assert expect_html == a.decode(formatter="html")
assert markup == a.decode(formatter=None)
expect_upper = 'E'
assert expect_upper == a.decode(formatter=lambda x: x.upper())
def test_formatter_skips_script_tag_for_html_documents(self):
doc = """
"""
encoded = BeautifulSoup(doc, 'html.parser').encode()
assert b"< < hey > >" in encoded
def test_formatter_skips_style_tag_for_html_documents(self):
doc = """
"""
encoded = BeautifulSoup(doc, 'html.parser').encode()
assert b"< < hey > >" in encoded
def test_prettify_leaves_preformatted_text_alone(self):
soup = self.soup(" foo \tbar\n \n
baz ")
# Everything outside the tag is reformatted, but everything
# inside is left alone.
assert '\n foo\n \tbar\n \n
\n baz\n \n\n' == soup.div.prettify()
def test_prettify_handles_nested_string_literal_tags(self):
# Most of this markup is inside a tag, so prettify()
# only does three things to it:
# 1. Add a newline and a space between the and the
# 2. Add a newline after the
# 3. Add a newline at the end.
#
# The contents of the tag are left completely alone. In
# particular, we don't start adding whitespace again once we
# encounter the first tag, because we know it's not
# the one that put us into string literal mode.
markup = """some
for you
"""
expect = """
some
for you
"""
soup = self.soup(markup)
assert expect == soup.div.prettify()
def test_prettify_accepts_formatter_function(self):
soup = BeautifulSoup("foo", 'html.parser')
pretty = soup.prettify(formatter = lambda x: x.upper())
assert "FOO" in pretty
def test_prettify_outputs_unicode_by_default(self):
soup = self.soup("")
assert str == type(soup.prettify())
def test_prettify_can_encode_data(self):
soup = self.soup("")
assert bytes == type(soup.prettify("utf-8"))
def test_html_entity_substitution_off_by_default(self):
markup = "Sacr\N{LATIN SMALL LETTER E WITH ACUTE} bleu!"
soup = self.soup(markup)
encoded = soup.b.encode("utf-8")
assert encoded == markup.encode('utf-8')
def test_encoding_substitution(self):
# Here's the tag saying that a document is
# encoded in Shift-JIS.
meta_tag = ('')
soup = self.soup(meta_tag)
# Parse the document, and the charset apprears unchanged.
assert soup.meta['content'] == 'text/html; charset=x-sjis'
# Encode the document into some encoding, and the encoding is
# substituted into the meta tag.
utf_8 = soup.encode("utf-8")
assert b"charset=utf-8" in utf_8
euc_jp = soup.encode("euc_jp")
assert b"charset=euc_jp" in euc_jp
shift_jis = soup.encode("shift-jis")
assert b"charset=shift-jis" in shift_jis
utf_16_u = soup.encode("utf-16").decode("utf-16")
assert "charset=utf-16" in utf_16_u
def test_encoding_substitution_doesnt_happen_if_tag_is_strained(self):
markup = ('foo
')
# Beautiful Soup used to try to rewrite the meta tag even if the
# meta tag got filtered out by the strainer. This test makes
# sure that doesn't happen.
strainer = SoupStrainer('pre')
soup = self.soup(markup, parse_only=strainer)
assert soup.contents[0].name == 'pre'
class TestPersistence(SoupTest):
"Testing features like pickle and deepcopy."
def setup_method(self):
self.page = """
Beautiful Soup: We called him Tortoise because he taught us.
foo
bar
"""
self.tree = self.soup(self.page)
def test_pickle_and_unpickle_identity(self):
# Pickling a tree, then unpickling it, yields a tree identical
# to the original.
dumped = pickle.dumps(self.tree, 2)
loaded = pickle.loads(dumped)
assert loaded.__class__ == BeautifulSoup
assert loaded.decode() == self.tree.decode()
def test_deepcopy_identity(self):
# Making a deepcopy of a tree yields an identical tree.
copied = copy.deepcopy(self.tree)
assert copied.decode() == self.tree.decode()
def test_copy_deeply_nested_document(self):
# This test verifies that copy and deepcopy don't involve any
# recursive function calls. If they did, this test would
# overflow the Python interpreter stack.
limit = sys.getrecursionlimit() + 1
markup = "" * limit
soup = self.soup(markup)
copied = copy.copy(soup)
copied = copy.deepcopy(soup)
def test_copy_preserves_encoding(self):
soup = BeautifulSoup(b'
', 'html.parser')
encoding = soup.original_encoding
copy = soup.__copy__()
assert "
" == str(copy)
assert encoding == copy.original_encoding
def test_copy_preserves_builder_information(self):
tag = self.soup('').p
# Simulate a tag obtained from a source file.
tag.sourceline = 10
tag.sourcepos = 33
copied = tag.__copy__()
# The TreeBuilder object is no longer availble, but information
# obtained from it gets copied over to the new Tag object.
assert tag.sourceline == copied.sourceline
assert tag.sourcepos == copied.sourcepos
assert tag.can_be_empty_element == copied.can_be_empty_element
assert tag.cdata_list_attributes == copied.cdata_list_attributes
assert tag.preserve_whitespace_tags == copied.preserve_whitespace_tags
assert tag.interesting_string_types == copied.interesting_string_types
def test_unicode_pickle(self):
# A tree containing Unicode characters can be pickled.
html = "\N{SNOWMAN}"
soup = self.soup(html)
dumped = pickle.dumps(soup, pickle.HIGHEST_PROTOCOL)
loaded = pickle.loads(dumped)
assert loaded.decode() == soup.decode()
def test_copy_navigablestring_is_not_attached_to_tree(self):
html = "FooBar"
soup = self.soup(html)
s1 = soup.find(string="Foo")
s2 = copy.copy(s1)
assert s1 == s2
assert None == s2.parent
assert None == s2.next_element
assert None != s1.next_sibling
assert None == s2.next_sibling
assert None == s2.previous_element
def test_copy_navigablestring_subclass_has_same_type(self):
html = ""
soup = self.soup(html)
s1 = soup.string
s2 = copy.copy(s1)
assert s1 == s2
assert isinstance(s2, Comment)
def test_copy_entire_soup(self):
html = "end"
soup = self.soup(html)
soup_copy = copy.copy(soup)
assert soup == soup_copy
def test_copy_tag_copies_contents(self):
html = "end"
soup = self.soup(html)
div = soup.div
div_copy = copy.copy(div)
# The two tags look the same, and evaluate to equal.
assert str(div) == str(div_copy)
assert div == div_copy
# But they're not the same object.
assert div is not div_copy
# And they don't have the same relation to the parse tree. The
# copy is not associated with a parse tree at all.
assert None == div_copy.parent
assert None == div_copy.previous_element
assert None == div_copy.find(string='Bar').next_element
assert None != div.find(string='Bar').next_element