pyhtml2text 0.3.0__tar.gz → 0.3.1__tar.gz

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## 0.3.1 (2026-10-19)
+
+- The vendored html5ever now carries
+  [servo/html5ever#800](https://github.com/servo/html5ever/pull/800) by Josef
+  Guenther in place of our narrower patch. Every "in scope" check and the
+  other searches of the stack of open elements are now O(1), so unmatched
+  end tags and end tags inside SVG or MathML no longer make deeply nested
+  HTML quadratic: 32,000 nested `<span>` followed by as many unmatched
+  `</em>` went from 3.4 s to 0.02 s. Output is unchanged.
+- About 0.4 µs more per call, and 0–2% on real pages.
+
 ## 0.3.0 (2026-10-15)
 
 - New `html2markdown(html, *, base_url=None)`: HTML to CommonMark with

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/Cargo.lock

@@ -249,7 +249,7 @@ dependencies = [
 
 [[package]]
 name = "pyhtml2text"
-version = "0.3.0"
+version = "0.3.1"
 dependencies = [
  "html2text",
  "html5ever",

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyhtml2text"
-version = "0.3.0"
+version = "0.3.1"
 edition = "2021"
 license = "MIT"
 publish = false

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pyhtml2text
-Version: 0.3.0
+Version: 0.3.1
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: 3
 Classifier: Topic :: Text Processing :: Markup :: HTML
@@ -122,14 +122,18 @@ To plain text:
   profile-guided optimisation, which makes them about 25% faster. Other
   platforms get the same code without it.
 - Deeply nested HTML is parsed in linear time, so hostile input can't stall a
-  worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever.
+  worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever,
+  and 100,000 levels of nesting followed by unmatched end tags take under
+  0.1 s.
 
 ## Credits
 
 The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
 This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
-that adds `base_url` and some performance changes, and a patched copy of
-[html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
+that adds `base_url` and some performance changes, and a copy of
+[html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0) with
+[servo/html5ever#800](https://github.com/servo/html5ever/pull/800) by Josef
+Guenther applied, which makes parsing deeply nested HTML linear (see
 `vendor/`). Markdown output is checked with
 [pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
 isn't affiliated with or endorsed by any of these projects, so please report

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/README.md

@@ -106,14 +106,18 @@ To plain text:
   profile-guided optimisation, which makes them about 25% faster. Other
   platforms get the same code without it.
 - Deeply nested HTML is parsed in linear time, so hostile input can't stall a
-  worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever.
+  worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever,
+  and 100,000 levels of nesting followed by unmatched end tags take under
+  0.1 s.
 
 ## Credits
 
 The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
 This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
-that adds `base_url` and some performance changes, and a patched copy of
-[html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
+that adds `base_url` and some performance changes, and a copy of
+[html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0) with
+[servo/html5ever#800](https://github.com/servo/html5ever/pull/800) by Josef
+Guenther applied, which makes parsing deeply nested HTML linear (see
 `vendor/`). Markdown output is checked with
 [pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
 isn't affiliated with or endorsed by any of these projects, so please report

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pyhtml2text"
-version = "0.3.0"
+version = "0.3.1"
 description = "Fast HTML to plain text, with relative link resolution (Python bindings to rust-html2text)"
 readme = "README.md"
 license = "MIT"

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/tests/test_basic.py

@@ -74,16 +74,24 @@ def test_rejects_other_types():
 
 def test_deep_nesting_is_linear():
     # Parsing used to be quadratic in nesting depth (40,000 nested lists took
-    # 6s); with the patched html5ever this takes well under a second.
+    # 6s); with the patched html5ever each of these takes well under a second.
+    # pyhtml2text 0.3.0, whose patch didn't cover them, took 33s and 50s on the
+    # last two.
     import platform
     import sys
     import time
 
-    html = "<ul><li>" * 100_000 + "x"
-    start = time.perf_counter()
-    out = html2text(html)
-    elapsed = time.perf_counter() - start
-    assert out.rstrip().endswith("x")
-    # Only time it where tests run natively; emulated CI runners are slow.
-    if platform.machine().lower() in ("x86_64", "amd64") or sys.platform == "darwin":
-        assert elapsed < 10, f"took {elapsed:.1f}s"
+    cases = [
+        "<ul><li>" * 100_000 + "x",
+        "<div>" * 100_000 + "x</p></p>",
+        "<span>" * 100_000 + "x" + "</em>" * 100_000,
+        "<svg>" + "<g>" * 100_000 + "</x>" * 100_000 + "</svg>x",
+    ]
+    for html in cases:
+        start = time.perf_counter()
+        out = html2text(html)
+        elapsed = time.perf_counter() - start
+        assert out.rstrip().endswith("x")
+        # Only time it where tests run natively; emulated CI runners are slow.
+        if platform.machine().lower() in ("x86_64", "amd64") or sys.platform == "darwin":
+            assert elapsed < 10, f"took {elapsed:.1f}s on {html[:20]!r}..."

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/tests/test_markdown.py

@@ -171,6 +171,8 @@ def test_hostile_input_stays_linear():
         "<ul><li>" * 100_000 + "x",
         "<blockquote>" * 100_000 + "x",
         "<table><tr><td>" * 10_000 + "x",
+        "<span>" * 100_000 + "x" + "</em>" * 100_000,
+        "<svg>" + "<g>" * 100_000 + "</x>" * 100_000 + "</svg>x",
         "<table><tr><td rowspan=65534 colspan=1000>a</td><td>b</td></tr>" + "<tr></tr>" * 200_000 + "</table>",
     ]
     for html in cases:

pyhtml2text-0.3.1/vendor/README.md

@@ -0,0 +1,48 @@
+# Vendored crates
+
+## html5ever 0.39.0
+
+Copied from the crates.io release (`src/`, licences, README and manifest;
+examples, tests and benchmarks removed). Dual-licensed MIT / Apache-2.0,
+see `html5ever/LICENSE-*`.
+
+Patched with [servo/html5ever#800](https://github.com/servo/html5ever/pull/800)
+by Josef Guenther ("Fix quadratic parsing of deeply nested markup", commit
+`67c7e7c423c5d1acab1872c630cae61e2bc45211`), which is not merged yet. The
+PR is against html5ever 0.40.1; its `src/` changes apply unchanged to
+0.39.0, apart from line offsets. Like the rest of html5ever, they are
+available under MIT or Apache-2.0.
+
+Without it, tree building is quadratic in nesting depth (see
+[servo/html5ever#788](https://github.com/servo/html5ever/issues/788)): most
+start and end tags search the stack of open elements from the top, and on
+deeply nested input each search walks the whole stack. The PR keeps the
+positions of open elements by name and by the sets those searches look for,
+so "in scope" checks and the other lookups are O(1). Two cases stay
+quadratic: the adoption agency removing elements from the middle of the
+stack, and searches of the list of active formatting elements.
+
+This replaces an earlier, narrower patch of ours that only covered the
+"`p` in button scope" check. That patch left unmatched end tags and end tags
+in foreign content quadratic: at 32,000 levels, `html2text()` took 3.4 s on
+nested `<span>` followed by unmatched `</em>`, and 4.5 s on SVG `<g>`
+followed by unmatched `</x>`. Both now take 0.02 s or less.
+
+Checked against stock 0.39.0 before vendoring:
+
+- html5ever's own tests pass, including the PR's new test, and so do all
+  3,515 html5lib tree-construction tests, in release and debug builds.
+- The tree, quirks mode and exact parse errors are identical to stock 0.39.0
+  on every html5lib input (documents and fragments), 300,000 random tag-soup
+  records (30% of them fragments, in 23 contexts) and 28 real pages,
+  including the single-page HTML standard.
+- pyhtml2text's output is byte-for-byte unchanged on the benchmark corpus
+  (`bench/bench.py --check`).
+
+The extra bookkeeping costs about 0.4 µs per call, and 0–2% on real pages.
+
+Once the PR is in an html5ever release that rust-html2text uses, drop this
+directory and the `[patch.crates-io]` entry in `Cargo.toml`. Upgrading
+rust-html2text to one that uses a newer html5ever without updating this copy
+silently drops the patch (cargo only warns that it is unused), so
+`test_deep_nesting_is_linear` checks that it is still in effect.

{pyhtml2text-0.3.0 → pyhtml2text-0.3.1}/vendor/html5ever/src/tree_builder/mod.rs

@@ -14,7 +14,7 @@ pub use crate::interface::{AppendNode, AppendText, Attribute, NodeOrText};
 pub use crate::interface::{LimitedQuirks, NoQuirks, Quirks, QuirksMode};
 pub use markup5ever::interface::tree_builder::create_element_with_flags;
 
-use self::open_elems::{ElemKind, OpenElems};
+use self::open_elems::{OpenElems, TrackedSet};
 use self::types::*;
 
 use crate::tendril::StrTendril;
@@ -546,7 +546,8 @@ where
     }
 
     fn end(&self) {
-        for elem in self.open_elems.borrow_mut().drain_all().rev() {
+        let mut open_elems = self.open_elems.borrow_mut();
+        while let Some(elem) = open_elems.pop() {
             self.sink.pop(&elem);
         }
     }
@@ -743,12 +744,7 @@ where
                 });
             };
 
-            let Some(fmt_elem_stack_index) = self
-                .open_elems
-                .borrow()
-                .iter()
-                .rposition(|n| self.sink.same_node(n, &fmt_elem))
-            else {
+            let Some(fmt_elem_stack_index) = self.position_in_open_elems(&fmt_elem) else {
                 self.sink
                     .parse_error(Borrowed("Formatting element not open"));
                 self.active_formatting.borrow_mut().remove(fmt_elem_index);
@@ -756,7 +752,11 @@ where
             };
 
             // 7.
-            if !self.in_scope(default_scope, |n| self.sink.same_node(&n, &fmt_elem)) {
+            if !self
+                .open_elems
+                .borrow()
+                .in_scope(TrackedSet::DefaultScope, fmt_elem_stack_index)
+            {
                 self.sink
                     .parse_error(Borrowed("Formatting element not in scope"));
                 return;
@@ -778,7 +778,7 @@ where
                 .find(|&(_, open_element)| self.elem_in(open_element, special_tag))
                 .map(|(i, h)| (i, h.clone()));
 
-            let Some((furthest_block_index, furthest_block)) = maybe_furthest_block else {
+            let Some((mut furthest_block_index, furthest_block)) = maybe_furthest_block else {
                 // 10.
                 self.open_elems.borrow_mut().truncate(fmt_elem_stack_index);
                 self.active_formatting.borrow_mut().remove(fmt_elem_index);
@@ -816,12 +816,14 @@ where
                     self.position_in_active_formatting(&node)
                         .map(|position| self.active_formatting.borrow_mut().remove(position));
                     self.open_elems.borrow_mut().remove(node_index);
+                    furthest_block_index -= 1;
                     continue;
                 }
 
                 let Some(node_formatting_index) = self.position_in_active_formatting(&node) else {
                     // 13.6.
                     self.open_elems.borrow_mut().remove(node_index);
+                    furthest_block_index -= 1;
                     continue;
                 };
 
@@ -841,10 +843,11 @@ where
                     tag.attrs.clone(),
                     tag.had_duplicate_attributes,
                 );
-                let kind = self.open_elem_kind(&new_element);
-                self.open_elems
-                    .borrow_mut()
-                    .replace(node_index, new_element.clone(), kind);
+                self.open_elems.borrow_mut().replace(
+                    node_index,
+                    new_element.clone(),
+                    self.sink.elem_name(&new_element).expanded(),
+                );
                 self.active_formatting.borrow_mut()[node_formatting_index] =
                     FormatEntry::Element(new_element.clone(), tag);
                 node = new_element;
@@ -910,37 +913,28 @@ where
             }
 
             // 19.
-            self.remove_from_stack(&fmt_elem);
-            let new_furthest_block_index = self
-                .open_elems
-                .borrow()
-                .iter()
-                .position(|n| self.sink.same_node(n, &furthest_block))
-                .expect("furthest block missing from open element stack");
-            let kind = self.open_elem_kind(&new_element);
-            self.open_elems
-                .borrow_mut()
-                .insert(new_furthest_block_index + 1, new_element, kind);
+            // Move the formatting element up to just above the furthest block, which moves the
+            // elements in between down by one, and put the new element in its place.
+            {
+                let mut open_elems = self.open_elems.borrow_mut();
+                open_elems.move_up(fmt_elem_stack_index, furthest_block_index);
+                open_elems.replace(
+                    furthest_block_index,
+                    new_element.clone(),
+                    self.sink.elem_name(&new_element).expanded(),
+                );
+            }
+            self.sink.pop(&fmt_elem);
 
             // 20.
         }
     }
 
-    fn open_elem_kind(&self, elem: &Handle) -> ElemKind {
-        let name = self.sink.elem_name(elem);
-        let name = name.expanded();
-        if *name.ns == ns!(html) && *name.local == local_name!("p") {
-            ElemKind::P
-        } else if button_scope(name) {
-            ElemKind::ButtonScopeBoundary
-        } else {
-            ElemKind::Other
-        }
-    }
-
     fn push(&self, elem: &Handle) {
-        let kind = self.open_elem_kind(elem);
-        self.open_elems.borrow_mut().push(elem.clone(), kind);
+        let name = self.sink.elem_name(elem);
+        self.open_elems
+            .borrow_mut()
+            .push(elem.clone(), name.expanded());
     }
 
     fn pop(&self) -> Handle {
@@ -954,12 +948,24 @@ where
         elem
     }
 
-    fn remove_from_stack(&self, elem: &Handle) {
-        let position = self
-            .open_elems
-            .borrow()
+    /// The position of `elem` in the stack of open elements, if it is open.
+    fn position_in_open_elems(&self, elem: &Handle) -> Option<usize> {
+        let name = self.sink.elem_name(elem);
+        let open_elems = self.open_elems.borrow();
+        let positions = if *name.ns() == ns!(html) {
+            open_elems.positions_named(name.local_name())
+        } else {
+            open_elems.positions_foreign_named(name.local_name())
+        };
+        let position = positions
             .iter()
-            .rposition(|x| self.sink.same_node(elem, x));
+            .rev()
+            .find(|&position| self.sink.same_node(&open_elems[position], elem));
+        position
+    }
+
+    fn remove_from_stack(&self, elem: &Handle) {
+        let position = self.position_in_open_elems(elem);
         if let Some(position) = position {
             self.open_elems.borrow_mut().remove(position);
             self.sink.pop(elem);
@@ -969,12 +975,7 @@ where
     fn is_marker_or_open(&self, entry: &FormatEntry<Handle>) -> bool {
         match *entry {
             FormatEntry::Marker => true,
-            FormatEntry::Element(ref node, _) => self
-                .open_elems
-                .borrow()
-                .iter()
-                .rev()
-                .any(|n| self.sink.same_node(n, node)),
+            FormatEntry::Element(ref node, _) => self.position_in_open_elems(node).is_some(),
         }
     }
 
@@ -1075,48 +1076,22 @@ where
     /// Signal an error depending on the state of the stack of open elements at
     /// the end of the body.
     fn check_body_end(&self) {
-        declare_tag_set!(body_end_ok =
-            "dd" "dt" "li" "optgroup" "option" "p" "rp" "rt" "tbody" "td" "tfoot" "th"
-            "thead" "tr" "body" "html");
-
-        for elem in self.open_elems.borrow().iter() {
-            let error = {
-                let elem_name = self.sink.elem_name(elem);
-                let name = elem_name.expanded();
-                if body_end_ok(name) {
-                    continue;
-                }
-
-                if self.opts.exact_errors {
-                    Cow::from(format!("Unexpected open tag {name:?} at end of body"))
-                } else {
-                    Cow::from("Unexpected open tag at end of body")
-                }
-            };
-            self.sink.parse_error(error);
-            // FIXME: Do we keep checking after finding one bad tag?
-            // The spec suggests not.
+        let open_elems = self.open_elems.borrow();
+        // FIXME: Do we keep checking after finding one bad tag?
+        // The spec suggests not.
+        let Some(position) = open_elems.positions_in(TrackedSet::NotBodyEndOk).first() else {
             return;
-        }
-    }
-
-    fn in_scope<TagSet, Pred>(&self, scope: TagSet, pred: Pred) -> bool
-    where
-        TagSet: Fn(ExpandedName) -> bool,
-        Pred: Fn(Handle) -> bool,
-    {
-        for node in self.open_elems.borrow().iter().rev() {
-            if pred(node.clone()) {
-                return true;
-            }
-            if scope(self.sink.elem_name(node).expanded()) {
-                return false;
+        };
+        let error = {
+            let elem_name = self.sink.elem_name(&open_elems[position]);
+            let name = elem_name.expanded();
+            if self.opts.exact_errors {
+                Cow::from(format!("Unexpected open tag {name:?} at end of body"))
+            } else {
+                Cow::from("Unexpected open tag at end of body")
             }
-        }
-
-        // supposed to be impossible, because <html> is always in scope
-
-        false
+        };
+        self.sink.parse_error(error);
     }
 
     fn elem_in<TagSet>(&self, elem: &Handle, set: TagSet) -> bool
@@ -1132,21 +1107,22 @@ where
     }
 
     fn in_html_elem_named(&self, name: LocalName) -> bool {
-        self.open_elems
-            .borrow()
-            .iter()
-            .any(|elem| self.html_elem_named(elem, name.clone()))
+        !self.open_elems.borrow().positions_named(&name).is_empty()
     }
 
     fn current_node_named(&self, name: LocalName) -> bool {
         self.html_elem_named(&self.current_node(), name)
     }
 
-    fn in_scope_named<TagSet>(&self, scope: TagSet, name: LocalName) -> bool
-    where
-        TagSet: Fn(ExpandedName) -> bool,
-    {
-        self.in_scope(scope, |elem| self.html_elem_named(&elem, name.clone()))
+    fn in_scope_named(&self, scope: TrackedSet, name: LocalName) -> bool {
+        self.open_elems.borrow().named_in_scope(scope, &name)
+    }
+
+    fn in_scope_any_named(&self, scope: TrackedSet, names: &[LocalName]) -> bool {
+        let open_elems = self.open_elems.borrow();
+        names
+            .iter()
+            .any(|name| open_elems.named_in_scope(scope, name))
     }
 
     /// <https://html.spec.whatwg.org/#generate-implied-end-tags>
@@ -1234,15 +1210,8 @@ where
         self.expect_to_close(local_name!("p"));
     }
 
-    /// Whether the stack of open elements has a `p` element in button scope.
-    fn p_in_button_scope(&self) -> bool {
-        let result = self.open_elems.borrow().has_p_in_button_scope();
-        debug_assert_eq!(result, self.in_scope_named(button_scope, local_name!("p")));
-        result
-    }
-
     fn close_p_element_in_button_scope(&self) {
-        if self.p_in_button_scope() {
+        if self.in_scope_named(TrackedSet::ButtonScope, local_name!("p")) {
             self.close_p_element();
         }
     }
@@ -1290,7 +1259,15 @@ where
     // https://html.spec.whatwg.org/multipage/#reset-the-insertion-mode-appropriately
     fn reset_insertion_mode(&self) -> InsertionMode {
         let open_elems = self.open_elems.borrow();
-        for (i, mut node) in open_elems.iter().enumerate().rev() {
+        // The steps below stop at the topmost element in this set, unless it is the first one,
+        // which they look at with the context element in its place.
+        let topmost = open_elems
+            .positions_in(TrackedSet::ResetInsertionMode)
+            .last()
+            .filter(|&i| i > 0);
+        let first = (!open_elems.is_empty()).then_some(0);
+        for i in topmost.into_iter().chain(first) {
+            let mut node = &open_elems[i];
             let last = i == 0usize;
             let context_elem = self.context_elem.borrow();
             if let (true, Some(ctx)) = (last, context_elem.as_ref()) {
@@ -1587,20 +1564,20 @@ where
     }
 
     fn process_end_tag_in_body(&self, tag: Tag) {
-        // Look back for a matching open element.
-        let mut match_idx = None;
-        for (i, elem) in self.open_elems.borrow().iter().enumerate().rev() {
-            if self.html_elem_named(elem, tag.name.clone()) {
-                match_idx = Some(i);
-                break;
-            }
-
-            if self.elem_in(elem, special_tag) {
+        // Look back for a matching open element, as far as the topmost special element.
+        let match_idx = {
+            let open_elems = self.open_elems.borrow();
+            let topmost_match = open_elems.positions_named(&tag.name).last();
+            if topmost_match.is_some_and(|i| open_elems.in_scope(TrackedSet::Special, i)) {
+                topmost_match
+            } else if !open_elems.positions_in(TrackedSet::Special).is_empty() {
                 self.sink
                     .parse_error(Borrowed("Found special tag while closing generic tag"));
                 return;
+            } else {
+                None
             }
-        }
+        };
 
         let Some(match_idx) = match_idx else {
             // I believe this is impossible, because the root