pyhtml2text 0.2.0__tar.gz → 0.3.0__tar.gz

{pyhtml2text-0.2.0 → pyhtml2text-0.3.0}/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.3.0 (2026-10-15)
+
+- New `html2markdown(html, *, base_url=None)`: HTML to CommonMark with
+  GitHub's tables and strikethrough. Links and images keep their URLs (made
+  absolute with `base_url`), tables become pipe tables, code blocks are
+  fenced with their language, and characters are escaped only where Markdown
+  would misread them. 3–13× faster than markdownify, html2text and
+  html-to-markdown on real pages.
+- `html2text()` output is unchanged.
+
 ## 0.2.0 (2026-10-15)
 
 - 1.5–1.7× faster than 0.1.0 on real web pages, and 2× faster on short

{pyhtml2text-0.2.0 → pyhtml2text-0.3.0}/Cargo.lock

@@ -51,7 +51,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 [[package]]
 name = "html2text"
 version = "0.17.1"
-source = "git+https://github.com/jaredlynskey/rust-html2text?rev=9c109fac38728040cf3c32b411305cdd840bd511#9c109fac38728040cf3c32b411305cdd840bd511"
+source = "git+https://github.com/jaredlynskey/rust-html2text?rev=b42930f8e6688ea121daf392e27a7d9812e6192d#b42930f8e6688ea121daf392e27a7d9812e6192d"
 dependencies = [
  "html5ever",
  "tendril",
@@ -117,6 +117,12 @@ dependencies = [
  "web_atoms",
 ]
 
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
 [[package]]
 name = "memoffset"
 version = "0.9.1"
@@ -230,12 +236,25 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "pulldown-cmark"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
+dependencies = [
+ "bitflags",
+ "memchr",
+ "unicase",
+]
+
 [[package]]
 name = "pyhtml2text"
-version = "0.2.0"
+version = "0.3.0"
 dependencies = [
  "html2text",
+ "html5ever",
  "mimalloc",
+ "pulldown-cmark",
  "pyo3",
 ]
 
@@ -458,6 +477,12 @@ dependencies = [
  "syn 3.0.9",
 ]
 
+[[package]]
+name = "unicase"
+version = "2.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f"
+
 [[package]]
 name = "unicode-ident"
 version = "1.0.27"

{pyhtml2text-0.2.0 → pyhtml2text-0.3.0}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pyhtml2text"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 license = "MIT"
 publish = false
@@ -12,9 +12,13 @@ crate-type = ["cdylib"]
 
 [dependencies]
 pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] }
-# Fork of jugglerchris/rust-html2text with Config::base_url and some
-# performance changes (branch pyhtml2text-perf).
-html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "9c109fac38728040cf3c32b411305cdd840bd511" }
+# Fork of jugglerchris/rust-html2text with Config::base_url, some performance
+# changes and the DOM exports the Markdown output uses (branch pyhtml2text-perf).
+html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "b42930f8e6688ea121daf392e27a7d9812e6192d" }
+# The parser html2text uses; patched below. Used directly to read element attributes.
+html5ever = "0.39"
+# Reads the Markdown back to check that emphasis parses as intended.
+pulldown-cmark = { version = "0.13", default-features = false }
 
 # Much faster than the system allocators for html2text's many small allocations.
 # Not on glibc armv7/ppc64le/s390x: their manylinux2014 cross compilers are too

pyhtml2text-0.3.0/PKG-INFO

@@ -0,0 +1,137 @@
+Metadata-Version: 2.4
+Name: pyhtml2text
+Version: 0.3.0
+Classifier: Programming Language :: Rust
+Classifier: Programming Language :: Python :: 3
+Classifier: Topic :: Text Processing :: Markup :: HTML
+License-File: LICENSE
+Summary: Fast HTML to plain text, with relative link resolution (Python bindings to rust-html2text)
+Keywords: html,text,markdown,html2text
+Author-email: Jared Lynskey <jared@lynskey.co.nz>
+License-Expression: MIT
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
+Project-URL: Source, https://github.com/jaredlynskey/pyhtml2text
+Project-URL: Upstream, https://github.com/jugglerchris/rust-html2text
+
+# pyhtml2text
+
+Fast HTML to plain text and to Markdown for Python, built on
+[rust-html2text](https://github.com/jugglerchris/rust-html2text).
+
+Relative links can be resolved against the page's URL, so the output's links
+work on their own.
+
+## Markdown
+
+```python
+from pyhtml2text import html2markdown
+
+html = """<h2>Before you start</h2>
+<p>Read the <a href="setup.html">setup guide</a> and <b>back up</b> your data.</p>
+<img src="/img/diagram.png" alt="How it fits together">
+<table><tr><th>Plan</th><th>Price</th></tr><tr><td>Basic</td><td>Free</td></tr></table>"""
+
+print(html2markdown(html, base_url="https://example.com/docs/intro.html"))
+```
+
+```markdown
+## Before you start
+
+Read the [setup guide](https://example.com/docs/setup.html) and **back up** your data.
+
+![How it fits together](https://example.com/img/diagram.png)
+
+| Plan | Price |
+| --- | --- |
+| Basic | Free |
+```
+
+`html2markdown(html, *, base_url=None)` writes
+[CommonMark](https://commonmark.org/) with GitHub's tables and
+strikethrough, in the usual conventions:
+
+- `#` headings, `-` bullets, `_emphasis_` and `**strong**`, `~~strikethrough~~`.
+- Inline links and images, `[text](url)` and `![alt](src)`, made absolute when
+  `base_url` is given. A link whose only content is an icon uses its
+  `aria-label` or `title` as its text.
+- Fenced code blocks, tagged with the language when the page names one
+  (`class="language-python"`).
+- Pipe tables. Cells spanning rows or columns become empty cells, a table
+  without a header row gets an empty one, and tables used for page layout are
+  written as plain paragraphs.
+- Characters are escaped only where Markdown would otherwise read them as
+  syntax, so `snake_case`, `5 * 3` and `$5` stay as they are.
+- Lines aren't wrapped. Hidden content (`hidden`, `aria-hidden`,
+  `display: none`), scripts, styles and form controls are left out.
+
+The output is checked by rendering it back with a CommonMark parser and
+comparing every word, link and image with the original page, on real sites
+and on tens of thousands of random documents (`tests/markdown_check.py`).
+
+## Plain text
+
+```python
+from pyhtml2text import html2text
+
+html = '<p>Please <a href="/login">sign in</a> first.</p>'
+
+print(html2text(html, base_url="https://example.com/docs/page.html"))
+# Please [sign in][1] first.
+#
+# [1]: https://example.com/login
+```
+
+`html2text(html, width=80, *, ...)`
+
+| option | default | |
+|---|---|---|
+| `base_url` | `None` | resolve relative `href`s against this URL (RFC 3986). Absolute and `mailto:` etc. links are left alone |
+| `decorate` | `True` | markdown-ish `*em*`, `**strong**`, `` `code` `` |
+| `link_footnotes` | `True` | list link targets as `[1]: ...` footnotes |
+| `table_borders` | `True` | draw table borders |
+| `wrap_links` | `True` | allow long links to wrap |
+| `max_wrap_width` | `None` | wrap paragraphs narrower than `width` |
+| `raw` | `False` | flatten tables so cell text stays together |
+
+Both functions take a `str` or UTF-8 `bytes` (e.g. `response.content`).
+Requires Python 3.10 or later.
+
+## Performance
+
+Median time on an Apple Silicon Mac. To Markdown:
+
+| | Wikipedia article (1 MB) | BBC News front page (370 KB) | Python docs page (245 KB) |
+|---|---|---|---|
+| [markdownify](https://pypi.org/project/markdownify/) | 184 ms | 28 ms | 97 ms |
+| [html2text](https://pypi.org/project/html2text/) (pure Python) | 73 ms | 13 ms | 37 ms |
+| [html-to-markdown](https://pypi.org/project/html-to-markdown/) | 44 ms | 5.6 ms | 15 ms |
+| **pyhtml2text** | **14 ms** | **1.7 ms** | **4.5 ms** |
+
+To plain text:
+
+| | Wikipedia article (780 KB) | BBC News front page (370 KB) |
+|---|---|---|
+| [html2text](https://pypi.org/project/html2text/) (pure Python) | 71.5 ms | 14.9 ms |
+| [html2text-rs](https://pypi.org/project/html2text-rs/) | 15.2 ms | 2.5 ms |
+| **pyhtml2text** | **10.9 ms** | **1.6 ms** |
+
+- The GIL is released while converting, so threads scale across cores. On
+  free-threaded Python (3.14t) the module runs without re-enabling the GIL.
+- Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with
+  profile-guided optimisation, which makes them about 25% faster. Other
+  platforms get the same code without it.
+- Deeply nested HTML is parsed in linear time, so hostile input can't stall a
+  worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever.
+
+## Credits
+
+The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
+This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
+that adds `base_url` and some performance changes, and a patched copy of
+[html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
+`vendor/`). Markdown output is checked with
+[pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
+isn't affiliated with or endorsed by any of these projects, so please report
+issues here rather than upstream.
+

pyhtml2text-0.3.0/README.md

@@ -0,0 +1,120 @@
+# pyhtml2text
+
+Fast HTML to plain text and to Markdown for Python, built on
+[rust-html2text](https://github.com/jugglerchris/rust-html2text).
+
+Relative links can be resolved against the page's URL, so the output's links
+work on their own.
+
+## Markdown
+
+```python
+from pyhtml2text import html2markdown
+
+html = """<h2>Before you start</h2>
+<p>Read the <a href="setup.html">setup guide</a> and <b>back up</b> your data.</p>
+<img src="/img/diagram.png" alt="How it fits together">
+<table><tr><th>Plan</th><th>Price</th></tr><tr><td>Basic</td><td>Free</td></tr></table>"""
+
+print(html2markdown(html, base_url="https://example.com/docs/intro.html"))
+```
+
+```markdown
+## Before you start
+
+Read the [setup guide](https://example.com/docs/setup.html) and **back up** your data.
+
+![How it fits together](https://example.com/img/diagram.png)
+
+| Plan | Price |
+| --- | --- |
+| Basic | Free |
+```
+
+`html2markdown(html, *, base_url=None)` writes
+[CommonMark](https://commonmark.org/) with GitHub's tables and
+strikethrough, in the usual conventions:
+
+- `#` headings, `-` bullets, `_emphasis_` and `**strong**`, `~~strikethrough~~`.
+- Inline links and images, `[text](url)` and `![alt](src)`, made absolute when
+  `base_url` is given. A link whose only content is an icon uses its
+  `aria-label` or `title` as its text.
+- Fenced code blocks, tagged with the language when the page names one
+  (`class="language-python"`).
+- Pipe tables. Cells spanning rows or columns become empty cells, a table
+  without a header row gets an empty one, and tables used for page layout are
+  written as plain paragraphs.
+- Characters are escaped only where Markdown would otherwise read them as
+  syntax, so `snake_case`, `5 * 3` and `$5` stay as they are.
+- Lines aren't wrapped. Hidden content (`hidden`, `aria-hidden`,
+  `display: none`), scripts, styles and form controls are left out.
+
+The output is checked by rendering it back with a CommonMark parser and
+comparing every word, link and image with the original page, on real sites
+and on tens of thousands of random documents (`tests/markdown_check.py`).
+
+## Plain text
+
+```python
+from pyhtml2text import html2text
+
+html = '<p>Please <a href="/login">sign in</a> first.</p>'
+
+print(html2text(html, base_url="https://example.com/docs/page.html"))
+# Please [sign in][1] first.
+#
+# [1]: https://example.com/login
+```
+
+`html2text(html, width=80, *, ...)`
+
+| option | default | |
+|---|---|---|
+| `base_url` | `None` | resolve relative `href`s against this URL (RFC 3986). Absolute and `mailto:` etc. links are left alone |
+| `decorate` | `True` | markdown-ish `*em*`, `**strong**`, `` `code` `` |
+| `link_footnotes` | `True` | list link targets as `[1]: ...` footnotes |
+| `table_borders` | `True` | draw table borders |
+| `wrap_links` | `True` | allow long links to wrap |
+| `max_wrap_width` | `None` | wrap paragraphs narrower than `width` |
+| `raw` | `False` | flatten tables so cell text stays together |
+
+Both functions take a `str` or UTF-8 `bytes` (e.g. `response.content`).
+Requires Python 3.10 or later.
+
+## Performance
+
+Median time on an Apple Silicon Mac. To Markdown:
+
+| | Wikipedia article (1 MB) | BBC News front page (370 KB) | Python docs page (245 KB) |
+|---|---|---|---|
+| [markdownify](https://pypi.org/project/markdownify/) | 184 ms | 28 ms | 97 ms |
+| [html2text](https://pypi.org/project/html2text/) (pure Python) | 73 ms | 13 ms | 37 ms |
+| [html-to-markdown](https://pypi.org/project/html-to-markdown/) | 44 ms | 5.6 ms | 15 ms |
+| **pyhtml2text** | **14 ms** | **1.7 ms** | **4.5 ms** |
+
+To plain text:
+
+| | Wikipedia article (780 KB) | BBC News front page (370 KB) |
+|---|---|---|
+| [html2text](https://pypi.org/project/html2text/) (pure Python) | 71.5 ms | 14.9 ms |
+| [html2text-rs](https://pypi.org/project/html2text-rs/) | 15.2 ms | 2.5 ms |
+| **pyhtml2text** | **10.9 ms** | **1.6 ms** |
+
+- The GIL is released while converting, so threads scale across cores. On
+  free-threaded Python (3.14t) the module runs without re-enabling the GIL.
+- Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with
+  profile-guided optimisation, which makes them about 25% faster. Other
+  platforms get the same code without it.
+- Deeply nested HTML is parsed in linear time, so hostile input can't stall a
+  worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever.
+
+## Credits
+
+The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
+This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
+that adds `base_url` and some performance changes, and a patched copy of
+[html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
+`vendor/`). Markdown output is checked with
+[pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
+isn't affiliated with or endorsed by any of these projects, so please report
+issues here rather than upstream.

{pyhtml2text-0.2.0 → pyhtml2text-0.3.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "pyhtml2text"
-version = "0.2.0"
+version = "0.3.0"
 description = "Fast HTML to plain text, with relative link resolution (Python bindings to rust-html2text)"
 readme = "README.md"
 license = "MIT"

pyhtml2text-0.3.0/python/pyhtml2text/__init__.py

@@ -0,0 +1,3 @@
+from ._pyhtml2text import html2markdown, html2text
+
+__all__ = ["html2markdown", "html2text"]

{pyhtml2text-0.2.0 → pyhtml2text-0.3.0}/python/pyhtml2text/__init__.pyi

@@ -10,3 +10,4 @@ def html2text(
     max_wrap_width: int | None = None,
     raw: bool = False,
 ) -> str: ...
+def html2markdown(html: str | bytes, *, base_url: str | None = None) -> str: ...

{pyhtml2text-0.2.0 → pyhtml2text-0.3.0}/src/lib.rs

@@ -5,6 +5,8 @@ use pyo3::exceptions::{PyTypeError, PyValueError};
 use pyo3::prelude::*;
 use pyo3::types::{PyBytes, PyString};
 
+mod markdown;
+
 #[cfg(not(all(
     target_env = "gnu",
     any(target_arch = "arm", target_arch = "powerpc64", target_arch = "s390x")
@@ -12,6 +14,43 @@ use pyo3::types::{PyBytes, PyString};
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
+/// Borrow the input without copying it; bytes are taken to be UTF-8.
+fn input_bytes<'a>(html: &'a Bound<'_, PyAny>) -> PyResult<&'a [u8]> {
+    if let Ok(s) = html.downcast::<PyString>() {
+        Ok(s.to_str()?.as_bytes())
+    } else if let Ok(b) = html.downcast::<PyBytes>() {
+        Ok(b.as_bytes())
+    } else {
+        Err(PyTypeError::new_err(format!(
+            "html must be str or bytes, not {}",
+            html.get_type().name()?
+        )))
+    }
+}
+
+/// Run a conversion. It is pure Rust, so other Python threads run meanwhile.
+/// A panic would otherwise surface as PanicException, which `except
+/// Exception` doesn't catch, so it is reported as a ValueError instead.
+fn convert<F>(py: Python<'_>, f: F) -> PyResult<String>
+where
+    F: FnOnce() -> Result<String, html2text::Error> + Send,
+{
+    match py.detach(|| catch_unwind(AssertUnwindSafe(|| f().map_err(|e| e.to_string())))) {
+        Ok(Ok(text)) => Ok(text),
+        Ok(Err(e)) => Err(PyValueError::new_err(e)),
+        Err(panic) => {
+            let msg = panic
+                .downcast_ref::<&str>()
+                .copied()
+                .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
+                .unwrap_or("unknown error");
+            Err(PyValueError::new_err(format!(
+                "html2text failed on this document: {msg}"
+            )))
+        }
+    }
+}
+
 /// Render HTML as plain text.
 #[pyfunction]
 #[pyo3(name = "html2text", signature = (
@@ -39,17 +78,7 @@ fn render(
     max_wrap_width: Option<usize>,
     raw: bool,
 ) -> PyResult<String> {
-    // Borrow the input without copying it; bytes are taken to be UTF-8.
-    let input: &[u8] = if let Ok(s) = html.downcast::<PyString>() {
-        s.to_str()?.as_bytes()
-    } else if let Ok(b) = html.downcast::<PyBytes>() {
-        b.as_bytes()
-    } else {
-        return Err(PyTypeError::new_err(format!(
-            "html must be str or bytes, not {}",
-            html.get_type().name()?
-        )));
-    };
+    let input = input_bytes(html)?;
 
     let mut conf = if decorate {
         config::plain()
@@ -73,27 +102,29 @@ fn render(
         conf = conf.raw_mode(true);
     }
 
-    // Rendering is pure Rust, so let other Python threads run meanwhile. A
-    // panic would otherwise surface as PanicException, which `except
-    // Exception` doesn't catch, so report it as a ValueError instead.
-    match py.detach(|| catch_unwind(AssertUnwindSafe(|| conf.string_from_read(input, width)))) {
-        Ok(Ok(text)) => Ok(text),
-        Ok(Err(e)) => Err(PyValueError::new_err(e.to_string())),
-        Err(panic) => {
-            let msg = panic
-                .downcast_ref::<&str>()
-                .copied()
-                .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
-                .unwrap_or("unknown error");
-            Err(PyValueError::new_err(format!(
-                "html2text failed on this document: {msg}"
-            )))
-        }
-    }
+    convert(py, || conf.string_from_read(input, width))
+}
+
+/// Convert HTML to Markdown.
+///
+/// The output is CommonMark with GitHub's pipe tables and strikethrough:
+/// ATX headings, `-` bullets, fenced code blocks and inline links and images.
+/// Lines are not wrapped. Relative links and image sources are resolved
+/// against `base_url` when one is given.
+#[pyfunction]
+#[pyo3(name = "html2markdown", signature = (html, *, base_url = None))]
+fn to_markdown(
+    py: Python<'_>,
+    html: &Bound<'_, PyAny>,
+    base_url: Option<String>,
+) -> PyResult<String> {
+    let input = input_bytes(html)?;
+    convert(py, || markdown::from_read(input, base_url.as_deref()))
 }
 
 #[pymodule(gil_used = false)]
 fn _pyhtml2text(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(render, m)?)?;
+    m.add_function(wrap_pyfunction!(to_markdown, m)?)?;
     Ok(())
 }