pyhtml2text 0.1.0__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. pyhtml2text-0.3.0/CHANGELOG.md +31 -0
  2. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/Cargo.lock +68 -4
  3. pyhtml2text-0.3.0/Cargo.toml +36 -0
  4. pyhtml2text-0.3.0/PKG-INFO +137 -0
  5. pyhtml2text-0.3.0/README.md +120 -0
  6. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/pyproject.toml +15 -2
  7. pyhtml2text-0.3.0/python/pyhtml2text/__init__.py +3 -0
  8. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/python/pyhtml2text/__init__.pyi +4 -5
  9. pyhtml2text-0.3.0/src/lib.rs +130 -0
  10. pyhtml2text-0.3.0/src/markdown.rs +2442 -0
  11. pyhtml2text-0.3.0/tests/markdown_check.py +464 -0
  12. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/tests/test_basic.py +31 -0
  13. pyhtml2text-0.3.0/tests/test_markdown.py +207 -0
  14. pyhtml2text-0.3.0/vendor/README.md +13 -0
  15. pyhtml2text-0.3.0/vendor/html5ever/Cargo.toml +61 -0
  16. pyhtml2text-0.3.0/vendor/html5ever/LICENSE-APACHE +201 -0
  17. pyhtml2text-0.3.0/vendor/html5ever/LICENSE-MIT +25 -0
  18. pyhtml2text-0.3.0/vendor/html5ever/README.md +60 -0
  19. pyhtml2text-0.3.0/vendor/html5ever/src/driver.rs +162 -0
  20. pyhtml2text-0.3.0/vendor/html5ever/src/encoding.rs +173 -0
  21. pyhtml2text-0.3.0/vendor/html5ever/src/lib.rs +31 -0
  22. pyhtml2text-0.3.0/vendor/html5ever/src/macros.rs +34 -0
  23. pyhtml2text-0.3.0/vendor/html5ever/src/serialize/mod.rs +255 -0
  24. pyhtml2text-0.3.0/vendor/html5ever/src/tokenizer/char_ref/mod.rs +438 -0
  25. pyhtml2text-0.3.0/vendor/html5ever/src/tokenizer/interface.rs +141 -0
  26. pyhtml2text-0.3.0/vendor/html5ever/src/tokenizer/mod.rs +2324 -0
  27. pyhtml2text-0.3.0/vendor/html5ever/src/tokenizer/states.rs +131 -0
  28. pyhtml2text-0.3.0/vendor/html5ever/src/tree_builder/data.rs +170 -0
  29. pyhtml2text-0.3.0/vendor/html5ever/src/tree_builder/mod.rs +1917 -0
  30. pyhtml2text-0.3.0/vendor/html5ever/src/tree_builder/open_elems.rs +134 -0
  31. pyhtml2text-0.3.0/vendor/html5ever/src/tree_builder/rules.rs +1690 -0
  32. pyhtml2text-0.3.0/vendor/html5ever/src/tree_builder/tag_sets.rs +109 -0
  33. pyhtml2text-0.3.0/vendor/html5ever/src/tree_builder/types.rs +111 -0
  34. pyhtml2text-0.3.0/vendor/html5ever/src/util/str.rs +52 -0
  35. pyhtml2text-0.1.0/.github/workflows/CI.yml +0 -298
  36. pyhtml2text-0.1.0/Cargo.toml +0 -21
  37. pyhtml2text-0.1.0/PKG-INFO +0 -56
  38. pyhtml2text-0.1.0/README.md +0 -39
  39. pyhtml2text-0.1.0/python/pyhtml2text/__init__.py +0 -3
  40. pyhtml2text-0.1.0/src/lib.rs +0 -62
  41. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/.gitignore +0 -0
  42. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/LICENSE +0 -0
  43. {pyhtml2text-0.1.0 → pyhtml2text-0.3.0}/python/pyhtml2text/py.typed +0 -0
@@ -0,0 +1,31 @@
1
+ # Changelog
2
+
3
+ ## 0.3.0 (2026-10-15)
4
+
5
+ - New `html2markdown(html, *, base_url=None)`: HTML to CommonMark with
6
+ GitHub's tables and strikethrough. Links and images keep their URLs (made
7
+ absolute with `base_url`), tables become pipe tables, code blocks are
8
+ fenced with their language, and characters are escaped only where Markdown
9
+ would misread them. 3–13× faster than markdownify, html2text and
10
+ html-to-markdown on real pages.
11
+ - `html2text()` output is unchanged.
12
+
13
+ ## 0.2.0 (2026-10-15)
14
+
15
+ - 1.5–1.7× faster than 0.1.0 on real web pages, and 2× faster on short
16
+ snippets. Output is unchanged.
17
+ - Wheels for Linux x86_64/aarch64, macOS and Windows x64 are built with
18
+ profile-guided optimisation.
19
+ - Uses mimalloc as the allocator (except on glibc armv7, ppc64le and s390x).
20
+ - Style rules are shared rather than rebuilt and copied for every call.
21
+ - Deeply nested HTML no longer takes quadratic time to parse: 40,000 nested
22
+ `<ul>` lists went from 6s to 0.14s (patched html5ever, see `vendor/`).
23
+ - `html` can be UTF-8 `bytes` as well as `str`.
24
+ - An internal error now raises `ValueError` rather than `PanicException`
25
+ (which `except Exception` doesn't catch).
26
+ - Supports free-threaded Python without re-enabling the GIL.
27
+ - Requires Python 3.10 or later (3.9 is end-of-life).
28
+
29
+ ## 0.1.0 (2026-10-15)
30
+
31
+ - First release: `html2text()` with `base_url` support.
@@ -14,6 +14,16 @@ version = "2.13.2"
14
14
  source = "registry+https://github.com/rust-lang/crates.io-index"
15
15
  checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06"
16
16
 
17
+ [[package]]
18
+ name = "cc"
19
+ version = "1.8.0"
20
+ source = "registry+https://github.com/rust-lang/crates.io-index"
21
+ checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
22
+ dependencies = [
23
+ "find-msvc-tools",
24
+ "shlex",
25
+ ]
26
+
17
27
  [[package]]
18
28
  name = "cfg-if"
19
29
  version = "1.0.5"
@@ -26,6 +36,12 @@ version = "2.5.0"
26
36
  source = "registry+https://github.com/rust-lang/crates.io-index"
27
37
  checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223"
28
38
 
39
+ [[package]]
40
+ name = "find-msvc-tools"
41
+ version = "0.1.14"
42
+ source = "registry+https://github.com/rust-lang/crates.io-index"
43
+ checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
44
+
29
45
  [[package]]
30
46
  name = "heck"
31
47
  version = "0.5.0"
@@ -35,7 +51,7 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
35
51
  [[package]]
36
52
  name = "html2text"
37
53
  version = "0.17.1"
38
- source = "git+https://github.com/jaredlynskey/rust-html2text?rev=ee049006a900891e9df46299004c38ff59c41d65#ee049006a900891e9df46299004c38ff59c41d65"
54
+ source = "git+https://github.com/jaredlynskey/rust-html2text?rev=b42930f8e6688ea121daf392e27a7d9812e6192d#b42930f8e6688ea121daf392e27a7d9812e6192d"
39
55
  dependencies = [
40
56
  "html5ever",
41
57
  "tendril",
@@ -46,8 +62,6 @@ dependencies = [
46
62
  [[package]]
47
63
  name = "html5ever"
48
64
  version = "0.39.0"
49
- source = "registry+https://github.com/rust-lang/crates.io-index"
50
- checksum = "46a1761807faccc9a19e86944bbf40610014066306f96edcdedc2fb714bcb7b8"
51
65
  dependencies = [
52
66
  "log",
53
67
  "markup5ever",
@@ -68,6 +82,15 @@ version = "0.2.190"
68
82
  source = "registry+https://github.com/rust-lang/crates.io-index"
69
83
  checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78"
70
84
 
85
+ [[package]]
86
+ name = "libmimalloc-sys"
87
+ version = "0.1.49"
88
+ source = "registry+https://github.com/rust-lang/crates.io-index"
89
+ checksum = "6a45a52f43e1c16f667ccfe4dd8c85b7f7c204fd5e3bf46c5b0db9a5c3c0b8e9"
90
+ dependencies = [
91
+ "cc",
92
+ ]
93
+
71
94
  [[package]]
72
95
  name = "lock_api"
73
96
  version = "0.4.14"
@@ -94,6 +117,12 @@ dependencies = [
94
117
  "web_atoms",
95
118
  ]
96
119
 
120
+ [[package]]
121
+ name = "memchr"
122
+ version = "2.8.3"
123
+ source = "registry+https://github.com/rust-lang/crates.io-index"
124
+ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
125
+
97
126
  [[package]]
98
127
  name = "memoffset"
99
128
  version = "0.9.1"
@@ -103,6 +132,15 @@ dependencies = [
103
132
  "autocfg",
104
133
  ]
105
134
 
135
+ [[package]]
136
+ name = "mimalloc"
137
+ version = "0.1.52"
138
+ source = "registry+https://github.com/rust-lang/crates.io-index"
139
+ checksum = "2d4139bb28d14ad1facf21d5eb8825051b326e172d216b39f6d31df53cc97862"
140
+ dependencies = [
141
+ "libmimalloc-sys",
142
+ ]
143
+
106
144
  [[package]]
107
145
  name = "new_debug_unreachable"
108
146
  version = "1.0.6"
@@ -198,11 +236,25 @@ dependencies = [
198
236
  "unicode-ident",
199
237
  ]
200
238
 
239
+ [[package]]
240
+ name = "pulldown-cmark"
241
+ version = "0.13.4"
242
+ source = "registry+https://github.com/rust-lang/crates.io-index"
243
+ checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e"
244
+ dependencies = [
245
+ "bitflags",
246
+ "memchr",
247
+ "unicase",
248
+ ]
249
+
201
250
  [[package]]
202
251
  name = "pyhtml2text"
203
- version = "0.1.0"
252
+ version = "0.3.0"
204
253
  dependencies = [
205
254
  "html2text",
255
+ "html5ever",
256
+ "mimalloc",
257
+ "pulldown-cmark",
206
258
  "pyo3",
207
259
  ]
208
260
 
@@ -326,6 +378,12 @@ dependencies = [
326
378
  "syn 3.0.9",
327
379
  ]
328
380
 
381
+ [[package]]
382
+ name = "shlex"
383
+ version = "2.0.1"
384
+ source = "registry+https://github.com/rust-lang/crates.io-index"
385
+ checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
386
+
329
387
  [[package]]
330
388
  name = "siphasher"
331
389
  version = "1.0.4"
@@ -419,6 +477,12 @@ dependencies = [
419
477
  "syn 3.0.9",
420
478
  ]
421
479
 
480
+ [[package]]
481
+ name = "unicase"
482
+ version = "2.10.0"
483
+ source = "registry+https://github.com/rust-lang/crates.io-index"
484
+ checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f"
485
+
422
486
  [[package]]
423
487
  name = "unicode-ident"
424
488
  version = "1.0.27"
@@ -0,0 +1,36 @@
1
+ [package]
2
+ name = "pyhtml2text"
3
+ version = "0.3.0"
4
+ edition = "2021"
5
+ license = "MIT"
6
+ publish = false
7
+ readme = "README.md"
8
+
9
+ [lib]
10
+ name = "_pyhtml2text"
11
+ crate-type = ["cdylib"]
12
+
13
+ [dependencies]
14
+ pyo3 = { version = "0.26", features = ["extension-module", "abi3-py310"] }
15
+ # Fork of jugglerchris/rust-html2text with Config::base_url, some performance
16
+ # changes and the DOM exports the Markdown output uses (branch pyhtml2text-perf).
17
+ html2text = { git = "https://github.com/jaredlynskey/rust-html2text", rev = "b42930f8e6688ea121daf392e27a7d9812e6192d" }
18
+ # The parser html2text uses; patched below. Used directly to read element attributes.
19
+ html5ever = "0.39"
20
+ # Reads the Markdown back to check that emphasis parses as intended.
21
+ pulldown-cmark = { version = "0.13", default-features = false }
22
+
23
+ # Much faster than the system allocators for html2text's many small allocations.
24
+ # Not on glibc armv7/ppc64le/s390x: their manylinux2014 cross compilers are too
25
+ # old to build it (keep in sync with the cfg in src/lib.rs).
26
+ [target.'cfg(not(all(target_env = "gnu", any(target_arch = "arm", target_arch = "powerpc64", target_arch = "s390x"))))'.dependencies]
27
+ mimalloc = { version = "0.1", default-features = false, features = ["local_dynamic_tls"] }
28
+
29
+ [patch.crates-io]
30
+ # Fixes quadratic parsing of deeply nested HTML, see vendor/README.md.
31
+ html5ever = { path = "vendor/html5ever" }
32
+
33
+ [profile.release]
34
+ lto = true
35
+ codegen-units = 1
36
+ strip = true
@@ -0,0 +1,137 @@
1
+ Metadata-Version: 2.4
2
+ Name: pyhtml2text
3
+ Version: 0.3.0
4
+ Classifier: Programming Language :: Rust
5
+ Classifier: Programming Language :: Python :: 3
6
+ Classifier: Topic :: Text Processing :: Markup :: HTML
7
+ License-File: LICENSE
8
+ Summary: Fast HTML to plain text, with relative link resolution (Python bindings to rust-html2text)
9
+ Keywords: html,text,markdown,html2text
10
+ Author-email: Jared Lynskey <jared@lynskey.co.nz>
11
+ License-Expression: MIT
12
+ Requires-Python: >=3.10
13
+ Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
14
+ Project-URL: Source, https://github.com/jaredlynskey/pyhtml2text
15
+ Project-URL: Upstream, https://github.com/jugglerchris/rust-html2text
16
+
17
+ # pyhtml2text
18
+
19
+ Fast HTML to plain text and to Markdown for Python, built on
20
+ [rust-html2text](https://github.com/jugglerchris/rust-html2text).
21
+
22
+ Relative links can be resolved against the page's URL, so the output's links
23
+ work on their own.
24
+
25
+ ## Markdown
26
+
27
+ ```python
28
+ from pyhtml2text import html2markdown
29
+
30
+ html = """<h2>Before you start</h2>
31
+ <p>Read the <a href="setup.html">setup guide</a> and <b>back up</b> your data.</p>
32
+ <img src="/img/diagram.png" alt="How it fits together">
33
+ <table><tr><th>Plan</th><th>Price</th></tr><tr><td>Basic</td><td>Free</td></tr></table>"""
34
+
35
+ print(html2markdown(html, base_url="https://example.com/docs/intro.html"))
36
+ ```
37
+
38
+ ```markdown
39
+ ## Before you start
40
+
41
+ Read the [setup guide](https://example.com/docs/setup.html) and **back up** your data.
42
+
43
+ ![How it fits together](https://example.com/img/diagram.png)
44
+
45
+ | Plan | Price |
46
+ | --- | --- |
47
+ | Basic | Free |
48
+ ```
49
+
50
+ `html2markdown(html, *, base_url=None)` writes
51
+ [CommonMark](https://commonmark.org/) with GitHub's tables and
52
+ strikethrough, in the usual conventions:
53
+
54
+ - `#` headings, `-` bullets, `_emphasis_` and `**strong**`, `~~strikethrough~~`.
55
+ - Inline links and images, `[text](url)` and `![alt](src)`, made absolute when
56
+ `base_url` is given. A link whose only content is an icon uses its
57
+ `aria-label` or `title` as its text.
58
+ - Fenced code blocks, tagged with the language when the page names one
59
+ (`class="language-python"`).
60
+ - Pipe tables. Cells spanning rows or columns become empty cells, a table
61
+ without a header row gets an empty one, and tables used for page layout are
62
+ written as plain paragraphs.
63
+ - Characters are escaped only where Markdown would otherwise read them as
64
+ syntax, so `snake_case`, `5 * 3` and `$5` stay as they are.
65
+ - Lines aren't wrapped. Hidden content (`hidden`, `aria-hidden`,
66
+ `display: none`), scripts, styles and form controls are left out.
67
+
68
+ The output is checked by rendering it back with a CommonMark parser and
69
+ comparing every word, link and image with the original page, on real sites
70
+ and on tens of thousands of random documents (`tests/markdown_check.py`).
71
+
72
+ ## Plain text
73
+
74
+ ```python
75
+ from pyhtml2text import html2text
76
+
77
+ html = '<p>Please <a href="/login">sign in</a> first.</p>'
78
+
79
+ print(html2text(html, base_url="https://example.com/docs/page.html"))
80
+ # Please [sign in][1] first.
81
+ #
82
+ # [1]: https://example.com/login
83
+ ```
84
+
85
+ `html2text(html, width=80, *, ...)`
86
+
87
+ | option | default | |
88
+ |---|---|---|
89
+ | `base_url` | `None` | resolve relative `href`s against this URL (RFC 3986). Absolute and `mailto:` etc. links are left alone |
90
+ | `decorate` | `True` | markdown-ish `*em*`, `**strong**`, `` `code` `` |
91
+ | `link_footnotes` | `True` | list link targets as `[1]: ...` footnotes |
92
+ | `table_borders` | `True` | draw table borders |
93
+ | `wrap_links` | `True` | allow long links to wrap |
94
+ | `max_wrap_width` | `None` | wrap paragraphs narrower than `width` |
95
+ | `raw` | `False` | flatten tables so cell text stays together |
96
+
97
+ Both functions take a `str` or UTF-8 `bytes` (e.g. `response.content`).
98
+ Requires Python 3.10 or later.
99
+
100
+ ## Performance
101
+
102
+ Median time on an Apple Silicon Mac. To Markdown:
103
+
104
+ | | Wikipedia article (1 MB) | BBC News front page (370 KB) | Python docs page (245 KB) |
105
+ |---|---|---|---|
106
+ | [markdownify](https://pypi.org/project/markdownify/) | 184 ms | 28 ms | 97 ms |
107
+ | [html2text](https://pypi.org/project/html2text/) (pure Python) | 73 ms | 13 ms | 37 ms |
108
+ | [html-to-markdown](https://pypi.org/project/html-to-markdown/) | 44 ms | 5.6 ms | 15 ms |
109
+ | **pyhtml2text** | **14 ms** | **1.7 ms** | **4.5 ms** |
110
+
111
+ To plain text:
112
+
113
+ | | Wikipedia article (780 KB) | BBC News front page (370 KB) |
114
+ |---|---|---|
115
+ | [html2text](https://pypi.org/project/html2text/) (pure Python) | 71.5 ms | 14.9 ms |
116
+ | [html2text-rs](https://pypi.org/project/html2text-rs/) | 15.2 ms | 2.5 ms |
117
+ | **pyhtml2text** | **10.9 ms** | **1.6 ms** |
118
+
119
+ - The GIL is released while converting, so threads scale across cores. On
120
+ free-threaded Python (3.14t) the module runs without re-enabling the GIL.
121
+ - Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with
122
+ profile-guided optimisation, which makes them about 25% faster. Other
123
+ platforms get the same code without it.
124
+ - Deeply nested HTML is parsed in linear time, so hostile input can't stall a
125
+ worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever.
126
+
127
+ ## Credits
128
+
129
+ The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
130
+ This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
131
+ that adds `base_url` and some performance changes, and a patched copy of
132
+ [html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
133
+ `vendor/`). Markdown output is checked with
134
+ [pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
135
+ isn't affiliated with or endorsed by any of these projects, so please report
136
+ issues here rather than upstream.
137
+
@@ -0,0 +1,120 @@
1
+ # pyhtml2text
2
+
3
+ Fast HTML to plain text and to Markdown for Python, built on
4
+ [rust-html2text](https://github.com/jugglerchris/rust-html2text).
5
+
6
+ Relative links can be resolved against the page's URL, so the output's links
7
+ work on their own.
8
+
9
+ ## Markdown
10
+
11
+ ```python
12
+ from pyhtml2text import html2markdown
13
+
14
+ html = """<h2>Before you start</h2>
15
+ <p>Read the <a href="setup.html">setup guide</a> and <b>back up</b> your data.</p>
16
+ <img src="/img/diagram.png" alt="How it fits together">
17
+ <table><tr><th>Plan</th><th>Price</th></tr><tr><td>Basic</td><td>Free</td></tr></table>"""
18
+
19
+ print(html2markdown(html, base_url="https://example.com/docs/intro.html"))
20
+ ```
21
+
22
+ ```markdown
23
+ ## Before you start
24
+
25
+ Read the [setup guide](https://example.com/docs/setup.html) and **back up** your data.
26
+
27
+ ![How it fits together](https://example.com/img/diagram.png)
28
+
29
+ | Plan | Price |
30
+ | --- | --- |
31
+ | Basic | Free |
32
+ ```
33
+
34
+ `html2markdown(html, *, base_url=None)` writes
35
+ [CommonMark](https://commonmark.org/) with GitHub's tables and
36
+ strikethrough, in the usual conventions:
37
+
38
+ - `#` headings, `-` bullets, `_emphasis_` and `**strong**`, `~~strikethrough~~`.
39
+ - Inline links and images, `[text](url)` and `![alt](src)`, made absolute when
40
+ `base_url` is given. A link whose only content is an icon uses its
41
+ `aria-label` or `title` as its text.
42
+ - Fenced code blocks, tagged with the language when the page names one
43
+ (`class="language-python"`).
44
+ - Pipe tables. Cells spanning rows or columns become empty cells, a table
45
+ without a header row gets an empty one, and tables used for page layout are
46
+ written as plain paragraphs.
47
+ - Characters are escaped only where Markdown would otherwise read them as
48
+ syntax, so `snake_case`, `5 * 3` and `$5` stay as they are.
49
+ - Lines aren't wrapped. Hidden content (`hidden`, `aria-hidden`,
50
+ `display: none`), scripts, styles and form controls are left out.
51
+
52
+ The output is checked by rendering it back with a CommonMark parser and
53
+ comparing every word, link and image with the original page, on real sites
54
+ and on tens of thousands of random documents (`tests/markdown_check.py`).
55
+
56
+ ## Plain text
57
+
58
+ ```python
59
+ from pyhtml2text import html2text
60
+
61
+ html = '<p>Please <a href="/login">sign in</a> first.</p>'
62
+
63
+ print(html2text(html, base_url="https://example.com/docs/page.html"))
64
+ # Please [sign in][1] first.
65
+ #
66
+ # [1]: https://example.com/login
67
+ ```
68
+
69
+ `html2text(html, width=80, *, ...)`
70
+
71
+ | option | default | |
72
+ |---|---|---|
73
+ | `base_url` | `None` | resolve relative `href`s against this URL (RFC 3986). Absolute and `mailto:` etc. links are left alone |
74
+ | `decorate` | `True` | markdown-ish `*em*`, `**strong**`, `` `code` `` |
75
+ | `link_footnotes` | `True` | list link targets as `[1]: ...` footnotes |
76
+ | `table_borders` | `True` | draw table borders |
77
+ | `wrap_links` | `True` | allow long links to wrap |
78
+ | `max_wrap_width` | `None` | wrap paragraphs narrower than `width` |
79
+ | `raw` | `False` | flatten tables so cell text stays together |
80
+
81
+ Both functions take a `str` or UTF-8 `bytes` (e.g. `response.content`).
82
+ Requires Python 3.10 or later.
83
+
84
+ ## Performance
85
+
86
+ Median time on an Apple Silicon Mac. To Markdown:
87
+
88
+ | | Wikipedia article (1 MB) | BBC News front page (370 KB) | Python docs page (245 KB) |
89
+ |---|---|---|---|
90
+ | [markdownify](https://pypi.org/project/markdownify/) | 184 ms | 28 ms | 97 ms |
91
+ | [html2text](https://pypi.org/project/html2text/) (pure Python) | 73 ms | 13 ms | 37 ms |
92
+ | [html-to-markdown](https://pypi.org/project/html-to-markdown/) | 44 ms | 5.6 ms | 15 ms |
93
+ | **pyhtml2text** | **14 ms** | **1.7 ms** | **4.5 ms** |
94
+
95
+ To plain text:
96
+
97
+ | | Wikipedia article (780 KB) | BBC News front page (370 KB) |
98
+ |---|---|---|
99
+ | [html2text](https://pypi.org/project/html2text/) (pure Python) | 71.5 ms | 14.9 ms |
100
+ | [html2text-rs](https://pypi.org/project/html2text-rs/) | 15.2 ms | 2.5 ms |
101
+ | **pyhtml2text** | **10.9 ms** | **1.6 ms** |
102
+
103
+ - The GIL is released while converting, so threads scale across cores. On
104
+ free-threaded Python (3.14t) the module runs without re-enabling the GIL.
105
+ - Wheels for Linux (x86_64, aarch64), macOS and Windows x64 are built with
106
+ profile-guided optimisation, which makes them about 25% faster. Other
107
+ platforms get the same code without it.
108
+ - Deeply nested HTML is parsed in linear time, so hostile input can't stall a
109
+ worker: 40,000 nested lists take 0.14 s, against 6 s with stock html5ever.
110
+
111
+ ## Credits
112
+
113
+ The plain-text rendering is done by rust-html2text by Chris Emerson (MIT).
114
+ This package uses [a fork](https://github.com/jaredlynskey/rust-html2text/tree/pyhtml2text-perf)
115
+ that adds `base_url` and some performance changes, and a patched copy of
116
+ [html5ever](https://github.com/servo/html5ever) (MIT/Apache-2.0, see
117
+ `vendor/`). Markdown output is checked with
118
+ [pulldown-cmark](https://github.com/raphlinus/pulldown-cmark) (MIT). It
119
+ isn't affiliated with or endorsed by any of these projects, so please report
120
+ issues here rather than upstream.
@@ -4,12 +4,12 @@ build-backend = "maturin"
4
4
 
5
5
  [project]
6
6
  name = "pyhtml2text"
7
- version = "0.1.0"
7
+ version = "0.3.0"
8
8
  description = "Fast HTML to plain text, with relative link resolution (Python bindings to rust-html2text)"
9
9
  readme = "README.md"
10
10
  license = "MIT"
11
11
  license-files = ["LICENSE"]
12
- requires-python = ">=3.9"
12
+ requires-python = ">=3.10"
13
13
  authors = [{ name = "Jared Lynskey", email = "jared@lynskey.co.nz" }]
14
14
  keywords = ["html", "text", "markdown", "html2text"]
15
15
  classifiers = [
@@ -25,6 +25,19 @@ Upstream = "https://github.com/jugglerchris/rust-html2text"
25
25
  [tool.maturin]
26
26
  python-source = "python"
27
27
  module-name = "pyhtml2text._pyhtml2text"
28
+ # cargo leaves out vendor/html5ever (it has its own Cargo.toml), but the
29
+ # [patch] in Cargo.toml needs it; the PGO data and CI files aren't needed.
30
+ include = [
31
+ { path = "vendor/html5ever/Cargo.toml", format = "sdist" },
32
+ { path = "vendor/html5ever/LICENSE-*", format = "sdist" },
33
+ { path = "vendor/html5ever/README.md", format = "sdist" },
34
+ { path = "vendor/html5ever/src/**/*.rs", format = "sdist" },
35
+ ]
36
+ exclude = [
37
+ { path = "pgo/**/*", format = "sdist" },
38
+ { path = "bench/**/*", format = "sdist" },
39
+ { path = ".github/**/*", format = "sdist" },
40
+ ]
28
41
 
29
42
  [tool.maturin.generate-ci.github]
30
43
  pytest = true
@@ -0,0 +1,3 @@
1
+ from ._pyhtml2text import html2markdown, html2text
2
+
3
+ __all__ = ["html2markdown", "html2text"]
@@ -1,14 +1,13 @@
1
- from typing import Optional
2
-
3
1
  def html2text(
4
- html: str,
2
+ html: str | bytes,
5
3
  width: int = 80,
6
4
  *,
7
- base_url: Optional[str] = None,
5
+ base_url: str | None = None,
8
6
  decorate: bool = True,
9
7
  link_footnotes: bool = True,
10
8
  table_borders: bool = True,
11
9
  wrap_links: bool = True,
12
- max_wrap_width: Optional[int] = None,
10
+ max_wrap_width: int | None = None,
13
11
  raw: bool = False,
14
12
  ) -> str: ...
13
+ def html2markdown(html: str | bytes, *, base_url: str | None = None) -> str: ...
@@ -0,0 +1,130 @@
1
+ use std::panic::{catch_unwind, AssertUnwindSafe};
2
+
3
+ use html2text::config;
4
+ use pyo3::exceptions::{PyTypeError, PyValueError};
5
+ use pyo3::prelude::*;
6
+ use pyo3::types::{PyBytes, PyString};
7
+
8
+ mod markdown;
9
+
10
+ #[cfg(not(all(
11
+ target_env = "gnu",
12
+ any(target_arch = "arm", target_arch = "powerpc64", target_arch = "s390x")
13
+ )))]
14
+ #[global_allocator]
15
+ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
16
+
17
+ /// Borrow the input without copying it; bytes are taken to be UTF-8.
18
+ fn input_bytes<'a>(html: &'a Bound<'_, PyAny>) -> PyResult<&'a [u8]> {
19
+ if let Ok(s) = html.downcast::<PyString>() {
20
+ Ok(s.to_str()?.as_bytes())
21
+ } else if let Ok(b) = html.downcast::<PyBytes>() {
22
+ Ok(b.as_bytes())
23
+ } else {
24
+ Err(PyTypeError::new_err(format!(
25
+ "html must be str or bytes, not {}",
26
+ html.get_type().name()?
27
+ )))
28
+ }
29
+ }
30
+
31
+ /// Run a conversion. It is pure Rust, so other Python threads run meanwhile.
32
+ /// A panic would otherwise surface as PanicException, which `except
33
+ /// Exception` doesn't catch, so it is reported as a ValueError instead.
34
+ fn convert<F>(py: Python<'_>, f: F) -> PyResult<String>
35
+ where
36
+ F: FnOnce() -> Result<String, html2text::Error> + Send,
37
+ {
38
+ match py.detach(|| catch_unwind(AssertUnwindSafe(|| f().map_err(|e| e.to_string())))) {
39
+ Ok(Ok(text)) => Ok(text),
40
+ Ok(Err(e)) => Err(PyValueError::new_err(e)),
41
+ Err(panic) => {
42
+ let msg = panic
43
+ .downcast_ref::<&str>()
44
+ .copied()
45
+ .or_else(|| panic.downcast_ref::<String>().map(String::as_str))
46
+ .unwrap_or("unknown error");
47
+ Err(PyValueError::new_err(format!(
48
+ "html2text failed on this document: {msg}"
49
+ )))
50
+ }
51
+ }
52
+ }
53
+
54
+ /// Render HTML as plain text.
55
+ #[pyfunction]
56
+ #[pyo3(name = "html2text", signature = (
57
+ html,
58
+ width = 80,
59
+ *,
60
+ base_url = None,
61
+ decorate = true,
62
+ link_footnotes = true,
63
+ table_borders = true,
64
+ wrap_links = true,
65
+ max_wrap_width = None,
66
+ raw = false,
67
+ ))]
68
+ #[allow(clippy::too_many_arguments)]
69
+ fn render(
70
+ py: Python<'_>,
71
+ html: &Bound<'_, PyAny>,
72
+ width: usize,
73
+ base_url: Option<String>,
74
+ decorate: bool,
75
+ link_footnotes: bool,
76
+ table_borders: bool,
77
+ wrap_links: bool,
78
+ max_wrap_width: Option<usize>,
79
+ raw: bool,
80
+ ) -> PyResult<String> {
81
+ let input = input_bytes(html)?;
82
+
83
+ let mut conf = if decorate {
84
+ config::plain()
85
+ } else {
86
+ config::plain_no_decorate()
87
+ };
88
+ conf = conf.link_footnotes(link_footnotes).allow_width_overflow();
89
+ if let Some(base) = base_url {
90
+ conf = conf.base_url(base);
91
+ }
92
+ if !table_borders {
93
+ conf = conf.no_table_borders();
94
+ }
95
+ if !wrap_links {
96
+ conf = conf.no_link_wrapping();
97
+ }
98
+ if let Some(w) = max_wrap_width {
99
+ conf = conf.max_wrap_width(w);
100
+ }
101
+ if raw {
102
+ conf = conf.raw_mode(true);
103
+ }
104
+
105
+ convert(py, || conf.string_from_read(input, width))
106
+ }
107
+
108
+ /// Convert HTML to Markdown.
109
+ ///
110
+ /// The output is CommonMark with GitHub's pipe tables and strikethrough:
111
+ /// ATX headings, `-` bullets, fenced code blocks and inline links and images.
112
+ /// Lines are not wrapped. Relative links and image sources are resolved
113
+ /// against `base_url` when one is given.
114
+ #[pyfunction]
115
+ #[pyo3(name = "html2markdown", signature = (html, *, base_url = None))]
116
+ fn to_markdown(
117
+ py: Python<'_>,
118
+ html: &Bound<'_, PyAny>,
119
+ base_url: Option<String>,
120
+ ) -> PyResult<String> {
121
+ let input = input_bytes(html)?;
122
+ convert(py, || markdown::from_read(input, base_url.as_deref()))
123
+ }
124
+
125
+ #[pymodule(gil_used = false)]
126
+ fn _pyhtml2text(m: &Bound<'_, PyModule>) -> PyResult<()> {
127
+ m.add_function(wrap_pyfunction!(render, m)?)?;
128
+ m.add_function(wrap_pyfunction!(to_markdown, m)?)?;
129
+ Ok(())
130
+ }