justhtml 3.3.0__tar.gz → 3.5.0__tar.gz

{justhtml-3.3.0 → justhtml-3.5.0}/.github/workflows/ci.yml

@@ -20,10 +20,10 @@ jobs:
       PYTHONHASHSEED: "0"
 
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
           cache: "pip"
@@ -45,8 +45,6 @@ jobs:
         run: |
           cd ..
           git clone --filter=blob:none --sparse https://github.com/web-platform-tests/wpt.git
-          git -C wpt fetch origin 75b089d70448e1da425ab8cf38971b6af41f3cea
-          git -C wpt checkout 75b089d70448e1da425ab8cf38971b6af41f3cea
           git -C wpt sparse-checkout set html/syntax/parsing/resources
           git -C wpt rev-parse HEAD
 
@@ -84,15 +82,15 @@ jobs:
   test-pyodide:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
 
       - name: Set up Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v5
         with:
           node-version: "20"
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.12"
 

{justhtml-3.3.0 → justhtml-3.5.0}/.github/workflows/publish.yml

@@ -17,10 +17,10 @@ jobs:
       contents: read
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@v5
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v6
         with:
           python-version: "3.x"
 

{justhtml-3.3.0 → justhtml-3.5.0}/.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
     hooks:
       - id: tests-coverage
         name: Tests & Coverage
-        entry: bash -c 'coverage run run_tests.py && coverage report'
+        entry: bash -c 'coverage run run_tests.py --no-write-summary && coverage report'
         language: system
         pass_filenames: false
         always_run: true

justhtml-3.5.0/AGENTS.md

@@ -0,0 +1,10 @@
+# Agent Contribution Guide
+
+Read these files before making changes:
+
+- [CONTRIBUTING.md](CONTRIBUTING.md) for development, tests, documentation
+  examples, changelog, and release expectations.
+- [SECURITY.md](SECURITY.md) for the untrusted-HTML threat model and security
+  boundaries.
+- [PERFORMANCE.md](PERFORMANCE.md) for parser-speed benchmarking and
+  optimization work.

{justhtml-3.3.0 → justhtml-3.5.0}/CHANGELOG.md

@@ -7,6 +7,48 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.5.0] - 2026-07-13
+
+### Changed
+- Add contributor guidance for documentation examples, security and availability reviews, releases, and a new `PERFORMANCE.md` workflow for measuring and testing performance improvements.
+- Add TurboHTML to the correctness benchmarks, and refresh the comparison guide with current Python-parser and browser-engine scores. Many dropped because of the new tests around processing instructions.
+- Run CI against the latest WPT tree-construction fixtures rather than a pinned revision.
+
+### Fixed
+Note: The below are all minor parsing differences found when diffing against Chromium output, that is not covered in WPT's html5 parsing tests.
+
+- Ignore null-plus-whitespace character runs in the post-doctype document shell before creating the body. Previously, removing the null after classification could leave a spurious newline in `<body>`.
+- Ignore late `<head>` start tags after body text even when the fast text path has not marked body mode explicitly. Previously, following metadata could be moved from the body back into the head.
+- Limit doctype tokenization whitespace to the HTML ASCII whitespace set. Previously, non-ASCII spaces such as NBSP could be discarded instead of becoming part of the doctype name.
+- Preserve the `frameset-ok` state across empty active-formatting elements. Previously, markup such as `<i><frameset>` could retain an empty body instead of replacing it with the frameset.
+- Keep comments following implied head metadata in the head. Previously, `<meta><!--...-->` could place the comment before the document element.
+- Exit column-group mode before reprocessing malformed end tags. Previously, `<table><colgroup></p>` could leave a paragraph inside the colgroup instead of foster-parenting it before the table.
+- Keep foreign elements named `colgroup` out of HTML table recovery. Previously, text following SVG or MathML `<colgroup>` could become a sibling instead of its child.
+- Keep form controls in template contents when a template appears inside `<select>`. Previously, an `<input>` could prematurely close the outer select.
+- Keep initial-template recovery in the head insertion mode after `</template>`. Previously, following head whitespace could be discarded.
+- Match the latest WPT processing-instruction tree expectations, including empty data fields and the `<?#` bogus-comment case.
+- Close an open template `<colgroup>` before processing following script and style tags. Previously, those tags could be incorrectly inserted inside the column group.
+- Recover table-body start tags correctly inside templates. Previously, a following `<colgroup>` could be discarded instead of closing the current table body.
+- Close template table bodies before handling `</table>`. Previously, following content could remain incorrectly nested inside `<tbody>`.
+- Close template rows before handling `</table>`. Previously, following nested templates could remain incorrectly nested inside `<tr>`.
+- Ignore nested table starts in template table modes that have no open table. Previously, malformed input could create a spurious empty `<table>`.
+- Resume template body recovery after `<title>`. Previously, invalid table-structure tags following a template title could create incorrect table nodes.
+- Resume template body recovery after base and noframes head-content tags. Previously, invalid following table-structure tags could create incorrect table nodes.
+- Apply phantom paragraph recovery inside templates. Previously, an unmatched `</p>` could be incorrectly ignored.
+- Keep unmatched `</p>` ignored in the template initial insertion mode.
+- Keep template table-body recovery scoped to an open table. Previously, a nested table token could close and duplicate synthetic table bodies.
+- Preserve open template tables when closing a select. Previously, following option content could be placed outside the select.
+
+## [3.4.0] - 2026-07-11
+
+### Security
+- (Severity: Low) Bound named character-reference lookup to the finite HTML entity tables. Previously, a malformed `&` followed by a long alphanumeric run caused repeated shrinking-prefix allocations and lookups, making parsing quadratic and allowing denial of service before sanitization.
+- (Severity: Low) Cap diagnostics retained by `JustHTML(...)` with the new `max_errors` option (default: 1000), including parser, sanitizer, raw-text, and transform findings. Previously, input containing many independently malformed characters or unsafe constructs could allocate an unbounded number of `ParseError` objects.
+- (Severity: Low) Index active-formatting entries by signature within each HTML5 Noah's Ark marker segment, retiring superseded entries lazily and compacting in bounded batches. Previously, repeated fourth occurrences across many formatting signatures rescanned and shifted the active-formatting list, allowing quadratic parsing of crafted HTML.
+- (Severity: Low) Avoid unbounded integer conversion for untrusted `srcset` width descriptors, URL host validation, and `Linkify(...)` URL ports. Previously, inputs with more decimal digits than Python permits in `int()` could raise an uncaught `ValueError` during sanitization or linkification.
+- (Severity: Low) Bound pretty-print indentation and avoid repeatedly copying nested serialized output. Previously, serializing deeply nested untrusted HTML with the default `to_html()` settings could consume quadratic time and memory.
+- (Severity: Low) Compact sibling lists when applying a tag-only `Drop(...)` transform. Previously, dropping many matching untrusted sibling elements shifted the remaining list after every removal, making transform execution quadratic.
+
 ## [3.3.0] - 2026-07-04
 
 ### Changed

{justhtml-3.3.0 → justhtml-3.5.0}/CONTRIBUTING.md

@@ -49,6 +49,28 @@ python -c 'from justhtml import JustHTML, to_test_format; print(to_test_format(J
 
 **Coverage is required to be 100%.** All new code must be fully tested.
 
+### Documentation examples
+
+Runnable documentation examples use an explicit output marker. Place it between
+a self-contained Python fence and the expected-output fence:
+
+````text
+```python
+from justhtml import JustHTML
+print(JustHTML("<p>x</p>", fragment=True).to_html())
+```
+
+<!-- justhtml: output -->
+
+```html
+<p>x</p>
+```
+````
+
+The docs test executes each marked Python block independently and compares its
+stdout with the following fence. Keep ordinary HTML fences unmarked so they
+remain explanatory examples rather than tests.
+
 ## Pre-commit Hooks
 
 Pre-commit runs automatically on every commit and checks:
@@ -85,12 +107,25 @@ After making changes, verify performance impact:
 
 ```bash
 # Quick benchmark
-python benchmarks/performance.py --iterations 1 --parser justhtml --no-mem
+python benchmarks/performance.py --parsers justhtml --iterations 1
 
 # Profile hotspots
 python benchmarks/profile.py
 ```
 
+For a parser-speed improvement, measure the relevant pipeline on the web100k
+corpus before and after the change; see [PERFORMANCE.md](PERFORMANCE.md). For
+an availability fix, add a regression that exercises the affected hostile-input
+shape and verify that growth is linear or otherwise bounded.
+
+## Releases
+
+Use `python scripts/release.py --version X.Y.Z --yes` for releases. The script
+runs release checks, bumps `pyproject.toml`, commits, tags, pushes, and creates
+the GitHub release. Before invoking it, ensure the worktree is clean and move
+the relevant Unreleased changelog entries into a dated `## [X.Y.Z]` section;
+the helper requires that section to exist.
+
 ## Architecture Notes
 
 - **Parser engine** (`src/justhtml/parser/engine.py`): Plan-driven tokenization, tree construction, and sanitizer projection
@@ -109,8 +144,9 @@ Golden rules:
 
 1. Fork the repository
 2. Create a feature branch
-3. Make your changes with tests
-4. Ensure pre-commit passes
-5. Submit a pull request
+3. Add an entry under the relevant section of `CHANGELOG.md`
+4. Make your changes with tests
+5. Ensure pre-commit passes
+6. Submit a pull request
 
 Questions? Open an issue on GitHub. For security vulnerabilities, please see our [Security Policy](SECURITY.md).

justhtml-3.5.0/PERFORMANCE.md

@@ -0,0 +1,84 @@
+# Performance Guide
+
+This guide is for improving JustHTML's parsing speed on real-world HTML while
+preserving its HTML5 recovery and sanitization behavior.
+
+## Benchmark data and setup
+
+The main benchmark uses the [web100k](https://github.com/EmilStenstrom/web100k)
+corpus. By default it expects the dataset beside this repository:
+
+```text
+../web100k/
+├── html.dict
+└── batches/
+    └── web100k-batch-001.tar.zst
+```
+
+Set `WEB100K_DIR` to use another location, or pass `--batches-dir` and
+`--dict` explicitly. Install the benchmark dependencies first:
+
+```bash
+pip install -e ".[benchmark]"
+```
+
+## Measure the right pipeline
+
+The benchmark has separate modes for parsing alone and parsing followed by
+serialization. Start with the narrowest mode that covers the change:
+
+```bash
+# Parser throughput on 100 real-world documents, without RSS sampling noise.
+python benchmarks/performance.py --parsers justhtml --iterations 5
+
+# Parse plus default HTML serialization.
+python benchmarks/performance.py --parsers justhtml_to_html --iterations 5
+
+# Compare selected installed parsers on a larger corpus sample.
+python benchmarks/performance.py --parsers justhtml html5lib lxml --limit 1000 --iterations 3
+```
+
+Use `--all-batches` for a corpus-wide measurement, `--batch PATH` for one
+archive, or `--downloaded DIR` for a directory of decompressed-source files.
+The benchmark reports total throughput and per-document timing. Compare the
+same command before and after a change on the same machine; do not compare
+absolute timings across machines.
+
+## Find the hot path
+
+Profile a representative web100k batch before optimizing:
+
+```bash
+python benchmarks/profile.py --mode parse
+```
+
+Available modes are `parse`, `fragment`, `compact-html`, `pretty-html`, and
+`text`. Use the mode closest to the public operation you are improving. Focus
+on functions that dominate cumulative work across many documents, not an
+isolated slow malformed sample.
+
+## Make a speed improvement
+
+- Preserve parser and sanitizer semantics. Run the parser differential suite
+  before relying on a throughput result.
+- Optimize work on the normal path: token scanning, tree construction,
+  attribute projection, and common serialization cases.
+- Prefer direct local data access, reused compiled plans, and fewer temporary
+  allocations in per-character and per-node loops.
+- Measure realistic HTML before and after the change. Include a focused
+  microbenchmark only when it explains the real-world result.
+- Keep benchmarks honest: warm up once, run multiple iterations, and report
+  the exact command, corpus size, and pipeline mode in the pull request.
+
+## Validate and document
+
+Add correctness tests for behavior affected by an optimization. Run the full
+project gate before submitting:
+
+```bash
+pre-commit run --all-files
+```
+
+Update `CHANGELOG.md` with a `Performance` entry that states the affected
+pipeline and the avoided work. If the change also prevents disproportionate
+resource use from hostile input, add a `Security` entry as well.

{justhtml-3.3.0 → justhtml-3.5.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: justhtml
-Version: 3.3.0
+Version: 3.5.0
 Summary: A pure Python HTML5 parser that just works.
 Project-URL: Homepage, https://github.com/emilstenstrom/justhtml
 Project-URL: Issues, https://github.com/emilstenstrom/justhtml/issues
@@ -40,8 +40,8 @@ Requires-Dist: html5-parser; extra == 'benchmark'
 Requires-Dist: html5lib; extra == 'benchmark'
 Requires-Dist: lxml; extra == 'benchmark'
 Requires-Dist: markupever; extra == 'benchmark'
-Requires-Dist: psutil; extra == 'benchmark'
 Requires-Dist: selectolax>=0.4.8; extra == 'benchmark'
+Requires-Dist: turbohtml>=1.3.1; extra == 'benchmark'
 Requires-Dist: zstandard; extra == 'benchmark'
 Provides-Extra: dev
 Requires-Dist: build; extra == 'dev'

{justhtml-3.3.0 → justhtml-3.5.0}/SECURITY.md

@@ -19,6 +19,25 @@
 | 1.x     | :x:                                      |
 | < 1.0   | :x:                                      |
 
+## Threat Model
+
+JustHTML's primary security boundary is **untrusted HTML input**: markup,
+bytes, and HTML-derived values supplied to parsing, sanitization, and
+serialization APIs. The promises below apply when callers use the documented
+safe defaults or a deliberately safe policy.
+
+Availability is part of that boundary. Small hostile inputs must not trigger
+disproportionate work or uncaught conversion errors while parsing, sanitizing,
+transforming, or serializing. Applications still need their own limits for
+total input size, execution time, and memory use.
+
+Library configuration is trusted application code. This includes custom
+sanitization policies, transform specifications and callbacks, selectors,
+glob patterns, CLI flags, and any policy or transform values assembled from
+configuration. Applications that accept those values from untrusted users must
+validate and constrain them before passing them to JustHTML; transform APIs are
+not a sandbox.
+
 ## Security Domains
 
 This section describes the security areas we use when reviewing `justhtml`.
@@ -42,7 +61,8 @@ Promise:
 - JustHTML decodes bytes using HTML-style encoding detection and rejects UTF-7.
 - JustHTML parses malformed HTML into a DOM using browser-style HTML5 recovery.
 - The parser handles entity and character-reference edge cases.
-- The parser avoids known small-input patterns that can cause excessive work.
+- The parser avoids known small-input patterns that can cause excessive work or
+  uncaught conversion errors.
 - JustHTML reports parse errors as data unless strict mode is enabled.
 - Fragment parsing and full-document parsing keep their different assumptions
   explicit.
@@ -126,6 +146,8 @@ Promise:
 - Serialization rejects unsafe element and attribute names created through
   custom DOM changes.
 - Serialization neutralizes rawtext end-tag sequences.
+- Pretty serialization bounds indentation and avoids repeatedly copying nested
+  output, so deeply nested input does not cause disproportionate work.
 - `to_text()` returns text content rather than markup.
 - `to_markdown()` escapes Markdown-sensitive text by default.
 - `to_markdown()` escapes line-start Markdown markers that could change block
@@ -165,6 +187,8 @@ Promise:
   sanitization step after later transforms.
 - Built-in URL-aware transforms reuse sanitizer URL cleaning where they can.
 - Linkification operates on DOM text nodes rather than raw HTML strings.
+- Built-in structural transforms avoid repeated sibling-list shifting for
+  common bulk operations on untrusted documents.
 
 Out of scope:
 - Safety of arbitrary user callbacks passed to transform APIs.

{justhtml-3.3.0 → justhtml-3.5.0}/benchmarks/correctness.py

@@ -22,7 +22,17 @@ from justhtml.parser.context import FragmentContext
 from justhtml.serializer import to_test_format
 
 # Available parsers
-PARSERS = ["justhtml", "html5lib", "html5_parser", "lxml", "bs4", "html.parser", "selectolax", "markupever"]
+PARSERS = [
+    "justhtml",
+    "html5lib",
+    "html5_parser",
+    "lxml",
+    "bs4",
+    "html.parser",
+    "selectolax",
+    "markupever",
+    "turbohtml",
+]
 
 
 class Status(Enum):
@@ -61,6 +71,7 @@ PARSER_CAPABILITIES = {
     "html.parser": ParserCapabilities(),
     "selectolax": ParserCapabilities(fragment_context=True, foreign_fragment_context=True),
     "markupever": ParserCapabilities(),
+    "turbohtml": ParserCapabilities(fragment_context=True, foreign_fragment_context=True),
 }
 
 
@@ -146,6 +157,15 @@ def check_parser_available(parser_name):
             return True
         except ImportError:
             return False
+    if parser_name == "turbohtml":
+        try:
+            from turbohtml import parse, parse_fragment
+
+            parse("<p></p>")
+            parse_fragment("<p></p>", "div")
+            return True
+        except (ImportError, TypeError, ValueError, AttributeError):
+            return False
     return False
 
 
@@ -174,6 +194,33 @@ def parse_dat_file(path):
     return tests
 
 
+def decode_escapes(text):
+    """Decode the \"\\xXX\" and \"\\uXXXX\" sequences used by tree fixtures."""
+    if "\\x" not in text and "\\u" not in text:
+        return text
+
+    result = []
+    i = 0
+    while i < len(text):
+        if text[i : i + 2] == "\\x" and i + 3 < len(text):
+            try:
+                result.append(chr(int(text[i + 2 : i + 4], 16)))
+                i += 4
+                continue
+            except ValueError:
+                pass
+        elif text[i : i + 2] == "\\u" and i + 5 < len(text):
+            try:
+                result.append(chr(int(text[i + 2 : i + 6], 16)))
+                i += 6
+                continue
+            except ValueError:
+                pass
+        result.append(text[i])
+        i += 1
+    return "".join(result)
+
+
 def parse_single_test(lines):
     """Parse a single test from lines."""
     data = []
@@ -209,7 +256,7 @@ def parse_single_test(lines):
 
     if data or document:
         return {
-            "data": "\n".join(data),
+            "data": decode_escapes("\n".join(data)),
             "document": "\n".join(document),
             "fragment_context": fragment_context,
             "script_directive": script_directive,
@@ -502,6 +549,23 @@ def run_test_markupever(html, fragment_context, expected, xml_coercion=False, if
         return _error(e)
 
 
+def run_test_turbohtml(html, fragment_context, expected, xml_coercion=False, iframe_srcdoc=False):
+    """Run a single test with TurboHTML's public DOM API."""
+    from turbohtml import parse, parse_fragment
+
+    try:
+        if fragment_context:
+            namespace, tag_name = fragment_context
+            context = f"{namespace} {tag_name}" if namespace else tag_name
+            node = parse_fragment(html, context)
+            actual = _turbohtml_to_test_format(node.children, 0)
+        else:
+            actual = _turbohtml_to_test_format(parse(html).children, 0)
+        return _compare_result(expected, actual)
+    except Exception as e:
+        return _error(e)
+
+
 # =============================================================================
 # Test format conversion helpers
 # =============================================================================
@@ -839,6 +903,56 @@ def _markupever_to_test_format(nodes):
     return "".join(line for node in nodes for line in process(node, 0))
 
 
+def _turbohtml_to_test_format(nodes, indent):
+    """Convert TurboHTML's public node objects to html5lib test format."""
+    from turbohtml import Comment, Doctype, Element, Namespace, Text
+
+    def qualified_name(namespace, name):
+        if namespace is Namespace.SVG:
+            return f"svg {name}"
+        if namespace is Namespace.MATHML:
+            return f"math {name}"
+        return name
+
+    def attribute_name(name):
+        if name.startswith("xlink:"):
+            return f"xlink {name.removeprefix('xlink:')}"
+        if name.startswith("xml:"):
+            return f"xml {name.removeprefix('xml:')}"
+        if name.startswith("xmlns:"):
+            return f"xmlns {name.removeprefix('xmlns:')}"
+        return name
+
+    def process(node, node_indent):
+        prefix = " " * node_indent
+        if isinstance(node, Doctype):
+            if node.public_id is not None or node.system_id is not None:
+                public_id = node.public_id or ""
+                system_id = node.system_id or ""
+                return [f'| <!DOCTYPE {node.name} "{public_id}" "{system_id}">']
+            return [f"| <!DOCTYPE {node.name}>"]
+        if isinstance(node, Comment):
+            return [f"| {prefix}<!-- {node.data} -->"]
+        if isinstance(node, Text):
+            return [f'| {prefix}"{node.text}"']
+        if not isinstance(node, Element):
+            return [line for child in node.children for line in process(child, node_indent)]
+
+        lines = [f"| {prefix}<{qualified_name(node.namespace, node.tag)}>"]
+        for name, value in sorted(node.attrs.items()):
+            lines.append(f'| {prefix}  {attribute_name(name)}="{value}"')
+
+        if node.tag == "template" and node.namespace is Namespace.HTML:
+            lines.append(f"| {prefix}  content")
+            content = node.children[0]
+            lines.extend(line for child in content.children for line in process(child, node_indent + 4))
+        else:
+            lines.extend(line for child in node.children for line in process(child, node_indent + 2))
+        return lines
+
+    return "\n".join(line for node in nodes for line in process(node, indent))
+
+
 # Parser dispatch
 PARSER_RUNNERS = {
     "justhtml": run_test_justhtml,
@@ -849,6 +963,7 @@ PARSER_RUNNERS = {
     "html.parser": run_test_html_parser,
     "selectolax": run_test_selectolax,
     "markupever": run_test_markupever,
+    "turbohtml": run_test_turbohtml,
 }