justhtml 3.1.0__tar.gz → 3.3.0__tar.gz

{justhtml-3.1.0 → justhtml-3.3.0}/.github/workflows/ci.yml

@@ -36,20 +36,31 @@ jobs:
       - name: Checkout html5lib-tests
         run: |
           cd ..
-          git clone https://github.com/html5lib/html5lib-tests.git
+          git clone --no-checkout https://github.com/html5lib/html5lib-tests.git
+          git -C html5lib-tests fetch origin 9fb614afaa42ce8787840f057b32084308e76549
+          git -C html5lib-tests checkout 9fb614afaa42ce8787840f057b32084308e76549
           git -C html5lib-tests rev-parse HEAD
 
-      - name: Setup test symlinks
+      - name: Checkout WPT tree-construction tests
         run: |
-          cd tests
-          ln -s ../../html5lib-tests/tokenizer html5lib-tests-tokenizer
-          ln -s ../../html5lib-tests/tree-construction html5lib-tests-tree
-          ln -s ../../html5lib-tests/serializer html5lib-tests-serializer
-          ln -s ../../html5lib-tests/encoding html5lib-tests-encoding
+          cd ..
+          git clone --filter=blob:none --sparse https://github.com/web-platform-tests/wpt.git
+          git -C wpt fetch origin 75b089d70448e1da425ab8cf38971b6af41f3cea
+          git -C wpt checkout 75b089d70448e1da425ab8cf38971b6af41f3cea
+          git -C wpt sparse-checkout set html/syntax/parsing/resources
+          git -C wpt rev-parse HEAD
+
+      - name: Setup test fixtures
+        run: |
+          cp -a ../html5lib-tests/tokenizer tests/html5lib-tests-tokenizer
+          cp -a ../wpt/html/syntax/parsing/resources tests/html5lib-tests-tree
+          cp -a ../html5lib-tests/serializer tests/html5lib-tests-serializer
+          cp -a ../html5lib-tests/encoding tests/html5lib-tests-encoding
 
       - name: Run pre-commit
         id: precommit
-        uses: pre-commit/action@v3.0.1
+        if: matrix.python-version == '3.14'
+        run: pre-commit run --show-diff-on-failure --color=always --all-files
         env:
           SKIP: mypy
 
@@ -60,14 +71,14 @@ jobs:
       - name: Diagnose test failures (verbose)
         if: always()
         run: |
-          if [ "${{ steps.precommit.outcome }}" = "success" ] && [ "${{ steps.run_tests.outcome }}" = "success" ]; then
+          if [ "${{ steps.precommit.outcome }}" != "failure" ] && [ "${{ steps.run_tests.outcome }}" = "success" ]; then
             echo "No failures detected; skipping verbose rerun."
             exit 0
           fi
           python run_tests.py -v --no-write-summary
 
       - name: Run mypy
-        if: matrix.python-version == '3.12'
+        if: matrix.python-version == '3.14'
         run: mypy
 
   test-pyodide:

{justhtml-3.1.0 → justhtml-3.3.0}/.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
     hooks:
       - id: tests-coverage
         name: Tests & Coverage
-        entry: bash -c 'coverage run run_tests.py --no-write-summary && coverage report'
+        entry: bash -c 'coverage run run_tests.py && coverage report'
         language: system
         pass_filenames: false
         always_run: true

{justhtml-3.1.0 → justhtml-3.3.0}/CHANGELOG.md

@@ -7,6 +7,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.3.0] - 2026-07-04
+
+### Changed
+- Document that `Linkify(...)` must run before an explicit `Sanitize(...)` in a transform list, not after, and correct the `clean_url_value()` example to escape the returned value with `escape_attr_value()` before embedding it in HTML. Previously, the docs suggested orderings that silently skip URL-policy validation on Linkify-generated links, and separately claimed `clean_url_value()` output needed no further escaping for HTML attribute embedding.
+
+### Fixed
+- Copy `override_attrs` when cloning a node instead of storing the caller's dict by reference. Previously, calling `clone_node(override_attrs=...)` more than once with the same dict (or holding onto that dict after cloning) produced clones that aliased the same attribute dictionary, so mutating one clone's attributes silently mutated another clone or the caller's own dict.
+
+### Performance
+- Avoid rescanning the entire open-elements stack for every unmatched end tag inside foreign (SVG/MathML) content in the streaming parser (`stream()`), by tracking open-element name counts the same way the main parser engine does. Previously, a run of unmatched end tags deep inside foreign content was quadratic in the depth of nesting.
+- Bound `Linkify(...)`'s email-candidate pattern to RFC 5321's 64-octet local-part limit instead of an unbounded match. Previously, the greedy local-part match followed by a required (and often absent) `@` forced a full-length backtrack at every candidate start position, making long "@"-free runs of otherwise email-local-part-legal text (such as repeated dotted-numeric tokens) quadratic to scan.
+- Share cached sibling/ancestor lookups across nodes when matching structural selectors (`:nth-child`, sibling combinators) in custom `EditAttrs(...)`/`SetAttrs(...)`-style transform pipelines, the same way `query()` already does. Previously, each node rebuilt this data from scratch, making structural-selector transforms quadratic over documents with many siblings. Sharing is only enabled when every transform in the pipeline is provably attribute-only; a pipeline that also uses `Edit(...)`, `Decide(...)`, `Drop(...)`, `Linkify(...)`, or similar structure-changing transforms is unaffected and keeps its previous (already correct) behavior.
+- Access the parser's open-elements stack through a plain attribute again instead of a property, removing a Python-level function call from one of the hottest paths in tree construction.
+
+### Security
+- (Severity: Low) HTML-escape a literal `&` left over after percent-encoding a URL for the `HTMLContext.URL` serialization context (used for `href`/`src`-style attributes). Previously, a URL containing a legacy (no-semicolon) named HTML entity immediately after a decoded `&` could be interpreted differently once embedded in an HTML attribute and re-parsed by a browser than what was validated.
+
+## [3.2.0] - 2026-07-04
+
+### Changed
+- Use the web platform html5 treebuilder tests from WPT for tree-construction compliance fixtures.
+
+### Fixed
+- Preserve HTML processing-instruction nodes in unsanitized parsing while keeping default-safe sanitization comment-like and conservative, and neutralize `>` in serialized processing-instruction data so a hand-built DOM or a policy that preserves comment-like nodes cannot serialize a processing instruction that a browser parses as a bogus comment terminating early and exposing following markup.
+- Keep raw-text, RCDATA, doctype, and foreign-content CDATA scans aligned after Unicode characters whose Python lowercase mapping changes length or becomes ASCII. Previously, content such as `İ` before a later `<script>` close tag could make the parser miss the real end tag and treat following markup as raw text.
+
+### Performance
+- Short-circuit open-elements-stack scope checks (`<p>`, `<li>`, `<dd>`/`<dt>`, table-cell scoping, and similar) when the target element isn't open anywhere on the stack, avoiding quadratic parse time for deeply nested common block elements such as `<div>`, `<table>`, `<section>`, and heading tags.
+
+### Security
+- (Severity: Low) Reject numeric character references with more digits than any valid Unicode code point could have before converting them to an integer. Previously, a decimal reference such as `&#` followed by thousands of digits could crash parsing with an uncaught `ValueError` before sanitization, since Python limits decimal string-to-integer conversion length.
+- (Severity: Moderate) Prioritize a byte-order mark over a caller-declared transport encoding when decoding byte input, matching the HTML encoding-sniffing algorithm. Previously, passing an explicit `encoding=` alongside untrusted bytes that began with a BOM for a different encoding could make JustHTML decode and sanitize content differently than a browser does, letting markup such as `<script>` pass through undetected while a browser executes it under the BOM-indicated encoding.
+
 ## [3.1.0] - 2026-07-02
 
 ### Changed

{justhtml-3.1.0 → justhtml-3.3.0}/CONTRIBUTING.md

@@ -24,9 +24,9 @@ Thanks for considering contributing to JustHTML! This document explains how to s
 
 ## Running Tests
 
-The test suite uses the html5lib test cases plus additional tests for selector functionality.
+The test suite uses the web platform html5 treebuilder tests plus additional tests for selector functionality.
 
-If you want to run the full html5lib test suite locally, clone `html5lib-tests` next to this repository and create the symlinks described in [tests/README.md](tests/README.md) (tokenizer, tree-construction, and serializer).
+If you want to run the full external fixture suite locally, clone the web platform html5 treebuilder tests and `html5lib-tests` serializer/encoding fixtures next to this repository, then create the symlinks described in [tests/README.md](tests/README.md).
 
 ```bash
 # Run all tests
@@ -58,7 +58,7 @@ Pre-commit runs automatically on every commit and checks:
 - **Ruff check** - linting with auto-fix
 - **Ruff format** - code formatting
 - **Tests & Coverage** - full test suite with 100% coverage requirement
-- **Parser Differential** - exact agreement with the reference parser path across scored html5lib tree cases
+- **Parser Differential** - exact agreement with the reference parser path across scored web platform html5 treebuilder cases
 
 Run manually:
 ```bash

{justhtml-3.1.0 → justhtml-3.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: justhtml
-Version: 3.1.0
+Version: 3.3.0
 Summary: A pure Python HTML5 parser that just works.
 Project-URL: Homepage, https://github.com/emilstenstrom/justhtml
 Project-URL: Issues, https://github.com/emilstenstrom/justhtml/issues
@@ -82,7 +82,7 @@ Most Python HTML libraries optimize for one part of the problem.
 
 JustHTML is for applications that want a boring, inspectable, pure-Python default:
 
-- **Correct parsing:** browser-style HTML5 recovery, tested against the official html5lib fixtures.
+- **Correct parsing:** browser-style HTML5 recovery, tested against the web platform html5 treebuilder tests.
 - **Safe by default:** `JustHTML(html)` sanitizes before you query or serialize.
 - **One DOM:** parse once, then sanitize, query, transform, serialize, extract text, or produce Markdown.
 - **Easy deployment:** zero runtime dependencies, no compiler, works on PyPy and Pyodide.
@@ -164,10 +164,10 @@ justhtml index.html --selector "article" --format markdown
 
 ## Correctness
 
-JustHTML is tested against the official html5lib tree-construction, serializer, and encoding fixtures, plus project-specific sanitizer, selector, transform, CLI, and regression tests.
+JustHTML is tested against the web platform html5 treebuilder tests, plus html5lib serializer/encoding fixtures and project-specific sanitizer, selector, transform, CLI, and regression tests.
 
 The current test suite enforces 100% combined line and branch coverage, including the parser engine.
-The parser engine additionally requires exact agreement with the reference path across the html5lib tree suite.
+The parser engine additionally requires exact agreement with the reference path across the web platform html5 treebuilder suite.
 See [Correctness Testing](docs/correctness.md) for details.
 
 ## Documentation

{justhtml-3.1.0 → justhtml-3.3.0}/README.md

@@ -28,7 +28,7 @@ Most Python HTML libraries optimize for one part of the problem.
 
 JustHTML is for applications that want a boring, inspectable, pure-Python default:
 
-- **Correct parsing:** browser-style HTML5 recovery, tested against the official html5lib fixtures.
+- **Correct parsing:** browser-style HTML5 recovery, tested against the web platform html5 treebuilder tests.
 - **Safe by default:** `JustHTML(html)` sanitizes before you query or serialize.
 - **One DOM:** parse once, then sanitize, query, transform, serialize, extract text, or produce Markdown.
 - **Easy deployment:** zero runtime dependencies, no compiler, works on PyPy and Pyodide.
@@ -110,10 +110,10 @@ justhtml index.html --selector "article" --format markdown
 
 ## Correctness
 
-JustHTML is tested against the official html5lib tree-construction, serializer, and encoding fixtures, plus project-specific sanitizer, selector, transform, CLI, and regression tests.
+JustHTML is tested against the web platform html5 treebuilder tests, plus html5lib serializer/encoding fixtures and project-specific sanitizer, selector, transform, CLI, and regression tests.
 
 The current test suite enforces 100% combined line and branch coverage, including the parser engine.
-The parser engine additionally requires exact agreement with the reference path across the html5lib tree suite.
+The parser engine additionally requires exact agreement with the reference path across the web platform html5 treebuilder suite.
 See [Correctness Testing](docs/correctness.md) for details.
 
 ## Documentation

{justhtml-3.1.0 → justhtml-3.3.0}/docs/api.md

@@ -602,7 +602,7 @@ To override the encoding for byte input, pass `encoding=...`.
 | `"end"` | `tag_name` | Closing tag |
 | `"text"` | `text_content` | Text content |
 | `"comment"` | `comment_text` | HTML comment |
-| `"doctype"` | `doctype_name` | DOCTYPE declaration |
+| `"doctype"` | `(name, public_id, system_id)` | DOCTYPE declaration fields |
 
 ---
 

{justhtml-3.1.0 → justhtml-3.3.0}/docs/correctness.md

@@ -2,18 +2,18 @@
 
 # Correctness Testing
 
-JustHTML is the only pure-Python HTML5 parser that passes 100% of the official html5lib test suite. This page explains how we verify and maintain that compliance.
+JustHTML is tested against the web platform html5 treebuilder tests. This page explains how we verify and maintain that compliance.
 
-## The html5lib Test Suite
+## The Web Platform Tests
 
-The [html5lib-tests](https://github.com/html5lib/html5lib-tests) repository is the gold standard for HTML5 parsing compliance. It's used by browser vendors to verify their implementations against the [WHATWG HTML5 specification](https://html.spec.whatwg.org/).
+The web platform html5 treebuilder tests live in [web-platform-tests/wpt](https://github.com/web-platform-tests/wpt/tree/master/html/syntax/parsing/resources). Serializer and encoding fixtures remain in [html5lib-tests](https://github.com/html5lib/html5lib-tests).
 
-Our checked-in test inputs contain:
+The external fixture inputs contain:
 
-- **56 tree-construction test files** - Testing how the parser builds the DOM tree
+- **61 treebuilder test files** - Testing how the parser builds the DOM tree
 - **5 serializer fixture files** - Testing how token streams are serialized back to HTML
 - **Encoding sniffing tests** - Testing BOM/meta charset/transport overrides and legacy fallbacks
-- **1,791 enabled tree-construction cases** - Covering edge cases, error recovery, and spec compliance
+- **1,914 enabled treebuilder cases** - Covering edge cases, error recovery, and spec compliance
 
 ### What the Tests Cover
 
@@ -54,12 +54,11 @@ This tests the adoption agency algorithm - when `</b>` is encountered inside `<p
 ## Compliance Comparison
 
 We run the same test suite against other Python parsers to compare compliance.
-The cross-parser snapshot below used the 1,743 cases available when it was
-recorded; the current JustHTML gate covers 1,791 enabled cases.
+The cross-parser snapshot below used the 1,743 cases available when it was recorded.
 
 | Parser | Tests Passed | Compliance | Notes |
 |--------|-------------|------------|-------|
-| **JustHTML** | 1743/1743 | **100%** | Full spec compliance in this comparison snapshot; current gate: 1791/1791 |
+| **JustHTML** | 1743/1743 | **100%** | Full spec compliance in this comparison snapshot |
 | selectolax | 1743/1743 | 100% | C-based (Lexbor), fast and spec-compliant with dev `html5test` output API |
 | markupever | 1545/1743 | 89% | Rust-based (html5ever), mostly correct |
 | html5lib | 1496/1743 | 86% | Reference implementation, but incomplete |
@@ -68,9 +67,9 @@ recorded; the current JustHTML gate covers 1,791 enabled cases.
 | html.parser | 6/1743 | <1% | Python stdlib, basic error recovery only |
 | lxml | 5/1743 | <1% | XML-based, not HTML5 compliant |
 
-*Run `python benchmarks/correctness.py` to reproduce these results. The selectolax score requires its dev `html5test` output and fragment-context APIs. These scores were refreshed against html5lib-tests `e446320`.*
+*Run `python benchmarks/correctness.py` to reproduce these results. The selectolax score requires its dev `html5test` output and fragment-context APIs. These historical scores were refreshed against html5lib-tests `e446320`.*
 
-These numbers come from a strict tree comparison against the expected output in the `html5lib-tests` tree-construction fixtures (excluding `#script-on` / `#script-off` cases). Unsupported parser capabilities count as failures for this compliance table. The numbers will not match the `html5lib` project’s own reported totals, because `html5lib` runs the suite in multiple configurations and also has its own skip/xfail lists.
+These numbers come from a strict tree comparison against the expected output in the treebuilder fixtures available at the time (excluding `#script-on` / `#script-off` cases). Unsupported parser capabilities count as failures for this compliance table. The numbers will not match the `html5lib` project’s own reported totals, because `html5lib` runs the suite in multiple configurations and also has its own skip/xfail lists.
 
 ## Our Testing Strategy
 
@@ -123,7 +122,7 @@ PYTHONPATH=src python benchmarks/html5lib_engine_diff.py \
   --fail-on-current-exceptions
 ```
 
-This requires exact agreement with the reference parser path across every scored html5lib tree-construction case.
+This requires exact agreement with the reference parser path across every scored web platform html5 treebuilder case.
 
 ### 3. Fuzz Testing (millions of cases)
 
@@ -169,14 +168,18 @@ We maintain additional tests in `tests/justhtml-tests/` for:
 ### Quick Start
 
 ```bash
-# Clone the test suite (one-time setup)
+# Clone the test suites (one-time setup)
+cd ..
+git clone --filter=blob:none --sparse https://github.com/web-platform-tests/wpt.git
+cd wpt
+git sparse-checkout set html/syntax/parsing/resources
 cd ..
 git clone https://github.com/html5lib/html5lib-tests.git
 cd justhtml
 
 # Create symlinks
 cd tests
-ln -s ../../html5lib-tests/tree-construction html5lib-tests-tree
+ln -s ../../wpt/html/syntax/parsing/resources html5lib-tests-tree
 ln -s ../../html5lib-tests/serializer html5lib-tests-serializer
 ln -s ../../html5lib-tests/encoding html5lib-tests-encoding
 cd ..

{justhtml-3.1.0 → justhtml-3.3.0}/docs/linkify.md

@@ -125,6 +125,33 @@ This matters for Linkify because sanitization policies can remove or rewrite att
 - Schemes not allowed for `a[href]` are stripped (the `<a>` remains, but `href` is removed).
 - Protocol-relative `//example.com` is resolved according to policy (default: `https://example.com`).
 
+**Put `Linkify()` before `Sanitize()`, not after.** It is tempting to reach for `transforms=[Sanitize(policy), Linkify()]`, reasoning "sanitize first, then add links to the clean text." That order does the opposite of what it looks like: your explicit `Sanitize()` position becomes the sanitize point (see above), so the `<a href="...">` elements Linkify creates afterward are never checked against your `UrlPolicy`/`UrlRule` restrictions.
+
+```python
+from justhtml import JustHTML, Linkify, Sanitize, SanitizationPolicy, UrlPolicy, UrlRule
+
+# A policy that only allows https:// links on <a href>.
+policy = SanitizationPolicy(
+    allowed_tags=["a"],
+    allowed_attributes={"a": ["href"]},
+    url_policy=UrlPolicy(allow_rules={("a", "href"): UrlRule(allowed_schemes={"https"})}),
+)
+text = "See http://example.com"
+
+# Wrong: Linkify's generated <a href> is created after the sanitize point,
+# so the https-only restriction never sees it.
+doc = JustHTML(text, transforms=[Sanitize(policy), Linkify()])
+print(doc.to_html(pretty=False))
+# => See <a href="http://example.com">http://example.com</a>
+
+# Right: Linkify runs first, so its output is covered by the same sanitize pass.
+doc = JustHTML(text, transforms=[Linkify(), Sanitize(policy)])
+print(doc.to_html(pretty=False))
+# => See http://example.com
+```
+
+If you don't need other transforms between them, the simplest safe option is to omit `Sanitize()` entirely and let `sanitize=True` (the default) append it automatically after `Linkify()`.
+
 If you want Linkify output without any sanitization changes (trusted input only), use `sanitize=False` and avoid adding `Sanitize(...)` in transforms.
 
 ## Provenance

{justhtml-3.1.0 → justhtml-3.3.0}/docs/playground/app.js

@@ -65,6 +65,7 @@ async function installJusthtmlFromLocalRepo(pyodideInstance) {
 		"parser/encoding.py",
 		"core/entities.py",
 		"core/errors.py",
+		"core/text.py",
 		"core/types.py",
 		"core/doctype.py",
 		"serializer/markdown.py",

{justhtml-3.1.0 → justhtml-3.3.0}/docs/sanitization.md

@@ -124,10 +124,12 @@ rule = UrlRule(allowed_schemes={"https", "mailto"})
 cleaned_url = JustHTML.clean_url_value(value=user_url, url_rule=rule)
 
 if cleaned_url:
-    # URL is safe to use.
-    # Note: clean_url_value performs URL encoding, so no extra escape needed if using double quotes in HTML.
-    # <a href="${cleaned_url}">
-    pass
+    # cleaned_url is a validated, percent-encoded URL, not an HTML-attribute-safe
+    # string: it deliberately keeps "&" literal so multi-parameter query strings
+    # stay usable as plain text (e.g. in a JS string). Escape it like any other
+    # attribute value before embedding it in HTML.
+    safe_url = JustHTML.escape_attr_value(cleaned_url)
+    # <a href="${safe_url}">
 ```
 
 ## Configuration

{justhtml-3.1.0 → justhtml-3.3.0}/docs/streaming.md

@@ -15,7 +15,6 @@ The streaming parser is:
 
 ```python
 from justhtml import stream
-from pathlib import Path
 
 html = "<html><body><p>Hello, world!</p></body></html>"
 
@@ -23,12 +22,24 @@ for event, data in stream(html):
     print(event, data)
 ```
 
+Output:
+```text
+start ('html', {})
+start ('body', {})
+start ('p', {})
+text Hello, world!
+end p
+end body
+end html
+```
+
 ## Byte Input and Encodings
 
 `stream()` also accepts bytes (`bytes`, `bytearray`, `memoryview`). In that case, the input is decoded using HTML encoding sniffing (including a `windows-1252` fallback for legacy documents).
 
 ```python
 from justhtml import stream
+from pathlib import Path
 
 data = Path("page.html").read_bytes()
 for event, data in stream(data):
@@ -48,19 +59,6 @@ for event, data in stream(data, encoding="utf-8"):
 
 See [Encoding & Byte Input](encoding.md) for details.
 
-Output:
-```
-start ('html', {})
-start ('head', {})
-end head
-start ('body', {})
-start ('p', {})
-text Hello, world!
-end p
-end body
-end html
-```
-
 ## Events
 
 | Event | Data Type | Description |
@@ -69,7 +67,7 @@ end html
 | `"end"` | `tag_name` | Closing tag encountered |
 | `"text"` | `str` | Text content |
 | `"comment"` | `str` | HTML comment content |
-| `"doctype"` | `str` | DOCTYPE name (usually `"html"`) |
+| `"doctype"` | `(name, public_id, system_id)` | DOCTYPE declaration fields |
 
 ## Examples
 

{justhtml-3.1.0 → justhtml-3.3.0}/pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "justhtml"
 authors = [{ name = "Emil Stenström", email = "emil@emilstenstrom.se" }]
-version = "3.1.0"
+version = "3.3.0"
 description = "A pure Python HTML5 parser that just works."
 readme = "README.md"
 license = { file = "LICENSE" }

{justhtml-3.1.0 → justhtml-3.3.0}/run_tests.py

@@ -114,7 +114,7 @@ def parse_args():
         "--check-errors",
         action="store_true",
         help=(
-            "Enable additional error validation. For html5lib tree-construction .dat tests, "
+            "Enable additional error validation. For web platform html5 treebuilder .dat tests, "
             "validates the number of parse errors; for tests/justhtml-tests, validates the exact "
             "ordered list of error codes."
             "parse errors (code+line+col) when provided by the fixture."
@@ -295,7 +295,7 @@ def main():
     run_encoding = suite in {"all", "encoding"}
     run_unit = suite in {"all", "unit"}
 
-    # Check that html5lib-tests symlinks exist (only for the selected suites)
+    # Check that external fixture symlinks exist (only for the selected suites)
     tree_tests = test_dir / "html5lib-tests-tree"
     serializer_tests = test_dir / "html5lib-tests-serializer"
     encoding_tests = test_dir / "html5lib-tests-encoding"
@@ -307,13 +307,19 @@ def main():
     if run_encoding and not encoding_tests.exists():
         missing.append(str(encoding_tests))
     if len(missing) > 0:
-        print("ERROR: html5lib-tests not found. Please create symlinks:", file=sys.stderr)
+        print("ERROR: external test fixtures not found. Please create symlinks:", file=sys.stderr)
         for path in missing:
             print(f"  {path}", file=sys.stderr)
-        print("\nTo set up, clone html5lib-tests and create symlinks:", file=sys.stderr)
-        print("  git clone https://github.com/html5lib/html5lib-tests.git ../html5lib-tests", file=sys.stderr)
+        print("\nTo set up, clone the fixture repositories and create symlinks:", file=sys.stderr)
         if run_tree:
-            print("  ln -s ../../html5lib-tests/tree-construction tests/html5lib-tests-tree", file=sys.stderr)
+            print(
+                "  git clone --filter=blob:none --sparse https://github.com/web-platform-tests/wpt.git ../wpt",
+                file=sys.stderr,
+            )
+            print("  (cd ../wpt && git sparse-checkout set html/syntax/parsing/resources)", file=sys.stderr)
+            print("  ln -s ../../wpt/html/syntax/parsing/resources tests/html5lib-tests-tree", file=sys.stderr)
+        if run_serializer or run_encoding:
+            print("  git clone https://github.com/html5lib/html5lib-tests.git ../html5lib-tests", file=sys.stderr)
         if run_serializer:
             print("  ln -s ../../html5lib-tests/serializer tests/html5lib-tests-serializer", file=sys.stderr)
         if run_encoding:
@@ -343,7 +349,7 @@ def main():
         combined_results.update(runner.file_results)
 
     if run_justhtml_tree:
-        # Run JustHTML-specific tree-construction tests (custom .dat fixtures).
+        # Run JustHTML-specific treebuilder tests (custom .dat fixtures).
         # These live outside the upstream html5lib-tests checkout.
         justhtml_tree_tests = test_dir / "justhtml-tests"
         justhtml_runner = TestRunner(justhtml_tree_tests, config)

{justhtml-3.1.0 → justhtml-3.3.0}/src/justhtml/__init__.py

@@ -1,5 +1,16 @@
 from .core.types import ParseError
-from .dom import Comment, Document, DocumentFragment, Element, Node, NodeType, QueryMatch, Template, Text
+from .dom import (
+    Comment,
+    Document,
+    DocumentFragment,
+    Element,
+    Node,
+    NodeType,
+    ProcessingInstruction,
+    QueryMatch,
+    Template,
+    Text,
+)
 from .parser import JustHTML, StrictModeError
 from .parser.stream import stream
 from .sanitizer import (
@@ -70,6 +81,7 @@ __all__ = [
     "Node",
     "NodeType",
     "ParseError",
+    "ProcessingInstruction",
     "PruneEmpty",
     "QueryMatch",
     "SanitizationPolicy",

{justhtml-3.1.0 → justhtml-3.3.0}/src/justhtml/core/entities.py

@@ -199,7 +199,16 @@ def decode_numeric_entity(
         The decoded character, or None if invalid
     """
     base = 16 if is_hex else 10
-    codepoint = int(text, base)
+
+    # A numeric character reference this long is already guaranteed to exceed
+    # the valid Unicode range, so treat it as out-of-range without calling
+    # int() on an attacker-controlled digit string long enough to exceed
+    # Python's integer-string conversion limit (see sys.set_int_max_str_digits).
+    digits = text.lstrip("0") or "0"
+    if len(digits) > 16:
+        return "\ufffd"  # REPLACEMENT CHARACTER
+
+    codepoint = int(digits, base)
 
     # Invalid ranges per HTML5 spec
     if codepoint > 0x10FFFF:

{justhtml-3.1.0 → justhtml-3.3.0}/src/justhtml/core/rawtext.py

@@ -1,13 +1,14 @@
 from __future__ import annotations
 
 from .constants import HTML_SPACE_OR_TAG_END_CHARACTERS
+from .text import ascii_lower
 
 
 def neutralize_rawtext_end_tag_sequences(text: str, tag_name: str) -> tuple[str, bool]:
     if not text:
         return text, False
 
-    lower_text = text.lower()
+    lower_text = ascii_lower(text)
     needle = f"</{tag_name}"
     needle_len = len(needle)
     out: list[str] = []

justhtml-3.3.0/src/justhtml/core/text.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import string
+
+_ASCII_LOWER_TABLE = str.maketrans(string.ascii_uppercase, string.ascii_lowercase)
+
+
+def ascii_lower(text: str) -> str:
+    """Fold A-Z for ASCII marker searches that reuse original offsets.
+
+    Some parser and serializer paths search a folded copy of full text for
+    ASCII HTML markers such as ``</script``, ``<!doctype``, and ``<![cdata[``.
+    Match offsets are then reused against the original string, so the folded
+    copy must preserve length, and it must not invent ASCII marker characters
+    from non-ASCII text content. HTML markup matching is ASCII
+    case-insensitive; it is not Unicode case-insensitive.
+
+    ``str.lower()`` is still the fast path when it is provably equivalent for
+    these searches. In Python's Unicode data, U+0130 LATIN CAPITAL LETTER I
+    WITH DOT ABOVE is the only character whose lowercase mapping changes
+    length, and U+212A KELVIN SIGN is the only non-ASCII character whose
+    lowercase mapping becomes an ASCII letter (``k``). Inputs containing either
+    shape take the strict A-Z translation path. That path deliberately keeps
+    U+212A as U+212A; it is not a tag-name fold.
+    """
+    if text.isascii():
+        return text.lower()
+    if "\u0130" in text or "\u212a" in text:
+        return text.translate(_ASCII_LOWER_TABLE)
+    return text.lower()

{justhtml-3.1.0 → justhtml-3.3.0}/src/justhtml/dom/__init__.py

@@ -172,7 +172,7 @@ class Node:
 
         if name.startswith("#") or name == "!doctype":
             self.namespace = namespace
-            if name == "#comment" or name == "!doctype":
+            if name in {"#comment", "#processing-instruction"} or name == "!doctype":
                 self.children = None
                 self.attrs = None
             else:
@@ -422,7 +422,7 @@ class Node:
         Returns:
             A new node that is a copy of this node.
         """
-        attrs = override_attrs if override_attrs is not None else (self.attrs.copy() if self.attrs else None)
+        attrs = override_attrs.copy() if override_attrs is not None else (self.attrs.copy() if self.attrs else None)
         clone = Node(
             self.name,
             attrs,
@@ -491,6 +491,25 @@ class Comment(Node):
         return clone
 
 
+class ProcessingInstruction(Node):
+    __slots__ = ()
+
+    def __init__(self, data: str | None = None) -> None:
+        super().__init__("#processing-instruction", data=data)
+
+    def clone_node(
+        self, deep: bool = False, override_attrs: dict[str, str | None] | None = None
+    ) -> ProcessingInstruction:
+        _ = override_attrs
+        _ = deep
+        clone = ProcessingInstruction(self.data if isinstance(self.data, str) else None)
+        clone._source_html = self._source_html
+        clone._origin_pos = self._origin_pos
+        clone._origin_line = self._origin_line
+        clone._origin_col = self._origin_col
+        return clone
+
+
 class Element(Node):
     __slots__ = (
         "_end_tag_end",
@@ -532,7 +551,7 @@ class Element(Node):
         self._self_closing = False
 
     def clone_node(self, deep: bool = False, override_attrs: dict[str, str | None] | None = None) -> Element:
-        attrs = override_attrs if override_attrs is not None else (self.attrs.copy() if self.attrs else {})
+        attrs = override_attrs.copy() if override_attrs is not None else (self.attrs.copy() if self.attrs else {})
         clone = Element(self.name, attrs, self.namespace)
         clone._source_html = self._source_html
         clone._origin_pos = self._origin_pos
@@ -567,7 +586,7 @@ class Template(Element):
             self.template_content = None
 
     def clone_node(self, deep: bool = False, override_attrs: dict[str, str | None] | None = None) -> Template:
-        attrs = override_attrs if override_attrs is not None else (self.attrs.copy() if self.attrs else {})
+        attrs = override_attrs.copy() if override_attrs is not None else (self.attrs.copy() if self.attrs else {})
         clone = Template(
             self.name,
             attrs,

{justhtml-3.1.0 → justhtml-3.3.0}/src/justhtml/parser/encoding.py

@@ -345,15 +345,16 @@ def _prescan_for_meta_charset(data: bytes) -> str | None:
 
 
 def sniff_html_encoding(data: bytes, transport_encoding: str | None = None) -> tuple[str, int]:
-    # Transport overrides everything.
-    transport = normalize_encoding_label(transport_encoding)
-    if transport:
-        return transport, 0
-
+    # Per the HTML encoding-sniffing algorithm, a byte-order mark takes
+    # priority over a transport-layer declared encoding.
     bom_enc, bom_len = _sniff_bom(data)
     if bom_enc:
         return bom_enc, bom_len
 
+    transport = normalize_encoding_label(transport_encoding)
+    if transport:
+        return transport, 0
+
     meta_enc = _prescan_for_meta_charset(data)
     if meta_enc:
         return meta_enc, 0