semdiff-html 0.1.0__py3-none-any.whl

semdiff/__init__.py

@@ -0,0 +1,32 @@
+"""SemDiff: semantic HTML change detection."""
+
+from semdiff.config import Config, NormalizationConfig, ParserBackend
+from semdiff.errors import (
+    AlignmentBudgetExceeded,
+    ErrorRecord,
+    InputTooLargeError,
+    ParseError,
+    SemDiffError,
+)
+from semdiff.normalize.api import normalize
+from semdiff.parse import PARSER_ID, PARSER_VERSION, ParsedDoc, parse, parser_fingerprint
+
+__version__ = "0.1.0"  # the one source; Hatchling reads it at build time (D-050)
+
+__all__ = [
+    "PARSER_ID",
+    "PARSER_VERSION",
+    "AlignmentBudgetExceeded",
+    "Config",
+    "ErrorRecord",
+    "InputTooLargeError",
+    "NormalizationConfig",
+    "ParseError",
+    "ParsedDoc",
+    "ParserBackend",
+    "SemDiffError",
+    "__version__",
+    "normalize",
+    "parse",
+    "parser_fingerprint",
+]

semdiff/__main__.py

@@ -0,0 +1,5 @@
+"""``python -m semdiff`` — same entry point as the installed ``semdiff`` command."""
+
+from semdiff.cli import main
+
+raise SystemExit(main())

semdiff/_models.py

@@ -0,0 +1,9 @@
+"""Shared pydantic base for every SemDiff model."""
+
+from pydantic import BaseModel, ConfigDict
+
+
+class FrozenModel(BaseModel):
+    """Immutable, and unknown keys fail loud."""
+
+    model_config = ConfigDict(frozen=True, extra="forbid")

semdiff/cli.py

@@ -0,0 +1,147 @@
+"""Command line interface (T-20, FR-29): ``semdiff normalize file.html``.
+
+A thin shell over the T-19 API. stdout carries normalized HTML and nothing else, so it
+can always be redirected into a file; diagnostics and the ``--report`` summary go to
+stderr. Every typed error from T-02 maps to its own exit code, so a script can tell an
+oversized input from a parse failure without reading the message.
+
+Rule provenance comes from ``apply_rules`` as D-033 anticipated — there is no second
+reporting model.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+from collections import Counter
+from collections.abc import Callable
+from pathlib import Path
+
+from semdiff import Config, NormalizationConfig, SemDiffError, __version__
+from semdiff.normalize import BUILTIN_RULES, NormalizedDoc, apply_rules
+from semdiff.parse import parse, parser_fingerprint
+
+EXIT_OK = 0
+EXIT_ERROR = 1
+EXIT_USAGE = 2  # argparse's own code for a bad command line
+EXIT_PARSE = 3
+EXIT_TOO_LARGE = 4
+EXIT_CONFIG = 5
+EXIT_IO = 6
+
+_EXIT_BY_ERROR_TYPE = {"parse_error": EXIT_PARSE, "input_too_large": EXIT_TOO_LARGE}
+
+
+def _read_stdin() -> bytes:
+    """Raw bytes, so the parser's encoding detection (T-10) still applies."""
+    return sys.stdin.buffer.read()
+
+
+def _write_stdout(text: str) -> None:
+    """Always UTF-8, whatever the console claims to be: the output is a document."""
+    sys.stdout.buffer.write(text.encode("utf-8"))
+    sys.stdout.buffer.flush()
+
+
+def _fail(message: str, code: int) -> int:
+    print(f"semdiff: {message}", file=sys.stderr)
+    return code
+
+
+def _load_config(args: argparse.Namespace) -> Config:
+    config = Config()
+    if args.config is not None:
+        config = Config.from_json(Path(args.config).read_text(encoding="utf-8"))
+    normalization = NormalizationConfig(
+        disabled_rules=config.normalization.disabled_rules | frozenset(args.disable_rule),
+        enabled_rules=config.normalization.enabled_rules | frozenset(args.enable_rule),
+    )
+    return config.model_copy(update={"normalization": normalization})
+
+
+def _report(result: NormalizedDoc) -> str:
+    """One line per rule that fired, in the order the rules ran, with application counts.
+
+    The parser build is named first: it is the one input to a result that the config does not
+    cover (D-046), so a report that omitted it could not be reproduced from what it printed.
+    """
+    header = f"parser {parser_fingerprint()}"
+    counts = Counter(application.rule_id for application in result.applied_rules)
+    if not counts:
+        return f"{header}\nno rules fired\n"
+    width = max(len(rule_id) for rule_id in counts)
+    lines = [header, f"{len(counts)} rules fired, {sum(counts.values())} applications"]
+    lines += [f"  {rule_id:<{width}}  {count}" for rule_id, count in counts.items()]
+    return "\n".join(lines) + "\n"
+
+
+def _normalize(args: argparse.Namespace) -> int:
+    try:
+        config = _load_config(args)
+    except OSError as exc:
+        return _fail(f"cannot read config: {exc}", EXIT_IO)
+    except (ValueError, TypeError) as exc:
+        return _fail(f"invalid config: {exc}", EXIT_CONFIG)
+
+    try:
+        source = _read_stdin() if args.file in (None, "-") else Path(args.file).read_bytes()
+    except OSError as exc:
+        return _fail(f"cannot read input: {exc}", EXIT_IO)
+
+    try:
+        doc = parse(source, encoding=args.encoding, config=config)
+        result = apply_rules(doc, BUILTIN_RULES, config.normalization)
+    except SemDiffError as exc:
+        return _fail(f"{exc.type_id}: {exc.message}", _EXIT_BY_ERROR_TYPE.get(exc.type_id, EXIT_ERROR))
+    except ValueError as exc:  # unknown rule id in a toggle
+        return _fail(f"invalid config: {exc}", EXIT_CONFIG)
+
+    html = result.tree.html or ""
+    try:
+        if args.output is None:
+            _write_stdout(html)
+        else:
+            Path(args.output).write_bytes(html.encode("utf-8"))
+    except OSError as exc:
+        return _fail(f"cannot write output: {exc}", EXIT_IO)
+
+    if args.report:
+        sys.stderr.write(_report(result))
+    return EXIT_OK
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="semdiff", description="Semantic HTML change detection.")
+    parser.add_argument(
+        "--version", action="version", version=f"{__version__} ({parser_fingerprint()})"
+    )
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    normalize_parser = commands.add_parser(
+        "normalize",
+        help="strip presentational noise from an HTML document and canonicalize it",
+        description="Write the normalized document to stdout, or to --output.",
+    )
+    normalize_parser.add_argument("file", nargs="?", help="input file; omit or pass - to read stdin")
+    normalize_parser.add_argument("-o", "--output", help="write to this file instead of stdout")
+    normalize_parser.add_argument("--report", action="store_true", help="list the rules that fired, on stderr")
+    normalize_parser.add_argument("--config", help="JSON config file (see Config.to_canonical_json)")
+    normalize_parser.add_argument(
+        "--disable-rule", action="append", default=[], metavar="ID", help="turn one rule off; repeatable"
+    )
+    normalize_parser.add_argument(
+        "--enable-rule", action="append", default=[], metavar="ID", help="turn one opt-in rule on; repeatable"
+    )
+    normalize_parser.add_argument("--encoding", help="override the input encoding instead of detecting it")
+    normalize_parser.set_defaults(handler=_normalize)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    handler: Callable[[argparse.Namespace], int] = args.handler
+    return handler(args)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

semdiff/config.py

@@ -0,0 +1,74 @@
+"""Immutable, serializable pipeline configuration and its deterministic hash (T-01).
+
+A ``Config`` plus the two input snapshots fully determines a diff's output (NFR-1).
+``config_hash`` covers every config field and the engine-owned built-in ruleset
+version, so a change to either the user's settings or a shipped rule's definition
+is visible in the output (DOMAIN_MODEL §12-J).
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from enum import StrEnum
+from typing import Any
+
+from pydantic import field_serializer, model_validator
+
+from semdiff._models import FrozenModel
+
+class ParserBackend(StrEnum):
+    """L0 parser selection (FR-2)."""
+
+    SELECTOLAX = "selectolax"
+    LXML = "lxml"
+
+
+class NormalizationConfig(FrozenModel):
+    """Per-rule toggles for L1 (FR-11). Built-in rules are enabled unless listed here."""
+
+    disabled_rules: frozenset[str] = frozenset()
+    enabled_rules: frozenset[str] = frozenset()
+
+    @field_serializer("disabled_rules", "enabled_rules")
+    def _sorted(self, value: frozenset[str]) -> list[str]:
+        return sorted(value)
+
+    @model_validator(mode="after")
+    def _no_overlap(self) -> NormalizationConfig:
+        overlap = self.disabled_rules & self.enabled_rules
+        if overlap:
+            raise ValueError(f"rules both enabled and disabled: {sorted(overlap)}")
+        return self
+
+
+class Config(FrozenModel):
+    """The single configuration object threaded through every pipeline layer."""
+
+    parser: ParserBackend = ParserBackend.SELECTOLAX
+    max_input_bytes: int = 10 * 1024 * 1024  # NFR-7 ceiling; larger input raises InputTooLargeError
+    normalization: NormalizationConfig = NormalizationConfig()
+
+    def to_canonical_json(self) -> str:
+        """Compact JSON with sorted keys and sorted sets; the only serialized form."""
+        return _canonical_json(self.model_dump(mode="json"))
+
+    @classmethod
+    def from_json(cls, text: str) -> Config:
+        return cls.model_validate_json(text)
+
+    @property
+    def config_hash(self) -> str:
+        # Local import: the registry depends on Config, so the dependency runs this way only here.
+        from semdiff.normalize.registry import builtin_ruleset_version
+
+        payload = {
+            "ruleset_version": builtin_ruleset_version(),
+            "config": self.model_dump(mode="json"),
+        }
+        digest = hashlib.sha256(_canonical_json(payload).encode("utf-8")).hexdigest()
+        return f"sha256:{digest}"
+
+
+def _canonical_json(data: Any) -> str:
+    return json.dumps(data, sort_keys=True, separators=(",", ":"), ensure_ascii=False)

semdiff/errors.py

@@ -0,0 +1,76 @@
+"""Typed error hierarchy (T-02).
+
+Every failure inside the pipeline is a ``SemDiffError`` subclass. Layers raise them;
+the public API boundary converts them into ``ErrorRecord`` entries in ``errors[]``
+so a caller never receives a partial diff presented as complete (FR-3, ARCHITECTURE
+"Fail loud, fail typed").
+"""
+
+from __future__ import annotations
+
+from typing import ClassVar, Literal
+
+from pydantic import field_serializer
+
+from semdiff._models import FrozenModel
+
+SnapshotLabel = Literal["old", "new"]
+DetailValue = str | int | None
+
+
+class ErrorRecord(FrozenModel):
+    """One element of ``DiffResult.errors``."""
+
+    type: str
+    message: str
+    snapshot: SnapshotLabel | None = None
+    detail: dict[str, DetailValue] = {}
+
+    @field_serializer("detail")
+    def _sorted_detail(self, value: dict[str, DetailValue]) -> dict[str, DetailValue]:
+        return dict(sorted(value.items()))
+
+
+class SemDiffError(Exception):
+    """Root of the hierarchy. ``type_id`` is the stable id used in ``errors[].type``."""
+
+    type_id: ClassVar[str] = "semdiff_error"
+
+    def __init__(self, message: str, *, snapshot: SnapshotLabel | None = None) -> None:
+        super().__init__(message)
+        self.message = message
+        self.snapshot = snapshot
+
+    def detail(self) -> dict[str, DetailValue]:
+        return {}
+
+    def to_record(self) -> ErrorRecord:
+        return ErrorRecord(
+            type=self.type_id, message=self.message, snapshot=self.snapshot, detail=self.detail()
+        )
+
+
+class ParseError(SemDiffError):
+    """A snapshot could not be parsed into a tree (FR-3)."""
+
+    type_id = "parse_error"
+
+
+class InputTooLargeError(SemDiffError):
+    """A snapshot exceeds the documented input size ceiling (NFR-7)."""
+
+    type_id = "input_too_large"
+
+    def __init__(self, size: int, limit: int, *, snapshot: SnapshotLabel | None = None) -> None:
+        super().__init__(f"input is {size} bytes; ceiling is {limit} bytes", snapshot=snapshot)
+        self.size = size
+        self.limit = limit
+
+    def detail(self) -> dict[str, DetailValue]:
+        return {"size": self.size, "limit": self.limit}
+
+
+class AlignmentBudgetExceeded(SemDiffError):
+    """Tree-edit-distance alignment ran past its budget (v0.3, T-75). Fields arrive with T-75."""
+
+    type_id = "alignment_budget_exceeded"

semdiff/locator.py

@@ -0,0 +1,141 @@
+"""``Locator`` — a pointer into a Document (DOMAIN_MODEL §7). Stage-relative, never absolute."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+
+from selectolax.lexbor import LexborNode
+
+
+@dataclass(frozen=True)
+class Locator:
+    xpath: str | None
+    css: str | None
+    node_hash: str  # the node itself, not its subtree (D-044); survives reordering
+
+    @classmethod
+    def for_node(cls, node: LexborNode, cache: PathCache | None = None) -> Locator:
+        """Locator for an element or text node as it is *right now* (pre-mutation).
+
+        Both fields are computed here and now, cache or no cache: a locator records the node as
+        the rule found it, which is why it is taken before the rewrite. ``cache`` only changes
+        how the css path is *arrived at* — see ``PathCache`` for the lifetime it is valid for.
+        """
+        target = node if node.tag not in ("-text", "-comment") or node.parent is None else node.parent
+        css = _css_path(target) if cache is None else cache.path(target)
+        return cls(xpath=None, css=css, node_hash=_node_hash(node))
+
+
+def _segment(node: LexborNode) -> str:
+    """``p`` or ``p:nth-child(2)`` — this element's own step, positional among element siblings."""
+    position = 1
+    sibling = node.prev
+    while sibling is not None:
+        if sibling.is_element_node:
+            position += 1
+        sibling = sibling.prev
+    tag = node.tag or ""
+    return tag if position == 1 else f"{tag}:nth-child({position})"
+
+
+def _at_root(node: LexborNode | None) -> bool:
+    """Whether the walk upwards has run out of path to build."""
+    return node is None or node.tag in ("", "#document") or node.is_document_node
+
+
+def _css_path(node: LexborNode) -> str:
+    """``html > body > div:nth-child(2) > p`` — positional among element siblings, 1-based."""
+    parts: list[str] = []
+    current: LexborNode | None = node
+    while not _at_root(current):
+        assert current is not None
+        parts.append(_segment(current))
+        current = current.parent
+    return " > ".join(reversed(parts))
+
+
+class PathCache:
+    """Element css paths already computed during **one pass over one tree** (T-27).
+
+    ``_css_path`` climbs to the root and counts previous siblings at every level: ~14 parent
+    hops but ~114 sibling hops per call on the corpus, and the engine makes one call per
+    application. Sibling applications share almost all of that walk, so an ancestor's path is
+    computed once and the walk stops at the first element already known.
+
+    Keyed by ``mem_id``, the underlying DOM node's address, **not** by the node: two wrappers
+    for the same node compare equal, but ``LexborNode.__eq__`` is proportional to the
+    serialized subtree — 765 µs for a 58 KB ``<html>`` against 0.6 µs for a leaf — so a
+    node-keyed cache spends more on dict probes than the walk it saves (F-032).
+
+    That key is an address, so a cache is only valid while every element it has seen is still
+    alive. Two rules keep it so, and both are the caller's to honour:
+
+    * one cache per pass, never reused across passes, so nothing survives a rule that may
+      reshape the tree between passes; and
+    * no cache at all for a pass that can destroy an element — ``DROP_NODE`` (``_apply_node``)
+      and user-supplied ``CANONICALIZE`` transforms, which the engine never hands one to.
+
+    Attribute rewrites, text replacement and comment removal cannot invalidate an entry: they
+    free text and comment nodes, which are never keys here, and they leave every element in
+    place. ``Locator.for_node`` resolves a text or comment node to its parent element before
+    asking for a path, so only elements are ever cached or looked up.
+
+    **Neither rule is enforced at runtime**, and that is a measured choice rather than an
+    oversight (T-27). Counting the tree's elements once per pass to detect a breach costs 58 ms
+    against the 192 ms this cache saves on the corpus, and reports the breach only after the
+    wrong paths have been handed out. Re-checking the queried node's own step on every hit
+    costs 59 ms and still misses the case that matters: when an *ancestor* shifts, the check
+    passes and the cache answers ``div:nth-child(3)`` for a node that is now
+    ``div:nth-child(2)``. Both spend about 30% of the gain to buy a partial guard, so the
+    invariant stays a convention of this module and the engine's. What would catch a violation
+    is the evidence that already exists: ``tests/corpus/test_fused_attribute_equivalence.py``
+    and the per-rule reference engine recompute every locator with no cache at all, and
+    ``tests/corpus/parser_contract.json`` pins every ``css`` value of all 40 snapshots.
+    """
+
+    __slots__ = ("_paths",)
+
+    def __init__(self) -> None:
+        self._paths: dict[int, str] = {}
+
+    def path(self, node: LexborNode) -> str:
+        """The same string ``_css_path`` returns, reusing any ancestor already computed."""
+        chain: list[LexborNode] = []
+        current: LexborNode | None = node
+        prefix = ""
+        while not _at_root(current):
+            assert current is not None
+            known = self._paths.get(current.mem_id)
+            if known is not None:
+                prefix = known
+                break
+            chain.append(current)
+            current = current.parent
+        for element in reversed(chain):
+            part = _segment(element)
+            prefix = f"{prefix} > {part}" if prefix else part
+            self._paths[element.mem_id] = prefix
+        return prefix
+
+
+def _node_hash(node: LexborNode) -> str:
+    """Fingerprint of the node *itself* — never of its descendants (D-044, F-030).
+
+    Hashing ``node.html`` made provenance order-dependent: the same rewrite of the same
+    attribute recorded a different hash depending on whether a descendant had already been
+    rewritten, which differs between one traversal per rule and the fused attribute pass. A
+    locator points at a node, so its fingerprint is a property of that node: the tag and
+    attributes of an element, the content of a text or comment node. It still survives the node
+    moving, and it still changes when the node itself changes.
+    """
+    if node.tag == "-text":
+        raw = node.text_content or ""
+    elif node.is_element_node:
+        # ``repr`` of the tag and the sorted (name, value) pairs: deterministic, and quoted, so
+        # two different attribute sets cannot collide the way a flat join would. A boolean
+        # attribute hashes like an empty one, as the canonical serializer unifies them (D-032).
+        raw = repr((node.tag or "", sorted((n, v or "") for n, v in node.attributes.items())))
+    else:
+        raw = node.html or ""  # a comment or doctype has no children to pull in
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()

semdiff/normalize/__init__.py

@@ -0,0 +1,39 @@
+"""L1 — normalization: rule model, phase-ordered registry (T-12) and built-in families (T-13+)."""
+
+from semdiff.normalize.api import normalize
+from semdiff.normalize.model import (
+    PLACEHOLDER,
+    Action,
+    EntropyMatcher,
+    GroupMatcher,
+    Matcher,
+    NodePredicate,
+    NormalizationRule,
+    NormalizedDoc,
+    RegexMatcher,
+    RuleApplication,
+    RuleFamily,
+    Target,
+)
+from semdiff.normalize.registry import BUILTIN_RULES, RuleRegistry, apply_rules
+
+import semdiff.normalize.rules  # noqa: E402, F401  (registers the built-in families; must stay last)
+
+__all__ = [
+    "BUILTIN_RULES",
+    "PLACEHOLDER",
+    "Action",
+    "EntropyMatcher",
+    "GroupMatcher",
+    "Matcher",
+    "NodePredicate",
+    "NormalizationRule",
+    "NormalizedDoc",
+    "RegexMatcher",
+    "RuleApplication",
+    "RuleFamily",
+    "RuleRegistry",
+    "Target",
+    "apply_rules",
+    "normalize",
+]

semdiff/normalize/api.py

@@ -0,0 +1,25 @@
+"""T-19: the public normalization entry point (FR-10).
+
+``semdiff.normalize()`` is L1 on its own: parse (T-10) → the phase-ordered engine over the
+built-in families (T-12..T-17) → canonical serialization (D-032). It is deterministic — the
+same input and config always produce the same string — and idempotent, because the
+canonicalization phase runs last and reaches a fixpoint.
+"""
+
+from __future__ import annotations
+
+from semdiff.config import Config
+from semdiff.normalize.registry import BUILTIN_RULES, apply_rules
+from semdiff.parse import parse
+
+
+def normalize(html: bytes | str, config: Config | None = None, *, encoding: str | None = None) -> str:
+    """Return ``html`` with presentational noise removed and the tree canonicalized.
+
+    ``bytes`` are decoded as T-10 decides (declared ``encoding`` → BOM → ``<meta charset>``
+    → UTF-8); the result is always ``str``. Raises ``InputTooLargeError`` (NFR-7),
+    ``ParseError`` (FR-3), or ``ValueError`` if the config toggles an unknown rule id.
+    """
+    config = config or Config()
+    doc = parse(html, encoding=encoding, config=config)
+    return apply_rules(doc, BUILTIN_RULES, config.normalization).tree.html or ""