ocr-foundry 0.2.0__py3-none-any.whl

khocr_gen/benchmark.py

@@ -0,0 +1,888 @@
+"""Frozen, verifiable OCR benchmark suites around the offline scorer.
+
+See docs/EVALUATION.md ("Frozen benchmark suites"). Stdlib only: this module hashes
+supplied bytes, validates their relationships, and scores supplied prediction files
+with ``evaluation.score_records``. It never runs OCR, decodes images, downloads data
+or fabricates results. Source and crop files are treated as opaque bytes.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import platform
+import re
+import unicodedata
+from dataclasses import dataclass
+from pathlib import Path, PurePosixPath, PureWindowsPath
+from typing import TYPE_CHECKING, Any
+
+from . import __version__
+from .evaluation import score_records
+
+if TYPE_CHECKING:
+    import os
+    from collections.abc import Mapping, Sequence
+
+SCHEMA_VERSION = 1
+LOCK_KIND = "khocr-gen.benchmark.lock"
+DEFAULT_LOCK = "benchmark.lock.json"
+SPLITS = ("train", "dev", "test")
+ROLES = ("baseline", "recipe", "ablation")
+DATA_KINDS = ("fixture", "real_document")
+STRATA = ("language", "script", "domain", "resolution")
+METADATA_KEYS = ("model", "checkpoint", "training", "recipe", "inference", "seed", "environment")
+STATUSES = ("ok", "failed")
+_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*\Z")
+_EVIDENCE_NOTE = {
+    "fixture": "fixture data: exercises the harness and is not a real-document OCR result",
+    "real_document": "scores apply only to the frozen reference crops of this release",
+}
+
+
+# --------------------------------------------------------------------------- parsing
+
+
+def _no_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise ValueError(f"duplicate JSON key {key!r}")
+        result[key] = value
+    return result
+
+
+def _invalid_constant(value: str) -> None:
+    raise ValueError(f"invalid JSON constant {value}")
+
+
+def _loads(text: str, context: str) -> Any:
+    try:
+        return json.loads(text, object_pairs_hook=_no_duplicates, parse_constant=_invalid_constant)
+    except (ValueError, RecursionError) as exc:
+        raise ValueError(f"{context}: invalid JSON: {exc}") from exc
+
+
+def _string(value: object, field: str, context: str, *, nonempty: bool = True) -> str:
+    if not isinstance(value, str) or (nonempty and not value.strip()):
+        qualifier = "nonempty " if nonempty else ""
+        raise ValueError(f"{context}: {field} must be a {qualifier}string")
+    try:
+        value.encode("utf-8")
+    except UnicodeEncodeError as exc:
+        raise ValueError(f"{context}: {field} contains an unpaired Unicode surrogate") from exc
+    return value
+
+
+def _fields(
+    value: object, context: str, required: tuple[str, ...], optional: tuple[str, ...] = ()
+) -> dict[str, Any]:
+    if not isinstance(value, dict):
+        raise ValueError(f"{context}: must be a JSON object")
+    typed_value: dict[str, Any] = {k: v for k, v in value.items() if isinstance(k, str)}
+    if len(typed_value) != len(value):
+        raise ValueError(f"{context}: must be a JSON object")
+    missing = [key for key in required if key not in typed_value]
+    unknown = sorted(set(typed_value) - set(required) - set(optional))
+    if missing or unknown:
+        raise ValueError(f"{context}: missing fields {missing!r}; unknown fields {unknown!r}")
+    return typed_value
+
+
+def _choice(value: object, field: str, context: str, choices: tuple[str, ...]) -> str:
+    if value not in choices:
+        raise ValueError(f"{context}: {field} must be one of {', '.join(choices)}")
+    return str(value)
+
+
+def _name(value: object, field: str, context: str) -> str:
+    text = _string(value, field, context)
+    if not _NAME.match(text):
+        raise ValueError(
+            f"{context}: {field} {text!r} must use only letters, digits, '.', '_', '-'"
+        )
+    return text
+
+
+def _rel_path(value: object, field: str, context: str) -> str:
+    """Syntactically validate a manifest-root-relative POSIX path."""
+    text = _string(value, field, context)
+    parts = text.split("/")
+    if (
+        "\\" in text
+        or "\x00" in text
+        or PureWindowsPath(text).drive
+        or PurePosixPath(text).is_absolute()
+        or any(part in ("", ".", "..") for part in parts)
+    ):
+        raise ValueError(f"{context}: {field} {text!r} must be a relative path inside the root")
+    return text
+
+
+def _locate(root: Path, rel: str, context: str) -> Path:
+    """Resolve *rel* below *root* (symlinks included) and require a regular file."""
+    path = root.joinpath(*rel.split("/")).resolve()
+    if not path.is_relative_to(root):
+        raise ValueError(f"{context}: {rel!r} escapes the suite root")
+    if not path.is_file():
+        raise ValueError(f"{context}: missing or non-regular file {rel!r}")
+    return path
+
+
+def _sha256_bytes(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _sha256_file(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1 << 20), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _jsonl(data: bytes, context: str) -> list[tuple[str, dict[str, Any]]]:
+    try:
+        text = data.decode("utf-8")
+    except UnicodeDecodeError as exc:
+        raise ValueError(f"{context}: invalid UTF-8: {exc}") from exc
+    lines = text.split("\n")
+    if lines and lines[-1] == "":
+        lines.pop()
+    records: list[tuple[str, dict[str, Any]]] = []
+    for number, line in enumerate(lines, 1):
+        where = f"{context}:{number}"
+        record = _loads(line, where)
+        if not isinstance(record, dict):
+            raise ValueError(f"{where}: each record must be a JSON object")
+        records.append((where, record))
+    if not records:
+        raise ValueError(f"{context}: empty corpus")
+    return records
+
+
+@dataclass(frozen=True)
+class Reference:
+    id: str
+    text: str
+    language: str
+    script: str
+    domain: str
+    document_id: str
+    crop_path: str
+    bounds: tuple[int, int, int, int]
+    resolution: str = "unknown"
+
+def _parse_references(data: bytes, context: str) -> dict[str, Reference]:
+    references: dict[str, Reference] = {}
+    for where, record in _jsonl(data, context):
+        sample_id = _string(record.get("id"), "id", where)
+        if sample_id in references:
+            raise ValueError(f"{where}: duplicate id {sample_id!r}")
+        text = _string(record.get("text"), "text", where, nonempty=False)
+        metadata = record.get("metadata")
+        if not isinstance(metadata, dict):
+            raise ValueError(f"{where}: metadata must be a JSON object")
+        crop = metadata.get("crop")
+        if not isinstance(crop, dict):
+            raise ValueError(f"{where}: metadata.crop must be an object with path and bounds")
+        bounds = crop.get("bounds")
+        if (
+            not isinstance(bounds, list)
+            or len(bounds) != 4
+            or not all(isinstance(v, int) and not isinstance(v, bool) and v >= 0 for v in bounds)
+            or bounds[2] <= bounds[0]
+            or bounds[3] <= bounds[1]
+        ):
+            raise ValueError(
+                f"{where}: metadata.crop.bounds must be [x0, y0, x1, y1] "
+                "nonnegative integers with x1 > x0 and y1 > y0"
+            )
+        res = metadata.get("resolution")
+        if res is None:
+            res = metadata.get("dpi")
+        res_str = str(res) if res is not None else "unknown"
+        references[sample_id] = Reference(
+            id=sample_id,
+            text=text,
+            language=_string(metadata.get("language"), "metadata.language", where),
+            script=_string(metadata.get("script"), "metadata.script", where),
+            domain=_string(metadata.get("domain"), "metadata.domain", where),
+            document_id=_string(metadata.get("document_id"), "metadata.document_id", where),
+            crop_path=_rel_path(crop.get("path"), "metadata.crop.path", where),
+            bounds=(bounds[0], bounds[1], bounds[2], bounds[3]),
+            resolution=res_str,
+        )
+    return references
+
+
+@dataclass(frozen=True)
+class Prediction:
+    text: str
+    status: str
+
+
+def _parse_predictions(data: bytes, context: str) -> dict[str, Prediction]:
+    predictions: dict[str, Prediction] = {}
+    for where, record in _jsonl(data, context):
+        sample_id = _string(record.get("id"), "id", where)
+        if sample_id in predictions:
+            raise ValueError(f"{where}: duplicate id {sample_id!r}")
+        text = _string(record.get("text"), "text", where, nonempty=False)
+        status = _choice(record.get("status"), "status", where, STATUSES)
+        reason = record.get("reason")
+        if status == "failed":
+            _string(reason, "reason (required for failed rows)", where)
+            if text != "":
+                raise ValueError(f"{where}: failed rows must have empty text")
+        elif reason is not None:
+            raise ValueError(f"{where}: reason is only allowed for failed rows")
+        predictions[sample_id] = Prediction(text, status)
+    return predictions
+
+
+# ----------------------------------------------------------------------- spec / lock
+
+
+_MAX_METADATA_DEPTH = 32
+
+
+def _depth(value: object) -> int:
+    """Nesting depth of JSON containers, computed iteratively."""
+    deepest = 0
+    stack = [(value, 1)]
+    while stack:
+        item, level = stack.pop()
+        if isinstance(item, dict | list):
+            deepest = max(deepest, level)
+            stack.extend(
+                (child, level + 1) for child in (item.values() if isinstance(item, dict) else item)
+            )
+    return deepest
+
+
+def _canonical(value: object) -> bytes:
+    return json.dumps(
+        value, sort_keys=True, ensure_ascii=False, allow_nan=False, separators=(",", ":")
+    ).encode("utf-8")
+
+
+def _parse_spec(raw: object, context: str) -> dict[str, Any]:
+    """Validate manifest-shaped input (also reconstructed from a lock) and sort it."""
+    spec = _fields(
+        raw,
+        context,
+        ("schema_version", "release_id", "data_kind", "references", "documents", "runs"),
+    )
+    if spec["schema_version"] != SCHEMA_VERSION or isinstance(spec["schema_version"], bool):
+        raise ValueError(f"{context}: schema_version must be {SCHEMA_VERSION}")
+    _name(spec["release_id"], "release_id", context)
+    _choice(spec["data_kind"], "data_kind", context, DATA_KINDS)
+    _rel_path(spec["references"], "references", context)
+    documents, runs = spec["documents"], spec["runs"]
+    if not isinstance(documents, list) or not documents:
+        raise ValueError(f"{context}: documents must be a nonempty list")
+    if not isinstance(runs, list) or not runs:
+        raise ValueError(f"{context}: runs must be a nonempty list")
+
+    seen_ids: set[str] = set()
+    seen_paths: dict[str, str] = {}
+    group_split: dict[str, str] = {}
+    typed_documents: list[dict[str, Any]] = []
+    for index, doc in enumerate(documents):
+        where = f"{context}: documents[{index}]"
+        doc_fields = _fields(
+            doc, where, ("id", "source_group", "split", "source", "provenance", "rights")
+        )
+        doc_id = _string(doc_fields["id"], "id", where)
+        if doc_id in seen_ids:
+            raise ValueError(f"{where}: duplicate document id {doc_id!r}")
+        seen_ids.add(doc_id)
+        group = _string(doc_fields["source_group"], "source_group", where)
+        split = _choice(doc_fields["split"], "split", where, SPLITS)
+        if group_split.setdefault(group, split) != split:
+            raise ValueError(
+                f"{where}: source_group {group!r} is assigned to splits "
+                f"{group_split[group]!r} and {split!r}"
+            )
+        path = _rel_path(doc_fields["source"], "source", where)
+        if seen_paths.setdefault(path, doc_id) != doc_id:
+            raise ValueError(f"{where}: source {path!r} is shared with another document")
+        _string(doc_fields["provenance"], "provenance", where)
+        _string(doc_fields["rights"], "rights", where)
+        typed_documents.append(doc_fields)
+
+    run_names: set[str] = set()
+    run_paths: set[str] = set()
+    typed_runs: list[dict[str, Any]] = []
+    for index, run in enumerate(runs):
+        where = f"{context}: runs[{index}]"
+        run_fields = _fields(run, where, ("name", "role", "predictions", "metadata"))
+        name = _name(run_fields["name"], "name", where)
+        if name in run_names:
+            raise ValueError(f"{where}: duplicate run name {name!r}")
+        run_names.add(name)
+        _choice(run_fields["role"], "role", where, ROLES)
+        path = _rel_path(run_fields["predictions"], "predictions", where)
+        if path in run_paths:
+            raise ValueError(f"{where}: predictions {path!r} is shared with another run")
+        run_paths.add(path)
+        _fields(run_fields["metadata"], f"{where}: metadata", (), METADATA_KEYS)
+        if _depth(run_fields["metadata"]) > _MAX_METADATA_DEPTH:
+            raise ValueError(f"{where}: metadata nests deeper than {_MAX_METADATA_DEPTH} levels")
+        _canonical(run_fields["metadata"])  # rejects non-JSON-representable values early
+        typed_runs.append(run_fields)
+    return {
+        **spec,
+        "documents": sorted(typed_documents, key=lambda d: d["id"]),
+        "runs": sorted(typed_runs, key=lambda r: r["name"]),
+    }
+
+
+@dataclass
+class Suite:
+    root: Path
+    lock: dict[str, Any]
+    references: dict[str, Reference]
+    document_split: dict[str, str]
+    predictions: dict[str, dict[str, Prediction]]
+
+
+def _build(root: Path, spec: dict[str, Any]) -> tuple[dict[str, Any], Suite]:
+    """Read every file once, validate closure/relationships, and derive the lock."""
+    ref_rel = spec["references"]
+    ref_data = _locate(root, ref_rel, "references").read_bytes()
+    references = _parse_references(ref_data, ref_rel)
+
+    documents = {doc["id"]: doc for doc in spec["documents"]}
+    used = {ref.document_id for ref in references.values()}
+    unknown = sorted(used - documents.keys())
+    if unknown:
+        raise ValueError(f"references name unknown documents {unknown!r}")
+    unused = sorted(documents.keys() - used)
+    if unused:
+        raise ValueError(f"documents without reference crops {unused!r}")
+    split_of = {doc_id: doc["split"] for doc_id, doc in documents.items()}
+
+    leaks: dict[tuple[str, str], dict[str, set[str]]] = {}
+
+    def note(kind: str, digest: str, split: str, owner: str) -> None:
+        by_split = leaks.setdefault((kind, digest), {})
+        by_split.setdefault(split, set()).add(owner)
+
+    doc_entries = []
+    for doc_id, doc in documents.items():
+        digest = _sha256_file(_locate(root, doc["source"], f"document {doc_id!r} source"))
+        note("source", digest, doc["split"], doc_id)
+        doc_entries.append({**doc, "source": {"path": doc["source"], "sha256": digest}})
+
+    crop_entries = []
+    crop_paths: set[str] = set()
+    for ref in sorted(references.values(), key=lambda r: r.id):
+        if ref.crop_path in crop_paths:
+            raise ValueError(f"crop path {ref.crop_path!r} is used by more than one reference")
+        crop_paths.add(ref.crop_path)
+        digest = _sha256_file(_locate(root, ref.crop_path, f"reference {ref.id!r} crop"))
+        note("crop", digest, split_of[ref.document_id], ref.id)
+        crop_entries.append(
+            {
+                "id": ref.id,
+                "document_id": ref.document_id,
+                "path": ref.crop_path,
+                "sha256": digest,
+                "bounds": list(ref.bounds),
+            }
+        )
+    for (kind, digest), by_split in sorted(leaks.items()):
+        if len(by_split) > 1:
+            owners = {s: sorted(o) for s, o in sorted(by_split.items())}
+            raise ValueError(
+                f"split leakage: identical {kind} bytes (sha256 {digest}) appear in "
+                f"multiple splits {owners!r}"
+            )
+
+    run_entries = []
+    predictions: dict[str, dict[str, Prediction]] = {}
+    for run in spec["runs"]:
+        pred_rel = run["predictions"]
+        data = _locate(root, pred_rel, f"run {run['name']!r} predictions").read_bytes()
+        parsed = _parse_predictions(data, f"run {run['name']!r} {pred_rel}")
+        missing = sorted(references.keys() - parsed.keys())
+        extra = sorted(parsed.keys() - references.keys())
+        if missing or extra:
+            raise ValueError(
+                f"run {run['name']!r}: sample ID mismatch: missing prediction IDs {missing!r}; "
+                f"extra prediction IDs {extra!r}"
+            )
+        predictions[run["name"]] = parsed
+        run_entries.append(
+            {
+                "name": run["name"],
+                "role": run["role"],
+                "metadata": run["metadata"],
+                "predictions": {
+                    "path": pred_rel,
+                    "sha256": _sha256_bytes(data),
+                    "records": len(parsed),
+                },
+            }
+        )
+
+    dataset = {
+        "schema_version": SCHEMA_VERSION,
+        "release_id": spec["release_id"],
+        "data_kind": spec["data_kind"],
+        "reference": {
+            "path": ref_rel,
+            "sha256": _sha256_bytes(ref_data),
+            "records": len(references),
+        },
+        "documents": doc_entries,
+        "crops": crop_entries,
+    }
+    dataset_fp = _sha256_bytes(_canonical(dataset))
+    suite_fp = _sha256_bytes(_canonical({"dataset_fingerprint": dataset_fp, "runs": run_entries}))
+    lock = {
+        "kind": LOCK_KIND,
+        **dataset,
+        "runs": run_entries,
+        "dataset_fingerprint": dataset_fp,
+        "suite_fingerprint": suite_fp,
+    }
+    return lock, Suite(root, lock, references, split_of, predictions)
+
+
+def _lock_bytes(lock: dict[str, Any]) -> bytes:
+    text = json.dumps(lock, sort_keys=True, indent=2, ensure_ascii=False, allow_nan=False)
+    return (text + "\n").encode("utf-8")
+
+
+def _summary(lock: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "release_id": lock["release_id"],
+        "data_kind": lock["data_kind"],
+        "dataset_fingerprint": lock["dataset_fingerprint"],
+        "suite_fingerprint": lock["suite_fingerprint"],
+        "documents": len(lock["documents"]),
+        "references": lock["reference"]["records"],
+        "runs": [run["name"] for run in lock["runs"]],
+    }
+
+
+def freeze(manifest_path: str | os.PathLike[str], lock_name: str = DEFAULT_LOCK) -> dict[str, Any]:
+    """Validate a manifest and write its deterministic lock beside the inputs.
+
+    The manifest directory is the suite root. The lock must be a new file directly in
+    that directory (verify treats the lock's directory as the root); an existing lock
+    is never overwritten. Returns a summary dict.
+    """
+    manifest = Path(manifest_path)
+    root = manifest.parent.resolve()
+    lock_rel = _rel_path(lock_name, "lock", "freeze")
+    if "/" in lock_rel:
+        raise ValueError(
+            f"freeze: lock {lock_rel!r} must be a file name directly beside the manifest"
+        )
+    lock_path = root / lock_rel
+    if lock_path.exists() or lock_path.is_symlink():
+        raise ValueError(f"freeze: refusing to overwrite existing lock {lock_rel!r}")
+    spec = _parse_spec(
+        _loads(manifest.read_text(encoding="utf-8"), manifest.name), f"manifest {manifest.name}"
+    )
+    lock, _ = _build(root, spec)
+    data = _lock_bytes(lock)
+    try:
+        with lock_path.open("xb") as stream:
+            stream.write(data)
+    except FileExistsError as exc:
+        raise ValueError(f"freeze: refusing to overwrite existing lock {lock_rel!r}") from exc
+    except BaseException:
+        lock_path.unlink(missing_ok=True)
+        raise
+    return {"lock": lock_rel, **_summary(lock)}
+
+
+def _first_difference(locked: object, actual: object, path: str = "lock") -> str | None:
+    if isinstance(locked, dict) and isinstance(actual, dict):
+        locked_map: dict[str, Any] = {str(k): v for k, v in locked.items()}
+        actual_map: dict[str, Any] = {str(k): v for k, v in actual.items()}
+        all_keys = sorted(set(locked_map) | set(actual_map))
+        for key in all_keys:
+            if key not in locked_map or key not in actual_map:
+                return f"{path}.{key} is {'absent' if key not in locked_map else 'unexpected'}"
+            found = _first_difference(locked_map[key], actual_map[key], f"{path}.{key}")
+            if found:
+                return found
+        return None
+    if isinstance(locked, list) and isinstance(actual, list):
+        if len(locked) != len(actual):
+            return f"{path} has {len(locked)} entries, expected {len(actual)}"
+        for index, (a, b) in enumerate(zip(locked, actual, strict=True)):
+            found = _first_difference(a, b, f"{path}[{index}]")
+            if found:
+                return found
+        return None
+    if type(locked) is not type(actual) or locked != actual:
+        return f"{path} is {str(locked)[:80]!r} but actual is {str(actual)[:80]!r}"
+    return None
+
+
+def _spec_from_lock(lock: object) -> dict[str, Any]:
+    try:
+        assert isinstance(lock, dict)
+        lock_dict: dict[str, Any] = {k: v for k, v in lock.items() if isinstance(k, str)}
+        if lock_dict.get("kind") != LOCK_KIND:
+            raise ValueError("not a khocr-gen benchmark lock")
+        ref = lock_dict["reference"]
+        if not isinstance(ref, dict):
+            raise TypeError
+        raw_docs = lock_dict["documents"]
+        if not isinstance(raw_docs, list):
+            raise TypeError
+        raw_runs = lock_dict["runs"]
+        if not isinstance(raw_runs, list):
+            raise TypeError
+        docs: list[dict[str, Any]] = []
+        for doc in raw_docs:
+            if not isinstance(doc, dict):
+                raise TypeError
+            doc_src = doc["source"]
+            if not isinstance(doc_src, dict):
+                raise TypeError
+            doc_entry: dict[str, Any] = {k: v for k, v in doc.items() if k != "source"}
+            doc_entry["source"] = doc_src["path"]
+            docs.append(doc_entry)
+        runs: list[dict[str, Any]] = []
+        for run in raw_runs:
+            if not isinstance(run, dict):
+                raise TypeError
+            run_preds = run["predictions"]
+            if not isinstance(run_preds, dict):
+                raise TypeError
+            runs.append(
+                {
+                    "name": run["name"],
+                    "role": run["role"],
+                    "metadata": run["metadata"],
+                    "predictions": run_preds["path"],
+                }
+            )
+        spec = {
+            "schema_version": lock_dict["schema_version"],
+            "release_id": lock_dict["release_id"],
+            "data_kind": lock_dict["data_kind"],
+            "references": ref["path"],
+            "documents": docs,
+            "runs": runs,
+        }
+    except (KeyError, TypeError, AttributeError, AssertionError) as exc:
+        raise ValueError("lock: malformed structure") from exc
+    return _parse_spec(spec, "lock")
+
+
+def load_suite(
+    lock_path: str | os.PathLike[str],
+    *,
+    expect_dataset_fingerprint: str | None = None,
+    expect_suite_fingerprint: str | None = None,
+) -> Suite:
+    """Fully verify a lock against the files beside it; raise ValueError/OSError."""
+    path = Path(lock_path)
+    root = path.parent.resolve()
+    locked = _loads(path.read_text(encoding="utf-8"), path.name)
+    lock, suite = _build(root, _spec_from_lock(locked))
+    difference = _first_difference(locked, lock)
+    if difference:
+        raise ValueError(f"frozen suite does not match its files: {difference}")
+    for label, expected, actual in (
+        ("dataset", expect_dataset_fingerprint, lock["dataset_fingerprint"]),
+        ("suite", expect_suite_fingerprint, lock["suite_fingerprint"]),
+    ):
+        if expected is not None and expected != actual:
+            raise ValueError(f"{label} fingerprint {actual} does not match expected {expected}")
+    return suite
+
+
+def verify(
+    lock_path: str | os.PathLike[str],
+    *,
+    check_near_duplicates: bool = False,
+    near_duplicate_threshold: float = 0.8,
+    **expected: str | None,
+) -> dict[str, Any]:
+    """Verify closure, hashes, relationships and leakage; return a summary."""
+    suite = load_suite(lock_path, **expected)
+    if check_near_duplicates:
+        records_by_split: dict[str, list[tuple[str, str]]] = {}
+        for ref_id, ref in suite.references.items():
+            split = suite.document_split.get(ref.document_id, "unknown")
+            records_by_split.setdefault(split, []).append((ref_id, ref.text))
+        leaks = check_near_duplicate_leakage(
+            records_by_split, threshold=near_duplicate_threshold
+        )
+        if leaks:
+            raise ValueError(
+                f"near-duplicate cross-split leakage detected ({len(leaks)} incident(s)): {leaks[:3]}"
+            )
+    summary = _summary(suite.lock)
+    summary["near_duplicates_checked"] = bool(check_near_duplicates)
+    return {"status": "ok", **summary}
+
+def check_near_duplicate_leakage(
+    records_by_split: Mapping[str, Sequence[str | tuple[str, str]]],
+    *,
+    threshold: float = 0.8,
+    ngram_size: int = 3,
+) -> list[dict[str, Any]]:
+    """Detect cross-split leakage of nearly identical lines using character n-gram Jaccard similarity.
+
+    records_by_split maps split name to a sequence of texts or (id, text) tuples.
+    Returns a list of leak incidents:
+        [{"splits": (split_a, split_b), "similarity": float, "text_a": str, "text_b": str, ...}]
+    """
+    def _ngrams(s: str) -> set[str]:
+        cleaned = unicodedata.normalize("NFC", s).strip().lower()
+        if len(cleaned) < ngram_size:
+            return {cleaned} if cleaned else set()
+        return {cleaned[i : i + ngram_size] for i in range(len(cleaned) - ngram_size + 1)}
+
+    split_items: list[tuple[str, str, str, set[str]]] = []
+    for split_name, items in records_by_split.items():
+        for idx, item in enumerate(items):
+            if isinstance(item, tuple):
+                item_id, item_text = item[0], item[1]
+            else:
+                item_id, item_text = f"{split_name}:{idx}", str(item)
+            ngs = _ngrams(item_text)
+            if ngs:
+                split_items.append((split_name, item_id, item_text, ngs))
+
+    leaks: list[dict[str, Any]] = []
+    for i in range(len(split_items)):
+        split_a, id_a, text_a, ngs_a = split_items[i]
+        for j in range(i + 1, len(split_items)):
+            split_b, id_b, text_b, ngs_b = split_items[j]
+            if split_a == split_b:
+                continue
+            intersection = len(ngs_a & ngs_b)
+            union = len(ngs_a | ngs_b)
+            sim = (intersection / union) if union else 0.0
+            if sim >= threshold:
+                leaks.append({
+                    "splits": (split_a, split_b),
+                    "similarity": round(sim, 4),
+                    "id_a": id_a,
+                    "text_a": text_a,
+                    "id_b": id_b,
+                    "text_b": text_b,
+                })
+    return leaks
+
+
+def format_comparison_markdown(report: dict[str, Any], *, variant: str = "raw") -> str:
+    """Format a benchmark compare report into a Markdown table with CER, line accuracy, and deltas."""
+    baseline = report.get("baseline")
+    runs = report.get("runs", [])
+    deltas = report.get("deltas", {})
+    data_kind = report.get("data_kind")
+    evidence_note = report.get("evidence_note")
+
+    lines = [
+        f"# Benchmark Comparison ({report.get('release_id', 'unknown')})",
+        "",
+        f"- **Splits**: {', '.join(report.get('splits', []))}",
+        f"- **Baseline**: {baseline}",
+        f"- **Variant**: {variant}",
+    ]
+    if data_kind:
+        lines.append(f"- **Data Kind**: {data_kind}")
+    if evidence_note:
+        lines.append(f"- **Evidence Note**: {evidence_note}")
+    lines.extend([
+        "",
+        "| Run | Role | CER | Line Accuracy | Δ CER | Δ Line Accuracy |",
+        "| :--- | :--- | :---: | :---: | :---: | :---: |",
+    ])
+
+    for run in runs:
+        name = run["name"]
+        role = run["role"]
+        metrics = run.get(variant, {}).get("overall", {}).get("metrics", {})
+        cer = metrics.get("cer")
+        cer_str = f"{cer:.4f}" if cer is not None else "N/A"
+        acc = metrics.get("line_accuracy")
+        acc_str = f"{acc:.4f}" if acc is not None else "N/A"
+
+        if name == baseline:
+            d_cer_str = "-"
+            d_acc_str = "-"
+        else:
+            d_block = deltas.get(name, {}).get(variant, {}).get("overall", {})
+            d_cer = d_block.get("cer")
+            d_cer_str = f"{d_cer:+.4f}" if d_cer is not None else "N/A"
+            d_acc = d_block.get("line_accuracy")
+            d_acc_str = f"{d_acc:+.4f}" if d_acc is not None else "N/A"
+
+        lines.append(f"| {name} | {role} | {cer_str} | {acc_str} | {d_cer_str} | {d_acc_str} |")
+
+    return "\n".join(lines)
+
+# ------------------------------------------------------------------------ comparison
+
+_DELTA_METRICS = ("line_accuracy", "cer", "wer")
+
+
+def _counts(ids: list[str], predictions: dict[str, Prediction]) -> dict[str, int]:
+    chosen = [predictions[i] for i in ids]
+    return {
+        "attempted": len(chosen),
+        "succeeded": sum(p.status == "ok" for p in chosen),
+        "failed": sum(p.status == "failed" for p in chosen),
+        "empty_output": sum(p.status == "ok" and p.text == "" for p in chosen),
+    }
+
+
+def _score_run(
+    suite: Suite,
+    ids: list[str],
+    predictions: dict[str, Prediction],
+    normalization: str,
+    word_tokenization: str | None,
+) -> dict[str, Any]:
+    """Return per-variant ``{scope: {"counts", "metrics"}}`` blocks for one run."""
+    texts = {i: predictions[i].text for i in ids}
+    variants: dict[str, dict[str, Any]] = {}
+    for stratum in STRATA:
+        labels = {i: getattr(suite.references[i], stratum) for i in ids}
+        report = score_records(
+            {i: (suite.references[i].text, labels[i]) for i in ids},
+            texts,
+            normalization=normalization,
+            word_tokenization=word_tokenization,
+        )
+        members = {}
+        for sample_id, label in labels.items():
+            members.setdefault(label, []).append(sample_id)
+        for variant in ("raw", "normalized"):
+            if variant not in report:
+                continue
+            blocks = variants.setdefault(variant, {})
+            blocks.setdefault(
+                "overall",
+                {"counts": _counts(ids, predictions), "metrics": report[variant]["overall"]},
+            )
+            blocks[f"by_{stratum}"] = {
+                label: {
+                    "counts": _counts(members[label], predictions),
+                    "metrics": metrics,
+                }
+                for label, metrics in report[variant]["by_language"].items()
+            }
+    return variants
+
+
+def _delta(candidate: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for metric in _DELTA_METRICS:
+        if metric in candidate["metrics"]:
+            a, b = candidate["metrics"][metric], baseline["metrics"][metric]
+            result[metric] = None if a is None or b is None else a - b
+    for count in ("failed", "empty_output"):
+        result[count] = candidate["counts"][count] - baseline["counts"][count]
+    return result
+
+
+def _deltas(candidate: dict[str, Any], baseline: dict[str, Any]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for scope, block in candidate.items():
+        if scope == "overall":
+            result[scope] = _delta(block, baseline[scope])
+        else:  # by_<stratum>: labels are reference data and may be any string
+            result[scope] = {
+                label: _delta(entry, baseline[scope][label]) for label, entry in block.items()
+            }
+    return result
+
+
+def compare(
+    lock_path: str | os.PathLike[str],
+    *,
+    splits: list[str],
+    baseline: str | None = None,
+    normalization: str = "none",
+    word_tokenization: str | None = None,
+    expect_dataset_fingerprint: str | None = None,
+    expect_suite_fingerprint: str | None = None,
+) -> dict[str, Any]:
+    """Verify the suite, then score every named run on the selected frozen references.
+
+    Returns one JSON-serializable report; any failure raises ValueError/OSError so
+    callers never emit a partial report. Deltas are candidate minus baseline, with
+    undefined (zero-denominator) metrics reported as ``None``.
+    """
+    suite = load_suite(
+        lock_path,
+        expect_dataset_fingerprint=expect_dataset_fingerprint,
+        expect_suite_fingerprint=expect_suite_fingerprint,
+    )
+    lock = suite.lock
+    if not splits or any(s not in SPLITS for s in splits):
+        raise ValueError(f"splits must be a nonempty subset of {', '.join(SPLITS)}")
+    chosen_splits = sorted(set(splits), key=SPLITS.index)
+    names = [run["name"] for run in lock["runs"]]
+    if baseline is None:
+        baselines = [run["name"] for run in lock["runs"] if run["role"] == "baseline"]
+        if len(baselines) != 1:
+            raise ValueError(
+                f"--baseline is required unless exactly one run has role baseline "
+                f"(found {baselines!r})"
+            )
+        baseline = baselines[0]
+    elif baseline not in names:
+        raise ValueError(f"unknown baseline run {baseline!r}; runs: {names!r}")
+
+    ids = sorted(
+        sample_id
+        for sample_id, ref in suite.references.items()
+        if suite.document_split[ref.document_id] in chosen_splits
+    )
+    if not ids:
+        raise ValueError(f"no frozen references in splits {chosen_splits!r}")
+
+    scored: dict[str, dict[str, Any]] = {}
+    runs = []
+    for run in lock["runs"]:
+        scored[run["name"]] = _score_run(
+            suite, ids, suite.predictions[run["name"]], normalization, word_tokenization
+        )
+        runs.append({**run, **scored[run["name"]]})
+    deltas = {
+        name: {
+            "baseline": baseline,
+            **{variant: _deltas(blocks[variant], scored[baseline][variant]) for variant in blocks},
+        }
+        for name, blocks in scored.items()
+        if name != baseline
+    }
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "scorer_schema_version": 1,
+        "release_id": lock["release_id"],
+        "data_kind": lock["data_kind"],
+        "evidence_note": _EVIDENCE_NOTE[lock["data_kind"]],
+        "dataset_fingerprint": lock["dataset_fingerprint"],
+        "suite_fingerprint": lock["suite_fingerprint"],
+        "reference": lock["reference"],
+        "splits": chosen_splits,
+        "selection": {"references": len(ids), "sha256": _sha256_bytes(_canonical(ids))},
+        "options": {"normalization": normalization, "word_tokenization": word_tokenization},
+        "baseline": baseline,
+        "runtime": {
+            "python": platform.python_version(),
+            "implementation": platform.python_implementation(),
+            "unicode_version": unicodedata.unidata_version,
+            "khocr_gen": __version__,
+        },
+        "runs": runs,
+        "deltas": deltas,
+    }