verifydoc 0.1.0__py3-none-any.whl

verifydoc/calibration/conformal.py

@@ -0,0 +1,75 @@
+"""Split conformal abstention / conformal risk control (PROJECT.md §5.G).
+
+Chooses the acceptance threshold on the calibration split so that the
+*expected* selective risk of accepted fields is bounded by the target alpha
+under exchangeability (conformal risk control with the bounded miscoverage
+loss; Angelopoulos et al.; conformal factuality, Mohri & Hashimoto 2024).
+
+Threshold rule: the smallest confidence t such that on the calibration split
+
+    (#errors accepted at t + 1) / (#accepted at t + 1) <= alpha
+
+The +1 terms are the finite-sample correction (the future test point counted
+as a worst-case error). Per arXiv:2606.29054, meaningful certification can
+force heavy abstention — so this class also reports the abstention it forces
+(``abstention_rate_`` on the calibration split), and the benchmark reports
+both numbers side by side.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+
+import numpy as np
+
+
+class ConformalAbstention:
+    """Distribution-free accept/review policy at a target selective risk."""
+
+    threshold_: float
+    abstention_rate_: float
+    alpha: float
+
+    def __init__(self, alpha: float = 0.05) -> None:
+        if not 0.0 < alpha < 1.0:
+            raise ValueError("alpha must be in (0, 1)")
+        self.alpha = alpha
+        self._fitted = False
+
+    def fit(self, conf: Sequence[float], correct: Sequence[int]) -> ConformalAbstention:
+        """Fit the threshold on the CALIBRATION split (never test)."""
+        c = np.asarray(conf, dtype=float)
+        y = np.asarray(correct, dtype=float)
+        if c.shape != y.shape or c.ndim != 1 or c.size == 0:
+            raise ValueError("conf and correct must be 1-D, equal-length, non-empty")
+
+        order = np.argsort(-c, kind="stable")
+        c_sorted, y_sorted = c[order], y[order]
+        n_accepted = np.arange(1, c.size + 1)
+        n_errors = np.cumsum(1.0 - y_sorted)
+        bound = (n_errors + 1.0) / (n_accepted + 1.0)
+        boundary: np.ndarray = np.append(c_sorted[:-1] > c_sorted[1:], True)
+        ok = (bound <= self.alpha) & boundary
+        if ok.any():
+            k = int(np.flatnonzero(ok)[-1])
+            self.threshold_ = float(c_sorted[k])
+            self.abstention_rate_ = float(1.0 - (k + 1) / c.size)
+        else:
+            self.threshold_ = float("inf")  # certification impossible: review everything
+            self.abstention_rate_ = 1.0
+        self._fitted = True
+        return self
+
+    def accept(self, conf: Sequence[float]) -> np.ndarray:
+        """Boolean accept mask for new confidences (True = auto-accept)."""
+        if not self._fitted:
+            raise RuntimeError("ConformalAbstention must be fit on the calibration split first")
+        return np.asarray(conf, dtype=float) >= self.threshold_
+
+    @property
+    def guarantee(self) -> str:
+        return (
+            f"E[selective risk] <= {self.alpha:.3f} under exchangeability "
+            f"(split conformal, finite-sample corrected); forced abstention on "
+            f"calibration split: {self.abstention_rate_:.1%}"
+        )

verifydoc/calibration/histogram.py

@@ -0,0 +1,34 @@
+"""Histogram binning (Zadrozny & Elkan 2001): equal-width bins over [0, 1].
+
+Each bin's calibrated confidence is its empirical accuracy on the calibration
+split; empty bins fall back to the global accuracy.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from verifydoc.calibration.base import Calibrator
+
+
+class HistogramBinning(Calibrator):
+    def __init__(self, n_bins: int = 15) -> None:
+        self.n_bins = n_bins
+
+    bin_accuracy_: np.ndarray
+
+    def _fit(self, conf: np.ndarray, correct: np.ndarray) -> None:
+        edges = np.linspace(0.0, 1.0, self.n_bins + 1)
+        idx = np.clip(np.digitize(conf, edges[1:-1], right=True), 0, self.n_bins - 1)
+        global_acc = float(correct.mean())
+        acc = np.full(self.n_bins, global_acc)
+        for m in range(self.n_bins):
+            mask = idx == m
+            if mask.any():
+                acc[m] = float(correct[mask].mean())
+        self.bin_accuracy_ = acc
+
+    def _transform(self, conf: np.ndarray) -> np.ndarray:
+        edges = np.linspace(0.0, 1.0, self.n_bins + 1)
+        idx = np.clip(np.digitize(conf, edges[1:-1], right=True), 0, self.n_bins - 1)
+        return self.bin_accuracy_[idx]

verifydoc/calibration/isotonic.py

@@ -0,0 +1,23 @@
+"""Isotonic regression calibration (Zadrozny & Elkan 2002).
+
+Non-parametric monotone mapping from confidence to empirical accuracy;
+the strongest of the classic calibrators given enough calibration data.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from sklearn.isotonic import IsotonicRegression
+
+from verifydoc.calibration.base import Calibrator
+
+
+class IsotonicCalibrator(Calibrator):
+    _model: IsotonicRegression
+
+    def _fit(self, conf: np.ndarray, correct: np.ndarray) -> None:
+        self._model = IsotonicRegression(y_min=0.0, y_max=1.0, out_of_bounds="clip")
+        self._model.fit(conf, correct)
+
+    def _transform(self, conf: np.ndarray) -> np.ndarray:
+        return np.asarray(self._model.predict(conf), dtype=float)

verifydoc/calibration/platt.py

@@ -0,0 +1,30 @@
+"""Platt scaling: logistic regression a*logit(p) + b (Platt 1999).
+
+Unlike temperature scaling it can also shift systematically biased scores,
+not just soften them.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from sklearn.linear_model import LogisticRegression
+
+from verifydoc.calibration.base import Calibrator, logit, sigmoid
+
+
+class PlattScaling(Calibrator):
+    a_: float
+    b_: float
+
+    def _fit(self, conf: np.ndarray, correct: np.ndarray) -> None:
+        if len(np.unique(correct)) < 2:
+            # degenerate split (all right or all wrong): identity mapping
+            self.a_, self.b_ = 1.0, 0.0
+            return
+        model = LogisticRegression(C=1e6, solver="lbfgs")
+        model.fit(logit(conf).reshape(-1, 1), correct)
+        self.a_ = float(model.coef_[0][0])
+        self.b_ = float(model.intercept_[0])
+
+    def _transform(self, conf: np.ndarray) -> np.ndarray:
+        return sigmoid(self.a_ * logit(conf) + self.b_)

verifydoc/calibration/splits.py

@@ -0,0 +1,44 @@
+"""Split management: the never-tune-on-test rule, enforced in code.
+
+Golden rule #4: calibration and abstention thresholds are fit only on a
+dedicated calibration split. These helpers create and verify disjoint splits
+by item id; every harness that fits a calibrator must call
+``assert_disjoint`` before scoring.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import TypeVar
+
+import numpy as np
+
+T = TypeVar("T")
+
+
+def split_calibration(
+    ids: Sequence[T], calibration_fraction: float = 0.5, seed: int = 0
+) -> tuple[list[T], list[T]]:
+    """Deterministically split ids into (calibration, test); disjoint and complete."""
+    if not 0.0 < calibration_fraction < 1.0:
+        raise ValueError("calibration_fraction must be in (0, 1)")
+    unique = list(dict.fromkeys(ids))
+    if len(unique) < 2:
+        raise ValueError("need at least two distinct ids to split")
+    rng = np.random.default_rng(seed)
+    order = rng.permutation(len(unique))
+    n_cal = max(1, min(len(unique) - 1, round(calibration_fraction * len(unique))))
+    cal = [unique[i] for i in sorted(order[:n_cal])]
+    test = [unique[i] for i in sorted(order[n_cal:])]
+    return cal, test
+
+
+def assert_disjoint(calibration_ids: Sequence[T], test_ids: Sequence[T]) -> None:
+    """Raise if any id appears in both splits (tuning-on-test guard)."""
+    overlap = set(calibration_ids) & set(test_ids)
+    if overlap:
+        sample = sorted(str(x) for x in overlap)[:5]
+        raise ValueError(
+            f"calibration and test splits overlap on {len(overlap)} ids "
+            f"(e.g. {sample}); calibration must never see test data"
+        )

verifydoc/calibration/temperature.py

@@ -0,0 +1,30 @@
+"""Temperature scaling (Guo et al. 2017) on the confidence's log-odds.
+
+Calibrated p' = sigmoid(logit(p) / T); the single scalar T is chosen to
+minimize NLL on the calibration split. T > 1 softens overconfident scores.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+from scipy.optimize import minimize_scalar
+
+from verifydoc.calibration.base import Calibrator, logit, sigmoid
+
+
+class TemperatureScaling(Calibrator):
+    temperature_: float
+
+    def _fit(self, conf: np.ndarray, correct: np.ndarray) -> None:
+        z = logit(conf)
+
+        def nll_at(log_t: float) -> float:
+            p = sigmoid(z / np.exp(log_t))
+            p = np.clip(p, 1e-12, 1.0 - 1e-12)
+            return float(-np.mean(correct * np.log(p) + (1.0 - correct) * np.log(1.0 - p)))
+
+        result = minimize_scalar(nll_at, bounds=(-4.0, 4.0), method="bounded")
+        self.temperature_ = float(np.exp(result.x))
+
+    def _transform(self, conf: np.ndarray) -> np.ndarray:
+        return sigmoid(logit(conf) / self.temperature_)

verifydoc/cli.py

@@ -0,0 +1,48 @@
+"""CLI: ``verifydoc extract doc.pdf --schema schema.json --target... ``."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import typer
+
+import verifydoc
+from verifydoc.adapters import get_adapter
+from verifydoc.pipeline import DEFAULT_THRESHOLD, verify
+
+app = typer.Typer(name="verifydoc", help="Trust layer for document -> JSON extraction.")
+
+
+@app.command()
+def extract(
+    source: Path = typer.Argument(..., help="Document to extract from (pdf/image/txt)."),
+    schema: Path = typer.Option(..., "--schema", "-s", help="JSON Schema with x-scoring rules."),
+    adapter: str = typer.Option("text-search", "--adapter", "-a", help="Extractor adapter."),
+    k: int = typer.Option(1, "--k", help="Self-consistency samples (k > 1 enables consensus)."),
+    threshold: float = typer.Option(
+        DEFAULT_THRESHOLD, "--threshold", "-t", help="Accept-decision confidence cutoff."
+    ),
+    out: Path | None = typer.Option(None, "--out", "-o", help="Write JSON here (else stdout)."),
+) -> None:
+    """Extract fields with confidence + grounding + accept/review decisions."""
+    result = verify(source, schema, adapter=get_adapter(adapter), k=k, threshold=threshold)
+    payload = json.dumps(result.to_dict(), indent=2, ensure_ascii=False)
+    if out is not None:
+        out.write_text(payload + "\n", encoding="utf-8")
+        typer.echo(
+            f"{result.doc_id}: {result.n_accepted} accepted, "
+            f"{result.n_review} to review -> {out}"
+        )
+    else:
+        typer.echo(payload)
+
+
+@app.command()
+def version() -> None:
+    """Print the installed VerifyDoc version."""
+    typer.echo(verifydoc.__version__)
+
+
+if __name__ == "__main__":  # pragma: no cover
+    app()

verifydoc/confidence/__init__.py

@@ -0,0 +1,18 @@
+"""Confidence signals (PROJECT.md §5.G): raw per-field scores before calibration."""
+
+from verifydoc.confidence.combined import combined_confidence
+from verifydoc.confidence.consensus import consensus
+from verifydoc.confidence.grounding_based import apply_grounding_confidence, grounding_confidence
+from verifydoc.confidence.token_prob import apply_token_prob, token_prob_confidence
+from verifydoc.confidence.verbalized import apply_verbalized, verbalized_confidence
+
+__all__ = [
+    "apply_grounding_confidence",
+    "apply_token_prob",
+    "apply_verbalized",
+    "combined_confidence",
+    "consensus",
+    "grounding_confidence",
+    "token_prob_confidence",
+    "verbalized_confidence",
+]

verifydoc/confidence/combined.py

@@ -0,0 +1,37 @@
+"""Combined confidence: fuse the available signals for one field.
+
+# DECISION: v0.1 uses a transparent weighted mean over whichever signals are
+# present (weights renormalized to the available subset). A learned combiner
+# (logistic regression on the calibration split) can replace this behind the
+# same function without touching other stages; the paper compares both.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+
+DEFAULT_WEIGHTS: dict[str, float] = {
+    "consensus": 0.5,
+    "grounding": 0.3,
+    "token_prob": 0.1,
+    "verbalized": 0.1,
+}
+
+
+def combined_confidence(
+    signals: Mapping[str, float | None],
+    weights: Mapping[str, float] | None = None,
+) -> float:
+    """Weighted mean of available signals; unknown signal names are rejected."""
+    w = dict(weights) if weights is not None else DEFAULT_WEIGHTS
+    unknown = set(signals) - set(w)
+    if unknown:
+        raise ValueError(f"unknown signals: {sorted(unknown)}")
+    available = {name: v for name, v in signals.items() if v is not None}
+    if not available:
+        raise ValueError("no signals available")
+    total_weight = sum(w[name] for name in available)
+    if total_weight <= 0:
+        raise ValueError("available signals have zero total weight")
+    value = sum(w[name] * v for name, v in available.items()) / total_weight
+    return min(1.0, max(0.0, value))

verifydoc/confidence/consensus.py

@@ -0,0 +1,71 @@
+"""Consensus (self-consistency) confidence — the black-box VerifyDoc default.
+
+Run the extractor k times (or run m different extractors) and use agreement as
+confidence: no logits needed, works for any adapter.
+
+# DECISION (consensus semantics, pinned by tests):
+# - Votes are normalized values (whitespace-collapsed, casefolded); a sample
+#   that omits the field (or predicts None) casts an explicit "omit" vote.
+# - The consensus value is the modal vote's first-seen raw value; confidence =
+#   modal vote count / number of samples. Ties break by first appearance.
+# - If the modal vote is "omit", the consensus prediction has value None with
+#   confidence = omit fraction (the abstention layer treats it as omitted).
+# - Grounding is taken from the first sample that cast the modal vote.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+
+from verifydoc.eval.extraction import normalize_text
+from verifydoc.types import FieldPrediction
+
+_OMIT = "<omit>"
+
+
+def _vote(value: Any) -> str:
+    return _OMIT if value is None else normalize_text(value).casefold()
+
+
+def consensus(samples: list[list[FieldPrediction]]) -> list[FieldPrediction]:
+    """Aggregate k sample runs into one prediction per field path."""
+    if not samples:
+        raise ValueError("need at least one sample run")
+    k = len(samples)
+
+    paths: list[str] = []
+    seen: set[str] = set()
+    for run in samples:
+        for sample_pred in run:
+            if sample_pred.path not in seen:
+                seen.add(sample_pred.path)
+                paths.append(sample_pred.path)
+
+    out: list[FieldPrediction] = []
+    for path in paths:
+        votes: list[str] = []
+        first_pred: dict[str, FieldPrediction] = {}
+        for run in samples:
+            by_path = {p.path: p for p in run}
+            pred = by_path.get(path)
+            vote = _vote(pred.value) if pred is not None else _OMIT
+            votes.append(vote)
+            if vote not in first_pred and pred is not None:
+                first_pred[vote] = pred
+        winner, count = Counter(votes).most_common(1)[0]
+        agreement = count / k
+        source = first_pred.get(winner)
+        if winner == _OMIT or source is None:
+            out.append(FieldPrediction(path=path, value=None, confidence=agreement))
+        else:
+            out.append(
+                FieldPrediction(
+                    path=path,
+                    value=source.value,
+                    confidence=agreement,
+                    grounding=source.grounding,
+                    meta=dict(source.meta),
+                )
+            )
+    return out

verifydoc/confidence/grounding_based.py

@@ -0,0 +1,29 @@
+"""Grounding-based confidence: provenance quality as a trust signal.
+
+The hypothesis (risk-controlled generative OCR, arXiv:2603.19790): values that
+cannot be traced to a tight source region are more likely hallucinated. The
+signal is the grounding's ``support`` score (e.g. string-match/IoU quality
+computed by the grounder); a field with no grounding at all gets
+``ungrounded_confidence`` (default 0).
+"""
+
+from __future__ import annotations
+
+from verifydoc.types import FieldPrediction
+
+
+def grounding_confidence(pred: FieldPrediction, ungrounded_confidence: float = 0.0) -> float:
+    """Confidence from provenance: the grounding's support, else the floor."""
+    if pred.grounding is None:
+        return ungrounded_confidence
+    return pred.grounding.support
+
+
+def apply_grounding_confidence(
+    predictions: list[FieldPrediction], ungrounded_confidence: float = 0.0
+) -> list[FieldPrediction]:
+    """Return copies with ``confidence`` set from grounding support."""
+    return [
+        pred.model_copy(update={"confidence": grounding_confidence(pred, ungrounded_confidence)})
+        for pred in predictions
+    ]

verifydoc/confidence/token_prob.py

@@ -0,0 +1,43 @@
+"""Token/sequence-probability confidence (requires logit access).
+
+Adapters that see logits stash per-field token log-probs in
+``FieldPrediction.meta["token_logprobs"]``; this signal turns them into a
+confidence: geometric-mean token probability (``mean``), worst token (``min``),
+or full sequence probability (``prod``).
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Literal
+
+from verifydoc.types import FieldPrediction
+
+Aggregate = Literal["mean", "min", "prod"]
+
+META_KEY = "token_logprobs"
+
+
+def token_prob_confidence(pred: FieldPrediction, aggregate: Aggregate = "mean") -> float | None:
+    """Confidence from token log-probs; ``None`` when the adapter gave none."""
+    logprobs = pred.meta.get(META_KEY)
+    if not logprobs:
+        return None
+    if aggregate == "mean":
+        value = math.exp(sum(logprobs) / len(logprobs))
+    elif aggregate == "min":
+        value = math.exp(min(logprobs))
+    else:
+        value = math.exp(sum(logprobs))
+    return min(1.0, max(0.0, value))
+
+
+def apply_token_prob(
+    predictions: list[FieldPrediction], aggregate: Aggregate = "mean"
+) -> list[FieldPrediction]:
+    """Return copies with ``confidence`` set from token log-probs where available."""
+    out = []
+    for pred in predictions:
+        conf = token_prob_confidence(pred, aggregate)
+        out.append(pred if conf is None else pred.model_copy(update={"confidence": conf}))
+    return out

verifydoc/confidence/verbalized.py

@@ -0,0 +1,30 @@
+"""Verbalized confidence: the extractor rates its own answer (0-1).
+
+Adapters that prompt the model for a per-field self-assessment stash it in
+``FieldPrediction.meta["verbalized_confidence"]``. Known failure mode
+(paper §5.G): verbalized scores can be inflated by RLHF — always compare
+against consensus and report calibration, never trust raw.
+"""
+
+from __future__ import annotations
+
+from verifydoc.types import FieldPrediction
+
+META_KEY = "verbalized_confidence"
+
+
+def verbalized_confidence(pred: FieldPrediction) -> float | None:
+    """Clamped self-reported confidence; ``None`` when the adapter gave none."""
+    value = pred.meta.get(META_KEY)
+    if value is None:
+        return None
+    return min(1.0, max(0.0, float(value)))
+
+
+def apply_verbalized(predictions: list[FieldPrediction]) -> list[FieldPrediction]:
+    """Return copies with ``confidence`` set from verbalized scores where available."""
+    out = []
+    for pred in predictions:
+        conf = verbalized_confidence(pred)
+        out.append(pred if conf is None else pred.model_copy(update={"confidence": conf}))
+    return out

verifydoc/eval/__init__.py

@@ -0,0 +1 @@
+"""Evaluation harness: scores anything that emits FieldPrediction."""

verifydoc/eval/calibration.py

@@ -0,0 +1,154 @@
+"""Calibration metrics (PROJECT.md §5.B).
+
+Implements ECE (15 equal-width bins by default), Adaptive ECE (equal-mass
+bins), MCE, Brier score, NLL, TCE (target calibration error at abstention
+operating points), and reliability-diagram data (Guo et al. 2017 conventions).
+
+All functions take a confidence vector ``conf`` in [0, 1] and a 0/1
+correctness vector ``correct`` — the (c_i, correct_i) pairs produced by
+``eval.extraction.score_fields``.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+
+import numpy as np
+
+
+def _validate(conf: Sequence[float], correct: Sequence[int]) -> tuple[np.ndarray, np.ndarray]:
+    c = np.asarray(conf, dtype=float)
+    y = np.asarray(correct, dtype=float)
+    if c.shape != y.shape or c.ndim != 1:
+        raise ValueError("conf and correct must be 1-D and the same length")
+    if c.size == 0:
+        raise ValueError("empty inputs")
+    if ((c < 0) | (c > 1)).any():
+        raise ValueError("confidences must lie in [0, 1]")
+    if not np.isin(y, (0.0, 1.0)).all():
+        raise ValueError("correct must be 0/1")
+    return c, y
+
+
+@dataclass
+class ReliabilityBin:
+    """One bin of a reliability diagram."""
+
+    lo: float
+    hi: float
+    mean_confidence: float
+    accuracy: float
+    count: int
+
+
+def reliability_bins(
+    conf: Sequence[float], correct: Sequence[int], n_bins: int = 15
+) -> list[ReliabilityBin]:
+    """Equal-width reliability-diagram bins over [0, 1] (empty bins skipped).
+
+    Bin edges follow Guo et al.: bin m covers ((m-1)/M, m/M]; conf == 0 falls
+    into the first bin.
+    """
+    c, y = _validate(conf, correct)
+    edges = np.linspace(0.0, 1.0, n_bins + 1)
+    idx = np.clip(np.digitize(c, edges[1:-1], right=True), 0, n_bins - 1)
+    bins = []
+    for m in range(n_bins):
+        mask = idx == m
+        if not mask.any():
+            continue
+        bins.append(
+            ReliabilityBin(
+                lo=float(edges[m]),
+                hi=float(edges[m + 1]),
+                mean_confidence=float(c[mask].mean()),
+                accuracy=float(y[mask].mean()),
+                count=int(mask.sum()),
+            )
+        )
+    return bins
+
+
+def ece(conf: Sequence[float], correct: Sequence[int], n_bins: int = 15) -> float:
+    """Expected Calibration Error: sum_m (|B_m|/N) * |acc(B_m) - conf(B_m)|."""
+    n = len(conf)
+    return sum(
+        (b.count / n) * abs(b.accuracy - b.mean_confidence)
+        for b in reliability_bins(conf, correct, n_bins)
+    )
+
+
+def adaptive_ece(conf: Sequence[float], correct: Sequence[int], n_bins: int = 15) -> float:
+    """Adaptive ECE: equal-mass bins (robust when confidences cluster)."""
+    c, y = _validate(conf, correct)
+    order = np.argsort(c, kind="stable")
+    c, y = c[order], y[order]
+    splits = np.array_split(np.arange(c.size), n_bins)
+    total = 0.0
+    for chunk in splits:
+        if chunk.size == 0:
+            continue
+        total += (chunk.size / c.size) * abs(float(y[chunk].mean()) - float(c[chunk].mean()))
+    return total
+
+
+def mce(conf: Sequence[float], correct: Sequence[int], n_bins: int = 15) -> float:
+    """Maximum Calibration Error: worst-bin |acc - conf| over non-empty bins."""
+    return max(abs(b.accuracy - b.mean_confidence) for b in reliability_bins(conf, correct, n_bins))
+
+
+def brier(conf: Sequence[float], correct: Sequence[int]) -> float:
+    """Brier score: mean squared error between confidence and correctness."""
+    c, y = _validate(conf, correct)
+    return float(np.mean((c - y) ** 2))
+
+
+def nll(conf: Sequence[float], correct: Sequence[int], eps: float = 1e-12) -> float:
+    """Negative log-likelihood; confidences clipped to [eps, 1-eps]."""
+    c, y = _validate(conf, correct)
+    c = np.clip(c, eps, 1.0 - eps)
+    return float(-np.mean(y * np.log(c) + (1.0 - y) * np.log(1.0 - c)))
+
+
+def tce(
+    conf: Sequence[float],
+    correct: Sequence[int],
+    alphas: Sequence[float] = (0.01, 0.02, 0.05, 0.10),
+    tune_conf: Sequence[float] | None = None,
+) -> float:
+    """Target Calibration Error: mean over alpha of |achieved risk - alpha|.
+
+    For each target risk ``alpha``, the acceptance threshold is chosen from the
+    confidences alone: accept the largest confidence-sorted prefix whose
+    *predicted* risk (mean of 1 - c over accepted) is <= alpha, cutting only at
+    confidence boundaries so the prefix corresponds to a real threshold.
+    The achieved selective risk on (conf, correct) is then compared to alpha.
+
+    # DECISION: the threshold is tuned from predicted risk (no labels), which
+    # is exactly how a deployed system must pick its operating point; pass
+    # ``tune_conf`` to select the threshold on a calibration split instead.
+    # Accepting nothing has selective risk 0 by convention.
+    """
+    c, y = _validate(conf, correct)
+    t = np.asarray(tune_conf, dtype=float) if tune_conf is not None else c
+    total = 0.0
+    for alpha in alphas:
+        threshold = _risk_threshold(t, float(alpha))
+        accepted = c >= threshold
+        achieved = float(1.0 - y[accepted].mean()) if accepted.any() else 0.0
+        total += abs(achieved - alpha)
+    return total / len(alphas)
+
+
+def _risk_threshold(conf: np.ndarray, alpha: float) -> float:
+    """Lowest confidence threshold whose predicted selective risk is <= alpha."""
+    c = np.sort(conf)[::-1]
+    predicted_risk = np.cumsum(1.0 - c) / np.arange(1, c.size + 1)
+    # valid cut points: where the next confidence is strictly lower (or the end)
+    boundary: np.ndarray = np.append(c[:-1] > c[1:], True)
+    ok = (predicted_risk <= alpha) & boundary
+    if not ok.any():
+        return float("inf")  # accept nothing
+    k = int(np.flatnonzero(ok)[-1])
+    return float(c[k])