modelpin 0.1.0__py3-none-any.whl

modelpin/__init__.py

@@ -0,0 +1,3 @@
+"""Modelpin — Dependabot for AI models. Know before the model breaks you."""
+
+__version__ = "0.1.0"

modelpin/cli.py

@@ -0,0 +1,336 @@
+"""Modelpin CLI (Typer). Wires the pipeline end-to-end:
+
+    mp scan      -> detector
+    mp init      -> scaffold modelpin.yaml + scenarios/
+    mp baseline  -> replay current model N times, persist traces
+    mp check     -> replay a new model, diff vs baseline, print the PR-style report
+
+Replays use the END USER's API key from the environment (BYO-key, spec section 9);
+`--provider fake --fixtures <file>` runs the whole flow offline for demos/tests.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import NoReturn, Optional
+
+import typer
+from rich.box import ASCII as ASCII_BOX
+from rich.console import Console
+from rich.table import Table
+
+from modelpin import __version__
+from modelpin.config import DEFAULT_PROVIDER, ConfigError, ModelpinConfig, load_config
+from modelpin.detector import scan_repo
+from modelpin.diff import diff_scenario
+from modelpin.models import DiffVerdict, Scenario
+from modelpin.providers import ProviderAdapter, ProviderError, get_adapter
+from modelpin.providers.fake import FakeProvider
+from modelpin.replay import replay
+from modelpin.report import render_cli, render_pr_comment
+from modelpin.scenarios import ScenarioError, load_scenarios
+from modelpin.storage import STORE_DIRNAME, BaselineError, load_baseline, save_baseline
+
+#: A behavioral diff compares run *distributions*; fewer than this can't form one.
+MIN_RUNS = 2
+#: Below this, the permutation test is underpowered — warn but proceed.
+RECOMMENDED_RUNS = 5
+
+app = typer.Typer(
+    help="Modelpin - Dependabot for AI models. Know before the model breaks you.",
+    no_args_is_help=True,
+    add_completion=False,
+    rich_markup_mode=None,  # plain Click help: ASCII-only, never crashes on cp1252
+)
+console = Console()
+
+_SAMPLE_SCENARIO = """{
+  "id": "greeting",
+  "name": "Simple greeting",
+  "kind": "single",
+  "input": {"messages": [{"role": "user", "content": "Say hello in one short sentence."}]},
+  "assertions": {"must_contain": ["hello"]}
+}
+"""
+
+_SAMPLE_CONFIG = """# modelpin.yaml - generated by `mp init`. See docs/ for the full reference.
+models:
+  - gpt-4o-mini            # the model your app currently depends on (set yours)
+scenarios_dir: scenarios
+providers:
+  - openai                 # uses YOUR OPENAI_API_KEY from the environment
+runs: 5                    # N replays per scenario (>=5 gives the diff real power)
+judge_model: gpt-4o-mini   # semantic LLM-judge (optional; extra calls). Remove to disable.
+"""
+
+
+def _fail(message: str) -> NoReturn:
+    console.print(f"[red]error:[/] {message}")
+    raise typer.Exit(code=1)
+
+
+def _adapter(provider: str, fixtures: Optional[str]) -> ProviderAdapter:
+    try:
+        if provider == "fake":
+            return FakeProvider.from_fixtures(fixtures) if fixtures else FakeProvider()
+        return get_adapter(provider)
+    except FileNotFoundError as exc:
+        _fail(f"fixtures file not found: {exc}")
+    except ValueError as exc:  # unknown provider
+        _fail(str(exc))
+
+
+def _unimplemented_msg(provider: str) -> str:
+    return (
+        f"the {provider!r} adapter isn't implemented yet. Try `--provider fake "
+        "--fixtures <file>` to run offline, or pick `--provider openai`."
+    )
+
+
+def _preflight_or_fail(adapter: ProviderAdapter, provider: str) -> None:
+    """Fail before any tokens are spent if the provider isn't ready (key/SDK)."""
+    try:
+        adapter.preflight()
+    except ProviderError as exc:
+        _fail(str(exc))
+
+
+def _guard_replay(provider: str, fn):
+    """Run a replay step, converting adapter failures into friendly CLI errors."""
+    try:
+        return fn()
+    except NotImplementedError:
+        _fail(_unimplemented_msg(provider))
+    except ProviderError as exc:
+        _fail(str(exc))
+
+
+def _build_judge(provider: str, cfg: ModelpinConfig):
+    """Construct + preflight the semantic LLM-judge if configured. Returns None when no
+    judge_model is set or the run is offline (fake), so the diff stays purely structural."""
+    if not cfg.judge_model or provider == "fake":
+        return None
+    try:
+        from modelpin.judge import build_judge
+
+        # The judge is independent of the models being compared (the judge model id picks
+        # its provider), so a cross-vendor check (e.g. google vs openai) can still judge.
+        judge = build_judge(cfg.judge_model)
+        judge.preflight()
+    except (ProviderError, ImportError) as exc:
+        _fail(f"semantic judge ({cfg.judge_model!r}): {exc}")
+    console.print(f"[dim]semantic judge: {cfg.judge_model}[/]")
+    return judge
+
+
+def _load_config_or_fail(config_path: str) -> ModelpinConfig:
+    try:
+        return load_config(config_path)
+    except ConfigError as exc:
+        _fail(str(exc))
+
+
+def _load_scenarios_or_fail(scenarios_dir: str) -> list[Scenario]:
+    try:
+        scenarios = load_scenarios(scenarios_dir)
+    except ScenarioError as exc:
+        _fail(str(exc))
+    if not scenarios:
+        _fail(f"no scenarios in {scenarios_dir!r}. Run `mp init` first.")
+    return scenarios
+
+
+def _resolve_provider(provider: Optional[str], cfg: ModelpinConfig) -> str:
+    return provider or (cfg.providers[0] if cfg.providers else DEFAULT_PROVIDER)
+
+
+def _resolve_runs(runs: Optional[int], cfg: ModelpinConfig) -> int:
+    n = cfg.runs if runs is None else runs  # an explicit --runs 0 must hit the floor, not coerce
+    if n < MIN_RUNS:
+        _fail(
+            f"--runs must be >= {MIN_RUNS}: the diff compares run *distributions*, and "
+            f"a single run can't form one (got {n})."
+        )
+    if n < RECOMMENDED_RUNS:
+        console.print(
+            f"[yellow]warning:[/] only {n} runs/scenario; {RECOMMENDED_RUNS}+ gives the "
+            "statistical diff real power and a lower false-positive rate."
+        )
+    return n
+
+
+@app.command()
+def version() -> None:
+    """Print the Modelpin version."""
+    console.print(f"modelpin {__version__}")
+
+
+@app.command()
+def scan(path: str = typer.Argument(".", help="Repo root to scan.")) -> None:
+    """Detect which AI models this repo depends on, and where."""
+    hits = scan_repo(path)
+    if not hits:
+        console.print("No model identifiers found.")
+        return
+    table = Table("model", "file", "line", title="Models this repo depends on", box=ASCII_BOX)
+    for h in sorted(hits, key=lambda x: (x["model"], x["file"], x["line"])):
+        table.add_row(h["model"], h["file"], str(h["line"]))
+    console.print(table)
+    console.print(f"[dim]{len({h['model'] for h in hits})} distinct model(s).[/]")
+
+
+@app.command()
+def init(directory: str = typer.Argument(".", help="Repo to scaffold.")) -> None:
+    """Create modelpin.yaml + scenarios/ in the current repo (never overwrites)."""
+    root = Path(directory)
+    cfg = root / "modelpin.yaml"
+    scenarios_dir = root / "scenarios"
+    created: list[str] = []
+    if not cfg.exists():
+        cfg.write_text(_SAMPLE_CONFIG, encoding="utf-8")
+        created.append(str(cfg))
+    scenarios_dir.mkdir(parents=True, exist_ok=True)
+    if not any(scenarios_dir.glob("*.json")):
+        (scenarios_dir / "greeting.json").write_text(_SAMPLE_SCENARIO, encoding="utf-8")
+        created.append(str(scenarios_dir / "greeting.json"))
+    if created:
+        console.print("[green]Scaffolded:[/]")
+        for c in created:
+            console.print(f"  - {c}")
+        console.print("\nNext: add scenarios, then run [bold]mp baseline[/].")
+    else:
+        console.print("Already initialised (modelpin.yaml + scenarios/ present).")
+
+
+@app.command()
+def baseline(
+    model: Optional[str] = typer.Option(
+        None, "--model", help="Model to baseline (default: config)."
+    ),
+    provider: Optional[str] = typer.Option(
+        None, "--provider", help="openai | google | anthropic | groq | openrouter | fake."
+    ),
+    fixtures: Optional[str] = typer.Option(
+        None, "--fixtures", help="Canned traces for --provider fake."
+    ),
+    runs: Optional[int] = typer.Option(None, "--runs", help="Replays per scenario."),
+    config_path: str = typer.Option("modelpin.yaml", "--config"),
+    scenarios_dir: Optional[str] = typer.Option(None, "--scenarios-dir"),
+    store_dir: str = typer.Option(STORE_DIRNAME, "--store-dir"),
+) -> None:
+    """Record current model behavior for your scenarios (N runs)."""
+    cfg = _load_config_or_fail(config_path)
+    scenarios = _load_scenarios_or_fail(scenarios_dir or cfg.scenarios_dir)
+    from_model = model or (cfg.models[0] if cfg.models else None)
+    if not from_model:
+        _fail("no model to baseline. Pass --model or set `models:` in modelpin.yaml.")
+    n = _resolve_runs(runs, cfg)
+    prov = _resolve_provider(provider, cfg)
+    adapter = _adapter(prov, fixtures)
+    console.print(f"[dim]provider={prov} model={from_model} runs={n}[/]")
+    _preflight_or_fail(adapter, prov)
+    traces = _guard_replay(
+        prov, lambda: {s.id: replay(s, from_model, adapter, runs=n) for s in scenarios}
+    )
+    path = save_baseline(traces, from_model, store_dir)
+    console.print(
+        f"[green]Baseline recorded[/] for [bold]{from_model}[/]: "
+        f"{len(scenarios)} scenario(s) x{n} runs -> {path}"
+    )
+
+
+@app.command()
+def check(
+    to: str = typer.Option(..., "--to", help="The new model id to test against your baseline."),
+    from_: Optional[str] = typer.Option(None, "--from", help="Baseline model (default: config)."),
+    provider: Optional[str] = typer.Option(
+        None, "--provider", help="openai | google | anthropic | groq | openrouter | fake."
+    ),
+    fixtures: Optional[str] = typer.Option(
+        None, "--fixtures", help="Canned traces for --provider fake."
+    ),
+    runs: Optional[int] = typer.Option(None, "--runs", help="Replays per scenario."),
+    mode: str = typer.Option(
+        "strict", "--match", help="Tool-call match mode: strict|unordered|subset|superset."
+    ),
+    config_path: str = typer.Option("modelpin.yaml", "--config"),
+    scenarios_dir: Optional[str] = typer.Option(None, "--scenarios-dir"),
+    store_dir: str = typer.Option(STORE_DIRNAME, "--store-dir"),
+) -> None:
+    """Replay scenarios on a new model and report behavioral regressions."""
+    cfg = _load_config_or_fail(config_path)
+    scenarios = _load_scenarios_or_fail(scenarios_dir or cfg.scenarios_dir)
+    from_model = from_ or (cfg.models[0] if cfg.models else None)
+    if not from_model:
+        _fail("no baseline model. Pass --from or set `models:` in modelpin.yaml.")
+    try:
+        base = load_baseline(from_model, store_dir)
+    except FileNotFoundError as e:
+        _fail(str(e))
+    except BaselineError as e:
+        _fail(str(e))
+    n = _resolve_runs(runs, cfg)
+    prov = _resolve_provider(provider, cfg)
+    adapter = _adapter(prov, fixtures)
+    console.print(f"[dim]provider={prov} from={from_model} to={to} runs={n} match={mode}[/]")
+    _preflight_or_fail(adapter, prov)
+    judge = _build_judge(prov, cfg)
+
+    results = []
+    skipped: list[str] = []
+
+    def _run_check() -> None:
+        for s in scenarios:
+            base_traces = base.get(s.id)
+            if not base_traces:
+                skipped.append(s.id)
+                continue
+            cand = replay(s, to, adapter, runs=n)
+            results.append(
+                diff_scenario(s.id, from_model, to, base_traces, cand, s, mode, judge=judge)
+            )
+
+    _guard_replay(prov, _run_check)
+
+    if not results:
+        _fail("nothing to compare. Record a baseline first with `mp baseline`.")
+
+    console.print(render_cli(results, from_model, to, n))
+
+    # Decide the CI exit code BEFORE any side effect, so a report-write failure
+    # can never silently mask a real regression.
+    has_regression = any(r.verdict == DiffVerdict.regression for r in results)
+
+    report_path = Path(store_dir) / "last-report.md"
+    try:
+        report_path.parent.mkdir(parents=True, exist_ok=True)
+        report_path.write_text(render_pr_comment(results, from_model, to, n), encoding="utf-8")
+        console.print(f"\n[dim]PR-style Markdown report written to {report_path}[/]")
+    except OSError as exc:
+        console.print(f"[yellow]warning:[/] could not write report to {report_path}: {exc}")
+
+    if skipped:
+        console.print(
+            f"[yellow]note:[/] {len(skipped)} scenario(s) had no baseline and were skipped: "
+            f"{', '.join(skipped)}. Re-run `mp baseline` to cover them."
+        )
+
+    if has_regression:
+        raise typer.Exit(code=1)  # fail CI on a real regression
+
+
+@app.command()
+def report() -> None:
+    """Run the public standard suite and draft a Modelpin Report. (Coming soon.)"""
+    console.print(
+        "[yellow]TODO[/]: the public Modelpin Report suite is not implemented yet. "
+        "See docs/Modelpin-Engineering-Context-Pack.md section 7."
+    )
+
+
+def main() -> None:
+    app()
+
+
+if __name__ == "__main__":
+    main()

modelpin/config.py

@@ -0,0 +1,46 @@
+"""Load and validate modelpin.yaml. See spec sections 3-4."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any, Optional
+
+import yaml
+from pydantic import BaseModel, Field, ValidationError
+
+DEFAULT_CONFIG_FILE = "modelpin.yaml"
+
+#: The default provider when none is given. OpenAI is the implemented adapter; the
+#: Anthropic adapter is still a stub, so zero-config must not route to it.
+DEFAULT_PROVIDER = "openai"
+
+
+class ConfigError(Exception):
+    """modelpin.yaml is malformed or fails validation. Carries a user-facing message."""
+
+
+class ModelpinConfig(BaseModel):
+    models: list[str] = Field(default_factory=list)
+    scenarios_dir: str = "scenarios"
+    providers: list[str] = Field(default_factory=lambda: [DEFAULT_PROVIDER])
+    runs: int = Field(default=3, ge=1)
+    judge_model: Optional[str] = None
+    regression_threshold: float = 0.2
+
+
+def load_config(path: str | Path = DEFAULT_CONFIG_FILE) -> ModelpinConfig:
+    p = Path(path)
+    if not p.exists():
+        return ModelpinConfig()
+    try:
+        data: Any = yaml.safe_load(p.read_text(encoding="utf-8"))
+    except yaml.YAMLError as exc:
+        raise ConfigError(f"{p} is not valid YAML: {exc}") from exc
+    if data is None:
+        return ModelpinConfig()
+    if not isinstance(data, dict):
+        raise ConfigError(f"{p} must be a YAML mapping (got {type(data).__name__}).")
+    try:
+        return ModelpinConfig(**data)
+    except ValidationError as exc:
+        raise ConfigError(f"{p} has invalid settings: {exc}") from exc

modelpin/detector/__init__.py

@@ -0,0 +1,49 @@
+"""Detector — scans a repo for AI model identifier strings. See spec section 4.2."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Iterable
+
+# Conservative patterns; extend as providers add families.
+MODEL_PATTERNS = [
+    re.compile(r"\bgpt-[0-9][\w.\-]*\b"),
+    re.compile(r"\bo[0-9][\w.\-]*\b"),
+    re.compile(r"\bclaude-[\w.\-]+\b"),
+    re.compile(r"\bgemini-[\w.\-]+\b"),
+]
+
+DEFAULT_EXTS = {".py", ".env", ".yaml", ".yml", ".json", ".toml", ".js", ".ts"}
+SKIP_DIRS = {".git", ".venv", "venv", "node_modules", "__pycache__", "dist", "build"}
+
+
+def _iter_files(root: Path, exts: set[str]) -> Iterable[Path]:
+    for p in root.rglob("*"):
+        if p.is_dir():
+            continue
+        if any(part in SKIP_DIRS for part in p.parts):
+            continue
+        if p.suffix.lower() in exts or p.name == ".env":
+            yield p
+
+
+def scan_repo(root: str | Path = ".", exts: set[str] | None = None) -> list[dict]:
+    """Return [{model, file, line}] for every model id found in the repo."""
+    root = Path(root)
+    exts = exts or DEFAULT_EXTS
+    hits: list[dict] = []
+    for f in _iter_files(root, exts):
+        try:
+            text = f.read_text(errors="ignore")
+        except OSError:
+            continue
+        for i, line in enumerate(text.splitlines(), start=1):
+            for pat in MODEL_PATTERNS:
+                for m in pat.findall(line):
+                    hits.append({"model": m, "file": str(f.relative_to(root)), "line": i})
+    return hits
+
+
+def models_used(root: str | Path = ".") -> set[str]:
+    return {h["model"] for h in scan_repo(root)}

modelpin/diff/__init__.py

@@ -0,0 +1,208 @@
+"""Behavioral diff orchestrator. See spec section 6.
+
+Combines structural per-run signals (``structural.py``) with the distributional
+permutation test (``stats.py``) into a single per-scenario verdict + confidence.
+
+Decision rule (tuned for a low FALSE-POSITIVE rate — the north-star metric):
+a signal counts as a regression only when the candidate distribution differs from
+baseline at p <= ALPHA *and* the effect clears a minimum size. A single odd run,
+or a majority that merely flips between two equally-likely behaviors, is NOT a
+regression — the permutation test treats it as noise. The semantic LLM-judge
+(``semantic.py``, spec 6B) is optional and injected: with ``judge=None`` this layer
+stays purely structural + statistical and never makes a network call; with a judge it
+adds a calibrated semantic-divergence signal through the same distributional test.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+from modelpin.diff.stats import (
+    permutation_pvalue_distribution,
+    permutation_pvalue_mean,
+    total_variation_distance,
+)
+from modelpin.diff.semantic import Judge, semantic_divergence_flags
+from modelpin.diff.structural import (
+    MatchMode,
+    assertion_violation_flags,
+    canonical_sequence,
+    modal_sequence,
+    refusal_rate,
+    refused_flags,
+    tool_call_sequence,
+)
+from modelpin.models import DiffResult, DiffSignals, DiffVerdict, Scenario, Trace
+
+#: Significance threshold for the permutation test. Lower = fewer false positives.
+ALPHA = 0.05
+#: A tool-call distribution must shift by at least this total-variation distance to
+#: count — guards against trivially-significant jitter once N grows large.
+MIN_TOOL_TVD = 0.5
+#: Ignore refusal-rate rises smaller than this even if "significant" (one run in three).
+MIN_REFUSAL_DELTA = 0.34
+#: Candidate semantic-divergence rate must exceed the baseline's by at least this much.
+#: CALIBRATED on examples/calibration/ (labeled set distinct from the held-out suite; see
+#: docs/STATUS.md): equivalent-but-reworded pairs land at delta 0.0, and the meaning changes
+#: that actually took effect land at delta >= 0.8 (one perturbation the model resisted scored
+#: 0.0 — a failed perturbation, not a missed regression), so 0.5 sits in the empty gap with 0
+#: false positives. The held-out suite re-validation stayed 0/8 with the promotion live. NOTE
+#: the calibration set is still small and the perturbations synthetic — see docs/STATUS.md for
+#: the limitations and the planned expansion to real migration traces + an independent judge.
+MIN_SEMANTIC_DELTA = 0.5
+
+
+def _scenario_task(scenario: Optional[Scenario]) -> Optional[str]:
+    """The user's request from a scenario (last user message) — context for the judge."""
+    if not scenario:
+        return None
+    for message in reversed(scenario.input.get("messages") or []):
+        if isinstance(message, dict) and message.get("role") == "user":
+            return str(message.get("content") or "")
+    return None
+
+
+def _mean(values: list[float]) -> float:
+    return sum(values) / len(values) if values else 0.0
+
+
+def diff_scenario(
+    scenario_id: str,
+    from_model: str,
+    to_model: str,
+    baseline_traces: list[Trace],
+    candidate_traces: list[Trace],
+    scenario: Optional[Scenario] = None,
+    mode: MatchMode = "strict",
+    judge: Optional[Judge] = None,
+) -> DiffResult:
+    """Compare baseline vs candidate trace distributions for one scenario.
+
+    With ``judge`` set, the semantic LLM-judge signal is evaluated (spec 6B); with it
+    ``None`` the diff is purely structural + statistical and makes no network call.
+    """
+    if not baseline_traces or not candidate_traces:
+        return DiffResult(
+            scenario_id=scenario_id,
+            from_model=from_model,
+            to_model=to_model,
+            verdict=DiffVerdict.unchanged,
+            confidence=0.0,
+            explanation="insufficient data: need baseline and candidate runs",
+        )
+
+    # --- tool-call trajectory distribution -------------------------------------
+    base_keys = [canonical_sequence(tool_call_sequence(t), mode) for t in baseline_traces]
+    cand_keys = [canonical_sequence(tool_call_sequence(t), mode) for t in candidate_traces]
+    tool_tvd = total_variation_distance(base_keys, cand_keys)
+    tool_p = permutation_pvalue_distribution(base_keys, cand_keys)
+    tool_regressed = tool_p <= ALPHA and tool_tvd >= MIN_TOOL_TVD
+
+    # --- refusal rate ----------------------------------------------------------
+    refusal_delta = refusal_rate(candidate_traces) - refusal_rate(baseline_traces)
+    refusal_p = permutation_pvalue_mean(
+        refused_flags(baseline_traces), refused_flags(candidate_traces)
+    )
+    refusal_regressed = refusal_p <= ALPHA and refusal_delta >= MIN_REFUSAL_DELTA
+
+    # --- output format / assertion drift (soft signal) -------------------------
+    fmt_p = 1.0
+    fmt_drift = False
+    if scenario and scenario.assertions:
+        a = scenario.assertions
+        base_v = assertion_violation_flags(baseline_traces, a.must_contain, a.must_not_contain)
+        cand_v = assertion_violation_flags(candidate_traces, a.must_contain, a.must_not_contain)
+        fmt_delta = _mean(cand_v) - _mean(base_v)
+        fmt_p = permutation_pvalue_mean(base_v, cand_v)
+        fmt_drift = fmt_p <= ALPHA and fmt_delta > 0
+
+    # --- semantic equivalence (LLM-as-judge; optional, only when a judge is given) ---
+    semantic_score: Optional[float] = None
+    semantic_p = 1.0
+    semantic_diverged = False
+    if judge is not None:
+        base_sem, cand_sem, semantic_score = semantic_divergence_flags(
+            baseline_traces, candidate_traces, judge, _scenario_task(scenario)
+        )
+        semantic_delta = _mean(cand_sem) - _mean(base_sem)
+        semantic_p = permutation_pvalue_mean(base_sem, cand_sem)
+        semantic_diverged = semantic_p <= ALPHA and semantic_delta >= MIN_SEMANTIC_DELTA
+
+    # --- cheap deltas (informational; not part of the verdict) -----------------
+    latency_delta = _mean([t.latency_ms for t in candidate_traces]) - _mean(
+        [t.latency_ms for t in baseline_traces]
+    )
+    token_delta = round(
+        _mean([t.tokens_out for t in candidate_traces])
+        - _mean([t.tokens_out for t in baseline_traces])
+    )
+
+    signals = DiffSignals(
+        tool_call_match=round(1.0 - tool_tvd, 3),  # 1.0 == identical distributions
+        format_valid=not fmt_drift,
+        refusal_delta=round(refusal_delta, 3),
+        semantic_score=semantic_score,
+        latency_delta_ms=round(latency_delta, 3),
+        token_delta=int(token_delta),
+    )
+
+    # --- verdict ---------------------------------------------------------------
+    reasons: list[str] = []
+    hard_pvalues: list[float] = []
+    verdict = DiffVerdict.unchanged
+
+    if tool_regressed:
+        verdict = DiffVerdict.regression
+        hard_pvalues.append(tool_p)
+        reasons.append(
+            f"tool-call behavior changed: {list(modal_sequence(baseline_traces, mode))} "
+            f"-> {list(modal_sequence(candidate_traces, mode))}"
+        )
+    if refusal_regressed:
+        verdict = DiffVerdict.regression
+        hard_pvalues.append(refusal_p)
+        reasons.append(
+            f"refusal rate {refusal_rate(baseline_traces):.0%} -> {refusal_rate(candidate_traces):.0%}"
+        )
+    minor_pvalues: list[float] = []
+    if fmt_drift:
+        if verdict != DiffVerdict.regression:
+            verdict = DiffVerdict.changed_minor
+        minor_pvalues.append(fmt_p)
+        reasons.append("output format drift: violates the scenario's text assertions")
+    if semantic_diverged:
+        # Calibrated (examples/calibration/): a consistent semantic divergence beyond the
+        # baseline's own spread, clearing MIN_SEMANTIC_DELTA at p <= ALPHA, is a hard,
+        # CI-failing regression. The labeled sweep separated equivalent (delta 0.0) from real
+        # meaning changes (delta >= 0.8) with 0 false positives at this floor, so this no
+        # longer over-fires on reworded-but-equivalent answers. A single divergent run, or a
+        # minority below the floor, still reads as noise (the permutation test + the floor).
+        verdict = DiffVerdict.regression
+        hard_pvalues.append(semantic_p)
+        reasons.append(
+            f"semantic drift: candidate answers diverge in meaning from baseline "
+            f"(equivalence {semantic_score:.0%})"
+        )
+
+    # confidence = how sure we are of the verdict.
+    #   regression/minor -> 1 - p of the firing signal (small p => high confidence);
+    #   unchanged        -> smallest p across signals (1.0 when distributions match,
+    #                       lower when something was a borderline near-miss).
+    if verdict == DiffVerdict.regression:
+        confidence = round(1.0 - min(hard_pvalues), 3)
+    elif verdict == DiffVerdict.changed_minor:
+        confidence = round(1.0 - min(minor_pvalues), 3)
+    else:
+        confidence = round(min(tool_p, refusal_p, fmt_p, semantic_p), 3)
+
+    explanation = "; ".join(reasons) if reasons else "no statistically significant behavior change"
+
+    return DiffResult(
+        scenario_id=scenario_id,
+        from_model=from_model,
+        to_model=to_model,
+        verdict=verdict,
+        signals=signals,
+        confidence=confidence,
+        explanation=explanation,
+    )