robotruth 0.1.6__py3-none-any.whl

robotruth/__init__.py

@@ -0,0 +1,12 @@
+"""robotruth: Robot CI for learned robot policies.
+
+Three truths, one report:
+
+- configuration truth: is the policy you evaluated the policy you deployed?
+  (`robotruth.contract`)
+- statistical truth: is checkpoint B really better than A? (`robotruth.stats`)
+- outcome truth: did the episode succeed, when did it fail, and why?
+  (`robotruth.schema`, `robotruth.judge`, later modules)
+"""
+
+__version__ = "0.1.6"

robotruth/_env.py

@@ -0,0 +1,35 @@
+"""Tiny .env loader that tolerates PowerShell's UTF-16 output. Never prints values."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+
+def load_env(path: Path | str = ".env") -> dict[str, str]:
+    p = Path(path)
+    if not p.exists():
+        return {}
+    raw = p.read_bytes()
+    for enc in ("utf-8-sig", "utf-16", "utf-16-le", "latin-1"):
+        try:
+            text = raw.decode(enc)
+            if "=" in text:
+                break
+        except UnicodeDecodeError:
+            continue
+    out: dict[str, str] = {}
+    for line in text.splitlines():
+        line = line.strip().lstrip("")
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        k, v = line.split("=", 1)
+        out[k.strip()] = v.strip().strip('"').strip("'")
+    for k, v in out.items():
+        os.environ.setdefault(k, v)
+    return out
+
+
+def hf_token() -> str | None:
+    env = load_env(Path(__file__).resolve().parents[2] / ".env")
+    return env.get("HF_TOKEN") or os.environ.get("HF_TOKEN")

robotruth/audit.py

@@ -0,0 +1,84 @@
+"""Claims audit: which published or internal comparisons survive an interval?
+
+Input: a CSV with columns `policy, task, successes, trials` (one row per policy and task, or
+task blank for a pooled number). Output: every pairwise comparison within a task with the
+difference, its interval, and whether it is resolved at the requested confidence. This is
+the tool behind the first public evidence artifact: run the numbers papers actually report
+through it and count how many claims hold up.
+"""
+
+from __future__ import annotations
+
+import csv
+from dataclasses import dataclass
+from itertools import combinations
+from pathlib import Path
+
+import numpy as np
+from scipy import stats as sps
+
+from robotruth.report import Report, Section
+from robotruth.stats.intervals import wilson
+
+
+@dataclass
+class Claim:
+    policy: str
+    task: str
+    successes: int
+    trials: int
+
+
+def read_claims(path: Path | str) -> list[Claim]:
+    with Path(path).open(newline="", encoding="utf-8") as f:
+        rows = list(csv.DictReader(f))
+    out = []
+    for r in rows:
+        out.append(Claim(r["policy"].strip(), (r.get("task") or "ALL").strip() or "ALL", int(float(r["successes"])), int(float(r["trials"]))))
+    return out
+
+
+def newcombe_diff_ci(k1: int, n1: int, k2: int, n2: int, alpha: float = 0.05) -> tuple[float, float, float]:
+    """Newcombe hybrid score interval for p2 - p1 (two independent proportions)."""
+    w1, w2 = wilson(k1, n1, alpha), wilson(k2, n2, alpha)
+    d = k2 / n2 - k1 / n1
+    lo = d - np.sqrt((w1.upper - w1.estimate) ** 2 + (w2.estimate - w2.lower) ** 2)
+    hi = d + np.sqrt((w1.estimate - w1.lower) ** 2 + (w2.upper - w2.estimate) ** 2)
+    return float(d), float(lo), float(hi)
+
+
+def fisher_p(k1: int, n1: int, k2: int, n2: int) -> float:
+    table = [[k1, n1 - k1], [k2, n2 - k2]]
+    return float(sps.fisher_exact(table)[1])
+
+
+def audit_claims(claims: list[Claim], alpha: float = 0.05, title: str = "robotruth claims audit") -> Report:
+    rep = Report(title)
+    by_task: dict[str, list[Claim]] = {}
+    for c in claims:
+        by_task.setdefault(c.task, []).append(c)
+
+    rate_rows = []
+    for c in claims:
+        w = wilson(c.successes, c.trials, alpha)
+        rate_rows.append({"task": c.task, "policy": c.policy, "trials": c.trials,
+                          "rate": f"{w.estimate:.2f} [{w.lower:.2f}, {w.upper:.2f}]", "interval width": round(w.width, 3)})
+    rep.add(Section("Reported rates with intervals", "What each number actually says once the interval is attached.", rate_rows))
+
+    comp_rows = []
+    n_resolved = 0
+    for task, cs in by_task.items():
+        for a, b in combinations(cs, 2):
+            d, lo, hi = newcombe_diff_ci(a.successes, a.trials, b.successes, b.trials, alpha)
+            p = fisher_p(a.successes, a.trials, b.successes, b.trials)
+            resolved = (lo > 0) or (hi < 0)
+            n_resolved += resolved
+            comp_rows.append({"task": task, "A": a.policy, "B": b.policy, "diff (B-A)": round(d, 3),
+                              "interval": f"[{lo:+.2f}, {hi:+.2f}]", "Fisher p": round(p, 4),
+                              "resolved": "yes" if resolved else "no"})
+    n_comp = len(comp_rows)
+    body = (f"{n_comp} pairwise comparisons within tasks; {n_resolved} resolved at {100*(1-alpha):.0f}% confidence, "
+            f"{n_comp - n_resolved} are inside the noise.")
+    rep.add(Section("Pairwise comparisons", body, comp_rows, verdict="INFO"))
+    rep.verdict = "WARN" if n_comp and n_resolved < n_comp else "PASS"
+    return rep

robotruth/cli.py

@@ -0,0 +1,272 @@
+"""robotruth command line."""
+
+from __future__ import annotations
+
+import csv
+import json
+from pathlib import Path
+from typing import Optional
+
+import typer
+from rich.console import Console
+from rich.table import Table
+
+from robotruth import __version__
+from robotruth.contract import ExecSpec, Severity, check_contract, extract_spec
+from robotruth.results import Results
+
+app = typer.Typer(help="Robot CI: contract checks, honest statistics, episode records, drift fingerprints, outcome judging and runtime monitoring for learned robot policies.", no_args_is_help=True)
+contract_app = typer.Typer(help="Module 1: executable-policy contract manifests.", no_args_is_help=True)
+stats_app = typer.Typer(help="Module 2: honest evaluation statistics.", no_args_is_help=True)
+episodes_app = typer.Typer(help="Module 3: episode records, fleet metrics, MCAP bridge.", no_args_is_help=True)
+fingerprint_app = typer.Typer(help="Module 4: cell and robot-unit fingerprints and drift.", no_args_is_help=True)
+app.add_typer(contract_app, name="contract")
+app.add_typer(stats_app, name="stats")
+app.add_typer(episodes_app, name="episodes")
+app.add_typer(fingerprint_app, name="fingerprint")
+from robotruth.judge.cli import judge_app  # noqa: E402
+from robotruth.guard.cli import guard_app  # noqa: E402
+app.add_typer(judge_app, name="judge")
+app.add_typer(guard_app, name="guard")
+console = Console()
+
+
+def _version_cb(value: bool):
+    if value:
+        console.print(f"robotruth {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def _main(version: bool = typer.Option(False, "--version", help="Show version and exit.", callback=_version_cb, is_eager=True)):
+    pass
+
+
+@contract_app.command("extract")
+def contract_extract(path: Path = typer.Argument(..., help="Checkpoint directory."),
+                     out: Path = typer.Option(Path("execspec.json"), "--out", "-o"),
+                     family: Optional[str] = typer.Option(None, help="lerobot | openpi | gr00t | custom (auto-detected if omitted)"),
+                     role: str = typer.Option("unspecified", help="evaluated | deployed"),
+                     overlay: Optional[Path] = typer.Option(None, help="JSON with fields the extractor cannot read (control_hz, gripper_convention, cameras...).")):
+    """Build an ExecSpec manifest from a checkpoint directory."""
+    ov = json.loads(overlay.read_text(encoding="utf-8")) if overlay else None
+    spec = extract_spec(path, family=family, overlay=ov, role=role)
+    spec.to_json(out)
+    console.print(f"[green]wrote[/] {out}  family={spec.policy.family}  fingerprint={spec.fingerprint()[:16]}")
+    if spec.unverified:
+        console.print(f"[yellow]unverified fields ({len(spec.unverified)}):[/] " + ", ".join(spec.unverified))
+        console.print("Supply them with --overlay or a robotruth.spec.json next to the checkpoint, or the check will fail closed.")
+
+
+@contract_app.command("check")
+def contract_check(evaluated: Path, deployed: Path,
+                   allow_unknown: bool = typer.Option(False, help="Downgrade unknown fatal fields to warnings (not recommended)."),
+                   out: Optional[Path] = typer.Option(None, "--out", "-o", help="Write a JSON report here.")):
+    """Compare the evaluated manifest against the deployed one. Exit code 1 on FAIL."""
+    res = check_contract(ExecSpec.from_json(evaluated), ExecSpec.from_json(deployed), allow_unknown=allow_unknown)
+    table = Table(title=res.summary())
+    for col in ("severity", "field", "evaluated", "deployed", "why"):
+        table.add_column(col, overflow="fold")
+    colours = {Severity.FATAL: "red", Severity.WARN: "yellow", Severity.INFO: "blue", Severity.OK: "green"}
+    for f in sorted(res.findings, key=lambda f: list(Severity).index(f.severity)):
+        table.add_row(f"[{colours[f.severity]}]{f.severity.value}[/]", f.field, _short(f.evaluated), _short(f.deployed), f.message)
+    console.print(table)
+    if out:
+        out.write_text(json.dumps({"passed": res.passed, "summary": res.summary(),
+                                   "findings": [f.__dict__ | {"severity": f.severity.value} for f in res.findings]},
+                                  indent=2, default=str), encoding="utf-8")
+    raise typer.Exit(code=0 if res.passed else 1)
+
+
+def _short(v) -> str:
+    s = str(v)
+    return s if len(s) <= 24 else s[:10] + "..." + s[-10:]
+
+
+@stats_app.command("plan")
+def stats_plan(p_a: float = typer.Option(..., help="Baseline success rate, e.g. 0.80"),
+               p_b: float = typer.Option(..., help="Rate you hope to detect, e.g. 0.90"),
+               alpha: float = 0.05, power: float = 0.8,
+               rho: float = typer.Option(0.4, help="Within-pair outcome correlation for the paired design.")):
+    """How many trials you need before you start."""
+    from robotruth.stats.planning import required_trials_two_proportions
+    console.print(str(required_trials_two_proportions(p_a, p_b, alpha, power, paired=False)))
+    console.print(str(required_trials_two_proportions(p_a, p_b, alpha, power, paired=True, rho=rho)))
+
+
+@stats_app.command("schedule")
+def stats_schedule(policies: str = typer.Option(..., help="Comma-separated policy names."),
+                   conditions: str = typer.Option(..., help="Comma-separated condition labels (object pose, lighting...)."),
+                   repeats: int = 1, seed: int = 0,
+                   out: Path = typer.Option(Path("schedule.csv")),
+                   key_out: Path = typer.Option(Path("schedule_key.json"), help="Blinding key; keep it away from the operator.")):
+    """Write a blinded, interleaved A/B/n schedule."""
+    from robotruth.stats.schedule import interleaved_schedule
+    s = interleaved_schedule([p.strip() for p in policies.split(",")], [c.strip() for c in conditions.split(",")], repeats, seed)
+    rows = s.to_rows()
+    with out.open("w", newline="", encoding="utf-8") as f:
+        w = csv.DictWriter(f, fieldnames=list(rows[0].keys()))
+        w.writeheader(); w.writerows(rows)
+    key_out.write_text(json.dumps(s.key, indent=2), encoding="utf-8")
+    console.print(f"[green]wrote[/] {out} ({len(rows)} slots) and blinding key {key_out}")
+
+
+@stats_app.command("compare")
+def stats_compare(results: Path = typer.Argument(..., help="Results CSV (see robotruth.results)."),
+                  a: str = typer.Option(..., help="Baseline policy name."),
+                  b: str = typer.Option(..., help="Candidate policy name."),
+                  alpha: float = 0.05,
+                  margin: float = typer.Option(0.0, help="Minimum improvement that counts, e.g. 0.02."),
+                  out_dir: Path = typer.Option(Path("reports"))):
+    """Is B better than A? Writes a Markdown and HTML report. Exit 1 if B is worse."""
+    from robotruth.compare import compare_policies
+    res = Results.read_csv(results)
+    rep = compare_policies(res, a, b, alpha=alpha, margin=margin)
+    md, html = rep.write(out_dir, stem=f"compare_{a}_vs_{b}")
+    console.print(rep.to_markdown())
+    console.print(f"[green]wrote[/] {md} and {html}")
+    raise typer.Exit(code=1 if rep.verdict == "FAIL" else 0)
+
+
+@stats_app.command("audit")
+def stats_audit(claims_csv: Path = typer.Argument(..., help="CSV: policy, task, successes, trials"),
+                alpha: float = 0.05, out_dir: Path = typer.Option(Path("reports"))):
+    """Which reported comparisons survive an interval? Pairwise Newcombe intervals and Fisher tests within each task."""
+    from robotruth.audit import audit_claims, read_claims
+    rep = audit_claims(read_claims(claims_csv), alpha)
+    md, html = rep.write(out_dir, stem="claims_audit")
+    console.print(rep.to_markdown())
+    console.print(f"[green]wrote[/] {md} and {html}")
+
+
+@stats_app.command("interval")
+def stats_interval(k: int, n: int, alpha: float = 0.05):
+    """Interval for k successes out of n. Because a rate without an interval is not a result."""
+    from robotruth.stats.intervals import clopper_pearson, wilson
+    console.print(str(wilson(k, n, alpha)))
+    console.print(str(clopper_pearson(k, n, alpha)))
+
+
+@episodes_app.command("validate")
+def episodes_validate(log: Path):
+    """Validate a JSONL episode log against the schema."""
+    from robotruth.schema import EpisodeLog
+    ok, errors = EpisodeLog(log).validate()
+    console.print(f"{ok} valid records, {len(errors)} errors")
+    for line, err in errors[:20]:
+        console.print(f"  [red]line {line}[/]: {err}")
+    raise typer.Exit(code=1 if errors else 0)
+
+
+@episodes_app.command("to-results")
+def episodes_to_results(log: Path, out: Path = typer.Option(Path("results.csv"), "--out", "-o")):
+    """Export episodes with an outcome to the results CSV used by `robotruth stats compare`."""
+    from robotruth.schema import EpisodeLog
+    p = EpisodeLog(log).to_results_csv(out)
+    console.print(f"[green]wrote[/] {p}")
+
+
+@episodes_app.command("metrics")
+def episodes_metrics(log: Path, alpha: float = 0.05, out: Optional[Path] = typer.Option(None, "--out", "-o"),
+                     out_dir: Optional[Path] = typer.Option(None, help="Write fleet_report.md and fleet_report.html here.")):
+    """Fleet metrics: success, autonomous fraction, interventions per hour, MTBI, failure Pareto. Markdown to stdout; --out-dir adds a styled HTML report."""
+    from robotruth.report import Report, Section
+    from robotruth.schema import EpisodeLog, fleet_metrics
+    fm = fleet_metrics(EpisodeLog(log), alpha)
+    md = fm.to_markdown()
+    console.print(md)
+    if out:
+        out.write_text(md, encoding="utf-8")
+    if out_dir:
+        rep = Report(f"robotruth fleet report: {log.name}")
+        rep.add(Section("Fleet metrics", md))
+        md_path, html_path = rep.write(out_dir, stem="fleet_report")
+        console.print(f"[green]wrote[/] {md_path} and {html_path}")
+
+
+@episodes_app.command("to-mcap")
+def episodes_to_mcap(log: Path, out: Path = typer.Option(Path("episodes.mcap"), "--out", "-o")):
+    """Write the log as JSON messages on /robotruth/episode for Foxglove or Rerun."""
+    from robotruth.schema import EpisodeLog
+    console.print(f"[green]wrote[/] {EpisodeLog(log).to_mcap(out)}")
+
+
+@episodes_app.command("from-mcap")
+def episodes_from_mcap(mcap_file: Path, out: Path = typer.Option(Path("episodes.jsonl"), "--out", "-o")):
+    """Extract /robotruth/episode messages from an MCAP file into a JSONL log."""
+    from robotruth.schema import EpisodeLog
+    log = EpisodeLog.from_mcap(mcap_file, out)
+    console.print(f"[green]wrote[/] {log.path} ({len(log.records())} records)")
+
+
+@episodes_app.command("from-lerobot")
+def episodes_from_lerobot(dataset_dir: Path, out: Path = typer.Option(Path("episodes.jsonl"), "--out", "-o"),
+                          policy: Optional[str] = typer.Option(None, help="Policy label; defaults to dataset:<repo_id>."),
+                          max_episodes: Optional[int] = None):
+    """Convert a LeRobot dataset directory into an episode log (success from next.success or next.reward, interventions from flag columns)."""
+    from robotruth.schema import EpisodeLog
+    from robotruth.schema.lerobot_ingest import LeRobotDataset
+    ds = LeRobotDataset(dataset_dir)
+    recs = ds.to_records(policy, max_episodes)
+    if out.exists():
+        out.unlink()
+    log = EpisodeLog(out)
+    log.extend(recs)
+    labelled = sum(r.outcome.success is not None for r in recs)
+    console.print(f"[green]wrote[/] {out}: {len(recs)} episodes from {ds.repo_id} ({ds.robot}, {ds.fps:.0f} fps, {len(ds.camera_keys())} cameras); {labelled} with success labels")
+
+
+@episodes_app.command("taxonomy")
+def episodes_taxonomy():
+    """Print the failure taxonomy."""
+    from robotruth.schema import FAILURE_SUBCLASSES
+    for cls, subs in FAILURE_SUBCLASSES.items():
+        console.print(f"[bold]{cls}[/]: " + ", ".join(subs))
+
+
+@fingerprint_app.command("unit")
+def fingerprint_unit(excitation_csv: Path, unit_id: str = typer.Option("unknown"), out: Path = typer.Option(Path("unit_fingerprint.json"), "--out", "-o")):
+    """Fingerprint a robot unit from a fixed excitation trajectory log (t, cmd_<j>, meas_<j> columns)."""
+    from robotruth.fingerprint.unit import read_excitation_csv, unit_fingerprint
+    t, cmd, meas = read_excitation_csv(excitation_csv)
+    fp = unit_fingerprint(t, cmd, meas, unit_id)
+    fp.to_json(out)
+    for j, st in fp.joints.items():
+        console.print(f"{j}: rmse={st.rmse:.4f} lag={1000*st.lag_s:.0f}ms backlash={st.backlash:.4f} offset={st.steady_error:+.4f} gain={st.gain:.3f}")
+    console.print(f"[green]wrote[/] {out} fingerprint={fp.fingerprint[:16]}")
+
+
+@fingerprint_app.command("cell")
+def fingerprint_cell(image: Path, camera: str = typer.Option("cam"), aruco_dict: str = typer.Option("4x4_50"),
+                     out: Path = typer.Option(Path("cell_fingerprint.json"), "--out", "-o")):
+    """Fingerprint a workspace camera view: ArUco marker positions, exposure, sharpness, colour balance."""
+    from robotruth.fingerprint import cell_fingerprint
+    fp = cell_fingerprint(image, camera, aruco_dict)
+    fp.to_json(out)
+    console.print(f"{len(fp.markers)} markers, luma={fp.photometrics.mean_luma:.0f}, sharpness={fp.photometrics.sharpness:.0f}")
+    console.print(f"[green]wrote[/] {out} fingerprint={fp.fingerprint[:16]}")
+
+
+@fingerprint_app.command("diff")
+def fingerprint_diff(reference: Path, current: Path, out: Optional[Path] = typer.Option(None, "--out", "-o")):
+    """Compare two fingerprints (unit or cell) against drift tolerances. Exit 1 on FAIL."""
+    from robotruth.fingerprint import CellFingerprint, UnitFingerprint, diff_cell, diff_unit
+    ref_d = json.loads(reference.read_text(encoding="utf-8"))
+    if "joints" in ref_d:
+        rep = diff_unit(UnitFingerprint.from_json(reference), UnitFingerprint.from_json(current))
+    else:
+        rep = diff_cell(CellFingerprint.from_json(reference), CellFingerprint.from_json(current))
+    table = Table(title=rep.summary())
+    for col in ("severity", "metric", "reference", "current", "delta", "why"):
+        table.add_column(col, overflow="fold")
+    colours = {"fail": "red", "warn": "yellow", "ok": "green"}
+    for f in sorted(rep.findings, key=lambda f: ["fail", "warn", "ok"].index(f.severity)):
+        table.add_row(f"[{colours[f.severity]}]{f.severity}[/]", f.metric, _short(f.reference), _short(f.current), f"{f.delta:.4g}", f.message)
+    console.print(table)
+    if out:
+        out.write_text(json.dumps(rep.to_dict(), indent=2, default=str), encoding="utf-8")
+    raise typer.Exit(code=0 if rep.passed else 1)
+
+
+if __name__ == "__main__":
+    app()

robotruth/compare.py

@@ -0,0 +1,105 @@
+"""Compare two policies from a results table and build the report.
+
+This is the function behind `robotruth stats compare`. It applies every rule in module 2:
+intervals on every rate, paired analysis when pair ids exist, an anytime-valid sequential
+verdict, time-to-success with censoring, per-task breakdown, and an explicit statement of
+how many trials would have been needed if the comparison is inconclusive.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import numpy as np
+
+from robotruth.report import Report, Section
+from robotruth.results import Results
+from robotruth.stats.intervals import bootstrap_diff_ci, paired_diff_ci, wilson
+from robotruth.stats.planning import required_trials_two_proportions
+from robotruth.stats.sequential import sequential_paired_test
+from robotruth.stats.timing import logrank_test
+
+
+def compare_policies(res: Results, a: str, b: str, alpha: float = 0.05, margin: float = 0.0,
+                     title: Optional[str] = None) -> Report:
+    rep = Report(title or f"robotruth: is {b} better than {a}?")
+    ra, rb = res.subset(a), res.subset(b)
+    if not ra.rows or not rb.rows:
+        raise ValueError(f"no rows for one of the policies: {a}={len(ra.rows)}, {b}={len(rb.rows)}")
+
+    # 1. Rates with intervals, overall and per task.
+    rows = []
+    for task in [None] + res.tasks():
+        sa, sb = ra.subset(task=task), rb.subset(task=task)
+        if not sa.rows or not sb.rows:
+            continue
+        wa = wilson(int(sa.successes().sum()), len(sa.rows), alpha)
+        wb = wilson(int(sb.successes().sum()), len(sb.rows), alpha)
+        rows.append({
+            "task": task or "ALL", f"{a} n": wa.n, f"{a} rate": f"{wa.estimate:.2f} [{wa.lower:.2f}, {wa.upper:.2f}]",
+            f"{b} n": wb.n, f"{b} rate": f"{wb.estimate:.2f} [{wb.lower:.2f}, {wb.upper:.2f}]",
+            "intervals overlap": "yes" if (wa.lower <= wb.upper and wb.lower <= wa.upper) else "no",
+        })
+    rep.add(Section("Success rates with Wilson intervals",
+                    f"Every rate carries a {100*(1-alpha):.0f}% interval. Overlapping intervals do not prove equality; "
+                    "non-overlap is strong evidence of a difference.", rows))
+
+    # 2. Difference estimate: paired if possible.
+    paired = res.has_pairs()
+    if paired:
+        xa, xb, pairs = res.paired(a, b)
+        if xa.size >= 2:
+            ci = paired_diff_ci(xa, xb, alpha)
+            boot = bootstrap_diff_ci(xa, xb, alpha, paired=True)
+            n_pairs = xa.size
+            body = (f"{n_pairs} matched pairs (same pair_id and task).\n"
+                    f"Difference in success (B - A): {ci}\n"
+                    f"Bootstrap check: {boot}\n"
+                    f"Discordant pairs: A only {int(np.sum((xa==1)&(xb==0)))}, B only {int(np.sum((xa==0)&(xb==1)))}.")
+            verdict = "PASS" if ci.lower > margin else ("FAIL" if ci.upper < -margin else "WARN")
+            rep.add(Section("Paired difference", body, verdict=verdict))
+            # 3. Anytime-valid sequential verdict, in pair order.
+            seq = sequential_paired_test(xa, xb, alpha=alpha, margin=margin, stop_early=False)
+            first = next((t for t in seq.trace if (t[2] > margin or t[3] < -margin)), None)
+            stop_note = (f"An anytime-valid stopping rule would have decided at n={first[0]}."
+                         if first else "The sequence never excluded zero; more paired trials are needed.")
+            rep.add(Section("Anytime-valid sequential verdict",
+                            f"{seq}\n{stop_note}\nThis interval is valid at every sample size simultaneously "
+                            "(empirical Bernstein confidence sequence), so peeking after each rollout is allowed.",
+                            table=[{"n": t[0], "diff": t[1], "lower": t[2], "upper": t[3]} for t in seq.trace[:: max(1, len(seq.trace)//20)]],
+                            verdict={"B_better": "PASS", "A_better": "FAIL"}.get(seq.decision, "WARN")))
+        else:
+            paired = False
+    if not paired:
+        boot = bootstrap_diff_ci(ra.scores(), rb.scores(), alpha, paired=False)
+        verdict = "PASS" if boot.lower > margin else ("FAIL" if boot.upper < -margin else "WARN")
+        rep.add(Section("Unpaired difference",
+                        f"No pair_id column found, so trials are treated as independent.\n"
+                        f"Difference in mean score (B - A): {boot}\n"
+                        "Interleave and pair your trials next time: it needs fewer rollouts for the same power.",
+                        verdict=verdict))
+
+    # 4. Time to success with censoring.
+    ta, ea = ra.times()
+    tb, eb = rb.times()
+    if np.isfinite(ta).all() and np.isfinite(tb).all() and (ea.sum() + eb.sum()) > 0:
+        lr = logrank_test(ta, ea, tb, eb)
+        rep.add(Section("Time to success (censored)",
+                        f"{lr}\nFailures are treated as censored at their timeout. A faster policy at equal success "
+                        "rate is a better policy; binary success cannot see that.",
+                        verdict="INFO"))
+
+    # 5. If inconclusive, say what it would take.
+    pa = ra.successes().mean()
+    pb = rb.successes().mean()
+    if abs(pa - pb) > 1e-9 and 0 < pa < 1 and 0 < pb < 1:
+        plan_ind = required_trials_two_proportions(pa, pb, alpha, 0.8, paired=False)
+        plan_pair = required_trials_two_proportions(pa, pb, alpha, 0.8, paired=True, rho=0.4)
+        rep.add(Section("What it would take to resolve this",
+                        f"Observed rates {pa:.2f} vs {pb:.2f}. To detect that gap with 80% power:\n"
+                        f"- independent trials: {plan_ind}\n- paired interleaved trials (rho 0.4): {plan_pair}\n"
+                        f"You have {len(ra.rows)} and {len(rb.rows)} trials.", verdict="INFO"))
+
+    verdicts = [s.verdict for s in rep.sections if s.verdict in ("PASS", "FAIL", "WARN")]
+    rep.verdict = "FAIL" if "FAIL" in verdicts else ("PASS" if verdicts and all(v == "PASS" for v in verdicts) else "WARN")
+    return rep

robotruth/contract/__init__.py

@@ -0,0 +1,39 @@
+"""Module 1: executable-policy contract checker.
+
+A checkpoint is not a policy. The thing that runs on a robot is the tuple of weights,
+normalization statistics, action semantics, control rate, chunking, preprocessing, camera
+calibration and embodiment. Swap any one and the same weights become a different policy
+("Same Weights, Different Robot", arXiv 2606.03724: 28/28 to 2/28).
+
+This module extracts that tuple into an `ExecSpec` manifest and compares two manifests,
+failing closed on anything it cannot verify.
+"""
+
+from robotruth.contract.spec import (
+    ActionSemantics,
+    CameraSpec,
+    Embodiment,
+    ExecSpec,
+    Normalization,
+    ObservationSpec,
+    PolicyIdentity,
+    RuntimeSpec,
+)
+from robotruth.contract.check import CheckResult, Finding, Severity, check_contract
+from robotruth.contract.extract import extract_spec
+
+__all__ = [
+    "ActionSemantics",
+    "CameraSpec",
+    "CheckResult",
+    "Embodiment",
+    "ExecSpec",
+    "Finding",
+    "Normalization",
+    "ObservationSpec",
+    "PolicyIdentity",
+    "RuntimeSpec",
+    "Severity",
+    "check_contract",
+    "extract_spec",
+]