agentstats 0.3.0__py3-none-any.whl

agentstats/sample.py

@@ -0,0 +1,68 @@
+"""
+A deterministic, stratified sample of cases for a smoke tier.
+
+Every group (tag `<stratify_by>:<value>`) gets a share of the sample proportional to its size (largest
+remainder), and at least one case when the sample is big enough to cover every group. Cases carrying an
+`always_include` tag are always in. Same seed, same cases: a smoke result can be reproduced.
+"""
+from __future__ import annotations
+
+import json
+import random
+from collections import defaultdict
+from pathlib import Path
+
+
+def load_cases(path: str | Path) -> dict[str, list[str]]:
+    """case -> tags, from a JSONL of cases ({"case": ..., "tags": [...]}) or of attempt rows."""
+    out: dict[str, set[str]] = defaultdict(set)
+    for line in Path(path).read_text().splitlines():
+        if line.strip():
+            row = json.loads(line)
+            out[str(row["case"])].update(row.get("tags") or [])
+    return {c: sorted(t) for c, t in out.items()}
+
+
+def group_of(tags: list[str], stratify_by: str | None) -> str:
+    if not stratify_by:
+        return "all"
+    vals = sorted(t for t in tags if t.startswith(stratify_by + ":"))
+    return vals[0] if vals else f"{stratify_by}:(none)"
+
+
+def stratified(cases: dict[str, list[str]], n: int, stratify_by: str | None = None,
+               always_include: list[str] | None = None, seed: int = 0) -> dict:
+    rng = random.Random(seed)
+    always = sorted(c for c, tags in cases.items() if set(tags) & set(always_include or []))
+    notes = []
+    if len(always) >= n:
+        if len(always) > n:
+            notes.append(f"{len(always)} always-include cases exceed the sample size {n}; all are included")
+        return {"cases": always, "groups": {}, "notes": notes}
+
+    pool: dict[str, list[str]] = defaultdict(list)
+    for c in sorted(cases):
+        if c not in always:
+            pool[group_of(cases[c], stratify_by)].append(c)
+    left = n - len(always)
+    total = sum(len(v) for v in pool.values())
+    if left >= total:
+        return {"cases": sorted(cases), "groups": {g: len(v) for g, v in pool.items()}, "notes": notes}
+
+    quotas = {g: left * len(v) / total for g, v in pool.items()}
+    alloc = {g: int(q) for g, q in quotas.items()}
+    if left >= len(pool):
+        for g in pool:
+            alloc[g] = max(alloc[g], 1)
+    while sum(alloc.values()) > left:  # the minimum-one rule overshot: take from the largest
+        g = max(alloc, key=lambda k: (alloc[k] - quotas[k], alloc[k]))
+        alloc[g] -= 1
+    for g in sorted(pool, key=lambda k: quotas[k] - int(quotas[k]), reverse=True):
+        if sum(alloc.values()) >= left:
+            break
+        if alloc[g] < len(pool[g]):
+            alloc[g] += 1
+    picked = list(always)
+    for g, members in sorted(pool.items()):
+        picked += rng.sample(members, min(alloc[g], len(members)))
+    return {"cases": sorted(picked), "groups": {g: alloc[g] for g in sorted(pool)}, "notes": notes}

agentstats/sizing.py

@@ -0,0 +1,86 @@
+"""
+Size a gate before trusting it: how often would it pass, fail or be inconclusive for a range of true changes?
+
+Simulated from your baseline's own per-case rates, so the answer fits your eval set (many cases near 0% or 100%,
+a few that flip). For each simulation, each case's true pass rate is drawn from its baseline trials (Jeffreys prior,
+so a case that passed 4 of 4 still carries some uncertainty), shifted by `shift` on the log-odds scale for the
+candidate, and BOTH sides are simulated from it: a fresh baseline with the baseline's trial count and a candidate with
+`trials`. Comparing simulated candidates with the observed baseline instead would be biased: a case that passed 4 of 4
+has a true rate below 1 on average, so even no change would look like a drop. (Method from agent-platform's diagnosis
+gate v2 power analysis.)
+
+Rules:
+  drop            FAIL only when the whole interval is below -margin, else PASS   (gate requirement no_significant_drop)
+  noninferiority  PASS when the lower bound >= -margin, FAIL when the upper bound < -margin, else INCONCLUSIVE
+                                                                                   (gate requirement no_regression)
+
+Pick trials and margin so that "no change" passes often enough (e.g. >= 90%) and the drops you care about fail often
+enough (e.g. >= 80%).
+"""
+from __future__ import annotations
+
+import math
+import random
+from collections import defaultdict
+
+from agentstats.comparisons import fixed_cases
+from agentstats.records import Attempt
+
+RULES = ("drop", "noninferiority")
+DEFAULT_SHIFTS = (0.0, 0.25, 0.5, 0.75, 1.0, 1.5)
+
+
+def _per_case(baseline: list[Attempt]) -> dict[str, tuple[int, int]]:
+    out: dict[str, list[bool]] = defaultdict(list)
+    for a in baseline:
+        if a.valid:
+            out[a.case].append(bool(a.passed))
+    return {c: (len(v), sum(v)) for c, v in out.items()}
+
+
+def simulate(baseline: list[Attempt], trials: int, rule: str = "drop", margin: float = 0.0, conf: float = 0.90,
+             shifts=DEFAULT_SHIFTS, sims: int = 1000, seed: int = 0) -> dict:
+    if rule not in RULES:
+        raise ValueError(f"rule must be one of {RULES}")
+    counts = _per_case(baseline)
+    if not counts:
+        raise ValueError("baseline has no graded attempts")
+    m = min(n for n, _ in counts.values())
+    rng = random.Random(seed)
+    rows = []
+    for shift in shifts:
+        out = {"PASS": 0, "INCONCLUSIVE": 0, "FAIL": 0}
+        drop = 0.0
+        for _ in range(sims):
+            a, b, d = {}, {}, 0.0
+            for case, (n, s) in counts.items():
+                p = min(max(rng.betavariate(s + 0.5, n - s + 0.5), 1e-6), 1 - 1e-6)
+                q = 1 / (1 + math.exp(-(math.log(p / (1 - p)) - shift)))
+                d += p - q
+                a[case] = (m, sum(rng.random() < p for _ in range(m)))
+                b[case] = (trials, sum(rng.random() < q for _ in range(trials)))
+            drop += d / len(counts)
+            iv = fixed_cases(a, b, conf)
+            if rule == "drop":
+                out["FAIL" if iv.high < -margin else "PASS"] += 1
+            else:
+                out["PASS" if iv.low >= -margin else "FAIL" if iv.high < -margin else "INCONCLUSIVE"] += 1
+        rows.append({"shift": shift, "mean_drop": drop / sims, **{k: v / sims for k, v in out.items()}})
+    return {"cases": len(counts), "baseline_trials": m, "trials": trials, "rule": rule, "margin": margin,
+            "conf": conf, "sims": sims, "rows": rows}
+
+
+def render(r: dict) -> str:
+    title = f"{r['rule']} at a {100 * r['margin']:.0f}-point margin, {100 * r['conf']:.0f}% interval"
+    lines = [f"# agentstats size: {title}",
+             "", f"{r['cases']} cases, baseline {r['baseline_trials']} trial(s), candidate {r['trials']} trial(s), "
+             f"{r['sims']} simulations per row.", "",
+             "| true shift (log-odds) | mean drop | PASS | INCONCLUSIVE | FAIL |", "|---|---|---|---|---|"]
+    for row in r["rows"]:
+        lines.append(f"| {row['shift']} | {100 * row['mean_drop']:.1f} pts | {row['PASS']:.0%} | "
+                     f"{row['INCONCLUSIVE']:.0%} | {row['FAIL']:.0%} |")
+    zero = next((x for x in r["rows"] if x["shift"] == 0), None)
+    if zero:
+        lines += ["", f"No change: {zero['PASS']:.0%} pass" + (f", {zero['FAIL']:.0%} false alarms"
+                  if r["rule"] == "drop" else f", {zero['INCONCLUSIVE']:.0%} inconclusive, {zero['FAIL']:.0%} fail")]
+    return "\n".join(lines) + "\n"

agentstats/switch.py

@@ -0,0 +1,180 @@
+"""
+The switch decision: is the cheaper setup shown to be no worse than the current one, at no higher cost?
+
+`agentstats switch` doesn't run agents; the statistics are agentstats' own gate:
+- results come in as agentstats attempt rows (your runner, or `harness-evolve export` for an adapted harness);
+- agentstats does the statistics (non-inferiority on fixed cases, cost ratio, infrastructure failures);
+- this module applies the switch rule, in steps, and switch_record.py writes down what it decided and why.
+
+The rule, for one candidate vs the current setup, with a margin m chosen in advance:
+
+    lower CI bound of (candidate - current) >= -m, and cost ratio upper bound <= max   -> SWITCH
+    upper CI bound < -m (worse than the margin even at the optimistic end), or costs more -> KEEP
+    anything else (the interval crosses -m; too few runs; a run-wide shock)            -> NOT YET
+
+The steps:
+    1. the cheaper model as-is (same harness as the current setup);
+    2. if adapted results are given (the cheaper model with a harness adapted to it, e.g. by harness-evolve,
+       measured on held-out cases), the adapted setup. The better of the two decisions is the result.
+If step 1 is already SWITCH, adapted results are still compared and reported: switching and improving the new model
+are separate decisions (from the diagnosis replay, where adapting after a passing switch added ~9 points).
+
+Slices (--slice-by <tag prefix>): every comparison is repeated per slice. By default slices are reported, not
+enforced, because small slices give wide intervals (agentstats warns below 100 cases). With strict slices, the
+decision can't be better than its worst slice: one slice KEEP makes it KEEP, one slice NOT YET makes it NOT YET.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from agentstats import gate
+from agentstats.policy import parse
+from agentstats.records import Attempt
+
+SWITCH, KEEP, NOT_YET, UNHEALTHY = "SWITCH", "KEEP", "NOT YET", "UNHEALTHY"
+ORDER = {SWITCH: 0, NOT_YET: 1, KEEP: 2, UNHEALTHY: 3}       # worse decisions sort later
+EXIT = {SWITCH: 0, KEEP: 1, NOT_YET: 2, UNHEALTHY: 3}
+PROVENANCE = ("independent", "current-model", "unknown")
+
+
+@dataclass
+class Rule:
+    margin: float = 0.03            # how much worse is acceptable, as a fraction (0.03 = 3 points)
+    max_cost_ratio: float = 1.0     # candidate cost per attempt / current, upper CI bound
+    conf: float = 0.95
+    max_infra: float = 0.05
+    design: str = "curated"
+
+
+@dataclass
+class Comparison:
+    name: str
+    decision: str
+    reason: str
+    delta: dict | None = None        # agentstats' delta: estimate, low, high
+    cost_ratio: str | None = None
+    gate_rows: list[dict] = field(default_factory=list)
+    slices: dict[str, Comparison] = field(default_factory=dict)
+
+
+def _has_cost(attempts: list[Attempt]) -> bool:
+    return any("cost_usd" in a.metrics for a in attempts if a.valid)
+
+
+def compare(name: str, current: list[Attempt], candidate: list[Attempt], rule: Rule) -> Comparison:
+    """One candidate vs the current setup, through agentstats' gate, mapped to SWITCH / KEEP / NOT YET."""
+    require = {"no_regression": rule.margin, "max_infra": rule.max_infra}
+    with_cost = _has_cost(current) and _has_cost(candidate)
+    if with_cost:
+        require["max_cost_ratio"] = rule.max_cost_ratio
+    policy = parse({"design": rule.design, "switch": {"runs": 1, "require": require}})
+    g = gate.evaluate(policy.tiers["switch"], policy, candidate, current, rule.conf)
+    rows = {r["requirement"]: r for r in g["rows"]}
+    nr, cost = rows.get("no_regression"), rows.get("max_cost_ratio")
+    delta = (g.get("comparison") or {}).get("delta")
+    c = Comparison(name, NOT_YET, "", delta, cost["measured"] if cost else None, g["rows"])
+
+    if g["overall"] == gate.UNHEALTHY:
+        c.decision, c.reason = UNHEALTHY, "too many infrastructure failures to judge: rerun"
+    elif nr is None:
+        c.decision, c.reason = NOT_YET, "no comparison was possible"
+    elif nr["result"] == gate.FAIL:
+        c.decision, c.reason = KEEP, f"worse than the margin even at the optimistic end: {nr['measured']}"
+    elif cost and cost["result"] == gate.FAIL:
+        c.decision, c.reason = KEEP, f"not cheaper: cost ratio {cost['measured']}"
+    elif nr["result"] == gate.PASS and (cost is None or cost["result"] == gate.PASS):
+        c.decision = SWITCH
+        c.reason = f"shown no more than {100 * rule.margin:g} points worse: {nr['measured']}"
+        c.reason += f"; cost {cost['measured']}" if cost else "; no cost data, cost not checked"
+    else:
+        why = nr["detail"] if nr["result"] != gate.PASS else f"cost: {cost['detail'] or cost['measured']}"
+        c.decision, c.reason = NOT_YET, f"can't tell yet ({nr['measured']}): {why}"
+    return c
+
+
+def _slices(attempts: list[Attempt], prefix: str) -> dict[str, list[Attempt]]:
+    out: dict[str, list[Attempt]] = {}
+    for a in attempts:
+        for t in a.tags:
+            if t.startswith(prefix + ":"):
+                out.setdefault(t[len(prefix) + 1:], []).append(a)
+    return out
+
+
+def compare_with_slices(name: str, current: list[Attempt], candidate: list[Attempt], rule: Rule,
+                        slice_by: str | None) -> Comparison:
+    c = compare(name, current, candidate, rule)
+    if slice_by:
+        cur, cand = _slices(current, slice_by), _slices(candidate, slice_by)
+        for s in sorted(set(cur) & set(cand)):
+            c.slices[s] = compare(f"{name} [{slice_by}:{s}]", cur[s], cand[s], rule)
+    return c
+
+
+def cost_per_accepted(attempts: list[Attempt]) -> dict:
+    valid = [a for a in attempts if a.valid]
+    passed = sum(1 for a in valid if a.passed)
+    out = {"attempts": len(valid), "pass_rate": passed / len(valid) if valid else None}
+    if _has_cost(attempts):
+        cost = sum(a.metrics.get("cost_usd", 0.0) for a in valid)
+        out["cost_per_attempt_usd"] = cost / len(valid) if valid else None
+        out["cost_per_accepted_usd"] = cost / passed if passed else None
+    return out
+
+
+@dataclass
+class Decision:
+    decision: str
+    reason: str
+    steps: list[Comparison]
+    arms: dict[str, dict]
+    warnings: list[str]
+
+
+def decide(current: list[Attempt], candidate: list[Attempt], adapted: list[Attempt] | None, rule: Rule, *,
+           slice_by: str | None = None, strict_slices: bool = False, provenance: str = "unknown",
+           preregistered: bool = False) -> Decision:
+    if provenance not in PROVENANCE:
+        raise ValueError(f"provenance must be one of {PROVENANCE}")
+    steps = [compare_with_slices("as-is", current, candidate, rule, slice_by)]
+    if adapted is not None:
+        steps.append(compare_with_slices("adapted", current, adapted, rule, slice_by))
+
+    def effective(c: Comparison) -> tuple[str, str]:
+        """A step's decision; with strict slices, no better than its worst slice."""
+        if strict_slices and c.slices:
+            worst = max(c.slices.values(), key=lambda s: ORDER[s.decision])
+            if ORDER[worst.decision] > ORDER[c.decision]:
+                return worst.decision, f"strict slices: {worst.name}: {worst.reason}"
+        return c.decision, f"{c.name}: {c.reason}"
+
+    # The better of the two steps decides (the adapted step on a tie). Two chances to pass make a false SWITCH a
+    # little more likely than the nominal rate; with a 95% interval on each, at most about twice as likely.
+    results = [(effective(c), c) for c in steps]
+    (decision, reason), final = min(reversed(results), key=lambda x: ORDER[x[0][0]])
+
+    warnings = []
+    if not strict_slices and final.slices:
+        blocking = [s.name for s in final.slices.values() if ORDER[s.decision] > ORDER[decision]]
+        if blocking:
+            warnings.append(f"slices that don't support the decision (reported, not enforced): {', '.join(blocking)}")
+    if steps[0].decision == SWITCH and len(steps) > 1 and steps[1].delta and steps[0].delta:
+        gain = steps[1].delta["estimate"] - steps[0].delta["estimate"]
+        if gain > 0:
+            warnings.append(f"switching as-is is enough, but the adapted harness measured {100 * gain:+.1f} points "
+                            "more than the as-is model: consider shipping it too (a separate decision)")
+    if not (_has_cost(current) and _has_cost(candidate)):
+        warnings.append("cost not checked: the current or candidate rows have no cost_usd")
+    if provenance == "current-model":
+        warnings.append("the eval set was created, filtered or labelled by the current model, so the comparison "
+                        "favours it: a SWITCH still holds, a KEEP or NOT YET may overstate the gap")
+    elif provenance == "unknown":
+        warnings.append("eval-set provenance not stated: if the current model shaped the cases, the comparison "
+                        "favours it")
+    if not preregistered:
+        warnings.append("the margin was not declared before the results were seen")
+
+    arms = {"current": cost_per_accepted(current), "candidate": cost_per_accepted(candidate)}
+    if adapted is not None:
+        arms["adapted"] = cost_per_accepted(adapted)
+    return Decision(decision, reason, steps, arms, warnings)

agentstats/switch_cli.py

@@ -0,0 +1,140 @@
+"""
+    agentstats preregister --name triage-haiku-5-5 --margin 0.03 --provenance current-model \\
+        [--input labels=app/evals/triage_heldout.jsonl] [--slice-by template] --out decisions/triage-haiku-5-5
+    (commit the registration, then run both arms)
+
+    agentstats switch --prereg decisions/triage-haiku-5-5/prereg.json \\
+        --current current.jsonl --candidate cheap.jsonl [--adapted cheap-adapted.jsonl] --out decisions/triage-haiku-5-5
+
+    agentstats switch --current current.jsonl --candidate cheap.jsonl --margin 0.03 --provenance independent \\
+        --name my-switch --out decisions/my-switch           # without a registration: recorded as not pre-registered
+
+Exit codes for switch: 0 SWITCH, 1 KEEP, 2 NOT YET, 3 UNHEALTHY.
+"""
+from __future__ import annotations
+
+import argparse
+from pathlib import Path
+
+from agentstats import prereg, records
+from agentstats import switch as decide
+from agentstats import switch_record as record
+
+RULE_DEFAULTS = {"max_cost_ratio": 1.0, "conf": 0.95, "design": "curated"}
+
+
+def _rule_args(p: argparse.ArgumentParser) -> None:
+    p.add_argument("--margin", type=float, help="acceptable loss, e.g. 0.03 for 3 points")
+    p.add_argument("--max-cost-ratio", type=float)
+    p.add_argument("--conf", type=float)
+    p.add_argument("--design", choices=["curated", "sampled", "clustered"])
+    p.add_argument("--provenance", choices=decide.PROVENANCE,
+                   help="who made the eval set: independent of both models, or the current model")
+    p.add_argument("--slice-by", help="tag prefix to compare per slice, e.g. repo for repo:<name> tags")
+    p.add_argument("--strict-slices", action="store_true", help="the decision can't be better than its worst slice")
+
+
+def _rule(args) -> decide.Rule:
+    if args.margin is None:
+        raise SystemExit("--margin is required (or use --prereg)")
+    return decide.Rule(margin=args.margin, **{k: getattr(args, k) if getattr(args, k) is not None else v
+                                              for k, v in RULE_DEFAULTS.items()})
+
+
+def _from_prereg(args) -> tuple[prereg.Registration, list[str]]:
+    """The registered rule; flags that contradict it are an error, not an override."""
+    reg = prereg.load(args.prereg)
+    given = {"margin": args.margin, "max_cost_ratio": args.max_cost_ratio, "conf": args.conf, "design": args.design}
+    conflicts = [f"--{k.replace('_', '-')} {v} (registered {getattr(reg.rule, k)})"
+                 for k, v in given.items() if v is not None and v != getattr(reg.rule, k)]
+    if args.provenance and args.provenance != reg.provenance:
+        conflicts.append(f"--provenance {args.provenance} (registered {reg.provenance})")
+    if args.slice_by and args.slice_by != reg.slice_by:
+        conflicts.append(f"--slice-by {args.slice_by} (registered {reg.slice_by})")
+    if args.strict_slices and not reg.strict_slices:
+        conflicts.append("--strict-slices (registered without)")
+    return reg, conflicts
+
+
+def add_parsers(sub) -> None:
+    """`preregister` and `switch` subcommands, added to agentstats' own parser."""
+
+
+    p = sub.add_parser("preregister", help="write the switch rule down before any result exists")
+    _rule_args(p)
+    p.add_argument("--name", required=True)
+    p.add_argument("--input", action="append", default=[], metavar="NAME=PATH",
+                   help="a file the decision depends on (e.g. labels); its hash must match at decision time")
+    p.add_argument("--note", default="", help="the plan: arms, cases, trials, how they'll be run")
+    p.add_argument("--out", type=Path, required=True, help="directory for prereg.json and prereg.md")
+
+    d = sub.add_parser("switch", help="switch to the cheaper setup? SWITCH / KEEP / NOT YET, with a decision record")
+    d.add_argument("--prereg", type=Path, help="prereg.json from `agentstats preregister`: the rule comes from it")
+    d.add_argument("--current", type=Path, required=True, help="agentstats rows for the current setup")
+    d.add_argument("--candidate", type=Path, required=True, help="rows for the cheaper model, as-is")
+    d.add_argument("--adapted", type=Path, help="rows for the cheaper model with an adapted harness (held-out cases)")
+    _rule_args(d)
+    d.add_argument("--preregistered", action="store_true",
+                   help="(without --prereg) you assert the margin was set before any result; recorded as asserted")
+    d.add_argument("--name")
+    d.add_argument("--note", default="")
+    d.add_argument("--out", type=Path, required=True, help="directory for decision.json and decision.md")
+
+
+def run(args) -> int:
+
+    if args.cmd == "preregister":
+        if not args.provenance:
+            raise SystemExit("--provenance is required: say who made the eval set")
+        inputs = {}
+        for item in args.input:
+            name, _, path = item.partition("=")
+            if not path:
+                raise SystemExit(f"--input wants NAME=PATH, got {item!r}")
+            inputs[name] = Path(path)
+        try:
+            path = prereg.create(args.out, args.name, _rule(args), args.provenance, slice_by=args.slice_by,
+                                 strict_slices=args.strict_slices, inputs=inputs, note=args.note)
+        except (FileExistsError, ValueError) as exc:
+            print(exc)
+            return 1
+        print(f"wrote {path} and {path.with_suffix('.md')}. Commit them before running anything.")
+        return 0
+
+    results = [args.current, args.candidate] + ([args.adapted] if args.adapted else [])
+    prereg_info = None
+    if args.prereg:
+        reg, conflicts = _from_prereg(args)
+        if conflicts:
+            print("these flags contradict the registration (the rule comes from the file): " + "; ".join(conflicts))
+            return 1
+        rule, name, provenance = reg.rule, args.name or reg.name, reg.provenance
+        slice_by, strict = reg.slice_by, reg.strict_slices
+        ok, reasons, info = prereg.verify(args.prereg, results)
+        preregistered = ok
+        prereg_info = {"file": str(args.prereg), "verified": ok, "reasons": reasons, **info}
+    else:
+        if not args.name:
+            raise SystemExit("--name is required (or use --prereg)")
+        rule, name = _rule(args), args.name
+        provenance, slice_by, strict = args.provenance or "unknown", args.slice_by, args.strict_slices
+        preregistered = args.preregistered
+        if preregistered:
+            prereg_info = {"file": None, "verified": False, "reasons": ["asserted with --preregistered, not checked"]}
+
+    current = records.load(args.current, arm="current")
+    candidate = records.load(args.candidate, arm="candidate")
+    adapted = records.load(args.adapted, arm="adapted") if args.adapted else None
+    result = decide.decide(current, candidate, adapted, rule, slice_by=slice_by, strict_slices=strict,
+                           provenance=provenance, preregistered=preregistered)
+    if prereg_info and not prereg_info["verified"] and args.prereg:
+        result.warnings.append("pre-registration not verified: " + "; ".join(prereg_info["reasons"]))
+    inputs = {"current": args.current, "candidate": args.candidate}
+    if args.adapted:
+        inputs["adapted"] = args.adapted
+    r = record.build(result, rule, inputs, name=name, provenance=provenance, preregistered=preregistered,
+                     slice_by=slice_by, strict_slices=strict, note=args.note, prereg=prereg_info)
+    _, md = record.write(args.out, r)
+    print(md.read_text())
+    return decide.EXIT[result.decision]
+

agentstats/switch_record.py

@@ -0,0 +1,99 @@
+"""
+The decision record: what was compared, under which rule, what was decided and why, written next to the inputs'
+hashes so the decision can be checked later ("why are we still on the expensive model?").
+
+    <out>/decision.json    everything, machine-readable
+    <out>/decision.md      the same, for people
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import asdict
+from datetime import UTC, datetime
+from pathlib import Path
+
+import agentstats
+from agentstats.switch import Comparison, Decision, Rule
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(Path(path).read_bytes()).hexdigest()[:16]
+
+
+def _comparison(c: Comparison) -> dict:
+    return {"name": c.name, "decision": c.decision, "reason": c.reason, "delta": c.delta, "cost_ratio": c.cost_ratio,
+            "gate_rows": c.gate_rows, "slices": {k: _comparison(v) for k, v in c.slices.items()}}
+
+
+def build(d: Decision, rule: Rule, inputs: dict[str, Path], *, name: str, provenance: str, preregistered: bool,
+          slice_by: str | None, strict_slices: bool, note: str = "", prereg: dict | None = None) -> dict:
+    return {
+        "name": name,
+        "decision": d.decision,
+        "reason": d.reason,
+        "warnings": d.warnings,
+        "rule": asdict(rule),
+        "margin_declared_before_results": preregistered,
+        "preregistration": prereg,
+        "provenance": provenance,
+        "slices": {"by": slice_by, "strict": strict_slices},
+        "inputs": {arm: {"path": str(p), "sha256": _sha256(p)} for arm, p in inputs.items()},
+        "arms": d.arms,
+        "steps": [_comparison(c) for c in d.steps],
+        "note": note,
+        "versions": {"agentstats": agentstats.__version__},
+        "decided_at": datetime.now(UTC).isoformat(timespec="seconds"),
+    }
+
+
+def _pts(x):
+    return "n/a" if x is None else f"{100 * x:+.1f}"
+
+
+def render(r: dict) -> str:
+    lines = [f"# agentstats switch decision: {r['name']}", "", f"**{r['decision']}**: {r['reason']}", ""]
+    rule = r["rule"]
+    rule_line = (f"Rule: switch only if shown no more than {100 * rule['margin']:g} points worse "
+                 f"({100 * rule['conf']:g}% interval, {rule['design']} eval set) at a cost ratio of at most "
+                 f"{rule['max_cost_ratio']:g}x. Margin declared before the results: "
+                 f"{'yes' if r['margin_declared_before_results'] else 'no'}. Eval-set provenance: {r['provenance']}.")
+    lines += [rule_line, ""]
+    p = r.get("preregistration")
+    if p and p.get("file"):
+        status = (f"verified: committed {p.get('registered_at')} ({p.get('commit', '')[:7]}), before the results"
+                  if p["verified"] else "NOT verified: " + "; ".join(p["reasons"]))
+        lines += [f"Pre-registration `{p['file']}`: {status}.", ""]
+    elif p:
+        lines += ["Pre-registration: asserted with --preregistered, not checked.", ""]
+    lines += ["| step | decision | change (pts) | 95% CI | cost ratio |", "|---|---|---|---|---|"]
+    for s in r["steps"]:
+        d = s["delta"] or {}
+        lines.append(f"| {s['name']} | {s['decision']} | {_pts(d.get('estimate'))} | "
+                     f"{_pts(d.get('low'))} to {_pts(d.get('high'))} | {s['cost_ratio'] or 'n/a'} |")
+        for sl in s["slices"].values():
+            d = sl["delta"] or {}
+            lines.append(f"| &nbsp;&nbsp;{sl['name']} | {sl['decision']} | {_pts(d.get('estimate'))} | "
+                         f"{_pts(d.get('low'))} to {_pts(d.get('high'))} | {sl['cost_ratio'] or 'n/a'} |")
+    lines += ["", "| arm | attempts | pass rate | $ per attempt | $ per accepted result |", "|---|---|---|---|---|"]
+    for arm, a in r["arms"].items():
+        pr = f"{100 * a['pass_rate']:.1f}%" if a.get("pass_rate") is not None else "n/a"
+        cpa = f"{a['cost_per_attempt_usd']:.4f}" if a.get("cost_per_attempt_usd") is not None else "n/a"
+        cpr = f"{a['cost_per_accepted_usd']:.4f}" if a.get("cost_per_accepted_usd") is not None else "n/a"
+        lines.append(f"| {arm} | {a['attempts']} | {pr} | {cpa} | {cpr} |")
+    if r["warnings"]:
+        lines += ["", "Warnings:"] + [f"- {w}" for w in r["warnings"]]
+    if r["note"]:
+        lines += ["", f"Note: {r['note']}"]
+    lines += ["", "Inputs:"] + [f"- {arm}: `{i['path']}` (sha256 {i['sha256']})" for arm, i in r["inputs"].items()]
+    decided = f"Decided {r['decided_at']} with agentstats {r['versions']['agentstats']}."
+    lines += ["", decided]
+    return "\n".join(lines) + "\n"
+
+
+def write(out_dir: Path, r: dict) -> tuple[Path, Path]:
+    out_dir.mkdir(parents=True, exist_ok=True)
+    j, m = out_dir / "decision.json", out_dir / "decision.md"
+    j.write_text(json.dumps(r, indent=1))
+    m.write_text(render(r))
+    return j, m