consequence-ledger 0.3.0__py3-none-any.whl

ledger/pagerduty.py

@@ -0,0 +1,102 @@
+"""Pull incidents from PagerDuty's REST API into the ledger.
+
+Reads the key from the PAGERDUTY_API_KEY environment variable only; it is never accepted as an
+argument, written to disk, or printed. A read-only REST API key is enough.
+
+PagerDuty incidents carry no commit links, so these bring in timing and titles. Link them to code with
+an export that names suspect commits, or with --incident-linking on `ledger label`.
+"""
+import datetime as dt
+import json
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from .incidents import import_incidents_records
+
+BASE_URL = "https://api.pagerduty.com"  # EU accounts: set PAGERDUTY_URL=https://api.eu.pagerduty.com
+WINDOW_DAYS = 150                       # the API rejects since/until ranges over 6 months
+PAGE_LIMIT = 100
+MAX_PAGES_PER_WINDOW = 100              # classic pagination stops at offset + limit = 10,000
+
+
+class PagerDutyError(RuntimeError):
+    pass
+
+
+def key_from_env():
+    key = os.environ.get("PAGERDUTY_API_KEY", "").strip()
+    if not key:
+        raise PagerDutyError("PAGERDUTY_API_KEY is not set. Create a read-only REST API key and store it with: "
+                             'setx PAGERDUTY_API_KEY "<key>" (then open a new terminal).')
+    return key
+
+
+def _get(url, key, opener=urllib.request.urlopen):
+    req = urllib.request.Request(url, headers={
+        "Authorization": f"Token token={key}",
+        "Accept": "application/vnd.pagerduty+json;version=2",
+    })
+    try:
+        with opener(req, timeout=30) as resp:
+            return json.load(resp)
+    except urllib.error.HTTPError as e:
+        hint = {401: "the key was rejected (wrong, disabled or deleted)",
+                403: "the key lacks permission for incidents",
+                429: "rate limited by PagerDuty; try again later"}.get(e.code, e.reason)
+        e.close()
+        raise PagerDutyError(f"PagerDuty returned HTTP {e.code}: {hint}") from None
+    except urllib.error.URLError as e:
+        raise PagerDutyError(f"could not reach PagerDuty: {e.reason}") from None
+
+
+def windows(days, now=None, size=WINDOW_DAYS):
+    """(since, until) ISO pairs covering the last `days` days in chunks the API accepts."""
+    now = now or dt.datetime.now(dt.timezone.utc)
+    start = now - dt.timedelta(days=days)
+    out, cursor = [], start
+    while cursor < now:
+        end = min(cursor + dt.timedelta(days=size), now)
+        out.append((cursor.strftime("%Y-%m-%dT%H:%M:%SZ"), end.strftime("%Y-%m-%dT%H:%M:%SZ")))
+        cursor = end
+    return out
+
+
+def fetch_incidents(days=90, urgencies=("high",), key=None, base_url=None, opener=urllib.request.urlopen, now=None):
+    """Incidents created in the last `days` days. Duplicates across window edges are removed."""
+    key = key or key_from_env()
+    base = (base_url or os.environ.get("PAGERDUTY_URL") or BASE_URL).rstrip("/")
+    found = {}
+    for since, until in windows(days, now):
+        offset = 0
+        for _ in range(MAX_PAGES_PER_WINDOW):
+            params = [("since", since), ("until", until), ("limit", PAGE_LIMIT), ("offset", offset),
+                      ("sort_by", "created_at")]
+            params += [("urgencies[]", u) for u in urgencies]
+            page = _get(f"{base}/incidents?{urllib.parse.urlencode(params)}", key, opener)
+            for inc in page.get("incidents", []):
+                found[inc["id"]] = inc
+            if not page.get("more"):
+                break
+            offset += PAGE_LIMIT
+        else:
+            raise PagerDutyError(f"more than {MAX_PAGES_PER_WINDOW * PAGE_LIMIT} incidents in one window; "
+                                 "narrow --days or --urgencies")
+    return list(found.values())
+
+
+def to_records(incidents):
+    out = []
+    for i in incidents:
+        service = (i.get("service") or {}).get("summary", "")
+        title = i.get("title") or ""
+        out.append({"id": f"PD-{i.get('incident_number', i['id'])}", "started_at": i["created_at"],
+                    "title": f"{service}: {title}" if service else title, "commits": "", "files": ""})
+    return out
+
+
+def sync(con, days=90, urgencies=("high",), **kwargs):
+    incidents = fetch_incidents(days, urgencies, **kwargs)
+    import_incidents_records(con, to_records(incidents))
+    return {"fetched": len(incidents), "imported": len(incidents)}

ledger/plan.py

@@ -0,0 +1,62 @@
+"""Sample-size planning by simulation: how many finished decisions to detect a given difference.
+
+Simulates routing with the same policy `ledger decide` uses (softmax over scores plus an
+epsilon floor), draws outcomes from the base rates you give, and runs the real estimator on
+each simulated dataset. Reports power for both the ordinary 95% interval and the
+anytime-valid interval that is safe to check repeatedly.
+"""
+import random
+
+from .estimate import analytic
+from .policy import propensities
+
+
+def simulate(n, rng, rates, epsilon, prefer):
+    """n decisions over arms named by `rates`; the last arm is preferred by `prefer` score points."""
+    arms = list(rates)
+    scores = [0.0] * (len(arms) - 1) + [prefer]
+    probs = dict(zip(arms, propensities(scores, epsilon, 1.0)))
+    weights = [probs[a] for a in arms]
+    rows = []
+    for _ in range(n):
+        arm = rng.choices(arms, weights=weights)[0]
+        rows.append({"arm": arm, "y": int(rng.random() < rates[arm]), "p": probs, "s": "all"})
+    return rows
+
+
+def power_at(n, rates, epsilon, prefer, reps, seed, planned_n):
+    rng = random.Random(seed)
+    arms = list(rates)
+    base, tested = arms[0], arms[-1]
+    fixed = anytime = 0
+    for _ in range(reps):
+        d = analytic(simulate(n, rng, rates, epsilon, prefer), arms, base, t_star=planned_n)[tested]["dr"]
+        fixed += d["diff_lo"] > 0 or d["diff_hi"] < 0
+        anytime += d["cs_lo"] > 0 or d["cs_hi"] < 0
+    return fixed / reps, anytime / reps
+
+
+def plan(base_rate, effect, arms=2, epsilon=0.1, prefer=0.0, grid=None, reps=200, seed=0, target=0.8):
+    """Power table over `grid` decision counts; the second element is the smallest n reaching `target`."""
+    if not 0 < base_rate < 1 or not 0 < base_rate + effect < 1:
+        raise ValueError("base rate and base rate + effect must both lie strictly between 0 and 1")
+    if arms < 2:
+        raise ValueError("need at least two arms")
+    names = [f"arm{i}" for i in range(arms)]
+    rates = {a: base_rate for a in names}
+    rates[names[-1]] = base_rate + effect
+    grid = grid or [500, 1000, 2000, 4000, 8000, 16000]
+    if any(n < 10 for n in grid):
+        raise ValueError("every value in the grid must be at least 10 decisions")
+    if reps < 1:
+        raise ValueError("reps must be at least 1")
+    grid = sorted(set(grid))
+    planned = max(grid) // 2
+    table = []
+    for n in grid:
+        fixed, anytime = power_at(n, rates, epsilon, prefer, reps, seed, planned)
+        table.append({"n": n, "power_fixed": fixed, "power_anytime": anytime})
+    reach = lambda key: next((r["n"] for r in table if r[key] >= target), None)
+    return {"base_rate": base_rate, "effect": effect, "arms": arms, "epsilon": epsilon, "prefer": prefer,
+            "reps": reps, "target": target, "planned_n": planned, "table": table,
+            "n_for_target_fixed": reach("power_fixed"), "n_for_target_anytime": reach("power_anytime")}

ledger/policy.py

@@ -0,0 +1,52 @@
+"""Stochastic arm selection with logged propensities.
+
+Randomisation happens in the selection step, never in LLM sampling, so every
+arm has a known, strictly positive probability. That is what makes IPW and
+doubly-robust estimates of cause and effect possible later.
+"""
+import json
+import math
+import random
+import time
+import uuid
+
+
+def propensities(scores, epsilon=0.1, temperature=1.0):
+    """p(a) = (1 - eps) * softmax(score / temperature) + eps / k."""
+    if not scores:
+        raise ValueError("need at least one arm")
+    if not 0 < epsilon <= 1:
+        raise ValueError("epsilon must be in (0, 1] so every arm stays possible")
+    k = len(scores)
+    top = max(s / temperature for s in scores)
+    exp = [math.exp(s / temperature - top) for s in scores]
+    total = sum(exp)
+    return [(1 - epsilon) * e / total + epsilon / k for e in exp]
+
+
+def record_decision(con, arms, scores=None, context=None, epsilon=0.1, temperature=1.0,
+                    policy_version="v1", ts=None, rng=None, decision_id=None):
+    """Choose an arm, write the decision with all propensities, and return it."""
+    rng = rng or random.Random()
+    scores = scores if scores is not None else [0.0] * len(arms)
+    if len(scores) != len(arms):
+        raise ValueError("scores and arms must have the same length")
+    probs = propensities(scores, epsilon, temperature)
+    chosen = rng.choices(arms, weights=probs)[0]
+    decision = {
+        "id": decision_id or uuid.UUID(int=rng.getrandbits(128), version=4).hex,
+        "ts": int(ts if ts is not None else time.time()),
+        "policy_version": policy_version,
+        "context": context or {},
+        "arms": list(arms),
+        "propensities": dict(zip(arms, probs)),
+        "chosen": chosen,
+        "propensity": probs[arms.index(chosen)],
+    }
+    con.execute(
+        "INSERT INTO decisions VALUES (?,?,?,?,?,?,?,?)",
+        (decision["id"], decision["ts"], policy_version, json.dumps(decision["context"]),
+         json.dumps(decision["arms"]), json.dumps(decision["propensities"]), chosen,
+         decision["propensity"]),
+    )
+    return decision

ledger/report.py

@@ -0,0 +1,330 @@
+"""Render estimate results as a self-contained HTML report."""
+import datetime as dt
+import html
+import math
+
+NAMES = {"naive": "Naive", "ipw": "IPW", "snipw": "SNIPW", "dm": "Direct", "dr": "Doubly robust"}
+
+
+def pct(x, digits=1, sign=False):
+    if x is None or (isinstance(x, float) and math.isnan(x)):
+        return "–"
+    s = f"{x * 100:+.{digits}f}" if sign else f"{x * 100:.{digits}f}"
+    return s + (" pp" if sign else "%")
+
+
+def ci(e, sign=False):
+    return f'{pct(e["value"], sign=sign)} <span class="ci">[{pct(e["lo"], sign=sign)}, {pct(e["hi"], sign=sign)}]</span>'
+
+
+def anytime_note(est, arm):
+    """Whether the difference also holds under the anytime-valid confidence sequence."""
+    a = est.get("analytic", {}).get(arm, {}).get("dr", {})
+    if "cs_lo" not in a or math.isnan(a["cs_lo"]):
+        return ""
+    if a["cs_lo"] > 0 or a["cs_hi"] < 0:
+        return ' <span class="tag ok">holds even if you check this report every week</span>'
+    return (' <span class="tag">not yet safe to act on if you have been checking repeatedly '
+            f'(anytime-valid range {pct(a["cs_lo"], sign=True)} to {pct(a["cs_hi"], sign=True)})</span>')
+
+
+def verdicts(est, primary):
+    base = est["baseline"]
+    lines = []
+    for arm, entry in est["arms"].items():
+        if arm == base:
+            continue
+        d = entry["vs_baseline"][primary]
+        name, bname = html.escape(arm), html.escape(base)
+        if est["diagnostics"][arm]["n"] == 0:
+            lines.append(f'<li class="even"><b>{name}</b> was never chosen, so there is nothing to compare with {bname}.</li>')
+            continue
+        note = anytime_note(est, arm) if d["lo"] > 0 or d["hi"] < 0 else ""
+        size = f'{abs(d["value"]) * 100:.1f}&nbsp;pp'
+        interval = f'95% CI {pct(d["lo"], sign=True)} to {pct(d["hi"], sign=True)}'
+        if d["lo"] > 0:
+            lines.append(f'<li class="worse"><b>{name}</b> causes <b>{size}</b> more bad outcomes than {bname} '
+                         f'({interval}).{note}</li>')
+        elif d["hi"] < 0:
+            lines.append(f'<li class="better"><b>{name}</b> causes <b>{size}</b> fewer bad outcomes than {bname} '
+                         f'({interval}).{note}</li>')
+        else:
+            lines.append(f'<li class="even"><b>{name}</b> vs {bname}: no clear difference yet '
+                         f'({pct(d["value"], sign=True)}, 95% CI {pct(d["lo"], sign=True)} to {pct(d["hi"], sign=True)}).</li>')
+    return "\n".join(lines)
+
+
+def forest(est, primary, truth=None):
+    arms = list(est["arms"])
+    vals = []
+    unused = {a for a in arms if est["diagnostics"][a]["n"] == 0}
+    for a in arms:
+        if a in unused:
+            continue
+        for e in ("naive", primary):
+            x = est["arms"][a]["estimates"][e]
+            vals += [x["lo"], x["hi"], x["value"]]
+        if truth and a in truth:
+            vals.append(truth[a])
+    vals = [v for v in vals if not math.isnan(v)] or [0.0, 0.1]
+    lo, hi = max(0.0, min(vals) - 0.01), max(vals) + 0.01
+    W, left, right, row = 640, 110, 24, 56
+    H = 40 + row * len(arms)
+    sx = lambda v: left + (v - lo) / (hi - lo) * (W - left - right)
+    parts = [f'<svg viewBox="0 0 {W} {H}" role="img" aria-label="Estimated bad-outcome rate per arm">']
+    step = 0.01 if hi - lo < 0.12 else 0.02 if hi - lo < 0.25 else 0.05
+    t = math.ceil(lo / step) * step
+    while t <= hi + 1e-9:
+        x = sx(t)
+        parts.append(f'<line x1="{x:.1f}" x2="{x:.1f}" y1="16" y2="{H - 20}" class="grid"/>'
+                     f'<text x="{x:.1f}" y="{H - 4}" class="tick">{t * 100:.0f}%</text>')
+        t += step
+    for i, a in enumerate(arms):
+        y = 34 + i * row
+        parts.append(f'<text x="{left - 12}" y="{y + 10}" class="arm">{html.escape(a)}</text>')
+        if a in unused:
+            parts.append(f'<text x="{left + 8}" y="{y + 10}" class="tick" style="text-anchor:start">never chosen: no data</text>')
+            continue
+        for j, (e, cls) in enumerate((("naive", "naive"), (primary, "primary"))):
+            x = est["arms"][a]["estimates"][e]
+            yy = y + j * 18
+            parts.append(f'<line x1="{sx(x["lo"]):.1f}" x2="{sx(x["hi"]):.1f}" y1="{yy}" y2="{yy}" class="ci-{cls}"/>'
+                         f'<circle cx="{sx(x["value"]):.1f}" cy="{yy}" r="5" class="pt-{cls}">'
+                         f'<title>{NAMES[e]}: {x["value"] * 100:.1f}% [{x["lo"] * 100:.1f}, {x["hi"] * 100:.1f}]</title></circle>')
+        if truth and a in truth:
+            x = sx(truth[a])
+            parts.append(f'<path d="M{x:.1f} {y - 9} l5 -7 h-10 z" class="truth"><title>True rate: {truth[a] * 100:.1f}%</title></path>')
+    parts.append("</svg>")
+    return "".join(parts)
+
+
+CSS = """
+:root{--bg:#f7f6f2;--card:#fff;--ink:#1d1c1a;--muted:#6b6862;--line:#e3e0d8;--accent:#2f5bd3;--naive:#a8a39a;
+--good:#1f7a4d;--bad:#b3261e;--truth:#d98a00}
+@media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#151515;--card:#1e1e1e;--ink:#ecebe7;--muted:#a19d95;
+--line:#34322e;--accent:#7ea1ff;--naive:#77736b;--good:#5fc48d;--bad:#ff8a80;--truth:#ffb74d}}
+:root[data-theme="dark"]{--bg:#151515;--card:#1e1e1e;--ink:#ecebe7;--muted:#a19d95;--line:#34322e;--accent:#7ea1ff;
+--naive:#77736b;--good:#5fc48d;--bad:#ff8a80;--truth:#ffb74d}
+*{box-sizing:border-box}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.55 system-ui,-apple-system,"Segoe UI",sans-serif}
+main{max-width:920px;margin:0 auto;padding:32px 16px 64px}h1{font-size:28px;margin:0 0 4px;letter-spacing:-.01em}
+h2{font-size:17px;margin:32px 0 10px}.sub{color:var(--muted);margin:0 0 24px}
+.card{background:var(--card);border:1px solid var(--line);border-radius:12px;padding:18px 20px;margin:12px 0}
+ul.verdict{list-style:none;padding:0;margin:0}ul.verdict li{padding:8px 0 8px 14px;border-left:3px solid var(--line);margin:6px 0}
+li.worse{border-color:var(--bad)!important}li.better{border-color:var(--good)!important}
+.table{overflow-x:auto}table{border-collapse:collapse;width:100%;font-variant-numeric:tabular-nums;font-size:14px}
+th,td{text-align:right;padding:7px 10px;border-bottom:1px solid var(--line);white-space:nowrap}th:first-child,td:first-child{text-align:left}
+th{color:var(--muted);font-weight:600;font-size:12px;text-transform:uppercase;letter-spacing:.04em}
+.ci{color:var(--muted);font-size:12px}.kpis{display:grid;grid-template-columns:repeat(auto-fit,minmax(140px,1fr));gap:10px}
+.kpi{background:var(--card);border:1px solid var(--line);border-radius:10px;padding:12px 14px}
+.kpi b{display:block;font-size:22px}.kpi span{color:var(--muted);font-size:13px}
+svg{width:100%;height:auto;display:block}.grid{stroke:var(--line)}.tick{fill:var(--muted);font-size:11px;text-anchor:middle}
+.arm{fill:var(--ink);font-size:13px;text-anchor:end;font-weight:600}
+.ci-naive{stroke:var(--naive);stroke-width:2}.pt-naive{fill:var(--card);stroke:var(--naive);stroke-width:2}
+.ci-primary{stroke:var(--accent);stroke-width:3}.pt-primary{fill:var(--accent)}.truth{fill:var(--truth)}
+.legend{display:flex;gap:18px;flex-wrap:wrap;color:var(--muted);font-size:13px;margin-top:6px}
+.dot{display:inline-block;width:10px;height:10px;border-radius:50%;margin-right:6px;vertical-align:-1px}
+.note{color:var(--muted);font-size:13.5px}code{font-size:13px}
+.tag{display:inline-block;margin-left:6px;padding:1px 8px;border-radius:999px;font-size:12px;color:var(--muted);
+border:1px solid var(--line)}.tag.ok{color:var(--good);border-color:var(--good)}
+.warn{border-left:4px solid var(--truth)}.warn ul{margin:8px 0 0;padding-left:20px}.warn li{margin:4px 0}
+"""
+
+
+def render_audit(result, repo_name="", title="Consequence Ledger audit"):
+    as_of = dt.datetime.fromtimestamp(result["as_of"], dt.timezone.utc).strftime("%Y-%m-%d")
+    tools = result["tools"]
+    rates = [v for t in tools.values() for v in (t["ci_lo"], t["ci_hi"], t["size_adjusted_rate"]) if not math.isnan(v)]
+    hi = max(rates + [0.05]) * 1.1
+    W, left, right, row = 640, 110, 24, 40
+    H = 30 + row * len(tools)
+    sx = lambda v: left + v / hi * (W - left - right)
+    svg = [f'<svg viewBox="0 0 {W} {H}" role="img" aria-label="Bad-outcome rate per tool">']
+    step = 0.01 if hi < 0.12 else 0.02 if hi < 0.25 else 0.05
+    t = 0.0
+    while t <= hi:
+        svg.append(f'<line x1="{sx(t):.1f}" x2="{sx(t):.1f}" y1="8" y2="{H - 20}" class="grid"/>'
+                   f'<text x="{sx(t):.1f}" y="{H - 4}" class="tick">{t * 100:.0f}%</text>')
+        t += step
+    for i, (name, d) in enumerate(tools.items()):
+        y = 24 + i * row
+        svg.append(f'<text x="{left - 12}" y="{y + 4}" class="arm">{html.escape(name)}</text>'
+                   f'<line x1="{sx(d["ci_lo"]):.1f}" x2="{sx(d["ci_hi"]):.1f}" y1="{y}" y2="{y}" class="ci-primary"/>'
+                   f'<circle cx="{sx(d["bad_rate"]):.1f}" cy="{y}" r="5" class="pt-primary">'
+                   f'<title>{d["bad_rate"] * 100:.1f}% of {d["commits"]} commits</title></circle>')
+        if not math.isnan(d["size_adjusted_rate"]):
+            svg.append(f'<circle cx="{sx(d["size_adjusted_rate"]):.1f}" cy="{y}" r="5" class="pt-naive">'
+                       f'<title>Size-adjusted: {d["size_adjusted_rate"] * 100:.1f}%</title></circle>')
+    svg.append("</svg>")
+    rows = "".join(
+        f"<tr><td>{html.escape(n)}</td><td>{d['commits']}</td><td>{d['reverted']}</td><td>{d['fixed']}</td>"
+        f"<td>{pct(d['bad_rate'])} <span class=ci>[{pct(d['ci_lo'])}, {pct(d['ci_hi'])}]</span></td>"
+        f"<td>{pct(d['size_adjusted_rate'])}</td></tr>" for n, d in tools.items())
+    detection = "".join(f'<div class="card note"><b>Possible undercount.</b> {html.escape(n)}</div>'
+                        for n in result.get("detection_notes", ()))
+    return f"""<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1"><title>{html.escape(title)}</title>
+<style>{CSS}</style></head><body><main>
+<h1>{html.escape(title)}</h1>
+<p class="sub">Share of each tool's commits reverted or bug-fixed within {result['horizon_days']} days.
+As of {as_of}{' · ' + html.escape(repo_name) if repo_name else ''} · {result['mature_commits']} mature commits.</p>
+<div class="card note"><b>This is a baseline, not proof.</b> Tools get used on different work, so these gaps mix the tool's
+effect with the task's difficulty. Size adjustment removes one confounder. To get cause and effect, route work with
+<code>ledger decide</code> and compare with <code>ledger estimate</code>.</div>
+{detection}<h2>Bad-outcome rate by tool</h2>
+<div class="card">{''.join(svg)}<div class="legend"><span><i class="dot" style="background:var(--accent)"></i>Raw rate, 95% CI</span>
+<span><i class="dot" style="background:var(--naive)"></i>Size-adjusted</span></div></div>
+<div class="card table"><table><thead><tr><th>Tool</th><th>Commits</th><th>Reverted</th><th>Fixed</th><th>Bad rate [95% CI]</th>
+<th>Size-adjusted</th></tr></thead><tbody>{rows}</tbody></table></div>
+</main></body></html>"""
+
+
+def warnings(est):
+    """Plain-language checks that would undermine the estimates if ignored."""
+    out = []
+    bal = est.get("merge_balance")
+    if bal and bal.get("p") is not None and bal["p"] < 0.05:
+        rates = ", ".join(f"{html.escape(a)} {pct(v['rate'])}" for a, v in bal["arms"].items())
+        out.append(f"<li><b>Unmerged work differs by arm</b> ({rates}; p = {bal['p']:.3f}). Estimates only cover merged "
+                   f"changes, so an arm whose weak changes get abandoned will look better than it is.</li>")
+    for a, d in est["diagnostics"].items():
+        n, bad, name = d["n"], d["bad"], html.escape(a)
+        if n == 0:
+            out.append(f"<li><b>{name}</b> was never chosen, so no estimate exists for it.</li>")
+            continue
+        if bad == 0:
+            out.append(f"<li><b>{name}</b> had no bad outcomes in {n} decisions. That is consistent with a true rate of up "
+                       f"to about {min(1.0, 3 / n) * 100:.0f}% (\"rule of three\"), so its interval is narrower than the data justify.</li>")
+        elif bad == n:
+            out.append(f"<li><b>{name}</b> had only bad outcomes in {n} decisions, so its interval is narrower than the "
+                       f"data justify.</li>")
+        elif bad < 5:
+            out.append(f"<li><b>{name}</b> has only {bad} bad outcomes, too few for reliable intervals.</li>")
+        if d["min_propensity"] < 0.02:
+            out.append(f"<li><b>{html.escape(a)}</b> was almost never chosen for some tasks (lowest probability "
+                       f"{d['min_propensity'] * 100:.1f}%), so a few decisions carry weights up to {d['max_weight']:.0f}. "
+                       f"Raise exploration (<code>--epsilon</code>) or cap weights (<code>--clip</code>).</li>")
+        if 0 < d["ess"] < 30:
+            out.append(f"<li><b>{html.escape(a)}</b> has an effective sample size of only {d['ess']:.0f}; "
+                       f"treat its numbers as rough.</li>")
+    if not out:
+        return ""
+    return f'<div class="card warn"><b>Check before trusting these numbers</b><ul>{"".join(out)}</ul></div>'
+
+
+def early_card(est, label_stats):
+    early = est.get("early")
+    if not early:
+        return ""
+    days = label_stats.get("early_days") or "early"
+    cal = early.get("calibration")
+    pairs = cal["pairs"] if cal else {}
+    head = "<th>Arm</th><th>Early difference [95% CI]</th>" + (
+        "<th>Same decisions: early</th><th>Same decisions: final</th>" if cal else "")
+    rows = "".join(
+        f"<tr><td>{html.escape(a)}</td><td>{pct(v['diff'], sign=True)} <span class=ci>[{pct(v['diff_lo'], sign=True)}, "
+        f"{pct(v['diff_hi'], sign=True)}]</span></td>"
+        + (f"<td>{pct(pairs[a]['early_diff'], sign=True)}</td><td>{pct(pairs[a]['final_diff'], sign=True)}</td>"
+           if a in pairs else "")
+        + "</tr>"
+        for a, v in early["arms"].items() if a != early["baseline"])
+    if cal:
+        share = cal["surfaced_early"]
+        weak = " That is too few for the early numbers to say much on their own." if share < 0.4 else ""
+        trust = (f"Only {pct(share, 0)} of the final bad outcomes had surfaced by day {days} on the {cal['n']} decisions "
+                 f"old enough to have both labels, so early differences are a muted version of final ones.{weak} "
+                 f"The last two columns show how they compared on those decisions; use them to judge the first column.")
+    else:
+        trust = "No decisions are old enough yet to check how well early signals predict the final outcome; treat this as a hint."
+    return (f'<h2>Early signal: bad within {days} days</h2><div class="card"><p class="note">Covers {early["n"]} decisions, '
+            f'including recent ones the main estimate must still exclude. Difference vs {html.escape(early["baseline"])}, '
+            f'doubly robust:</p><div class="table"><table><thead><tr>{head}</tr></thead>'
+            f'<tbody>{rows}</tbody></table></div><p class="note">{trust}</p></div>')
+
+
+def incident_rows(links):
+    names = {"suspect": "Suspect commit in the export", "files_off": "Files only, not linked (default)",
+             "files_latest": "Files only, latest matching deploy", "files_window": "Files only, every match in 14 days"}
+    return "".join(
+        f"<tr><td>{names.get(k, html.escape(k))}</td><td>{v['incidents']}</td><td>{pct(v['culprit_found'], 0)}</td>"
+        f"<td>{v['avg_commits_blamed']:.1f}</td></tr>" for k, v in links.items())
+
+
+def validation_card(v):
+    if not v:
+        return ""
+    links = v.get("incident_linking")
+    incident_table = (
+        '<p>How incidents were tied to the change that caused them:</p><div class="table"><table><thead><tr>'
+        '<th>Link</th><th>Incidents</th><th>Culprit found</th><th>Commits blamed</th></tr></thead>'
+        f'<tbody>{incident_rows(links)}</tbody></table></div>' if links else "")
+    oracle = ", ".join(f"{html.escape(a)} {pct(e['value'])}" for a, e in v["oracle_dr"].items())
+    return (f'<h2>Label check against the known truth</h2><div class="card note">'
+            f'<p>Of the decisions the labeler marked bad, <b>{pct(v["precision"], 0)}</b> really were (precision). '
+            f'Of the truly bad ones, it caught <b>{pct(v["recall"], 0)}</b> (recall). '
+            f'{v["fp"]} false alarms (fixes that touched these lines for unrelated reasons) and '
+            f'{v["fn"]} misses.</p>'
+            f'<p>Doubly robust estimates using perfect labels instead: {oracle}. The gap between these and the '
+            f'table above is labeling noise; the gap between these and the true rates is estimator error.</p>'
+            f'{incident_table}</div>')
+
+
+def render(est, label_stats, title="Consequence Ledger", primary="dr", truth=None, meta=None, validation=None):
+    meta = meta or {}
+    as_of = dt.datetime.fromtimestamp(label_stats["as_of"], dt.timezone.utc).strftime("%Y-%m-%d")
+    rows = []
+    for a, entry in est["arms"].items():
+        d = est["diagnostics"][a]
+        e = entry["estimates"]
+        vs = entry["vs_baseline"][primary]
+        rows.append(
+            f"<tr><td>{html.escape(a)}{' <span class=ci>(baseline)</span>' if a == est['baseline'] else ''}</td>"
+            f"<td>{d['n']}</td><td>{d['bad']}</td><td>{pct(e['naive']['value'])}</td>"
+            f"<td>{pct(e['ipw']['value'])}</td><td>{pct(e['snipw']['value'])}</td><td>{ci(e[primary])}</td>"
+            f"<td>{'–' if a == est['baseline'] else ci(vs, sign=True)}</td>"
+            f"<td>{d['ess']:.0f}</td>"
+            + (f"<td>{pct(truth[a])}</td>" if truth and a in truth else "")
+            + "</tr>")
+    truth_head = "<th>True</th>" if truth else ""
+    total = label_stats.get("mature", 0) + label_stats.get("immature", 0) + label_stats.get("no_commit", 0)
+    kpis = [
+        (total, "decisions logged"),
+        (label_stats.get("mature", 0), f"mature (≥{label_stats['horizon_days']} days old)"),
+        (label_stats.get("revert", 0), "reverted in horizon"),
+        (label_stats.get("fix", 0), "fixed in horizon (SZZ)"),
+        (label_stats.get("incident", 0), "linked to an incident"),
+        (label_stats.get("no_commit", 0), "never merged"),
+    ]
+    legend = ('<div class="legend"><span><i class="dot" style="background:var(--naive)"></i>Naive (biased by routing)</span>'
+              f'<span><i class="dot" style="background:var(--accent)"></i>{NAMES[primary]} (causal)</span>'
+              + ('<span><i class="dot" style="background:var(--truth)"></i>True rate (synthetic demo)</span>' if truth else "")
+              + "</div>")
+    source = html.escape(meta.get("repo", ""))
+    clusters = est.get("clusters", est["n"])
+    cluster_note = f", resampling {clusters} clusters of related decisions" if clusters < est["n"] else ""
+    return f"""<!doctype html><html lang="en"><head><meta charset="utf-8">
+<meta name="viewport" content="width=device-width,initial-scale=1"><title>{html.escape(title)}</title>
+<style>{CSS}</style></head><body><main>
+<h1>{html.escape(title)}</h1>
+<p class="sub">Which AI arm <i>causes</i> more reverted or bug-fixed code within {label_stats['horizon_days']} days of merge. As of {as_of}{' · ' + source if source else ''}.</p>
+<div class="card"><ul class="verdict">{verdicts(est, primary)}</ul></div>
+{warnings(est)}
+<div class="kpis">{''.join(f'<div class="kpi"><b>{v}</b><span>{k}</span></div>' for v, k in kpis)}</div>
+<h2>Bad-outcome rate if every task had gone to this arm</h2>
+<div class="card">{forest(est, primary, truth)}{legend}</div>
+<h2>Estimates</h2>
+<div class="card table"><table><thead><tr><th>Arm</th><th>Decisions</th><th>Bad</th><th>Naive</th><th>IPW</th><th>SNIPW</th>
+<th>{NAMES[primary]} [95% CI]</th><th>vs baseline</th><th>ESS</th>{truth_head}</tr></thead><tbody>{''.join(rows)}</tbody></table></div>
+{early_card(est, label_stats)}
+{validation_card(validation)}
+<h2>How to read this</h2>
+<div class="card note">
+<p><b>Bad</b> means the merged commit was reverted, or a later <code>fix:</code>/<code>hotfix:</code> commit changed its lines (SZZ via
+<code>git blame</code>), within {label_stats['horizon_days']} days. Only decisions whose full window has elapsed are counted, so recent
+work can't look clean just because its bugs haven't surfaced yet.</p>
+<p><b>Naive</b> compares raw rates and is misleading when harder tasks are routed to one tool. <b>IPW</b>, <b>SNIPW</b> and
+<b>doubly robust</b> use the probability logged at decision time to undo that routing, which turns correlation into cause and
+effect. Intervals are {est['boot']}-sample bootstrap 95% CIs{cluster_note}. <b>ESS</b> is the effective sample size after weighting; low values mean noisy estimates.</p>
+<p>SZZ labels are noisy (false positives from unrelated fixes), but that noise hits every arm alike, so differences stay meaningful.
+Decisions that were never merged ({label_stats.get('no_commit', 0)}) are excluded; check they are spread evenly across arms.</p>
+</div>
+</main></body></html>"""