swearbench 0.2.1__py3-none-any.whl

swearbench/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.2.1"

swearbench/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .cli import main
+
+sys.exit(main())

swearbench/card.py

@@ -0,0 +1,66 @@
+"""Leaderboard card, styled to match chart.svg. Model names and numbers only; never quotes."""
+from __future__ import annotations
+
+from html import escape
+
+from .chart import FAMILIES, PAD, STYLE, _text_w, family, pretty
+
+MODE_LABEL = {
+    "fabrication": "lies", "overreach": "overreach", "giving_up": "gave up", "regression": "breaks things",
+    "ignored": "ignores me", "repeat": "repeats", "incomplete": "half-done", "insult": "insults",
+    "profanity": "swearing", "shouting": "CAPS", "sarcasm": "sarcasm", "slow": "slow",
+}
+
+W = 760
+TOP = 120
+ROW_H = 40
+RANK_END = PAD + 16
+NAME_X = RANK_END + 14
+BAR_X = NAME_X + 160
+BAR_W = 180
+SCORE_END = BAR_X + BAR_W + 40
+STATS_X = SCORE_END + 24
+
+
+def svg(res, rows=8):
+    board = res["board"][:rows]
+    h = TOP + ROW_H * max(len(board), 1) + PAD + 20
+    hi = max([s["score"] for s in board] + [1])
+    o = [f'<svg xmlns="http://www.w3.org/2000/svg" class="c" width="{W}" height="{h}" viewBox="0 0 {W} {h}" '
+         'role="img" aria-label="SwearBench leaderboard">',
+         f"<style>{STYLE}</style>",
+         f'<rect width="{W}" height="{h}" rx="14" fill="var(--surface)"/>',
+         f'<text x="{PAD}" y="{PAD + 18}" font-size="20" font-weight="700" fill="var(--ink)">SwearBench</text>']
+
+    lx, ly = PAD, PAD + 52
+    for i, (name, _) in enumerate(FAMILIES):
+        o.append(f'<circle cx="{lx + 5}" cy="{ly - 4}" r="5" fill="var(--s{i + 1})"/>'
+                 f'<text x="{lx + 15}" y="{ly}" font-size="12" fill="var(--ink2)">{name}</text>')
+        lx += 15 + _text_w(name) + 24
+    o.append(f'<text x="{W - PAD}" y="{ly}" font-size="12" fill="var(--muted)" text-anchor="end">'
+             f'{res["n_reactions"]} reactions</text>')
+
+    for i, s in enumerate(board):
+        y = TOP + i * ROW_H
+        mid = y + ROW_H / 2
+        base = mid + 4.5
+        col = f"var(--s{family(s['model']) + 1})"
+        top_mode = max(s["modes"], key=s["modes"].get) if s["modes"] else None
+        o += [f'<line x1="{PAD}" x2="{W - PAD}" y1="{y}" y2="{y}" stroke="var(--grid)" stroke-width="1"/>',
+              f'<text x="{RANK_END}" y="{base}" font-size="12" fill="var(--muted)" text-anchor="end">{i + 1}</text>',
+              f'<text x="{NAME_X}" y="{base}" font-size="13" font-weight="600" fill="var(--ink)">'
+              f'{escape(pretty(s["model"]))}</text>',
+              f'<rect x="{BAR_X}" y="{mid - 3}" width="{BAR_W}" height="6" rx="3" fill="var(--grid)"/>',
+              f'<rect x="{BAR_X}" y="{mid - 3}" width="{max(6, BAR_W * max(s["score"], 0) / hi):.0f}" height="6" '
+              f'rx="3" fill="{col}"/>',
+              f'<text x="{SCORE_END}" y="{base}" font-size="13" font-weight="700" fill="var(--ink)" '
+              f'text-anchor="end" font-variant-numeric="tabular-nums">{s["score"]:.0f}</text>',
+              f'<text x="{STATS_X}" y="{base}" font-size="12" fill="var(--ink2)">'
+              f'ships {s["outcome"]:.0f}% · {s["angry_pct"]:.0f}% angry</text>',
+              f'<text x="{W - PAD}" y="{base}" font-size="12" fill="var(--muted)" text-anchor="end">'
+              f'{MODE_LABEL.get(top_mode, "")}</text>']
+    end = TOP + ROW_H * len(board)
+    o.append(f'<line x1="{PAD}" x2="{W - PAD}" y1="{end}" y2="{end}" stroke="var(--axis)" stroke-width="1"/>')
+    o.append(f'<text x="{PAD}" y="{h - PAD}" font-size="11" fill="var(--muted)">github.com/AlenHay/swearbench</text>')
+    o.append("</svg>")
+    return "\n".join(o)

swearbench/chart.py

@@ -0,0 +1,134 @@
+"""Swearing vs. result: one dot per model. x = rage per 100 turns, y = share of sessions that land."""
+from __future__ import annotations
+
+import math
+import re
+from html import escape
+
+FAMILIES = [("Claude", ("claude",)), ("GPT", ("gpt", "o1", "o3", "o4", "codex")), ("Other", ())]
+
+STYLE = """
+.c{--surface:#fcfcfb;--ink:#0b0b0b;--ink2:#52514e;--muted:#898781;--grid:#e1e0d9;--axis:#c3c2b7;
+--s1:#2a78d6;--s2:#eb6834;--s3:#1baf7a}
+@media (prefers-color-scheme:dark){.c{--surface:#1a1a19;--ink:#fff;--ink2:#c3c2b7;--grid:#2c2c2a;--axis:#383835;
+--s1:#3987e5;--s2:#d95926;--s3:#199e70}}
+text{font-family:ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}
+"""
+
+
+def pretty(model):
+    """claude-opus-5-5 -> Opus 5.5, gpt-5.6-sol -> GPT-5.6 Sol, gemini-3.8-flash-high -> Gemini 3.8 Flash."""
+    m = re.sub(r"-\d{8}$", "", model.split("/")[-1])
+    if m.startswith("claude-"):
+        parts = m[7:].split("-")
+        return f"{parts[0].title()} {'.'.join(parts[1:])}"
+    if m.startswith("gpt-"):
+        ver, *rest = m[4:].split("-")
+        return " ".join([f"GPT-{ver}"] + [r.title() for r in rest])
+    words = [w for w in m.split("-") if w not in ("high", "low", "medium", "free", "preview")]
+    return " ".join(w if any(c.isdigit() for c in w) else w.title() for w in words)
+
+
+def family(model):
+    for i, (_, prefixes) in enumerate(FAMILIES[:-1]):
+        if model.startswith(prefixes):
+            return i
+    return len(FAMILIES) - 1
+
+
+def _ticks(lo, hi, n=5):
+    step = 10 ** math.floor(math.log10((hi - lo) / n or 1))
+    for m in (1, 2, 2.5, 5, 10):
+        if (hi - lo) / (step * m) <= n:
+            step *= m
+            break
+    start = math.floor(lo / step) * step
+    return [start + i * step for i in range(int((hi - start) / step) + 2) if start + i * step <= hi + 1e-9]
+
+
+PAD = 32
+
+
+def _text_w(text, size=12):
+    return len(text) * size * 0.56
+
+
+def svg(res):
+    pts = res["board"]
+    W, H = 760, 540
+    L, R, T, B = PAD + 48, PAD, 120, 76
+    pw, ph = W - L - R, H - T - B
+    xs = [p["rage100"] for p in pts] or [0, 1]
+    x_lo, x_hi = math.floor(min(xs) * 0.9 / 10) * 10, math.ceil(max(xs) * 1.05 / 10) * 10
+    x_lo, x_hi = (x_lo, x_hi) if x_hi > x_lo else (0, 100)
+    X = lambda v: L + (v - x_lo) / (x_hi - x_lo) * pw  # noqa: E731
+    Y = lambda v: T + ph - v / 100 * ph  # noqa: E731
+    max_n = max([p["sessions"] for p in pts] + [1])
+
+    o = [f'<svg xmlns="http://www.w3.org/2000/svg" class="c" width="{W}" height="{H}" viewBox="0 0 {W} {H}" '
+         'role="img" aria-labelledby="t d">',
+         f"<style>{STYLE}</style>",
+         '<title id="t">SwearBench: how much swearing gets you what result</title>',
+         '<desc id="d">' + escape("; ".join(f'{p["model"]}: rage {p["rage100"]:.0f} per 100 turns, '
+                                            f'ships {p["outcome"]:.0f}%' for p in pts)) + "</desc>",
+         f'<rect width="{W}" height="{H}" rx="14" fill="var(--surface)"/>',
+         f'<text x="{PAD}" y="{PAD + 18}" font-size="20" font-weight="700" fill="var(--ink)">'
+         'Swearing vs. result</text>']
+
+    lx, ly = PAD, PAD + 52
+    for i, (name, _) in enumerate(FAMILIES):
+        o.append(f'<circle cx="{lx + 5}" cy="{ly - 4}" r="5" fill="var(--s{i + 1})"/>'
+                 f'<text x="{lx + 15}" y="{ly}" font-size="12" fill="var(--ink2)">{name}</text>')
+        lx += 15 + _text_w(name) + 24
+
+    for v in _ticks(x_lo, x_hi):
+        o.append(f'<line x1="{X(v):.1f}" x2="{X(v):.1f}" y1="{T}" y2="{T + ph}" stroke="var(--grid)" stroke-width="1"/>')
+        o.append(f'<text x="{X(v):.1f}" y="{T + ph + 22}" font-size="11" fill="var(--muted)" text-anchor="middle">{v:g}</text>')
+    for v in range(0, 101, 20):
+        o.append(f'<line x1="{L}" x2="{L + pw}" y1="{Y(v):.1f}" y2="{Y(v):.1f}" stroke="var(--grid)" stroke-width="1"/>')
+        o.append(f'<text x="{L - 12}" y="{Y(v) + 4:.1f}" font-size="11" fill="var(--muted)" text-anchor="end">{v}%</text>')
+    o.append(f'<line x1="{L}" x2="{L + pw}" y1="{T + ph}" y2="{T + ph}" stroke="var(--axis)" stroke-width="1"/>')
+    o.append(f'<text x="{L + pw / 2}" y="{H - PAD}" font-size="12" fill="var(--ink2)" text-anchor="middle">'
+             'Swearing →</text>')
+    o.append(f'<text transform="translate({PAD + 4} {T + ph / 2}) rotate(-90)" font-size="12" fill="var(--ink2)" '
+             'text-anchor="middle">Shipped →</text>')
+
+    dots = []
+    for p in sorted(pts, key=lambda p: -p["sessions"]):
+        cx, cy = X(p["rage100"]), Y(p["outcome"])
+        r = 5 + 9 * math.sqrt(p["sessions"] / max_n)
+        col = f"var(--s{family(p['model']) + 1})"
+        lo, hi = p.get("outcome_ci", (p["outcome"], p["outcome"]))
+        tip = escape(f'{p["model"]}\nrage {p["rage100"]:.0f}/100 turns · {p["angry_pct"]:.0f}% angry msgs\n'
+                     f'ships {p["outcome"]:.0f}% of {p["decided"]} decided sessions (90% CI {lo:.0f}–{hi:.0f}%)\n'
+                     f'SwearBench {p["score"]:.1f}')
+        o.append(f'<g><title>{tip}</title>'
+                 f'<line x1="{cx:.1f}" x2="{cx:.1f}" y1="{Y(hi):.1f}" y2="{Y(lo):.1f}" stroke="{col}" stroke-width="2" '
+                 'stroke-opacity="0.45" stroke-linecap="round"/>'
+                 f'<circle cx="{cx:.1f}" cy="{cy:.1f}" r="{r + 8:.1f}" fill="transparent"/>'
+                 f'<circle cx="{cx:.1f}" cy="{cy:.1f}" r="{r:.1f}" fill="{col}" stroke="var(--surface)" stroke-width="2"/></g>')
+        dots.append((p, cx, cy, r))
+
+    boxes = [(cx - r, cy - r, cx + r, cy + r) for _, cx, cy, r in dots]
+    for p, cx, cy, r in dots:
+        label = pretty(p["model"])
+        w = _text_w(label)
+        best = None
+        for dy in (0, -18, 18, -32, 32, -46, 46):
+            for side in (1, -1):
+                x0 = cx + r + 8 if side > 0 else cx - r - 8 - w
+                y = cy + 4 + dy
+                box = (x0, y - 11, x0 + w, y + 3)
+                if not (L + 2 <= box[0] and box[2] <= L + pw - 2 and T + 28 <= box[1] and box[3] <= T + ph - 20):
+                    continue
+                overlap = sum(max(0, min(box[2], b[2]) - max(box[0], b[0])) * max(0, min(box[3], b[3]) - max(box[1], b[1]))
+                              for b in boxes)
+                if best is None or overlap < best[0]:
+                    best = (overlap, x0, y, box)
+            if best and best[0] == 0:
+                break
+        _, x0, y, box = best or (0, cx + r + 6, cy + 4, (cx + r + 6, cy - 7, cx + r + 6 + w, cy + 7))
+        boxes.append(box)
+        o.append(f'<text x="{x0:.1f}" y="{y:.1f}" font-size="12" fill="var(--ink)">{escape(label)}</text>')
+    o.append("</svg>")
+    return "\n".join(o)

swearbench/cli.py

@@ -0,0 +1,77 @@
+"""swearbench: rank AI coding models by how much you got mad at them."""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import sys
+
+from . import card, chart, judge, score, sources
+
+
+def main(argv=None):
+    ap = argparse.ArgumentParser(prog="swearbench", description=__doc__)
+    ap.add_argument("--judge", choices=["claude-cli", "codex-cli", "anthropic", "openai", "command"],
+                    help="who labels your messages (default: first available of claude, codex, ANTHROPIC_API_KEY, OPENAI_API_KEY)")
+    ap.add_argument("--judge-model", help="model id for the judge backend")
+    ap.add_argument("--judge-cmd", help="with --judge command: shell command that reads a prompt on stdin and prints the reply")
+    ap.add_argument("--since", help="only reactions on/after this date (YYYY-MM-DD)")
+    ap.add_argument("--min-reactions", type=int, default=40, help="reactions a model needs to be ranked (default 40)")
+    ap.add_argument("--exclude", action="append", default=[], help="model id to leave out (repeatable)")
+    ap.add_argument("--no-t3", action="store_true", help="ignore T3 Code's database even if present")
+    ap.add_argument("--no-quotes", action="store_true", help="leave the hall of shame out of report.md")
+    ap.add_argument("--out", default="swearbench-out", help="output directory (default ./swearbench-out)")
+    ap.add_argument("--workers", type=int, default=8)
+    ap.add_argument("--dry-run", action="store_true", help="show what was found and what would be judged, then stop")
+    ap.add_argument("-y", "--yes", action="store_true", help="don't ask before sending messages to the judge")
+    a = ap.parse_args(argv)
+
+    corpus = sources.collect(use_t3=not a.no_t3, exclude_dirs=["swearbench"])
+    if a.since:
+        corpus.reactions = [r for r in corpus.reactions if r.ts >= a.since]
+        corpus.interrupts = [i for i in corpus.interrupts if i.ts >= a.since]
+    print("Found: " + ("; ".join(corpus.found) or "no logs"), file=sys.stderr)
+    if not corpus.reactions:
+        print("No human messages found. Supported: Claude Code, Codex, OpenCode, T3 Code.", file=sys.stderr)
+        return 1
+
+    backend = a.judge or judge.detect_backend()
+    if not backend:
+        print("No judge available: install claude or codex CLI, or set ANTHROPIC_API_KEY / OPENAI_API_KEY, "
+              "or use --judge command --judge-cmd '...'.", file=sys.stderr)
+        return 1
+    model = a.judge_model or judge.DEFAULT_MODELS.get(backend)
+    cached = judge.load_cache()
+    todo = len({judge.key(r) for r in corpus.reactions} - set(cached))
+    print(f"{len(corpus.reactions)} messages from you; {todo} not judged yet. Judge: {backend}"
+          f"{' / ' + model if model else ''}.", file=sys.stderr)
+    if a.dry_run:
+        return 0
+    if todo and not a.yes:
+        if input(f"Send {todo} of your messages (with the AI reply each one answers) to the judge? [y/N] ").lower() != "y":
+            return 1
+
+    labels = judge.label(corpus.reactions, backend, model, a.judge_cmd, workers=a.workers)
+    res = score.build(corpus, labels, a.min_reactions, set(a.exclude))
+    family = (model or backend).split("-")[0]
+    if any(s["model"].startswith(family) for s in res["board"]):
+        print(f"Note: the judge ({model or backend}) is from a family being ranked. "
+              "Try a different --judge to check it isn't playing favourites.", file=sys.stderr)
+
+    os.makedirs(a.out, exist_ok=True)
+    report = score.markdown(res, quotes=not a.no_quotes)
+    with open(os.path.join(a.out, "report.md"), "w") as f:
+        f.write(report)
+    with open(os.path.join(a.out, "card.svg"), "w") as f:
+        f.write(card.svg(res))
+    with open(os.path.join(a.out, "chart.svg"), "w") as f:
+        f.write(chart.svg(res))
+    with open(os.path.join(a.out, "results.json"), "w") as f:
+        json.dump({k: v for k, v in res.items()}, f, indent=1, default=str)
+    print(report.split("\n## ")[0])
+    print(f"Wrote {a.out}/report.md, card.svg, chart.svg, results.json", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

swearbench/judge.py

@@ -0,0 +1,149 @@
+"""LLM judge: labels each reaction with anger, anger modes and satisfaction. Results are cached on disk."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+import tempfile
+import urllib.request
+from concurrent.futures import ThreadPoolExecutor
+
+RUBRIC_VERSION = "2"
+CACHE = os.path.join(os.environ.get("XDG_CACHE_HOME", os.path.expanduser("~/.cache")), "swearbench", "labels.jsonl")
+
+RUBRIC = """You are labelling a developer's chat messages to AI coding agents, to measure how frustrated
+the developer was with the AI. Each item has `prev` (tail of the AI's last reply) and `msg` (the
+developer's next message). Judge `msg` as a reaction to the AI's work.
+
+Return ONLY a JSON array, one object per item, same order:
+{"id": str,
+ "agent_authored": bool,   // msg is clearly written by another AI/agent (long structured brief, "You are...", tool-dispatch prose), not a human
+ "anger": 0-4,             // 0 calm/neutral, 1 mild annoyance, 2 clearly irritated, 3 angry, 4 furious
+ "target": "model"|"external"|"self"|"none",  // who the frustration is aimed at; tools/vendors/infra/third parties = external
+ "modes": [..],            // zero or more, only when target=="model":
+    // "profanity"  swearing directed at the work or the model
+    // "insult"     calling the model stupid/useless/lazy/idiot etc.
+    // "shouting"   caps, !!!, ??? for emphasis
+    // "repeat"     had to repeat an instruction / "I already told you" / "again"
+    // "ignored"    model ignored or violated an explicit instruction or rule
+    // "fabrication" model claimed success/facts that were false, hallucinated, lied
+    // "incomplete" lazy, half-done, stopped early, left TODOs, didn't verify
+    // "regression" model broke something that worked
+    // "overreach"  did unrequested/destructive things, scope creep, touched what it shouldn't
+    // "slow"       too slow, too many questions, too verbose, wasted time
+    // "sarcasm"    sarcastic/passive-aggressive phrasing
+    // "giving_up"  abandons the model/approach, "I'll do it myself", "forget it", threatens to switch
+    // "taste"      subjective critique while iterating on look/feel/wording ("looks lame", "too much text");
+    //              the work does what was asked, the developer just doesn't like it yet
+ "blames_earlier": bool,   // msg complains that something done BEFORE the AI's last reply (earlier merged/shipped work,
+                           // a previous session) is broken, missing or wrong, rather than the last reply itself
+ "satisfaction": -2..2,    // -2 rejects the work, -1 wants fixes, 0 neutral/new task, 1 accepts, 2 explicit praise/delight
+ "quote": str              // <=80 char excerpt that best shows the anger (or "" if anger==0)
+}
+Profanity used positively ("fucking cool") is not anger. Terse instructions are not anger. A new
+unrelated task after a reply implies mild acceptance (satisfaction 0 or 1), not anger.
+
+ITEMS:
+"""
+
+DEFAULT_MODELS = {"claude-cli": "claude-sonnet-5-5", "codex-cli": None, "anthropic": "claude-sonnet-5-5", "openai": "gpt-5.5"}
+
+
+def key(r):
+    return hashlib.sha1(f"{RUBRIC_VERSION}\0{r.prev_reply[-500:]}\0{r.text[:1500]}".encode()).hexdigest()
+
+
+def load_cache():
+    out = {}
+    if os.path.exists(CACHE):
+        with open(CACHE) as f:
+            for line in f:
+                try:
+                    d = json.loads(line)
+                    out[d["key"]] = d
+                except (ValueError, KeyError):
+                    pass
+    return out
+
+
+def detect_backend():
+    if shutil.which("claude"):
+        return "claude-cli"
+    if shutil.which("codex"):
+        return "codex-cli"
+    if os.environ.get("ANTHROPIC_API_KEY"):
+        return "anthropic"
+    if os.environ.get("OPENAI_API_KEY"):
+        return "openai"
+    return None
+
+
+def _post(url, headers, body):
+    req = urllib.request.Request(url, json.dumps(body).encode(), {"content-type": "application/json", **headers})
+    with urllib.request.urlopen(req, timeout=600) as r:
+        return json.loads(r.read())
+
+
+def call(backend, model, cmd, prompt):
+    if backend == "command":
+        return subprocess.run(shlex.split(cmd), input=prompt, capture_output=True, text=True, timeout=900).stdout
+    if backend == "claude-cli":
+        args = ["claude", "-p", "--tools", ""] + (["--model", model] if model else [])
+        return subprocess.run(args, input=prompt, capture_output=True, text=True, timeout=900).stdout
+    if backend == "codex-cli":
+        with tempfile.NamedTemporaryFile("r", suffix=".txt") as out:
+            args = ["codex", "exec", "--skip-git-repo-check", "--sandbox", "read-only", "-o", out.name]
+            subprocess.run(args + (["-m", model] if model else []) + ["-"], input=prompt,
+                           capture_output=True, text=True, timeout=900)
+            return out.read()
+    if backend == "anthropic":
+        r = _post("https://api.anthropic.com/v1/messages",
+                  {"x-api-key": os.environ["ANTHROPIC_API_KEY"], "anthropic-version": "2023-06-01"},
+                  {"model": model, "max_tokens": 16000, "messages": [{"role": "user", "content": prompt}]})
+        return "".join(b.get("text", "") for b in r["content"])
+    if backend == "openai":
+        r = _post("https://api.openai.com/v1/chat/completions",
+                  {"authorization": "Bearer " + os.environ["OPENAI_API_KEY"]},
+                  {"model": model, "messages": [{"role": "user", "content": prompt}]})
+        return r["choices"][0]["message"]["content"]
+    raise ValueError(f"unknown judge backend {backend}")
+
+
+def _label_batch(batch, backend, model, cmd):
+    items = [{"id": str(i), "prev": r.prev_reply[-500:], "msg": r.text[:1500]} for i, r in enumerate(batch)]
+    prompt = RUBRIC + json.dumps(items, ensure_ascii=False)
+    err = "unparseable output"
+    for _ in range(3):
+        try:
+            txt = call(backend, model, cmd, prompt) or ""
+            arr = json.loads(txt[txt.index("["):txt.rindex("]") + 1])
+            got = {str(a.get("id")): a for a in arr if isinstance(a, dict)}
+            if len(got) >= len(batch) * 0.9:
+                return [(key(r), got[str(i)]) for i, r in enumerate(batch) if str(i) in got]
+        except Exception as e:  # noqa: BLE001
+            err = e
+    print(f"  judge batch failed after 3 tries: {err}", file=sys.stderr)
+    return []
+
+
+def label(reactions, backend, model, cmd=None, batch=30, workers=8):
+    cache = load_cache()
+    todo = [r for r in {key(r): r for r in reactions}.values() if key(r) not in cache]
+    if todo:
+        os.makedirs(os.path.dirname(CACHE), exist_ok=True)
+        batches = [todo[i:i + batch] for i in range(0, len(todo), batch)]
+        print(f"Judging {len(todo)} new messages in {len(batches)} batches with {backend}"
+              f"{' / ' + model if model else ''} ...", file=sys.stderr)
+        with open(CACHE, "a") as f, ThreadPoolExecutor(workers) as ex:
+            for n, res in enumerate(ex.map(lambda b: _label_batch(b, backend, model, cmd), batches), 1):
+                for k, lab in res:
+                    lab = {**lab, "key": k, "judge": f"{backend}:{model or 'default'}"}
+                    cache[k] = lab
+                    f.write(json.dumps(lab, ensure_ascii=False) + "\n")
+                f.flush()
+                print(f"  {n}/{len(batches)}", file=sys.stderr)
+    return {r.id: cache[key(r)] for r in reactions if key(r) in cache}

swearbench/score.py

@@ -0,0 +1,233 @@
+"""Turn labels into leaderboards."""
+from __future__ import annotations
+
+import random
+import re
+from collections import Counter, defaultdict
+
+MODE_WEIGHT = {
+    "fabrication": 3, "overreach": 3, "giving_up": 3, "regression": 2.5,
+    "ignored": 2, "repeat": 2, "incomplete": 1.5, "insult": 1.5,
+    "profanity": 1, "shouting": 0.5, "sarcasm": 0.5, "slow": 0.5, "taste": 0.5,
+}
+BREACH = {"fabrication", "overreach", "giving_up", "regression", "ignored", "repeat", "incomplete"}
+TASTE_DISCOUNT = 0.25
+FRICTION_K = 0.25
+REGRET_WINDOW_DAYS = 7
+INTERRUPT_RAGE = 1.5
+SWEAR = re.compile(r"\b(fuck\w*|shit\w*|wtf|damn\w*|crap\w*|bullshit|stupid|idiot\w*|dumb\w*|moron\w*|"
+                   r"bitch\w*|asshole|sucks?)\b", re.I)
+
+
+def rage(lab):
+    """Anger squared, so one blowup outweighs several grumbles, plus how you got mad.
+    Critique of taste while iterating, with no breach of trust, counts a quarter."""
+    if lab.get("agent_authored") or lab.get("target") != "model" or lab.get("anger", 0) < 1:
+        return 0.0
+    modes = set(lab.get("modes", []))
+    r = lab["anger"] ** 2 + sum(MODE_WEIGHT.get(m, 0) for m in modes)
+    return r * TASTE_DISCOUNT if "taste" in modes and not modes & BREACH else r
+
+
+def blame_shares(reactions, labels):
+    """For a complaint about earlier work, split it across the models that worked in the same workspace
+    in the week before, outside the current session, by how many turns each had there."""
+    import datetime as dt
+
+    def t(r):
+        return dt.datetime.fromisoformat(r.ts.replace("Z", "+00:00"))
+
+    by_ws = defaultdict(list)
+    for r in reactions:
+        if r.workspace and r.ts:
+            by_ws[r.workspace].append(r)
+    for rs in by_ws.values():
+        rs.sort(key=lambda r: r.ts)
+    shares = {}
+    window = dt.timedelta(days=REGRET_WINDOW_DAYS)
+    for r in reactions:
+        lab = labels.get(r.id)
+        if not lab or not lab.get("blames_earlier") or rage(lab) == 0 or not r.workspace:
+            continue
+        now = t(r)
+        prior = Counter(o.model for o in by_ws[r.workspace]
+                        if o.session != r.session and o.ts < r.ts and now - t(o) <= window and o.model)
+        total = sum(prior.values())
+        if total:
+            shares[r.id] = {m: n / total for m, n in prior.items()}
+    return shares
+
+
+def _own(r, l, shares):
+    return rage(l) * shares[r.id].get(r.model, 0.0) if r.id in shares else rage(l)
+
+
+def _friction(rows, n_int, shares, inbound_per_turn):
+    n = len(rows)
+    total = sum(_own(r, l, shares) for r, l in rows) + inbound_per_turn * n + INTERRUPT_RAGE * n_int
+    clean = sum(1 for _, l in rows if rage(l) == 0 and l.get("satisfaction", 0) >= 0)
+    sat = sum(l.get("satisfaction", 0) for _, l in rows) / n
+    rage100 = 100 * total / n
+    clean_pct = 100 * clean / n
+    return {
+        "n": n, "friction": 100 - FRICTION_K * rage100 + 10 * sat, "clean_pct": clean_pct, "rage100": rage100,
+        "angry_pct": 100 * sum(rage(l) > 0 for _, l in rows) / n, "sat": sat,
+        "int100": 100 * n_int / n,
+        "swears100": 100 * sum(len(SWEAR.findall(r.text)) for r, _ in rows) / n,
+    }
+
+
+def _verdict(rows):
+    """True = you accepted the work, False = you rejected it, None = it stopped or moved elsewhere without a verdict."""
+    last = rows[-1][1]
+    if rage(last) > 0 or last.get("satisfaction", 0) < 0:
+        return False
+    if last.get("satisfaction", 0) >= 1:
+        return True
+    return None
+
+
+def _stats(sessions, n_int, merged, shares, inbound_per_turn):
+    rows = [x for s in sessions for x in s]
+    st = _friction(rows, n_int, shares, inbound_per_turn)
+    st["sessions"] = len(sessions)
+    verdicts = [v for v in map(_verdict, sessions) if v is not None]
+    st["decided"] = len(verdicts)
+    st["outcome"] = 100 * sum(verdicts) / len(verdicts) if verdicts else 0.0
+    tracked = [s for s in sessions if merged["since"] and s[0][0].ts >= merged["since"]]
+    st["merged_n"] = len(tracked)
+    st["merged_pct"] = 100 * sum(s[0][0].session in merged["ids"] for s in tracked) / len(tracked) if tracked else None
+    st["score"] = (st["friction"] + st["outcome"]) / 2
+    return st
+
+
+def _ci(sessions, n_int, merged, shares, inbound_per_turn, field, k=300):
+    rng = random.Random(0)
+    xs = sorted(_stats(rng.choices(sessions, k=len(sessions)), n_int, merged, shares, inbound_per_turn)[field]
+                for _ in range(k))
+    return xs[int(k * .05)], xs[int(k * .95)]
+
+
+def modes_share(rows):
+    c = Counter()
+    for _, l in rows:
+        if rage(l) > 0:
+            for m in l.get("modes", []):
+                c[m] += MODE_WEIGHT.get(m, 0)
+    total = sum(c.values()) or 1
+    return {m: c[m] / total for m in MODE_WEIGHT if c[m]}
+
+
+def build(corpus, labels, min_reactions=40, exclude=()):
+    per = defaultdict(list)
+    for r in corpus.reactions:
+        lab = labels.get(r.id)
+        if lab and not lab.get("agent_authored") and r.model and r.model not in exclude:
+            per[r.model].append((r, lab))
+    ints = Counter(i.model for i in corpus.interrupts)
+    shares = blame_shares(corpus.reactions, labels)
+    inbound = Counter()
+    for r in corpus.reactions:
+        if r.id in shares:
+            for m, share in shares[r.id].items():
+                if m != r.model:
+                    inbound[m] += rage(labels[r.id]) * share
+
+    board = []
+    for model, rows in per.items():
+        if len(rows) < min_reactions:
+            continue
+        by_session = defaultdict(list)
+        for r, l in sorted(rows, key=lambda x: x[0].ts):
+            by_session[r.session].append((r, l))
+        sessions = list(by_session.values())
+        merged = {"ids": corpus.merged, "since": corpus.pr_tracking_since}
+        inb = inbound[model] / len(rows)
+        s = _stats(sessions, ints[model], merged, shares, inb)
+        s["model"], s["modes"] = model, modes_share(rows)
+        s["regret_in"], s["regret_out"] = inbound[model], sum(
+            rage(l) * (1 - shares[r.id].get(model, 0.0)) for r, l in rows if r.id in shares)
+        s["blowups"] = sum(1 for _, l in rows if rage(l) > 0 and l.get("anger", 0) >= 4)
+        s["ci"] = _ci(sessions, ints[model], merged, shares, inb, "score")
+        s["outcome_ci"] = _ci(sessions, ints[model], merged, shares, inb, "outcome")
+        s["worst"] = [l for _, l in sorted(rows, key=lambda x: -rage(x[1]))[:3] if rage(l) > 0]
+        board.append(s)
+    board.sort(key=lambda s: -s["score"])
+
+    tokens = []
+    for model, rows in per.items():
+        you = corpus.usage.tokens.get(model, {}).get("you")
+        start = corpus.usage.first_ts.get(model)
+        if not you or not you[1] or not start:
+            continue
+        win = [(r, l) for r, l in rows if r.ts >= start]
+        if len(win) < min_reactions:
+            continue
+        n_int = sum(1 for i in corpus.interrupts if i.model == model and i.ts >= start)
+        out_m = you[1] / 1e6
+        tokens.append({
+            "model": model, "out_m": out_m, "agents_out_m": corpus.usage.tokens[model].get("agents", [0, 0, 0])[1] / 1e6,
+            "total_b": sum(you) / 1e9, "n": len(win),
+            "rage_per_m": (sum(_own(r, l, shares) for r, l in win) + INTERRUPT_RAGE * n_int) / out_m,
+            "angry_per_m": sum(rage(l) > 0 for _, l in win) / out_m, "out_per_reaction": you[1] / len(win),
+        })
+    tokens.sort(key=lambda t: t["rage_per_m"])
+
+    sent = set(per)
+    agent_only = sorted(m for m, o in corpus.usage.tokens.items() if m not in sent)
+    small = sorted((m, len(r)) for m, r in per.items() if len(r) < min_reactions)
+    return {"board": board, "tokens": tokens, "agent_only": agent_only, "small": small,
+            "n_reactions": sum(len(r) for r in per.values()), "n_interrupts": len(corpus.interrupts)}
+
+
+def _merged_cell(s):
+    return "—" if s["merged_pct"] is None else f"{s['merged_pct']:.0f}% of {s['merged_n']}"
+
+
+def markdown(res, quotes=True):
+    o = ["# SwearBench\n",
+         f"{res['n_reactions']} of your reactions to AI replies, {res['n_interrupts']} interrupts.\n",
+         "| # | Model | SwearBench ↑ | 90% CI | Ships | Friction | Decided / sessions | Merged PR* | Reactions | Clean turns | "
+         "Rage /100 turns | Angry msgs | 4/4 blowups | Regret charged in / passed on | Interrupts /100 | Swears /100 | "
+         "Avg satisfaction |",
+         "|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|"]
+    for i, s in enumerate(res["board"], 1):
+        o.append(f"| {i} | {s['model']} | **{s['score']:.1f}** | {s['ci'][0]:.0f}–{s['ci'][1]:.0f} | {s['outcome']:.0f}% | {s['friction']:.1f} | "
+                 f"{s['decided']} / {s['sessions']} | "
+                 f"{_merged_cell(s)} | {s['n']} | "
+                 f"{s['clean_pct']:.0f}% | {s['rage100']:.1f} | {s['angry_pct']:.1f}% | {s['blowups']} | "
+                 f"{s['regret_in']:.0f} / {s['regret_out']:.0f} | {s['int100']:.1f} | "
+                 f"{s['swears100']:.1f} | {s['sat']:+.2f} |")
+    if res["tokens"]:
+        o += ["\n## Per token of work\n",
+              "Output tokens the model produced in sessions you drove; reactions counted from the first logged token on.\n",
+              "| # | Model | Your output tok | Agent-run output tok | Your total tok | Reactions | Rage per 1M out tok ↓ | "
+              "Angry msgs per 1M | Out tok per reaction |",
+              "|---|---|---|---|---|---|---|---|---|"]
+        for i, t in enumerate(res["tokens"], 1):
+            o.append(f"| {i} | {t['model']} | {t['out_m']:.1f}M | {t['agents_out_m']:.1f}M | {t['total_b']:.2f}B | {t['n']} | "
+                     f"**{t['rage_per_m']:.1f}** | {t['angry_per_m']:.1f} | {t['out_per_reaction'] / 1000:.1f}k |")
+    o += ["\n## How you get mad at each model (share of rage points)\n",
+          "| Model | " + " | ".join(MODE_WEIGHT) + " |", "|---|" + "---|" * len(MODE_WEIGHT)]
+    for s in res["board"]:
+        o.append(f"| {s['model']} | " + " | ".join(f"{100 * s['modes'][m]:.0f}%" if m in s["modes"] else "·"
+                                                    for m in MODE_WEIGHT) + " |")
+    if quotes:
+        o.append("\n## Hall of shame\n\n> Contains your own words. Review before sharing.\n")
+        for s in res["board"]:
+            o.append(f"**{s['model']}**")
+            o += [f"- [{l['anger']}/4, {', '.join(l.get('modes', [])) or '—'}] “{l.get('quote', '')}”" for l in s["worst"]]
+            o.append("")
+    if res["small"]:
+        o.append("Too few reactions to rank: " + ", ".join(f"{m} ({n})" for m, n in res["small"]))
+    if res["agent_only"]:
+        o.append("\nOnly ever ran as subagents / headless runs (never ranked): " + ", ".join(res["agent_only"]))
+    o.append(f"\nSwearBench = (Friction + Ships) / 2. Friction = 100 − {FRICTION_K} × rage per 100 turns + 10 × "
+             "avg satisfaction; rage = anger² + mode weights, only when aimed at the model, a quarter for taste-only "
+             "critique; each interrupt adds 1.5. Complaints about earlier work are charged to the models that worked "
+             f"in that workspace in the {REGRET_WINDOW_DAYS} days before. "
+             "Ships = accepted ÷ (accepted + rejected) sessions; sessions that stop or are handed off without a verdict "
+             "are left out, so limits and model switches don't count as failures. "
+             "*Merged PR is informational (T3 Code only, sessions after it started tracking PRs) and not in the score. "
+             "Intervals bootstrap over sessions. See chart.svg for swearing vs. result.")
+    return "\n".join(o) + "\n"

swearbench/sources.py

@@ -0,0 +1,317 @@
+"""Readers for local agent logs. Each yields human reactions, interrupts and token usage."""
+from __future__ import annotations
+
+import glob
+import json
+import os
+import sqlite3
+from collections import defaultdict
+from dataclasses import dataclass, field
+
+HOME = os.path.expanduser("~")
+
+
+@dataclass
+class Reaction:
+    id: str
+    source: str
+    session: str
+    ts: str
+    text: str
+    prev_reply: str
+    model: str
+    workspace: str = ""
+
+
+@dataclass
+class Interrupt:
+    source: str
+    session: str
+    ts: str
+    model: str
+
+
+@dataclass
+class Usage:
+    """Per model and origin ("you" = sessions a human drove, "agents" = subagents/headless runs)."""
+    tokens: dict = field(default_factory=lambda: defaultdict(lambda: defaultdict(lambda: [0, 0, 0])))
+    first_ts: dict = field(default_factory=dict)
+
+    def add(self, model, origin, ts, inp, out, cache):
+        model = norm(model)
+        t = self.tokens[model][origin]
+        t[0] += inp
+        t[1] += out
+        t[2] += cache
+        if origin == "you" and ts and (model not in self.first_ts or ts < self.first_ts[model]):
+            self.first_ts[model] = ts
+
+
+@dataclass
+class Corpus:
+    reactions: list = field(default_factory=list)
+    interrupts: list = field(default_factory=list)
+    usage: Usage = field(default_factory=Usage)
+    found: list = field(default_factory=list)
+    merged: set = field(default_factory=set)
+    pr_tracking_since: str | None = None
+
+
+def norm(model):
+    if not model:
+        return None
+    model = model.strip().lower()
+    if model.startswith("claude-"):
+        model = model.replace(".", "-")
+    return model
+
+
+def workspace_of(path):
+    """Repo-level key for a working directory; worktrees of one repo share it."""
+    if not path:
+        return ""
+    parts = path.rstrip("/").split("/")
+    if "worktrees" in parts and parts.index("worktrees") + 1 < len(parts):
+        return parts[parts.index("worktrees") + 1]
+    return parts[-1]
+
+
+def ms_to_iso(ms):
+    import datetime
+    return datetime.datetime.fromtimestamp(ms / 1000, datetime.timezone.utc).isoformat().replace("+00:00", "Z")
+
+
+def _jsonl(path):
+    with open(path, errors="replace") as f:
+        for line in f:
+            try:
+                yield json.loads(line)
+            except ValueError:
+                continue
+
+
+def _ro(path):
+    return sqlite3.connect(f"file:{path}?mode=ro", uri=True)
+
+
+# --- Claude Code -----------------------------------------------------------------------------
+
+HEADLESS_ENTRYPOINTS = {"sdk-cli", "sdk-py"}
+
+
+def _claude_text(content):
+    if isinstance(content, str):
+        return content
+    if any(isinstance(p, dict) and p.get("type") == "tool_result" for p in content):
+        return None
+    parts = [p.get("text", "") for p in content if isinstance(p, dict) and p.get("type") == "text"]
+    return "\n".join(parts) if parts else None
+
+
+def read_claude(corpus, root, skip_session=lambda entrypoint: False, exclude_dirs=()):
+    files = glob.glob(os.path.join(root, "**", "*.jsonl"), recursive=True)
+    if not files:
+        return
+    corpus.found.append(f"Claude Code: {len(files)} transcripts")
+    for path in files:
+        excluded = any(d in path for d in exclude_dirs)
+        subagent = "/subagents/" in path
+        entrypoint = None
+        model = None
+        reply = ""
+        cwd = ""
+        usage = {}
+        pending = []
+        for d in _jsonl(path):
+            kind = d.get("type")
+            msg = d.get("message") or {}
+            if kind == "assistant":
+                if msg.get("model") and msg["model"] != "<synthetic>":
+                    model = msg["model"]
+                    if msg.get("usage"):
+                        usage[msg.get("id") or d.get("uuid")] = (model, msg["usage"], d.get("isSidechain"), d.get("timestamp"))
+                text = _claude_text(msg.get("content") or [])
+                if text and text.strip():
+                    reply = text
+            elif kind == "user" and not d.get("isSidechain") and not d.get("isMeta"):
+                entrypoint = entrypoint or d.get("entrypoint")
+                cwd = cwd or d.get("cwd", "")
+                text = _claude_text(msg.get("content") or "")
+                if not text or d.get("promptSource") == "system":
+                    continue
+                text = text.strip()
+                if text.startswith("[Request interrupted by user"):
+                    if model:
+                        corpus.interrupts.append(Interrupt("claude", path, d.get("timestamp", ""), norm(model)))
+                    continue
+                if text.startswith(("<", "Caveat:")) or not model:
+                    continue
+                pending.append(Reaction(d.get("uuid", ""), "claude", path, d.get("timestamp", ""), text,
+                                        reply[-900:], norm(model), workspace_of(cwd)))
+                reply = ""
+        headless = entrypoint in HEADLESS_ENTRYPOINTS
+        human = not (subagent or headless or excluded)
+        if human and not skip_session(entrypoint):
+            corpus.reactions += pending
+        for model_id, u, side, ts in usage.values():
+            if excluded:
+                continue
+            origin = "you" if human and not side else "agents"
+            corpus.usage.add(model_id, origin, ts, u.get("input_tokens", 0), u.get("output_tokens", 0),
+                             u.get("cache_read_input_tokens", 0) + u.get("cache_creation_input_tokens", 0))
+
+
+# --- Codex -------------------------------------------------------------------------------------
+
+def read_codex(corpus, root, skip_originator=lambda originator: False):
+    files = glob.glob(os.path.join(root, "**", "*.jsonl"), recursive=True)
+    if not files:
+        return
+    corpus.found.append(f"Codex: {len(files)} sessions")
+    for path in files:
+        human, skip, model, reply, cwd = False, False, None, "", ""
+        for d in _jsonl(path):
+            p = d.get("payload") or {}
+            kind = d.get("type")
+            if kind == "session_meta":
+                source = p.get("source")
+                human = isinstance(source, str) and source != "exec" and p.get("originator") != "codex_exec"
+                skip = skip_originator(p.get("originator") or "")
+                cwd = p.get("cwd") or ""
+            elif kind == "turn_context":
+                model = p.get("model") or model
+            elif kind == "event_msg":
+                t = p.get("type")
+                if t == "agent_message":
+                    reply = p.get("message") or reply
+                elif t == "user_message" and human and not skip and model:
+                    text = (p.get("message") or "").strip()
+                    if text and not text.startswith("<"):
+                        corpus.reactions.append(Reaction(f"{path}:{d.get('timestamp')}", "codex", path,
+                                                         d.get("timestamp", ""), text, reply[-900:], norm(model),
+                                                         workspace_of(cwd)))
+                    reply = ""
+                elif t == "turn_aborted" and human and not skip and model:
+                    corpus.interrupts.append(Interrupt("codex", path, d.get("timestamp", ""), norm(model)))
+                elif t == "token_count" and model and (p.get("info") or {}).get("last_token_usage"):
+                    u = p["info"]["last_token_usage"]
+                    cached = u.get("cached_input_tokens", 0)
+                    corpus.usage.add(model, "you" if human else "agents", d.get("timestamp"),
+                                     u.get("input_tokens", 0) - cached, u.get("output_tokens", 0), cached)
+
+
+# --- OpenCode ----------------------------------------------------------------------------------
+
+def read_opencode(corpus, db, skip_sessions=frozenset()):
+    if not os.path.exists(db):
+        return
+    c = _ro(db)
+    try:
+        sessions = dict(c.execute("select id, parent_id from session"))
+        dirs = dict(c.execute("select id, directory from session"))
+    except sqlite3.Error:
+        return
+    corpus.found.append(f"OpenCode: {len(sessions)} sessions")
+    texts = defaultdict(list)
+    for mid, data in c.execute("select message_id, data from part order by time_created"):
+        p = json.loads(data)
+        if p.get("type") == "text" and not p.get("synthetic"):
+            texts[mid].append(p.get("text", ""))
+    state = {}
+    for mid, sid, created, data in c.execute(
+            "select id, session_id, time_created, data from message order by session_id, time_created"):
+        d = json.loads(data)
+        model, reply = state.get(sid, (None, ""))
+        human = sessions.get(sid) is None
+        ts = ms_to_iso(created)
+        text = "\n".join(texts.get(mid, [])).strip()
+        if d.get("role") == "assistant":
+            model = "opencode/" + d.get("modelID", "?")
+            t = d.get("tokens") or {}
+            corpus.usage.add(model, "you" if human else "agents", ts, t.get("input", 0),
+                             t.get("output", 0) + t.get("reasoning", 0), (t.get("cache") or {}).get("read", 0))
+            if (d.get("error") or {}).get("name") == "MessageAbortedError" and human and sid not in skip_sessions:
+                corpus.interrupts.append(Interrupt("opencode", sid, ts, norm(model)))
+            state[sid] = (model, text or reply)
+        elif d.get("role") == "user":
+            if human and model and text and sid not in skip_sessions:
+                corpus.reactions.append(Reaction(mid, "opencode", sid, ts, text, reply[-900:], norm(model),
+                                                 workspace_of(dirs.get(sid))))
+            state[sid] = (model, "")
+
+
+# --- T3 Code (optional; gives exact per-turn model attribution across providers) ----------------
+
+def t3_db():
+    base = os.path.join(HOME, ".config", "t3", "userdata")
+    for name in ("statev2.sqlite", "state.sqlite"):
+        path = os.path.join(base, name)
+        if os.path.exists(path):
+            return path
+    return None
+
+
+def t3_provider_sessions(db):
+    out = defaultdict(set)
+    for provider, cursor in _ro(db).execute("select provider_name, resume_cursor_json from provider_session_runtime"):
+        cur = json.loads(cursor or "{}")
+        out[provider].add(cur.get("sessionId") or cur.get("threadId"))
+    return out
+
+
+def read_t3(corpus, db):
+    c = _ro(db)
+    children = {r[0] for r in c.execute("select child_thread_id from orchestration_v2_projection_subagents "
+                                        "where child_thread_id is not null")} if _has(c, "orchestration_v2_projection_subagents") else set()
+    turn_model = {}
+    for mid, payload in c.execute("select json_extract(payload_json,'$.messageId'), payload_json from orchestration_events "
+                                  "where event_type='thread.turn-start-requested'"):
+        turn_model[mid] = (json.loads(payload).get("modelSelection") or {}).get("model")
+    project = dict(c.execute("select thread_id, project_id from projection_threads"))
+    by_thread = defaultdict(list)
+    for mid, tid, role, text, ts in c.execute("select message_id, thread_id, role, text, created_at from projection_thread_messages "
+                                              "where role in ('user','assistant') order by created_at"):
+        if tid not in children:
+            by_thread[tid].append((mid, role, text or "", ts))
+    corpus.found.append(f"T3 Code: {len(by_thread)} threads")
+    sent = defaultdict(list)
+    for tid, msgs in by_thread.items():
+        current, reply = None, ""
+        for mid, role, text, ts in msgs:
+            if role == "assistant":
+                reply = text if text.strip() else reply
+                continue
+            if current and text.strip():
+                corpus.reactions.append(Reaction(mid, "t3", tid, ts, text.strip(), reply[-900:], norm(current),
+                                                 project.get(tid) or ""))
+            current = turn_model.get(mid) or current
+            sent[tid].append((ts, current))
+            reply = ""
+    if _has(c, "projection_thread_pull_requests"):
+        corpus.merged |= {tid for tid, state in c.execute(
+            "select thread_id, json_extract(snapshot_json,'$.state') from projection_thread_pull_requests") if state == "merged"}
+        corpus.pr_tracking_since = c.execute("select min(linked_at) from projection_thread_pull_requests").fetchone()[0]
+    for tid, ts in c.execute("select json_extract(payload_json,'$.threadId'), occurred_at from orchestration_events "
+                             "where event_type='thread.turn-interrupt-requested'"):
+        before = [m for t, m in sent.get(tid, []) if t <= ts]
+        if before and before[-1]:
+            corpus.interrupts.append(Interrupt("t3", tid, ts, norm(before[-1])))
+
+
+def _has(c, table):
+    return c.execute("select 1 from sqlite_master where name=?", (table,)).fetchone() is not None
+
+
+def collect(use_t3=True, exclude_dirs=()):
+    corpus = Corpus()
+    db = t3_db() if use_t3 else None
+    t3_sessions = t3_provider_sessions(db) if db else {}
+    if db:
+        read_t3(corpus, db)
+    read_claude(corpus, os.path.join(HOME, ".claude", "projects"),
+                skip_session=(lambda ep: ep == "sdk-ts") if db else (lambda ep: False), exclude_dirs=exclude_dirs)
+    read_codex(corpus, os.path.join(HOME, ".codex", "sessions"),
+               skip_originator=(lambda o: "t3" in o.lower()) if db else (lambda o: False))
+    read_opencode(corpus, os.path.join(HOME, ".local", "share", "opencode", "opencode.db"),
+                  skip_sessions=frozenset(t3_sessions.get("opencode", ())))
+    return corpus

swearbench-0.2.1.dist-info/METADATA

@@ -0,0 +1,130 @@
+Metadata-Version: 2.5
+Name: swearbench
+Version: 0.2.1
+Summary: Rank AI coding models by how much they made you swear and how often their work shipped, from your own local chat logs.
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# SwearBench
+
+**Which AI coding model made you swear the least, and still shipped?**
+
+Public benchmarks measure what models can do. SwearBench measures how they made *you* feel: it reads your
+own local agent logs, finds every message you sent in reaction to a model's reply, has an LLM judge label how
+mad you were and why, checks whether the work ended up accepted, and ranks the models.
+
+Example from the author's own logs (~4,600 messages, July–October 2026):
+
+![swearing vs. result](docs/example-chart.svg)
+
+![leaderboard card](docs/example-card.svg)
+
+## Run it
+
+```sh
+uvx swearbench
+```
+
+(or `pipx run swearbench`, or `pip install swearbench`)
+
+It prints a leaderboard and writes `swearbench-out/report.md`, `chart.svg` (swearing vs. result), `card.svg` and `results.json`.
+Before anything leaves your machine it tells you how many messages it will send to the judge and asks.
+
+| Flag | |
+|---|---|
+| `--dry-run` | show what logs were found and how many messages would be judged |
+| `--judge claude-cli\|codex-cli\|anthropic\|openai\|command` | who labels your messages (default: first of `claude`, `codex`, `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`) |
+| `--judge-model ID` | model for the judge |
+| `--judge command --judge-cmd "ollama run qwen3"` | any command that reads a prompt on stdin and prints the reply; keeps everything local |
+| `--since 2026-09-01` | only recent reactions |
+| `--exclude MODEL` | drop a model from the ranking |
+| `--no-quotes` | leave the hall of shame out of the report |
+
+Labels are cached in `~/.cache/swearbench/`, so re-runs only judge new messages.
+
+## Where it looks
+
+| Tool | Location |
+|---|---|
+| Claude Code | `~/.claude/projects/**/*.jsonl` |
+| Codex CLI | `~/.codex/sessions/**/*.jsonl` |
+| OpenCode | `~/.local/share/opencode/opencode.db` |
+| T3 Code | `~/.config/t3/userdata/state*.sqlite` (used for exact per-turn model attribution when present) |
+
+Only messages **you** typed count. Subagent transcripts, headless runs (`claude -p`, `codex exec`) and messages
+the judge flags as agent-written are skipped. Models that only ever ran as subagents are listed, never ranked.
+
+## How it scores
+
+Every message you send is charged to the model whose reply you were answering. The judge labels it:
+
+- **anger** 0–4, and **who it's aimed at**: the model, or something external (swearing at your cloud
+  provider is not the model's fault; "this looks fucking cool" is not anger)
+- **how** you got mad, weighted by how bad it is:
+
+  | Mode | Weight | | Mode | Weight |
+  |---|---|---|---|---|
+  | fabrication (claimed success that wasn't) | 3 | | incomplete / lazy | 1.5 |
+  | overreach (did things you didn't ask) | 3 | | insult | 1.5 |
+  | giving up on it | 3 | | profanity | 1 |
+  | regression (broke what worked) | 2.5 | | shouting | 0.5 |
+  | ignored an instruction | 2 | | sarcasm | 0.5 |
+  | made you repeat yourself | 2 | | slow / verbose | 0.5 |
+  | | | | taste (critique while iterating on looks) | 0.5 |
+
+- **satisfaction** −2 (rejects the work) to +2 (praise)
+- whether it **blames earlier work** (something shipped before the last reply is broken or missing)
+
+Rage for a message is built to match how it felt, not how often it happened:
+
+- **Severity beats frequency.** Rage = anger² + mode weights, so one 4/4 blowup (16) outweighs four
+  1/4 grumbles (4). The report counts 4/4 blowups per model.
+- **Taste isn't failure.** "That looks lame" while iterating on a design, with no broken rule, lie,
+  regression or ignored instruction, counts a quarter.
+- **Regret goes to whoever caused it.** A complaint about earlier work ("why did X disappear") is charged to
+  the models that worked in the same repo in the week before, split by how many turns each had there, not
+  to the model that happens to be fixing it.
+- Each interrupt adds 1.5.
+
+The headline is half friction, half result:
+
+```
+Friction   = 100 − 0.25 × rage per 100 turns + 10 × average satisfaction
+Ships      = accepted ÷ (accepted + rejected) sessions
+SwearBench = (Friction + Ships) / 2
+```
+
+Friction deliberately ignores how *often* you were annoyed (a model you use for lots of quick "merge it"
+turns would look calm by volume alone); it only counts how much rage piled up per turn.
+A session's verdict is your last reaction in it. Sessions that stop or move to another model without a
+verdict (usage limits, "pick up the work" in a new thread) are left out of Ships rather than counted as
+failures; switching away in anger still counts as a rejection.
+Friction alone rewards a model that is pleasant but never finishes; Ships alone ignores what it cost you
+to get there. The chart plots the two against each other.
+
+Intervals are a 90% bootstrap over sessions; models with fewer than 40 reactions aren't ranked.
+With T3 Code, the report also shows the share of sessions that ended in a merged PR. It is informational
+only, since T3 records PRs only from when it started tracking them.
+
+**Per token of work.** If the logs carry token usage, the report adds a second ranking: rage per million output
+tokens the model produced in sessions you drove. A model that does twice the work per message gets credit for it.
+Subagent token use is shown separately.
+
+## Caveats
+
+- n = 1. It measures you, your tasks and your mood as much as the models. Models used in different months
+  did different work; the report tells you counts, not causes.
+- The judge is a model too. If it belongs to a family being ranked, SwearBench says so; re-run with another
+  `--judge` and compare.
+- Regret attribution is a heuristic: it blames whoever worked in the same repo during the previous week,
+  not the session that actually introduced the problem.
+- The weights are opinions. Rage, taste and severity weights live at the top of `score.py`; change them
+  and re-run, labels are cached.
+- Deleted or rotated logs mean missing data, especially for the per-token view.
+- `report.md` quotes your own messages. Read it before you share it. The card and chart have no quotes.
+
+## License
+
+MIT

swearbench-0.2.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+swearbench/__init__.py,sha256=HfjVOrpTnmZ-xVFCYSVmX50EXaBQeJteUHG-PD6iQs8,22
+swearbench/__main__.py,sha256=E6Gls0DNz8GQK2K-kOUIx8cYhgANW_CH54VKrfCfs14,52
+swearbench/card.py,sha256=xysCWH-6cRQ9fm71GIEKpnrLGp58e5rFShpQyDpaS08,3412
+swearbench/chart.py,sha256=EfyUCD8xmyt_vzeJDz-lWkDotW2yJcKVQrIWN3b5T-s,6595
+swearbench/cli.py,sha256=VzigcBt5oGTFVYKCQaXaoMJWaalHK-DUU8ku7Dbh0Bg,4108
+swearbench/judge.py,sha256=FxrpUHlj5LXJExWW3FJ8v4kKQuBz-yTbcfU6R0Ik3i8,7452
+swearbench/score.py,sha256=PJpV_yRiI2nfen0VcM7xS0pYGjYPqK0-fKdAshZ93sc,11518
+swearbench/sources.py,sha256=LJN1jBkFksvZja-h12Y2UxjmpdJuXVmlJWL6JCqpINI,13645
+swearbench-0.2.1.dist-info/METADATA,sha256=a-Sgjw_t4X6u89boms860vp9___6M7wXmVgvK9k4EfE,6293
+swearbench-0.2.1.dist-info/WHEEL,sha256=W3fkpkm7-wf9vBI5Z-7s0eWkeM-spu78I8Neb98DeEg,87
+swearbench-0.2.1.dist-info/entry_points.txt,sha256=qrmKvKi9_IF4klWqvGhS4o6RU6kt_CgCaahl67rev1s,51
+swearbench-0.2.1.dist-info/licenses/LICENSE,sha256=_OBJk5LyJLaZeO_rY8OZO6IKaDvmb0Wgf9Wc0PI_1Fg,1064
+swearbench-0.2.1.dist-info/RECORD,,

swearbench-0.2.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.32.4
+Root-Is-Purelib: true
+Tag: py3-none-any

swearbench-0.2.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+swearbench = swearbench.cli:main

swearbench-0.2.1.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 AlenHay
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.