ossa 0.1.0a1__py3-none-any.whl

ossa/__init__.py

@@ -0,0 +1,2 @@
+"""OSSA toolbox: the deterministic half of a safety-first open-source contribution crew."""
+__version__ = "0.1.0-alpha.1"

ossa/arena.py

@@ -0,0 +1,591 @@
+"""Breaker Arena: score Breaker on real merged fixes and their plausible-but-wrong twins. No LLM here.
+
+An instance is a merged PR that closes an issue opened after the models' training cutoff and adds
+tests (replay.mine_instances). The human fix is the reference, and its tests stay hidden.
+
+  mine      instances from a repo's merged PRs (hidden: runs/arena/hidden/<instance>/)
+  env       a frozen copy of the repo at the instance's base (history ends there, so the later fix
+            can't be found) and an env attempt; Sherlock builds the sandbox on it with `ossa env`
+  prepare   checks the hidden tests fail on base and pass with the human fix (F2P, P2P), then makes
+            wrong candidates: mutants of the human fix (flipped comparisons, booleans, constants,
+            arithmetic) and partial fixes (a hunk, a file or a line left out). A candidate counts
+            only if it is plausible (every P2P test still passes) and wrong (an F2P test fails).
+  next      one candidate at a time (the human fix or a wrong one, in a shuffled order) becomes a
+            blind duel attempt: base + candidate as the "verified fix", the issue text, no tests.
+            Breaker attacks it with the ordinary `ossa attack`, as in a real duel.
+  score     re-runs every attack against the hidden human fix (the verifiable reward):
+              caught       fails on a wrong candidate, passes on the human fix      +1
+              false_alarm  fails on the human fix                                   -1
+              missed       holds on a wrong candidate                                0
+              held         holds on the human fix (as it should)                     0
+              invalid      never exercised the code, or flaky                        0
+  solve     Patch's side: an attempt at base from the issue alone; `grade` runs the hidden tests on
+            its verified fix (resolved = every F2P passes and no P2P breaks)
+  report    catch rate (wrong candidates with at least one caught attack), false-alarm rate (human
+            fixes with at least one false alarm), by Breaker prompt version and model.
+The crew never reads runs/arena/hidden/ (the guard blocks it); everything there is for scoring.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import random
+import re
+import shutil
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+from ossa.schemas import Attempt, IssueRef, State
+from ossa.store import now
+from ossa.toolbox import ToolError, _attempt, _pristine
+
+WRONG_PER_INSTANCE = 3
+MAX_TRIES = 12
+HUNK = re.compile(r"^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@(.*)$")
+OPERATORS: list[tuple[str, str, Any]] = [   # (name, pattern, replacement): one change per mutant
+    ("== to !=", r"==", "!="), ("!= to ==", r"!=", "=="),
+    ("<= to <", r"<=", "<"), (">= to >", r">=", ">"),
+    ("< to <=", r"(?<![<\-=])<(?![<=\-])", "<="), ("> to >=", r"(?<![>\-=])>(?![>=])", ">="),
+    ("and to or", r"\band\b", "or"), ("or to and", r"\bor\b", "and"),
+    ("&& to ||", r"&&", "||"), ("|| to &&", r"\|\|", "&&"),
+    ("drop not", r"\bnot\s+", ""), ("drop !", r"!(?=[A-Za-z_(])", ""),
+    ("+ to -", r"(?<=\s)\+(?=\s)", "-"), ("- to +", r"(?<=\s)-(?=\s)", "+"),
+    ("True to False", r"\bTrue\b", "False"), ("False to True", r"\bFalse\b", "True"),
+    ("true to false", r"\btrue\b", "false"), ("false to true", r"\bfalse\b", "true"),
+    ("is None to is not None", r"\bis None\b", "is not None"), ("is not None to is None", r"\bis not None\b", "is None"),
+    ("n to n+1", r"(?<![\w.])(\d{1,4})(?![\w.])", lambda m: str(int(m.group(1)) + 1)),
+]
+COMMENT = re.compile(r"^\s*(#|//|/\*|\*)")
+OPENER = re.compile(r"(:|\{|\(|\[|,)\s*$|^\s*(\}|\)|\]|else\b|elif\b|except\b|finally\b|try\b|def |class |@|return\s*$)")
+
+
+def _hash(text: str) -> str:
+    return hashlib.sha1(text.encode()).hexdigest()[:12]
+
+
+def _arena(ctx) -> Path:
+    return ctx.config.path("runs") / "arena"
+
+
+def hidden_dir(ctx, iid: str) -> Path:
+    return _arena(ctx) / "hidden" / iid
+
+
+def _row(ctx, iid: str) -> dict:
+    rows = ctx.store.query("SELECT * FROM arena_instances WHERE id=?", (iid,))
+    if not rows:
+        raise ToolError(f"no Arena instance {iid} (`ossa arena status` lists them)")
+    d = dict(rows[0])
+    d["f2p"] = json.loads(d.pop("f2p_json") or "[]")
+    d["p2p"] = json.loads(d.pop("p2p_json") or "[]")
+    return d
+
+
+def _update(ctx, iid: str, **fields: Any) -> None:
+    fields["updated_at"] = now()
+    sets = ", ".join(f"{k}=?" for k in fields)
+    ctx.store.execute(f"UPDATE arena_instances SET {sets} WHERE id=?", (*fields.values(), iid))
+
+
+def _instance(ctx, iid: str) -> dict:
+    return json.loads((hidden_dir(ctx, iid) / "instance.json").read_text())
+
+
+def _blind_issue(inst: dict) -> IssueRef:
+    """The issue as it was reported: no thread, no URL (nothing that points at the fix)."""
+    i = IssueRef.model_validate(inst["issue"])
+    return i.model_copy(update={"comments": [], "url": ""})
+
+
+def _move(a: Attempt, dest: Path) -> None:
+    """Put an attempt's artifacts somewhere else (blind, hidden, solve folders) and remember it."""
+    old = Path(a.artifacts_dir)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    if old.exists() and old != dest:
+        shutil.move(str(old), str(dest))
+    dest.mkdir(parents=True, exist_ok=True)
+    a.artifacts_dir = str(dest)
+    a.ctx.store.save_attempt(a)
+
+
+# ------------------------------------------------------------------ mining
+def mine(ctx, repo: str, n: int = 5, allow_contaminated: bool = False) -> list[dict]:
+    """New instances from `repo`'s merged PRs (newest first). Already-mined ones are listed again."""
+    from ossa import replay
+    out = []
+    for inst in replay.mine_instances(ctx, repo, n, allow_contaminated=allow_contaminated):
+        iid = f"{repo.replace('/', '__')}-pr{inst['pr']}"
+        summary = {"instance": iid, "pr": inst["pr"], "issue": inst["issue"]["number"],
+                   "contaminated": inst["contaminated"]}
+        if ctx.store.query("SELECT id FROM arena_instances WHERE id=?", (iid,)):
+            out.append({**summary, "new": False})
+            continue
+        h = hidden_dir(ctx, iid)
+        h.mkdir(parents=True, exist_ok=True)
+        (h / "instance.json").write_text(json.dumps(inst, indent=1))
+        ctx.store.execute(
+            "INSERT INTO arena_instances(id, repo, pr, issue, base_sha, issue_created_at, contaminated, state, "
+            "created_at, updated_at) VALUES(?,?,?,?,?,?,?,?,?,?)",
+            (iid, repo, inst["pr"], inst["issue"]["number"], inst["base_sha"], inst.get("issue_created_at"),
+             int(bool(inst["contaminated"])), "mined", now(), now()))
+        out.append({**summary, "new": True})
+    return out
+
+
+# ------------------------------------------------------------------ the frozen repo and the env
+def freeze(ctx, repo: str, base_sha: str) -> Path:
+    """A repo whose only history ends at `base_sha`. It borrows the upstream clone's objects
+    (git alternates), so it costs no disk, but no ref in it leads past the base."""
+    from ossa import gitops
+    from ossa.stages import s03_workspace as W
+    dest = W.frozen_dir(ctx, repo, base_sha)
+    if (dest / ".git").exists():
+        return dest
+    upstream = W.clone_dir(ctx, repo)
+    if not ((upstream / ".git").exists() and W._has_commit(upstream, base_sha)):
+        gitops.ensure_clone(W.clone_url(repo), upstream)
+    if not W._has_commit(upstream, base_sha):
+        raise ToolError(f"{base_sha[:12]} isn't in {repo}'s history")
+    dest.mkdir(parents=True, exist_ok=True)
+    gitops.git(dest, "init", "-q")
+    (dest / ".git" / "objects" / "info" / "alternates").write_text(f"{(upstream / '.git' / 'objects').resolve()}\n")
+    gitops.git(dest, "update-ref", "refs/heads/base", base_sha)
+    gitops.git(dest, "checkout", "-q", "--detach", base_sha)
+    return dest
+
+
+def _copy_env(src: Attempt, dst: Attempt) -> None:
+    for name in ("env.json", "env/image.txt", "env/Dockerfile", "baseline.json"):
+        text = src.ctx.store.read_artifact(src, name)
+        if text is not None:
+            dst.ctx.store.write_artifact(dst, name, text)
+
+
+def _new(ctx, row: dict, kind: str, folder: Path, branch: str) -> Attempt:
+    a = ctx.store.new_attempt(row["repo"], f"{row['repo']}#{row['issue']}", kind=kind)
+    a.ctx, a.base_sha, a.branch = ctx, row["base_sha"], f"{branch}/{a.id[-10:].lower()}"
+    _move(a, folder / a.id)
+    ctx.store.write_artifact(a, "issue.json", _blind_issue(_instance(ctx, row["id"])).model_dump(mode="json"))
+    return a
+
+
+def env_attempt(ctx, iid: str) -> Attempt:
+    """The attempt Sherlock builds the instance's sandbox on (`ossa env <attempt> ...`)."""
+    from ossa.stages import s03_workspace
+    row = _row(ctx, iid)
+    if row["env_attempt"]:
+        return _attempt(ctx, row["env_attempt"])
+    freeze(ctx, row["repo"], row["base_sha"])
+    a = _new(ctx, row, "arena-env", _arena(ctx) / "env", "arena-env")
+    s03_workspace.prepare_worktree(a, _blind_issue(_instance(ctx, iid)))
+    a.state = State.ISSUE_SELECTED
+    ctx.store.save_attempt(a)
+    _update(ctx, iid, env_attempt=a.id)
+    return a
+
+
+def _env(ctx, row: dict) -> Attempt:
+    from ossa.stages import s03_workspace
+    if not row["env_attempt"]:
+        raise ToolError(f"no sandbox yet: `ossa arena env {row['id']}`, then `ossa env <attempt> ...`")
+    a = _attempt(ctx, row["env_attempt"])
+    try:
+        s03_workspace.load_env(a)
+    except Exception:  # noqa: BLE001 - Blocked: env.json missing
+        raise ToolError(f"build the sandbox first: `ossa env {a.id} --dockerfile ... --test ...`") from None
+    return a
+
+
+def _suite(env: Attempt, patches: list[str]):
+    """The full suite on base + patches in a clean worktree; None when a patch doesn't apply."""
+    from ossa import gitops
+    from ossa.stages import s03_workspace
+    spec, image = s03_workspace.load_env(env)
+    wt = _pristine(env, with_repro=False)
+    for p in patches:
+        if p.strip():
+            ok, _ = gitops.apply_patch(wt, p)
+            if not ok:
+                return None
+    try:
+        return env.ctx.sandbox.run_tests(wt, image, spec)
+    finally:
+        gitops.git(wt, "clean", "-fdq", check=False)
+
+
+# ------------------------------------------------------------------ wrong candidates
+def recount(patch: str) -> str:
+    """Fix every hunk header's line counts after lines were added or dropped."""
+    lines, out, i = patch.splitlines(keepends=True), [], 0
+    while i < len(lines):
+        m = HUNK.match(lines[i].rstrip("\n"))
+        if not m:
+            out.append(lines[i])
+            i += 1
+            continue
+        j, old, new = i + 1, 0, 0
+        while j < len(lines) and not lines[j].startswith(("@@", "diff --git")):
+            line = lines[j]
+            if line.startswith("\\"):
+                pass
+            elif line.startswith("+"):
+                new += 1
+            elif line.startswith("-"):
+                old += 1
+            else:
+                old += 1
+                new += 1
+            j += 1
+        out.append(f"@@ -{m.group(1)},{old} +{m.group(2)},{new} @@{m.group(3)}\n")
+        out.extend(lines[i + 1:j])
+        i = j
+    return "".join(out)
+
+
+def _sections(patch: str) -> list[str]:
+    """Per-file sections of a git diff."""
+    parts = re.split(r"(?m)^(?=diff --git )", patch)
+    return [p for p in parts if p.strip()]
+
+
+def _hunks(section: str) -> tuple[str, list[str]]:
+    """(file header, hunks) of one file section."""
+    parts = re.split(r"(?m)^(?=@@ )", section)
+    return parts[0], parts[1:]
+
+
+def _in_string(line: str, pos: int) -> bool:
+    before = line[:pos]
+    return (before.count('"') - before.count('\\"')) % 2 == 1 or (before.count("'") - before.count("\\'")) % 2 == 1
+
+
+def mutants(patch: str) -> list[tuple[str, str]]:
+    """(label, patch) for every single-change mutant and partial fix of `patch`, deduplicated."""
+    out: dict[str, str] = {}
+    sections = _sections(patch)
+    if len(sections) > 1:
+        for i, sec in enumerate(sections):
+            path = re.search(r"^diff --git a/(\S+)", sec, re.M)
+            out.setdefault("".join(sections[:i] + sections[i + 1:]),
+                           f"partial: leaves out {path.group(1) if path else f'file {i + 1}'}")
+    for si, sec in enumerate(sections):
+        head, hunks = _hunks(sec)
+        if len(hunks) > 1:
+            for hi in range(len(hunks)):
+                rest = head + "".join(hunks[:hi] + hunks[hi + 1:])
+                out.setdefault("".join(sections[:si] + [rest] + sections[si + 1:]), f"partial: leaves out hunk {hi + 1}")
+    lines = patch.splitlines(keepends=True)
+    in_hunk = False
+    for idx, line in enumerate(lines):
+        if line.startswith("@@"):
+            in_hunk = True
+            continue
+        if line.startswith("diff --git"):
+            in_hunk = False
+        if not in_hunk or not line.startswith("+") or line.startswith("+++"):
+            continue
+        body = line[1:]
+        if not body.strip() or COMMENT.match(body):
+            continue
+        for name, pattern, repl in OPERATORS:
+            m = next((m for m in re.finditer(pattern, body) if not _in_string(body, m.start())), None)
+            if m is None:
+                continue
+            new = body[:m.start()] + (repl(m) if callable(repl) else repl) + body[m.end():]
+            text = "".join(lines[:idx] + ["+" + new] + lines[idx + 1:])
+            out.setdefault(text, f"mutant: {name} on line {idx + 1}")
+        if not OPENER.search(body.rstrip("\n")):
+            text = recount("".join(lines[:idx] + lines[idx + 1:]))
+            out.setdefault(text, f"partial: drops line {idx + 1}")
+    out.pop(patch, None)
+    return [(label, text) for text, label in out.items() if not is_noop(text)]
+
+
+def is_noop(patch: str) -> bool:
+    """True when every hunk adds back exactly the lines it removes (a "fix" that changes nothing)."""
+    for sec in _sections(patch):
+        for hunk in _hunks(sec)[1]:
+            rows = hunk.splitlines()[1:]
+            if sorted(r[1:] for r in rows if r.startswith("-")) != sorted(r[1:] for r in rows if r.startswith("+")):
+                return False
+    return True
+
+
+def plausible_and_wrong(tests: dict[str, str], f2p: list[str], p2p: list[str]) -> bool:
+    return all(tests.get(t) == "passed" for t in p2p) and any(tests.get(t) != "passed" for t in f2p)
+
+
+def prepare(ctx, iid: str, wrong: int = WRONG_PER_INSTANCE, tries: int = MAX_TRIES) -> dict:
+    """Validate the instance in the sandbox and make its candidates (shuffled; labels stay hidden)."""
+    from ossa import replay
+    row = _row(ctx, iid)
+    env, inst = _env(ctx, row), _instance(ctx, iid)
+    human, tests = inst["human_patch"], inst["human_tests_patch"]
+    on_base, with_fix = _suite(env, [tests]), _suite(env, [human, tests])
+    if on_base is None or with_fix is None:
+        _update(ctx, iid, state="invalid", note="the human patch or tests don't apply to base")
+        return {"instance": iid, "valid": False, "why": "the human patch or tests don't apply to base"}
+    f2p, p2p = replay.f2p_p2p(on_base.tests, with_fix.tests)
+    if not f2p:
+        _update(ctx, iid, state="invalid", note="no hidden test fails on base and passes with the fix")
+        return {"instance": iid, "valid": False, "why": "no hidden test fails on base and passes with the fix"}
+    rng = random.Random(iid)
+    pool = mutants(human)
+    rng.shuffle(pool)
+    kept, tried = [], 0
+    for label, patch in pool[:tries]:
+        if len(kept) >= wrong:
+            break
+        tried += 1
+        result = _suite(env, [patch, tests])
+        if result is not None and plausible_and_wrong(result.tests, f2p, p2p):
+            kept.append((label, patch, "partial" if label.startswith("partial") else "mutant"))
+    cands = [("human fix", human, "human")] + kept
+    rng.shuffle(cands)
+    ctx.store.execute("DELETE FROM arena_candidates WHERE instance_id=? AND state='waiting'", (iid,))
+    cdir = hidden_dir(ctx, iid) / "candidates"
+    cdir.mkdir(parents=True, exist_ok=True)
+    for k, (label, patch, kind) in enumerate(cands, 1):
+        (cdir / f"{k}.patch").write_text(patch)
+        ctx.store.execute("INSERT OR REPLACE INTO arena_candidates(instance_id, k, kind, label, patch_hash, state) "
+                          "VALUES(?,?,?,?,?,?)", (iid, k, kind, label, _hash(patch), "waiting"))
+    _update(ctx, iid, state="ready", f2p_json=json.dumps(f2p), p2p_json=json.dumps(p2p), note="")
+    return {"instance": iid, "valid": True, "f2p": len(f2p), "p2p": len(p2p), "candidates": len(cands),
+            "mutants_tried": tried}
+
+
+def add_candidate(ctx, iid: str, patch_file: str, label: str) -> dict:
+    """Add a wrong fix by hand (for example one of Patch's failed candidates). It must be plausible
+    and wrong, like the generated ones."""
+    row = _row(ctx, iid)
+    if row["state"] != "ready":
+        raise ToolError("prepare the instance first: `ossa arena prepare`")
+    patch = Path(patch_file).read_text()
+    result = _suite(_env(ctx, row), [patch, _instance(ctx, iid)["human_tests_patch"]])
+    if result is None or not plausible_and_wrong(result.tests, row["f2p"], row["p2p"]):
+        raise ToolError("not a usable wrong candidate: it must apply, keep every P2P test passing and fail an F2P test")
+    k = ctx.store.query("SELECT COALESCE(MAX(k), 0) AS m FROM arena_candidates WHERE instance_id=?", (iid,))[0]["m"] + 1
+    (hidden_dir(ctx, iid) / "candidates" / f"{k}.patch").write_text(patch)
+    ctx.store.execute("INSERT INTO arena_candidates(instance_id, k, kind, label, patch_hash, state) VALUES(?,?,?,?,?,?)",
+                      (iid, k, "added", label[:200], _hash(patch), "waiting"))
+    return {"instance": iid, "added": True}
+
+
+# ------------------------------------------------------------------ rounds
+def next_candidate(ctx, iid: str = "") -> dict:
+    """Turn the next waiting candidate into a blind duel attempt for Breaker. One at a time."""
+    from ossa import gitops
+    from ossa.stages import s03_workspace
+    active = ctx.store.query("SELECT attempt_id FROM arena_candidates WHERE state='active'")
+    if active:
+        raise ToolError(f"score the current candidate first: `ossa arena score {active[0]['attempt_id']}`")
+    sql = ("SELECT c.instance_id, c.k FROM arena_candidates c JOIN arena_instances i ON i.id=c.instance_id "
+           "WHERE c.state='waiting' AND i.state='ready'" + (" AND c.instance_id=?" if iid else "")
+           + " ORDER BY i.created_at, c.k LIMIT 1")
+    rows = ctx.store.query(sql, (iid,) if iid else ())
+    if not rows:
+        return {"attempt": None, "note": "no candidates waiting: mine and prepare more instances"}
+    iid, k = rows[0]["instance_id"], rows[0]["k"]
+    row = _row(ctx, iid)
+    env = _env(ctx, row)
+    patch = (hidden_dir(ctx, iid) / "candidates" / f"{k}.patch").read_text()
+    a = _new(ctx, row, "arena", _arena(ctx) / "blind", "arena")
+    _copy_env(env, a)
+    for name in ("fix.patch", "final.patch"):
+        ctx.store.write_artifact(a, name, patch)
+    ctx.store.write_artifact(a, "verify.json", {"verified": True})
+    wt = gitops.add_worktree(s03_workspace.repo_dir(a), s03_workspace.worktree_for(a), a.branch, a.base_sha)
+    ok, msg = gitops.apply_patch(wt, patch)
+    if not ok:
+        raise ToolError(f"candidate {k} of {iid} no longer applies: {msg[:200]}")
+    a.state = State.FIXED
+    ctx.store.save_attempt(a)
+    ctx.store.execute("UPDATE arena_candidates SET state='active', attempt_id=? WHERE instance_id=? AND k=?",
+                      (a.id, iid, k))
+    return {"attempt": a.id, "worktree": str(wt), "artifacts": a.artifacts_dir,
+            "next": "start the breaker subagent with only the attempt id, the artifacts dir and the worktree; "
+                    f"when its duel is done, `ossa arena score {a.id}`"}
+
+
+def _attack_on(env: Attempt, patch: str, dest: str, text: str, command: str) -> str:
+    """passes | fails | bad: one attack test on base + `patch` (the hidden reference)."""
+    from ossa import gitops
+    from ossa.stages import s03_workspace
+    from ossa.stages.s06_reproduce import classify_failure
+    _, image = s03_workspace.load_env(env)
+    wt = _pristine(env, with_repro=False)
+    ok, _ = gitops.apply_patch(wt, patch)
+    if not ok:
+        return "bad"
+    (wt / dest).parent.mkdir(parents=True, exist_ok=True)
+    (wt / dest).write_text(text)
+    r = env.ctx.sandbox.run(wt, image, command, network=False, timeout=900)
+    gitops.git(wt, "clean", "-fdq", check=False)
+    return classify_failure(r.exit_code, r.output, getattr(r, "timed_out", False))[0]
+
+
+def verdict(kind: str, candidate_outcome: str, status: str, reference: str) -> str:
+    if status == "invalid" or candidate_outcome in ("invalid", "flaky") or reference == "bad":
+        return "invalid"
+    if reference == "fails":
+        return "false_alarm"
+    if kind == "human":
+        return "held"
+    return "caught" if candidate_outcome == "broke" else "missed"
+
+
+def score(ctx, attempt_id: str) -> dict:
+    """Score the active candidate's attacks against the hidden human fix, then hide the candidate."""
+    from ossa import breaker, gitops
+    from ossa.stages import s03_workspace
+    a = _attempt(ctx, attempt_id)
+    rows = ctx.store.query("SELECT * FROM arena_candidates WHERE attempt_id=?", (a.id,))
+    if not rows or rows[0]["state"] != "active":
+        raise ToolError(f"{a.id} is not an Arena candidate waiting to be scored")
+    cand = dict(rows[0])
+    attacks = breaker.attacks(ctx, a.id)
+    if not attacks:
+        raise ToolError("no attacks on this candidate yet: Breaker attacks first")
+    iid, kind = cand["instance_id"], cand["kind"]
+    env, human = _env(ctx, _row(ctx, iid)), _instance(ctx, iid)["human_patch"]
+    verdicts = []
+    for r in attacks:
+        reference = ""
+        if r["status"] != "invalid" and r["outcome"] not in ("invalid", "flaky"):
+            text = ctx.store.read_artifact(a, f"attacks/{r['n']}/{PurePosixPath(r['test_path']).name}") or ""
+            reference = _attack_on(env, human, r["test_path"], text, r["command"])
+        v = verdict(kind, r["outcome"], r["status"], reference)
+        verdicts.append(v)
+        ctx.store.execute(
+            "INSERT OR REPLACE INTO arena_scores(attempt_id, n, instance_id, k, kind, category, candidate_outcome, "
+            "reference_outcome, verdict, breaker_version, model, ts) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)",
+            (a.id, r["n"], iid, cand["k"], kind, r["category"], r["outcome"], reference, v,
+             r.get("breaker_version") or "", r.get("model") or "", now()))
+    if kind == "human":
+        result = "false_alarm" if "false_alarm" in verdicts else "clean"
+    else:
+        result = "caught" if "caught" in verdicts else "missed"
+    duel = breaker.duel(ctx, a.id)["verdict"]
+    ctx.store.execute("UPDATE arena_candidates SET state='scored', result=?, duel_verdict=? WHERE attempt_id=?",
+                      (result, duel, a.id))
+    for wt in (s03_workspace.worktree_for(a), s03_workspace.worktree_for(a, 0)):
+        gitops.remove_worktree(s03_workspace.repo_dir(a), wt)
+    _move(a, hidden_dir(ctx, iid) / "scored" / a.id)
+    a.state = State.CLOSED
+    ctx.store.save_attempt(a)
+    left = ctx.store.query("SELECT COUNT(*) AS n FROM arena_candidates WHERE instance_id=? AND state='waiting'",
+                           (iid,))[0]["n"]
+    return {"scored": a.id, "attacks": len(attacks), "candidates_left": int(left),
+            "next": "`ossa arena next`" if left else "`ossa arena report`"}
+
+
+# ------------------------------------------------------------------ Patch's side
+def solve(ctx, iid: str) -> Attempt:
+    """An attempt at the instance's base with only the issue: Sherlock reproduces, Patch fixes,
+    `ossa verify` checks it, and `ossa arena grade` scores it with the hidden tests."""
+    from ossa.stages import s03_workspace
+    row = _row(ctx, iid)
+    if row["state"] != "ready":
+        raise ToolError("prepare the instance first: `ossa arena prepare`")
+    env = _env(ctx, row)
+    a = _new(ctx, row, "arena-solve", _arena(ctx) / "solve", "arena-solve")
+    _copy_env(env, a)
+    s03_workspace.prepare_worktree(a, _blind_issue(_instance(ctx, iid)))
+    a.state = State.WORKSPACE_READY
+    ctx.store.save_attempt(a)
+    ctx.store.execute("INSERT INTO arena_solves(attempt_id, instance_id, ts) VALUES(?,?,?)", (a.id, iid, now()))
+    return a
+
+
+def grade(ctx, attempt_id: str) -> dict:
+    from ossa import replay
+    from ossa.gitops import diff_stats
+    a = _attempt(ctx, attempt_id)
+    if a.kind != "arena-solve":
+        raise ToolError(f"{a.id} is not an Arena solve attempt")
+    if not (ctx.store.read_json(a, "verify.json") or {}).get("verified"):
+        raise ToolError("not verified yet: `ossa verify` first")
+    iid = ctx.store.query("SELECT instance_id FROM arena_solves WHERE attempt_id=?", (a.id,))[0]["instance_id"]
+    row = _row(ctx, iid)
+    fix = ctx.store.read_artifact(a, "fix.patch") or ""
+    result = _suite(_env(ctx, row), [fix, _instance(ctx, iid)["human_tests_patch"]])
+    tests = result.tests if result else {}
+    resolved = bool(result) and replay.is_resolved(row["f2p"], row["p2p"], tests)
+    out = {"attempt": a.id, "instance": iid, "resolved": resolved,
+           "f2p_passed": sum(tests.get(t) == "passed" for t in row["f2p"]), "f2p_total": len(row["f2p"]),
+           "p2p_broken": sum(tests.get(t) != "passed" for t in row["p2p"]), "lines": diff_stats(fix)[0],
+           "applies": result is not None}
+    ctx.store.execute("INSERT OR REPLACE INTO arena_solves VALUES(?,?,?,?,?,?,?,?)",
+                      (a.id, iid, int(resolved), out["f2p_passed"], out["f2p_total"], out["p2p_broken"],
+                       out["lines"], now()))
+    return out
+
+
+# ------------------------------------------------------------------ status and report
+def status(ctx) -> list[dict]:
+    """Instances and how far each got (no labels: those stay hidden)."""
+    out = []
+    for r in ctx.store.query("SELECT * FROM arena_instances ORDER BY created_at"):
+        counts = {s: 0 for s in ("waiting", "active", "scored")}
+        for c in ctx.store.query("SELECT state, COUNT(*) AS n FROM arena_candidates WHERE instance_id=? GROUP BY state",
+                                 (r["id"],)):
+            counts[c["state"]] = c["n"]
+        out.append({"instance": r["id"], "repo": r["repo"], "pr": r["pr"], "issue": r["issue"], "state": r["state"],
+                    "env_attempt": r["env_attempt"], "contaminated": bool(r["contaminated"]), "note": r["note"],
+                    "candidates": counts})
+    return out
+
+
+def report(ctx) -> dict:
+    """Catch and false-alarm rates by Breaker version and model; Patch's resolve rate."""
+    cands = [dict(r) for r in ctx.store.query("SELECT * FROM arena_candidates WHERE state='scored'")]
+    scores = [dict(r) for r in ctx.store.query("SELECT * FROM arena_scores")]
+    by: dict[tuple[str, str], dict] = {}
+    for c in cands:
+        mine = [s for s in scores if s["attempt_id"] == c["attempt_id"]]
+        key = (mine[0]["breaker_version"] if mine else "", mine[0]["model"] if mine else "")
+        g = by.setdefault(key, {"wrong": 0, "caught": 0, "human": 0, "false_alarm": 0, "ship_on_wrong": 0,
+                                "attacks": 0, "verdicts": {}, "categories": {}})
+        g["attacks"] += len(mine)
+        for s in mine:
+            g["verdicts"][s["verdict"]] = g["verdicts"].get(s["verdict"], 0) + 1
+            cat = g["categories"].setdefault(s["category"] or "?", {"attacks": 0, "caught": 0, "false_alarm": 0})
+            cat["attacks"] += 1
+            if s["verdict"] in ("caught", "false_alarm"):
+                cat[s["verdict"]] += 1
+        if c["kind"] == "human":
+            g["human"] += 1
+            g["false_alarm"] += c["result"] == "false_alarm"
+        else:
+            g["wrong"] += 1
+            g["caught"] += c["result"] == "caught"
+            g["ship_on_wrong"] += c["duel_verdict"] == "SHIP"
+    rows = []
+    for (version, model), g in sorted(by.items()):
+        rows.append({"breaker_version": version, "model": model, **g,
+                     "catch_rate": round(g["caught"] / g["wrong"], 3) if g["wrong"] else None,
+                     "false_alarm_rate": round(g["false_alarm"] / g["human"], 3) if g["human"] else None})
+    solves = ctx.store.query("SELECT COUNT(*) AS n, COALESCE(SUM(resolved), 0) AS r FROM arena_solves "
+                             "WHERE resolved IS NOT NULL")[0]
+    inst = {s: n for s, n in ((r["state"], r["n"]) for r in ctx.store.query(
+        "SELECT state, COUNT(*) AS n FROM arena_instances GROUP BY state"))}
+    out = {"instances": inst, "breaker": rows,
+           "patch": {"solves": solves["n"], "resolved": solves["r"],
+                     "resolve_rate": round(solves["r"] / solves["n"], 3) if solves["n"] else None}}
+    _arena(ctx).mkdir(parents=True, exist_ok=True)
+    (_arena(ctx) / "REPORT.md").write_text(render(out))
+    return out
+
+
+def render(rep: dict) -> str:
+    pct = lambda x: "-" if x is None else f"{x:.0%}"  # noqa: E731
+    lines = ["# Breaker Arena", "", "Instances: " + (", ".join(f"{n} {s}" for s, n in rep["instances"].items()) or "none"),
+             "", "| Breaker version | model | wrong fixes | caught | catch rate | human fixes | false alarms "
+             "| false-alarm rate | SHIP on a wrong fix | attacks |", "|---|---|---|---|---|---|---|---|---|---|"]
+    for r in rep["breaker"]:
+        lines.append(f"| {r['breaker_version'] or '?'} | {r['model'] or '?'} | {r['wrong']} | {r['caught']} | "
+                     f"{pct(r['catch_rate'])} | {r['human']} | {r['false_alarm']} | {pct(r['false_alarm_rate'])} | "
+                     f"{r['ship_on_wrong']} | {r['attacks']} |")
+    p = rep["patch"]
+    lines += ["", f"Patch: {p['resolved']} of {p['solves']} solves resolved ({pct(p['resolve_rate'])})."]
+    return "\n".join(lines) + "\n"