whichagent 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
whichagent/report.py ADDED
@@ -0,0 +1,312 @@
1
+ """Leaderboard and pairwise verdicts: terminal text, Markdown, a self-contained HTML page and an SVG card."""
2
+ import collections
3
+ import datetime
4
+ import html
5
+ import itertools
6
+ import json
7
+
8
+ from . import git as g, stats, store
9
+
10
+ SMALL = 15 # under this many tasks, only large gaps are detectable
11
+ CAP = 100_000 # characters of each diff embedded in report.html
12
+ HEAD = ["agent", "solved", "95% CI", "every time", "$/run", "$/solved", "median time", "issues"]
13
+ MARK = {"solved": "✓", "failed": "✗", "timeout": "⏱", "agent_error": "!", "setup_error": "?"}
14
+
15
+
16
+ def label(r, by_version=False):
17
+ return f"{r['agent']} {r.get('agent_version') or '?'}" if by_version else r["agent"]
18
+
19
+
20
+ def analyze(tasks, results, by_version=False):
21
+ """(rows, pairs, per_agent): summaries best first, and every pairwise comparison."""
22
+ ids = {t["id"] for t in tasks}
23
+ per = {}
24
+ for r in results:
25
+ if r["task"] in ids:
26
+ per.setdefault(label(r, by_version), []).append(r)
27
+ rows = [(n, s) for n, rs in per.items() if (s := stats.summary(rs))]
28
+ rows.sort(key=lambda x: (-x[1]["rate"], x[1]["per_solved"] if x[1]["per_solved"] is not None else float("inf")))
29
+ pairs = [(a, b, *stats.compare(per[a], per[b])) for (a, _), (b, _) in itertools.combinations(rows, 2)]
30
+ return rows, pairs, per
31
+
32
+
33
+ def versus(per, baseline, by_version=False):
34
+ """Each agent against its own baseline results on the tasks both ran: [agent, change, record, verdict]."""
35
+ base = {}
36
+ for r in baseline:
37
+ base.setdefault(label(r, by_version), []).append(r)
38
+ out = []
39
+ for n, rs in per.items():
40
+ now, then = stats.summary(rs), stats.summary(base.get(n, []))
41
+ if now and then:
42
+ up, down, _, p = stats.compare(rs, base[n])
43
+ verdict = (f"{'better' if up > down else 'worse'} (p = {p:.2g})" if p < 0.05
44
+ else f"no real change (p = {p:.2g})")
45
+ out.append([n, f"{_pct(then['rate'])} -> {_pct(now['rate'])}", f"{up} up, {down} down", verdict])
46
+ return out
47
+
48
+
49
+ def _money(x):
50
+ return "-" if x is None else f"${x:.2f}"
51
+
52
+
53
+ def _pct(x):
54
+ return f"{100 * x:.0f}%"
55
+
56
+
57
+ def _issues(s):
58
+ parts = [f"{n} {k.replace('_', ' ')}" for k, n in s["status"].items() if k not in ("solved", "failed")]
59
+ parts += [f"{s[k]} {k}" for k in ("tampered", "regressions", "flagged") if s[k]]
60
+ return ", ".join(parts) or "-"
61
+
62
+
63
+ def _table(rows):
64
+ multi = any(s["every"] is not None for _, s in rows)
65
+ head = [h for h in HEAD if multi or h != "every time"]
66
+ body = []
67
+ for n, s in rows:
68
+ cells = [n, f"{s['solved']}/{s['runs']}", f"{_pct(s['ci'][0])}-{_pct(s['ci'][1])}"]
69
+ if multi:
70
+ cells.append("-" if s["every"] is None else _pct(s["every"]))
71
+ secs = "-" if s["secs"] is None else f"{int(s['secs'] // 60)}m {int(s['secs'] % 60):02d}s"
72
+ cells += [_money(s["per_run"]), _money(s["per_solved"]) + ("" if s["cost_complete"] else "*"), secs,
73
+ _issues(s)]
74
+ body.append(cells)
75
+ return head, body
76
+
77
+
78
+ def _verdict(a, b, wins, losses, ties, p):
79
+ record = f"{wins} win{'s' * (wins != 1)}, {losses} loss{'es' * (losses != 1)}"
80
+ record += f", {ties} tie{'s' * (ties != 1)}" if ties else ""
81
+ if p < 0.05:
82
+ return [f"{a} vs {b}", record, f"{a if wins > losses else b} is better (p = {p:.2g})"]
83
+ return [f"{a} vs {b}", record, f"no real difference (p = {p:.2g})"]
84
+
85
+
86
+ def _notes(rows, per, by_version):
87
+ notes = []
88
+ n = max((s["tasks"] for _, s in rows), default=0)
89
+ if n < SMALL:
90
+ notes.append(f"Only {n} task{'s' * (n != 1)}: only large gaps can be detected.")
91
+ if not all(s["cost_complete"] for _, s in rows):
92
+ notes.append("* Some runs reported no cost; the $ columns use the runs that did.")
93
+ for name, rs in per.items():
94
+ versions = collections.Counter(r.get("agent_version") for r in rs)
95
+ if len(versions) > 1 and not by_version:
96
+ mix = ", ".join(f"{v} ({k} runs)" for v, k in versions.items())
97
+ notes.append(f"{name} ran as {mix}: --by-version compares them.")
98
+ return notes
99
+
100
+
101
+ def _text(rows):
102
+ w = [max(len(str(r[i])) for r in rows) for i in range(len(rows[0]))]
103
+ return "\n".join(" ".join(str(x).ljust(n) for x, n in zip(r, w)).rstrip() for r in rows)
104
+
105
+
106
+ def terminal(tasks, results, baseline=None, by_version=False):
107
+ rows, pairs, per = analyze(tasks, results, by_version)
108
+ if not rows:
109
+ return "No graded runs yet."
110
+ head, body = _table(rows)
111
+ out = [_text([head, *body])]
112
+ if pairs:
113
+ out += ["", _text([_verdict(*p) for p in pairs])]
114
+ if baseline and (vs := versus(per, baseline, by_version)):
115
+ out += ["", "Against the baseline:", _text(vs)]
116
+ return "\n".join(out + [""] + _notes(rows, per, by_version)).rstrip()
117
+
118
+
119
+ def _md(rows):
120
+ rows = [[str(x).replace("|", "\\|") for x in r] for r in rows]
121
+ return "\n".join("| " + " | ".join(r) + " |" for r in [rows[0], ["---"] * len(rows[0]), *rows[1:]])
122
+
123
+
124
+ def _cell(rs):
125
+ return "".join(MARK[r["status"]] for r in sorted(rs, key=lambda r: r["attempt"]))
126
+
127
+
128
+ def _title(t, n=70):
129
+ line = t["statement"].strip().splitlines()[0] if t["statement"].strip() else ""
130
+ return line if len(line) <= n else line[: n - 3] + "..."
131
+
132
+
133
+ def markdown(tasks, results, name, baseline=None, by_version=False):
134
+ rows, pairs, per = analyze(tasks, results, by_version)
135
+ if not rows:
136
+ return f"# whichagent: {name}\n\nNo graded runs yet.\n"
137
+ head, body = _table(rows)
138
+ runs = sum(s["runs"] for _, s in rows)
139
+ md = [f"# whichagent: {name}", "", f"{len(tasks)} tasks, {runs} runs.", "", _md([head, *body])]
140
+ if pairs:
141
+ md += ["", "## Head to head", "", _md([["pair", "record", "verdict"], *[_verdict(*p) for p in pairs]])]
142
+ if baseline and (vs := versus(per, baseline, by_version)):
143
+ md += ["", "## Against the baseline", "", _md([["agent", "solve rate", "tasks", "verdict"], *vs])]
144
+ md += ["", *_notes(rows, per, by_version)]
145
+ names = [n for n, _ in rows]
146
+ grid = [[f"`{t['id']}`", _title(t), *[_cell([r for r in per[n] if r["task"] == t["id"]]) for n in names]]
147
+ for t in tasks]
148
+ md += ["", "## Tasks", "", " ".join(f"{v} {k.replace('_', ' ')}" for k, v in MARK.items()), "",
149
+ _md([["task", "statement", *names], *grid]), ""]
150
+ return "\n".join(md)
151
+
152
+
153
+ def card(tasks, results, name, by_version=False):
154
+ """A small SVG leaderboard to share or put in a README; follows the viewer's light or dark theme."""
155
+ rows, _, _ = analyze(tasks, results, by_version)
156
+ esc, w, h = html.escape, 520, 76 + 30 * len(rows)
157
+ out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}" role="img" '
158
+ f'aria-label="whichagent results for {esc(name)}"><style>'
159
+ "text{font:13px system-ui,-apple-system,'Segoe UI',sans-serif;fill:#1f2328}.m{fill:#656d76}"
160
+ ".t{font-size:15px;font-weight:600}.bg{fill:#fff;stroke:#d0d7de}.bar{fill:#0969da}.track{fill:#eaeef2}"
161
+ "@media (prefers-color-scheme:dark){text{fill:#e6edf3}.m{fill:#8d96a0}.bg{fill:#0d1117;stroke:#30363d}"
162
+ ".bar{fill:#4493f8}.track{fill:#21262d}}</style>"
163
+ f'<rect class="bg" x=".5" y=".5" width="{w - 1}" height="{h - 1}" rx="8"/>'
164
+ f'<text class="t" x="16" y="28">whichagent · {esc(name)}</text>'
165
+ f'<text class="m" x="16" y="48">{len(tasks)} tasks from this repo\'s history, graded by its own tests</text>']
166
+ for i, (n, s) in enumerate(rows):
167
+ y = 78 + 30 * i
168
+ out.append(f'<text x="16" y="{y}">{esc(n[:24])}</text>'
169
+ f'<rect class="track" x="200" y="{y - 11}" width="160" height="12" rx="3"/>'
170
+ f'<rect class="bar" x="200" y="{y - 11}" width="{160 * s["rate"]:.1f}" height="12" rx="3"/>'
171
+ f'<text x="370" y="{y}">{_pct(s["rate"])}</text>'
172
+ f'<text class="m" x="420" y="{y}">{_money(s["per_solved"])}/fix</text>')
173
+ return "".join(out) + "</svg>\n"
174
+
175
+
176
+ def _chart(rows):
177
+ pts = [(n, s["per_run"], s["rate"]) for n, s in rows if s["per_run"] is not None]
178
+ if not pts:
179
+ return '<p class="mute">No cost data to chart.</p>'
180
+ W, H, L, R, T, B = 640, 300, 56, 90, 20, 44
181
+ xmax = max(x for _, x, _ in pts) * 1.15 or 1.0
182
+
183
+ def X(x):
184
+ return L + x / xmax * (W - L - R)
185
+
186
+ def Y(y):
187
+ return H - B - y * (H - T - B)
188
+ front, best = [], -1.0
189
+ for _, x, y in sorted(pts, key=lambda p: (p[1], -p[2])):
190
+ if y > best:
191
+ front.append((x, y))
192
+ best = y
193
+ s = [f'<svg viewBox="0 0 {W} {H}" role="img" aria-label="Solve rate against cost per run">']
194
+ for v in (0, 0.5, 1):
195
+ s.append(f'<line class="grid" x1="{L}" x2="{W - R}" y1="{Y(v):.1f}" y2="{Y(v):.1f}"/>'
196
+ f'<text x="{L - 8}" y="{Y(v) + 4:.1f}" text-anchor="end">{v:.0%}</text>')
197
+ for v in (0, xmax / 2, xmax):
198
+ s.append(f'<text x="{X(v):.1f}" y="{H - B + 18}" text-anchor="middle">${v:.2f}</text>')
199
+ s.append(f'<text x="{(L + W - R) / 2}" y="{H - 6}" text-anchor="middle" class="mute">cost per run</text>')
200
+ if len(front) > 1:
201
+ s.append('<polyline class="front" points="' + " ".join(f"{X(x):.1f},{Y(y):.1f}" for x, y in front) + '"/>')
202
+ for n, x, y in pts:
203
+ s.append(f'<circle cx="{X(x):.1f}" cy="{Y(y):.1f}" r="5"><title>{html.escape(n)}: {y:.0%}, '
204
+ f'${x:.2f}/run</title></circle><text x="{X(x) + 8:.1f}" y="{Y(y) + 4:.1f}">{html.escape(n)}</text>')
205
+ return "".join(s) + "</svg>"
206
+
207
+
208
+ def page(cfg, tasks, results, by_version=False):
209
+ """report.html: leaderboard, cost chart with the efficient frontier, task grid with diffs side by side."""
210
+ rows, pairs, per = analyze(tasks, results, by_version)
211
+ root, esc = cfg["root"], html.escape
212
+ names = [n for n, _ in rows]
213
+ data = {"tasks": {}, "runs": {}}
214
+ for t in tasks:
215
+ try:
216
+ fix = g.git(root, "show", "--format=", "--no-color", "--no-ext-diff", t["fix"])
217
+ except RuntimeError:
218
+ fix = "(fix commit not found)"
219
+ data["tasks"][t["id"]] = {"statement": t["statement"], "fix": fix[:CAP]}
220
+ for n in names:
221
+ for r in sorted(per[n], key=lambda r: r["attempt"]):
222
+ f = store.run_dir(root, r["task"], r["agent"], r.get("agent_version"), r["attempt"]) / "agent.diff"
223
+ data["runs"].setdefault(f"{r['task']}|{n}", []).append(
224
+ {"attempt": r["attempt"], "status": r["status"],
225
+ "diff": f.read_text("utf-8", "replace")[:CAP] if f.exists() else ""})
226
+ head, body = _table(rows) if rows else (HEAD, [])
227
+ table = ("<tr>" + "".join(f"<th>{esc(h)}</th>" for h in head) + "</tr>"
228
+ + "".join("<tr>" + "".join(f"<td>{esc(str(x))}</td>" for x in r) + "</tr>" for r in body))
229
+ verdicts = "".join("<tr>" + "".join(f"<td>{esc(x)}</td>" for x in _verdict(*p)) + "</tr>" for p in pairs)
230
+ grid = "<tr><th>task</th>" + "".join(f"<th>{esc(n)}</th>" for n in names) + "</tr>"
231
+ for t in tasks:
232
+ grid += f'<tr><td><b>{esc(t["id"])}</b> <span class="mute">{esc(_title(t))}</span></td>'
233
+ for n in names:
234
+ rs = data["runs"].get(f"{t['id']}|{n}")
235
+ if not rs:
236
+ grid += "<td></td>"
237
+ continue
238
+ ok = sum(r["status"] == "solved" for r in rs)
239
+ cls = rs[0]["status"] if len(rs) == 1 else "solved" if ok == len(rs) else "failed" if not ok else "mixed"
240
+ mark = MARK[rs[0]["status"]] if len(rs) == 1 else f"{ok}/{len(rs)}"
241
+ grid += (f'<td class="c {cls}" tabindex="0" data-k="{esc(t["id"])}|{esc(n)}" '
242
+ f'title="{esc(n)}: {esc(rs[0]["status"])}">{mark}</td>')
243
+ grid += "</tr>"
244
+ notes = "".join(f"<p class='mute'>{esc(x)}</p>" for x in _notes(rows, per, by_version))
245
+ when = datetime.datetime.now().strftime("%Y-%m-%d %H:%M")
246
+ payload = json.dumps(data).replace("<", "\\u003c")
247
+ runs = sum(s["runs"] for _, s in rows)
248
+ return PAGE.format(name=esc(root.name), when=when, ntasks=len(tasks), nruns=runs, table=table,
249
+ notes=notes, verdicts=verdicts or "<tr><td>Only one agent: nothing to compare.</td></tr>",
250
+ chart=_chart(rows), grid=grid, data=payload, js=JS, css=CSS)
251
+
252
+
253
+ CSS = """:root{--bg:#fff;--fg:#1f2328;--mute:#656d76;--line:#d0d7de;--card:#f6f8fa;--ok:#1a7f37;--bad:#cf222e;
254
+ --warn:#9a6700;--add:#dafbe1;--del:#ffebe9;--acc:#0969da}
255
+ @media (prefers-color-scheme:dark){:root{--bg:#0d1117;--fg:#e6edf3;--mute:#8d96a0;--line:#30363d;--card:#161b22;
256
+ --ok:#3fb950;--bad:#f85149;--warn:#d29922;--add:#12261e;--del:#2d1517;--acc:#4493f8}}
257
+ *{box-sizing:border-box}body{font:14px/1.5 system-ui,-apple-system,"Segoe UI",sans-serif;background:var(--bg);
258
+ color:var(--fg);max-width:1120px;margin:0 auto;padding:24px 16px}h1{font-size:22px;margin:0 0 4px}
259
+ h2{font-size:16px;margin:32px 0 8px}h4{margin:12px 0 4px;font-size:13px}.mute{color:var(--mute)}
260
+ .wrap{overflow-x:auto}table{border-collapse:collapse;font-variant-numeric:tabular-nums}
261
+ th,td{padding:6px 12px;border-bottom:1px solid var(--line);text-align:left;white-space:nowrap}
262
+ td.c{text-align:center;cursor:pointer;font-weight:600}td.c:hover,td.c:focus{background:var(--card);outline:none}
263
+ .solved{color:var(--ok)}.failed{color:var(--bad)}.mixed,.timeout,.agent_error,.setup_error{color:var(--warn)}
264
+ svg{width:100%;max-width:640px;height:auto}svg text{fill:var(--fg);font-size:12px}svg text.mute{fill:var(--mute)}
265
+ svg .grid{stroke:var(--line)}svg .front{stroke:var(--acc);fill:none;stroke-width:1.5;stroke-dasharray:5 4}
266
+ svg circle{fill:var(--acc)}.stmt{white-space:pre-wrap;background:var(--card);padding:10px 12px;border-radius:6px}
267
+ .cols{display:grid;grid-template-columns:1fr 1fr;gap:16px}@media (max-width:820px){.cols{grid-template-columns:1fr}}
268
+ pre{background:var(--card);padding:10px;overflow:auto;font:12px/1.45 ui-monospace,Consolas,monospace;
269
+ max-height:520px;border-radius:6px;margin:0}pre span{display:block;white-space:pre}pre .add{background:var(--add)}
270
+ pre .del{background:var(--del)}pre .hunk{color:var(--acc)}"""
271
+
272
+ JS = """const D = JSON.parse(document.getElementById("data").textContent);
273
+ const el = (tag, text, cls) => { const e = document.createElement(tag);
274
+ if (text != null) e.textContent = text; if (cls) e.className = cls; return e; };
275
+ function pre(text) {
276
+ const p = el("pre");
277
+ for (const l of (text || "(no changes)").replace(/\\n$/, "").split("\\n"))
278
+ p.append(el("span", l || " ", l.startsWith("+") && !l.startsWith("+++") ? "add"
279
+ : l.startsWith("-") && !l.startsWith("---") ? "del" : l.startsWith("@@") ? "hunk" : ""));
280
+ return p;
281
+ }
282
+ function show(key) {
283
+ const i = key.indexOf("|"), task = key.slice(0, i), agent = key.slice(i + 1), box = document.getElementById("detail");
284
+ const cols = el("div", null, "cols"), fix = el("div"), runs = el("div");
285
+ fix.append(el("h4", "Real fix"), pre(D.tasks[task].fix));
286
+ for (const r of D.runs[key]) runs.append(el("h4", agent + ", attempt " + r.attempt + ": " + r.status), pre(r.diff));
287
+ cols.append(fix, runs);
288
+ box.replaceChildren(el("h2", "Task " + task), el("p", D.tasks[task].statement, "stmt"), cols);
289
+ box.scrollIntoView({behavior: "smooth"});
290
+ }
291
+ for (const td of document.querySelectorAll("td[data-k]")) {
292
+ td.addEventListener("click", () => show(td.dataset.k));
293
+ td.addEventListener("keydown", e => { if (e.key === "Enter") show(td.dataset.k); });
294
+ }"""
295
+
296
+ PAGE = """<!doctype html>
297
+ <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
298
+ <title>whichagent: {name}</title><style>{css}</style></head><body>
299
+ <h1>whichagent: {name}</h1>
300
+ <p class="mute">{ntasks} tasks, {nruns} runs, {when}. Each task is a past fix from this repo's history, graded by its own tests.</p>
301
+ <h2>Leaderboard</h2><div class="wrap"><table>{table}</table></div>{notes}
302
+ <h2>Head to head</h2><p class="mute">A wins a task when it solves it more often. Sign test on wins vs losses.</p>
303
+ <div class="wrap"><table>{verdicts}</table></div>
304
+ <h2>Solve rate vs cost</h2><p class="mute">The dashed line is the efficient frontier: nothing else is both cheaper and better.</p>
305
+ {chart}
306
+ <h2>Tasks</h2><p class="mute">Click a cell to see the agent's diff next to the real fix.</p>
307
+ <div class="wrap"><table>{grid}</table></div>
308
+ <div id="detail"></div>
309
+ <script type="application/json" id="data">{data}</script>
310
+ <script>{js}</script>
311
+ </body></html>
312
+ """
whichagent/run.py ADDED
@@ -0,0 +1,153 @@
1
+ """The matrix: tasks x agents x attempts, with jobs, timeouts, resume and a spend cap."""
2
+ import concurrent.futures as cf
3
+ import shutil
4
+ import statistics
5
+ import threading
6
+
7
+ from . import agents as ag, config as c, git as g, grade, sandbox as sb, store, workspace
8
+
9
+ NO_HISTORY = (0.10, 1.00) # USD per run before an agent has any results: a deliberately wide guess
10
+ ERRORS = ("agent_error", "setup_error")
11
+
12
+
13
+ def prompt(task):
14
+ text = f"Resolve the following task in this repository.\n\n{task['statement']}\n"
15
+ if task["mode"] == "visible":
16
+ text += ("\nThese tests fail now and must pass:\n"
17
+ + "".join(f"- {f}\n" for f in grade.runnable(task["test_files"])) + "Don't edit these files.\n")
18
+ return text
19
+
20
+
21
+ def run_one(cfg, task, agent, attempt, keep=False):
22
+ """One agent on one task in a fresh workspace, graded. Returns the result dict."""
23
+ root = cfg["root"]
24
+ argv = c.argv(agent)
25
+ version = sb.version(cfg, argv[0])
26
+ d = store.run_dir(root, task["id"], agent["name"], version, attempt)
27
+ d.mkdir(parents=True, exist_ok=True)
28
+ r = {"task": task["id"], "agent": agent["name"], "attempt": attempt, "status": "", "tests_pass": False,
29
+ "regression": None, "cost_usd": None, "tokens_in": None, "tokens_out": None, "secs": None, "turns": None,
30
+ "files_changed": 0, "extra_files": 0, "wrote_tests": False, "tampered": False, "flagged": [],
31
+ "agent_version": version, "model": None}
32
+ ws, box = workspace.create(cfg, task["id"], task["base"])
33
+ try:
34
+ exe = box.which(argv[0])
35
+ if not exe:
36
+ return {**r, "status": "agent_error", "error": f"{argv[0]} not found on PATH"}
37
+ if not workspace.prepare(cfg, ws, box, d):
38
+ return {**r, "status": "setup_error"}
39
+ for dst, src in agent["overlay"].items():
40
+ (ws / dst).parent.mkdir(parents=True, exist_ok=True)
41
+ (shutil.copytree(root / src, ws / dst, dirs_exist_ok=True) if (root / src).is_dir()
42
+ else shutil.copyfile(root / src, ws / dst))
43
+ if task["mode"] == "visible":
44
+ g.put(root, task["fix"], task["test_files"], ws)
45
+ start = g.snapshot(ws)
46
+ text = prompt(task)
47
+ pf = ws.parent / "prompt.txt" # outside the repo, so it isn't part of the agent's diff
48
+ pf.write_text(text, "utf-8")
49
+ (d / "prompt.txt").write_text(text, "utf-8")
50
+ argv = [exe, *(a.replace("{prompt_file}", box.path(pf)) for a in argv[1:])]
51
+ rc, secs = box.agent(argv, c.secs(agent["timeout"]), d / "stdout.log", d / "stderr.log", box.env(agent),
52
+ pf if agent["stdin"] else None)
53
+ usage = ag.parse(cfg, agent["parse"], (d / "stdout.log").read_text("utf-8", "replace"), argv)
54
+ r.update({k: v for k, v in usage.items() if k != "error"}, secs=round(secs, 1), exit_code=rc)
55
+ try:
56
+ r.update(grade.grade(cfg, task, ws, box, start, d))
57
+ except RuntimeError as e: # e.g. the agent broke the workspace's .git
58
+ return {**r, "status": "agent_error", "error": str(e)}
59
+ r["status"] = ("timeout" if rc is None else "solved" if r["tests_pass"] and not r["regression"]
60
+ else "agent_error" if rc or usage.get("error") else "failed")
61
+ return r
62
+ finally:
63
+ if keep:
64
+ box.close()
65
+ print(f" kept {ws}")
66
+ else:
67
+ workspace.remove(ws, box)
68
+
69
+
70
+ def run(cfg, agents, tasks, yes=False, keep=False, retry_errors=False, new_version=False, log=print):
71
+ """Run every (task, agent, attempt) not yet in results.jsonl. Returns the new results.
72
+
73
+ With new_version, a run done by another version of the agent doesn't count as done."""
74
+ sb.STOP.clear()
75
+ rc = cfg["run"]
76
+ path = store.state(cfg["root"]) / "results.jsonl"
77
+ old = store.read(path)
78
+ versions = {a["name"]: sb.version(cfg, c.argv(a)[0]) for a in agents}
79
+
80
+ def key(task_id, agent, attempt, version):
81
+ return task_id, agent, attempt, version if new_version else None
82
+ done = {key(x["task"], x["agent"], x["attempt"], x.get("agent_version")) for x in store.latest(old, new_version)
83
+ if not (retry_errors and x["status"] in ERRORS)}
84
+ todo = [(t, a, k) for t in tasks for k in range(1, rc["attempts"] + 1) for a in agents
85
+ if key(t["id"], a["name"], k, versions[a["name"]]) not in done] # interleaved: a stop stays comparable
86
+ for a in agents:
87
+ if a["parse"] not in ag.PARSERS:
88
+ raise SystemExit(f"agent {a['name']!r}: parse must be one of {', '.join(ag.PARSERS)}")
89
+ for src in a["overlay"].values():
90
+ if not (cfg["root"] / src).exists():
91
+ raise SystemExit(f"agent {a['name']!r}: overlay file {src} not found")
92
+ if not todo:
93
+ n = sum(x["status"] in ERRORS for x in store.latest(old))
94
+ log("Nothing to run: every run is already in results.jsonl."
95
+ + (f" {n} ended in an error; --retry-errors reruns them." if n else ""))
96
+ return []
97
+ costs = {a["name"]: [x["cost_usd"] for x in old if x["agent"] == a["name"] and x["cost_usd"] is not None
98
+ and x["status"] not in ERRORS] for a in agents} # a failed login costs $0 and says nothing
99
+ lo = sum(statistics.median(costs[a["name"]]) if costs[a["name"]] else NO_HISTORY[0] for _, a, _ in todo)
100
+ hi = sum(max(costs[a["name"]]) if costs[a["name"]] else NO_HISTORY[1] for _, a, _ in todo)
101
+ log(f"{len(todo)} runs, estimated ${lo:.2f}-${hi:.2f} (stops before passing max_cost ${rc['max_cost']:.2f})")
102
+ blind = [a["name"] for a in agents if a["parse"] == "none"]
103
+ if blind:
104
+ log(f"No cost data from {', '.join(blind)}: their spending isn't counted toward max_cost.")
105
+ if not yes and not c.ask("Continue?"):
106
+ return []
107
+ lock = threading.Lock()
108
+ budget = {"spent": 0.0, "inflight": 0.0, "over": False}
109
+ slots = {a["name"]: threading.Semaphore(rc["jobs_per_agent"] or rc["jobs"]) for a in agents}
110
+ new = []
111
+
112
+ def work(task, agent, attempt):
113
+ # ponytail: a worker waiting on a busy agent's slot idles; a per-agent queue would keep it busy
114
+ with slots[agent["name"]]:
115
+ with lock:
116
+ xs = costs[agent["name"]]
117
+ est = sum(xs) / len(xs) if xs else NO_HISTORY[1]
118
+ if sb.STOP.is_set() or budget["over"] or budget["spent"] + budget["inflight"] + est > rc["max_cost"]:
119
+ budget["over"] = not sb.STOP.is_set()
120
+ return
121
+ budget["inflight"] += est
122
+ try:
123
+ r = run_one(cfg, task, agent, attempt, keep)
124
+ except RuntimeError:
125
+ if sb.STOP.is_set():
126
+ return # interrupted: rerun on resume
127
+ raise
128
+ finally:
129
+ with lock:
130
+ budget["inflight"] -= est
131
+ if sb.STOP.is_set():
132
+ return
133
+ with lock:
134
+ budget["spent"] += r["cost_usd"] or 0
135
+ if r["cost_usd"] is not None:
136
+ xs.append(r["cost_usd"])
137
+ store.append(path, r)
138
+ new.append(r)
139
+ money = "-" if r["cost_usd"] is None else f"${r['cost_usd']:.2f}"
140
+ log(f" {r['status']:<11} {task['id']} {agent['name']} #{attempt} {money} {r['secs'] or 0:.0f}s")
141
+
142
+ with cf.ThreadPoolExecutor(rc["jobs"]) as ex:
143
+ futures = [ex.submit(work, *x) for x in todo]
144
+ try:
145
+ for f in cf.as_completed(futures):
146
+ f.result()
147
+ except BaseException:
148
+ sb.STOP.set()
149
+ sb.killall()
150
+ raise
151
+ if budget["over"]:
152
+ log(f"Stopped at max_cost: spent ${budget['spent']:.2f}. Raise --max-cost and rerun to continue.")
153
+ return new