whichagent 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
whichagent/__init__.py ADDED
@@ -0,0 +1,2 @@
1
+ """whichagent: benchmark coding agents on your repo's own git history."""
2
+ __version__ = "0.1.0"
whichagent/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ import sys
2
+
3
+ from .cli import main
4
+
5
+ sys.exit(main())
whichagent/agents.py ADDED
@@ -0,0 +1,199 @@
1
+ """Built-in adapters' output parsers, model prices and flag checks."""
2
+ import functools
3
+ import itertools
4
+ import json
5
+ import os
6
+ import re
7
+ import subprocess
8
+ import time
9
+ import tomllib
10
+ import urllib.request
11
+ from pathlib import Path
12
+
13
+ PRICES_URL = "https://models.dev/api.json"
14
+
15
+
16
+ def parse(cfg, kind, out, argv):
17
+ """Usage from an agent's stdout: cost_usd, tokens_in, tokens_out, turns, model, error (all optional)."""
18
+ return PARSERS[kind](cfg, out, argv)
19
+
20
+
21
+ def _json(out):
22
+ """The output as JSON: the whole text, else its last line that parses."""
23
+ for text in [out, *reversed(out.strip().splitlines())]:
24
+ try:
25
+ return json.loads(text)
26
+ except ValueError:
27
+ pass
28
+ return None
29
+
30
+
31
+ def claude(cfg, out, argv):
32
+ d = _json(out)
33
+ if isinstance(d, list):
34
+ d = next((x for x in reversed(d) if isinstance(x, dict) and x.get("type") == "result"), None)
35
+ if not isinstance(d, dict):
36
+ return {"error": True}
37
+ mu = d.get("modelUsage") or {}
38
+ u = d.get("usage") or {}
39
+ if mu:
40
+ tin = sum(m.get("inputTokens", 0) + m.get("cacheReadInputTokens", 0) + m.get("cacheCreationInputTokens", 0)
41
+ for m in mu.values())
42
+ tout = sum(m.get("outputTokens", 0) for m in mu.values())
43
+ else:
44
+ tin = u.get("input_tokens", 0) + u.get("cache_read_input_tokens", 0) + u.get("cache_creation_input_tokens", 0)
45
+ tout = u.get("output_tokens", 0)
46
+ usd = d.get("total_cost_usd")
47
+ return {"cost_usd": None if usd is None else round(usd, 6), "tokens_in": tin, "tokens_out": tout,
48
+ "turns": d.get("num_turns"),
49
+ "model": max(mu, key=lambda k: mu[k].get("costUSD", 0), default=None), "error": bool(d.get("is_error"))}
50
+
51
+
52
+ def _events(out):
53
+ """The JSON objects in JSONL output, skipping anything else."""
54
+ for line in out.splitlines():
55
+ try:
56
+ e = json.loads(line)
57
+ except ValueError:
58
+ continue
59
+ if isinstance(e, dict):
60
+ yield e
61
+
62
+
63
+ def codex(cfg, out, argv):
64
+ """`codex exec --json` events, token usage on turn.completed."""
65
+ tin = tout = cached = 0
66
+ error = False
67
+ for e in _events(out):
68
+ if e.get("type") == "turn.completed":
69
+ u = e.get("usage") or {}
70
+ tin += u.get("input_tokens", 0)
71
+ cached += u.get("cached_input_tokens", 0)
72
+ tout += u.get("output_tokens", 0)
73
+ error = error or e.get("type") in ("error", "turn.failed")
74
+ model = _flag(argv, "-m", "--model") or _codex_model()
75
+ return {"cost_usd": cost(cfg, model, tin, tout, cached), "tokens_in": tin, "tokens_out": tout,
76
+ "model": model, "error": error}
77
+
78
+
79
+ def gemini(cfg, out, argv):
80
+ """Unverified: `gemini --output-format json`, tokens per model under stats.models. Qwen Code prints a list of
81
+ events with the same stats on its "result" event."""
82
+ d = _json(out)
83
+ if isinstance(d, list):
84
+ d = next((x for x in reversed(d) if isinstance(x, dict) and x.get("type") == "result"), None)
85
+ if not isinstance(d, dict):
86
+ return {"error": True}
87
+ models = (d.get("stats") or {}).get("models") or {}
88
+ tin = tout = 0
89
+ total = 0.0 if models else None
90
+ for name, m in models.items():
91
+ t = m.get("tokens") or {}
92
+ i, o = t.get("prompt", 0), t.get("candidates", 0) + t.get("thoughts", 0)
93
+ tin, tout = tin + i, tout + o
94
+ p = cost(cfg, name, i, o, t.get("cached", 0))
95
+ total = None if p is None or total is None else total + p
96
+ model = max(models, key=lambda n: (models[n].get("tokens") or {}).get("total", 0), default=None)
97
+ return {"cost_usd": total, "tokens_in": tin, "tokens_out": tout, "model": model, "turns": d.get("num_turns"),
98
+ "error": bool(d.get("error") or d.get("is_error"))}
99
+
100
+
101
+ def aider(cfg, out, argv):
102
+ """aider's "Tokens: ... sent, ... received. Cost: ... $X session." lines; local models print no Cost."""
103
+ def n(s):
104
+ return float(s[:-1]) * {"k": 1e3, "M": 1e6}[s[-1]] if s[-1] in "kM" else float(s)
105
+ toks = re.findall(r"Tokens: ([\d.]+[kM]?) sent.*?([\d.]+[kM]?) received", out)
106
+ session = re.findall(r"\$([\d.]+) session", out)
107
+ m = re.search(r"^(?:Main model|Model): (\S+)", out, re.M)
108
+ model, tin, tout = m and m[1], round(sum(n(a) for a, _ in toks)), round(sum(n(b) for _, b in toks))
109
+ return {"cost_usd": float(session[-1]) if session else cost(cfg, model, tin, tout),
110
+ "tokens_in": tin, "tokens_out": tout, "model": model}
111
+
112
+
113
+ def opencode(cfg, out, argv):
114
+ """`opencode run --format json` events: tokens and cost on each step_finish; the model is the -m flag."""
115
+ tin = tout = steps = 0
116
+ usd, error = 0.0, False
117
+ for e in _events(out):
118
+ p = e.get("part") or {}
119
+ if e.get("type") == "step_finish":
120
+ t, cache = p.get("tokens") or {}, (p.get("tokens") or {}).get("cache") or {}
121
+ tin += t.get("input", 0) + cache.get("read", 0) + cache.get("write", 0)
122
+ tout += t.get("output", 0) + t.get("reasoning", 0)
123
+ usd += p.get("cost") or 0
124
+ steps += 1
125
+ error = error or e.get("type") == "error"
126
+ return {"cost_usd": round(usd, 6) if steps else None, "tokens_in": tin, "tokens_out": tout, "turns": steps,
127
+ "model": _flag(argv, "-m", "--model"), "error": error}
128
+
129
+
130
+ PARSERS = {"claude-json": claude, "codex-jsonl": codex, "gemini-json": gemini, "aider-text": aider,
131
+ "opencode-json": opencode, "none": lambda cfg, out, argv: {}}
132
+
133
+
134
+ def _flag(argv, short, long):
135
+ for i, a in enumerate(argv):
136
+ if a in (short, long) and i + 1 < len(argv):
137
+ return argv[i + 1]
138
+ if a.startswith(long + "="):
139
+ return a.split("=", 1)[1]
140
+ return None
141
+
142
+
143
+ def _codex_model():
144
+ try:
145
+ f = Path(os.environ.get("CODEX_HOME") or Path.home() / ".codex") / "config.toml"
146
+ return tomllib.loads(f.read_text("utf-8")).get("model")
147
+ except (OSError, ValueError):
148
+ return None
149
+
150
+
151
+ LOCAL = ("ollama", "ollama_chat", "lm_studio", "llamafile") # litellm prefixes for models on your machine
152
+
153
+
154
+ def cost(cfg, model, tin, tout, cached=0):
155
+ """USD from tokens with [prices] from whichagent.toml, else models.dev. None when the price is unknown."""
156
+ if not model:
157
+ return None
158
+ if model.split("/")[0] in LOCAL:
159
+ return 0.0
160
+ p = cfg["prices"].get(model) or _models_dev(cfg["root"]).get(model) or _models_dev(cfg["root"]).get(
161
+ model.split("/")[-1])
162
+ if not p or "input" not in p or "output" not in p:
163
+ return None
164
+ usd = (tin - cached) * p["input"] + cached * p.get("cache_read", p["input"]) + tout * p["output"]
165
+ return round(usd / 1e6, 6)
166
+
167
+
168
+ @functools.cache
169
+ def _models_dev(root):
170
+ """{model: cost} from models.dev, cached for 24 hours in .whichagent/prices.json."""
171
+ f = Path(root) / ".whichagent" / "prices.json"
172
+ if not f.exists() or time.time() - f.stat().st_mtime > 86400:
173
+ try:
174
+ with urllib.request.urlopen(PRICES_URL, timeout=15) as r:
175
+ f.write_bytes(r.read())
176
+ except OSError:
177
+ pass # offline: keep using the stale copy, if any
178
+ try:
179
+ data = json.loads(f.read_text("utf-8"))
180
+ except (OSError, ValueError):
181
+ return {}
182
+ out = {}
183
+ for provider in ("anthropic", "openai", "google", *data):
184
+ for mid, m in ((data.get(provider) or {}).get("models") or {}).items():
185
+ if m.get("cost"):
186
+ out.setdefault(mid, m["cost"])
187
+ return out
188
+
189
+
190
+ def missing_flags(exe, argv):
191
+ """Flags in argv that the CLI's --help doesn't mention (for `whichagent doctor`)."""
192
+ sub = list(itertools.takewhile(lambda a: not a.startswith("-"), argv[1:])) # e.g. `codex exec`
193
+ try:
194
+ r = subprocess.run([exe, *sub, "--help"], capture_output=True, text=True, encoding="utf-8",
195
+ errors="replace", timeout=60)
196
+ except (OSError, subprocess.TimeoutExpired):
197
+ return ["(--help failed)"]
198
+ flags = [a.split("=")[0] for a in argv[1:] if a.startswith("-") and a != "-"]
199
+ return [f for f in flags if not re.search(rf"(?<![\w-]){re.escape(f)}(?![\w-])", r.stdout + r.stderr)]
whichagent/cli.py ADDED
@@ -0,0 +1,244 @@
1
+ """whichagent command line: quickstart, init, doctor, mine, run, report, compare, show."""
2
+ import argparse
3
+ import shutil
4
+ import sys
5
+ import tempfile
6
+ from pathlib import Path
7
+
8
+ from . import __version__, agents as ag, config as c, git as g, grade, mine, report, run, sandbox as sb, stats
9
+ from . import store, workspace
10
+
11
+ SAFETY = """whichagent runs coding agents unattended, with their permission prompts off:
12
+ - each run happens in a throwaway clone in your temp dir; your working tree isn't touched
13
+ - agents get an allowlist of environment variables, not your whole shell
14
+ - runs are killed at their timeout, and no run starts that would pass max_cost
15
+ On the host this is not a sandbox: an agent can still read and write whatever your user account can,
16
+ and reach the network. --sandbox docker runs setup, agents and tests in a container that sees only the workspace."""
17
+
18
+
19
+ def _list(s):
20
+ return [x.strip() for x in s.split(",") if x.strip()]
21
+
22
+
23
+ def parser():
24
+ common = argparse.ArgumentParser(add_help=False)
25
+ flag = common.add_argument
26
+ flag("-C", dest="repo", metavar="DIR", default=argparse.SUPPRESS, help="the repo (default: current directory)")
27
+ flag("-y", "--yes", action="store_true", default=argparse.SUPPRESS, help="don't ask to confirm")
28
+ flag("--keep", action="store_true", default=argparse.SUPPRESS, help="keep agent workspaces for inspection")
29
+ flag("--retry-errors", action="store_true", default=argparse.SUPPRESS,
30
+ help="rerun runs that ended in agent_error or setup_error (e.g. after fixing a login)")
31
+ flag("--new-version", action="store_true", default=argparse.SUPPRESS,
32
+ help="rerun tasks an agent has only done with another version of itself")
33
+ flag("--by-version", action="store_true", default=argparse.SUPPRESS,
34
+ help="report each agent version separately")
35
+ flag("--tasks", metavar="IDS", type=_list, default=argparse.SUPPRESS,
36
+ help="only these tasks (ids or id prefixes, comma-separated), e.g. to try an agent on a free tier")
37
+ flag("--baseline", metavar="PATH", default=argparse.SUPPRESS,
38
+ help="compare against another results.jsonl (or a dir holding .whichagent/)")
39
+ for sec, keys in c.DEFAULTS.items(): # every config key is also a flag: --max-tasks 40
40
+ for key, v in keys.items():
41
+ kw = {"dest": f"{sec}.{key}", "default": argparse.SUPPRESS, "help": f"[{sec}] {key}"}
42
+ if isinstance(v, bool):
43
+ flag("--" + key.replace("_", "-"), action=argparse.BooleanOptionalAction, **kw)
44
+ else:
45
+ flag("--" + key.replace("_", "-"), type=_list if isinstance(v, list) else type(v), **kw)
46
+ p = argparse.ArgumentParser(prog="whichagent", parents=[common],
47
+ description="Find out which coding agent actually works on your repo.")
48
+ p.add_argument("--version", action="version", version=f"whichagent {__version__}")
49
+ sub = p.add_subparsers(dest="cmd", metavar="command")
50
+ for name, text in [("init", "write whichagent.toml from the detected stack"),
51
+ ("doctor", "check git, the test command at HEAD and each agent"),
52
+ ("mine", "find and validate tasks -> .whichagent/tasks.jsonl"),
53
+ ("run", "run the matrix (resumable) -> .whichagent/results.jsonl"),
54
+ ("report", "terminal table, report.html, report.md, card.svg")]:
55
+ sub.add_parser(name, parents=[common], help=text)
56
+ s = sub.add_parser("compare", parents=[common], help="paired comparison of two agents")
57
+ s.add_argument("a")
58
+ s.add_argument("b")
59
+ s = sub.add_parser("show", parents=[common], help="statement, real fix, each agent's diff")
60
+ s.add_argument("task")
61
+ return p
62
+
63
+
64
+ def main(argv=None):
65
+ if hasattr(sys.stdout, "reconfigure"):
66
+ sys.stdout.reconfigure(errors="replace") # a cp1252 pipe on Windows can't print everything
67
+ a = vars(parser().parse_args(argv))
68
+ root = Path(a.get("repo", ".")).resolve()
69
+ over = {}
70
+ for k, v in a.items():
71
+ if "." in k:
72
+ sec, key = k.split(".")
73
+ over.setdefault(sec, {})[key] = v
74
+ try:
75
+ g.git(root, "rev-parse", "--git-dir")
76
+ except RuntimeError:
77
+ raise SystemExit(f"{root} isn't a git repository")
78
+ cmd = a.get("cmd") or "quickstart"
79
+ try:
80
+ if cmd in ("init", "quickstart") and not (root / "whichagent.toml").exists():
81
+ init(c.load(root, over), a)
82
+ elif cmd == "init":
83
+ print("whichagent.toml already exists")
84
+ if cmd == "init":
85
+ return 0
86
+ cfg = c.load(root, over)
87
+ return COMMANDS[cmd](cfg, a) or 0
88
+ except KeyboardInterrupt:
89
+ sb.STOP.set()
90
+ sb.killall()
91
+ print("\nStopped. Run the same command again to resume.")
92
+ return 130
93
+
94
+
95
+ def init(cfg, a):
96
+ rp = cfg["repo"]
97
+ if not rp["test"]:
98
+ if a.get("yes") or not sys.stdin.isatty():
99
+ raise SystemExit("Can't detect the stack: pass --setup and --test, e.g. --test 'pytest -q {files}'")
100
+ rp["setup"] = input("Setup command (empty for none): ").strip()
101
+ rp["test"] = input("Test command, with {files} where the test files go: ").strip()
102
+ if not c.PLACEHOLDER.search(rp["test"]):
103
+ raise SystemExit("The test command needs {files}")
104
+ c.write_template(cfg["root"], rp)
105
+ print("Wrote whichagent.toml: " + ", ".join(f"{k} = {rp[k]!r}" for k in ("install", "setup", "test") if rp[k]))
106
+
107
+
108
+ def _need_test(cfg):
109
+ if not cfg["repo"]["test"]:
110
+ raise SystemExit("No test command: run `whichagent init` or pass --test")
111
+
112
+
113
+ def _data(cfg, a):
114
+ d = store.state(cfg["root"])
115
+ return store.read(d / "tasks.jsonl"), store.latest(store.read(d / "results.jsonl"), a.get("by_version", False))
116
+
117
+
118
+ def _baseline(a):
119
+ if not a.get("baseline"):
120
+ return None
121
+ p = Path(a["baseline"])
122
+ for f in (p, p / "results.jsonl", p / ".whichagent" / "results.jsonl"):
123
+ if f.is_file():
124
+ return store.latest(store.read(f), a.get("by_version", False))
125
+ raise SystemExit(f"No results.jsonl at {p}")
126
+
127
+
128
+ def cmd_quickstart(cfg, a):
129
+ cmd_mine(cfg, a)
130
+ return cmd_run(cfg, a)
131
+
132
+
133
+ def cmd_doctor(cfg, a):
134
+ root, ok = cfg["root"], True
135
+ print(g.git(root, "--version").strip())
136
+ if cfg["run"]["sandbox"] == "docker":
137
+ if not shutil.which("docker"):
138
+ raise SystemExit("docker mode: docker isn't on PATH")
139
+ print(f"docker image: {sb.image(root, cfg['docker']['image'])}")
140
+ if not cfg["repo"]["test"]:
141
+ print("test command: not set (run `whichagent init`)")
142
+ ok = False
143
+ else:
144
+ tmp = Path(tempfile.mkdtemp())
145
+ ws, logs = tmp / root.name, store.state(root)
146
+ g.git(tmp, "clone", "-q", "--depth=1", root.as_uri(), str(ws))
147
+ box = sb.executor(cfg, ws)
148
+ try:
149
+ if not workspace.prepare(cfg, ws, box, logs):
150
+ print(f"install or setup at HEAD: FAILED, see {logs / 'install.log'} and {logs / 'setup.log'}")
151
+ ok = False
152
+ else:
153
+ rc, secs, results = grade.test(cfg, box, ws, [], logs / "doctor.log", suite=True)
154
+ graded = (f"{len(results)} tests with per-test results" if results
155
+ else "no per-test results, so tasks are graded by exit code")
156
+ print(f"test suite at HEAD: {'passes' if rc == 0 else 'FAILS'} ({secs:.0f}s, {graded}, "
157
+ f"log in {logs / 'doctor.log'})")
158
+ if rc != 0:
159
+ tail = (logs / "doctor.log").read_text("utf-8", "replace").strip().splitlines()[-8:]
160
+ print(" " + "\n ".join(tail))
161
+ ok = ok and rc == 0
162
+ finally:
163
+ workspace.remove(ws, box)
164
+ for name, agent in cfg["agents"].items():
165
+ argv = c.argv(agent)
166
+ note = " (unverified adapter)" if agent.get("unverified") else ""
167
+ version = sb.version(cfg, argv[0])
168
+ if not version:
169
+ print(f"{name}: {argv[0]} not found{note}")
170
+ elif cfg["run"]["sandbox"] == "docker":
171
+ print(f"{name}: {version} (in the image){note}")
172
+ else:
173
+ missing = ag.missing_flags(shutil.which(argv[0]), argv)
174
+ print(f"{name}: {version}, " + (f"flags not in --help: {' '.join(missing)}" if missing
175
+ else "flags ok") + note)
176
+ return 0 if ok else 1
177
+
178
+
179
+ def cmd_mine(cfg, a):
180
+ _need_test(cfg)
181
+ mine.mine(cfg)
182
+
183
+
184
+ def cmd_run(cfg, a):
185
+ _need_test(cfg)
186
+ tasks = store.read(store.state(cfg["root"]) / "tasks.jsonl")
187
+ if not tasks:
188
+ raise SystemExit("No tasks: run `whichagent mine` first (or check .whichagent/rejected.jsonl)")
189
+ if a.get("tasks"):
190
+ tasks = [t for t in tasks if any(t["id"].startswith(x) for x in a["tasks"])] or sys.exit("No such tasks")
191
+ agents = c.selected(cfg)
192
+ if not agents:
193
+ raise SystemExit("No agents: none of the built-ins is on PATH and whichagent.toml defines none")
194
+ print("Agents: " + ", ".join(x["name"] + (" (unverified adapter)" if x.get("unverified") else "") for x in agents))
195
+ mark = Path.home() / ".whichagent" / "accepted"
196
+ if not mark.exists():
197
+ print(SAFETY)
198
+ if not a.get("yes"):
199
+ if not c.ask("Continue?"):
200
+ return 1
201
+ mark.parent.mkdir(exist_ok=True)
202
+ mark.write_text("yes\n")
203
+ run.run(cfg, agents, tasks, yes=a.get("yes", False), keep=a.get("keep", False),
204
+ retry_errors=a.get("retry_errors", False), new_version=a.get("new_version", False))
205
+ return cmd_report(cfg, a)
206
+
207
+
208
+ def cmd_report(cfg, a):
209
+ tasks, results = _data(cfg, a)
210
+ if not results:
211
+ raise SystemExit("No results yet: run `whichagent run` first")
212
+ d, name, by, base = store.state(cfg["root"]), cfg["root"].name, a.get("by_version", False), _baseline(a)
213
+ print(report.terminal(tasks, results, base, by))
214
+ (d / "report.md").write_text(report.markdown(tasks, results, name, base, by), "utf-8")
215
+ (d / "report.html").write_text(report.page(cfg, tasks, results, by), "utf-8")
216
+ (d / "card.svg").write_text(report.card(tasks, results, name, by), "utf-8")
217
+ print(f"\nReport: {d / 'report.html'} (also report.md and card.svg)")
218
+
219
+
220
+ def cmd_compare(cfg, a):
221
+ _, results = _data(cfg, a)
222
+ by = a.get("by_version", False)
223
+ ra, rb = ([r for r in results if report.label(r, by) == n] for n in (a["a"], a["b"]))
224
+ if not ra or not rb:
225
+ raise SystemExit(f"No results for {a['a'] if not ra else a['b']}")
226
+ print(" ".join(report._verdict(a["a"], a["b"], *stats.compare(ra, rb))))
227
+
228
+
229
+ def cmd_show(cfg, a):
230
+ tasks, results = _data(cfg, a)
231
+ t = next((t for t in tasks if t["id"].startswith(a["task"])), None)
232
+ if not t:
233
+ raise SystemExit(f"No task {a['task']}")
234
+ graded = f"{len(t['fail_to_pass'])} tests must flip" if t.get("fail_to_pass") else "graded by exit code"
235
+ print(f"Task {t['id']} ({t['mode']} tests, {graded}, statement from {t['statement_source']})\n\n{t['statement']}\n")
236
+ print("=== Real fix\n" + g.git(cfg["root"], "show", "--format=", "--no-color", "--no-ext-diff", t["fix"]))
237
+ for r in sorted((r for r in results if r["task"] == t["id"]), key=lambda r: (r["agent"], r["attempt"])):
238
+ f = store.run_dir(cfg["root"], t["id"], r["agent"], r.get("agent_version"), r["attempt"]) / "agent.diff"
239
+ print(f"=== {r['agent']}, attempt {r['attempt']}: {r['status']}\n"
240
+ + (f.read_text("utf-8", "replace") if f.exists() else "(no diff)\n"))
241
+
242
+
243
+ COMMANDS = {"quickstart": cmd_quickstart, "doctor": cmd_doctor, "mine": cmd_mine, "run": cmd_run,
244
+ "report": cmd_report, "compare": cmd_compare, "show": cmd_show}
whichagent/config.py ADDED
@@ -0,0 +1,157 @@
1
+ """whichagent.toml: defaults, built-in agents, path patterns."""
2
+ import functools
3
+ import json
4
+ import re
5
+ import shlex
6
+ import shutil
7
+ import tomllib
8
+ from pathlib import Path
9
+
10
+ from . import git, stacks
11
+
12
+ DEFAULTS = {
13
+ "repo": {
14
+ "install": "", # installs dependencies; with cache = true, cached by lockfile hash
15
+ "setup": "", # runs after install in every workspace: builds, codegen
16
+ "test": "", # runs the task's tests: {files}, {dirs} or {names}; {report} for per-test results
17
+ "suite": "", # runs the whole suite; default: test without the file placeholders
18
+ "report": "", # where the runner writes per-test results when test can't take {report}
19
+ "cache": False,
20
+ "test_patterns": ["tests/**", "test/**", "**/__tests__/**", "**/src/test/**", "**/testdata/**",
21
+ "test_*.py", "*_test.py", "*_test.go", "*.test.*", "*.spec.*",
22
+ "*.test-*.*", "*.spec-*.*"], # the last two: vitest type tests (x.test-d.ts) and such
23
+ "skip_if_changed": ["uv.lock", "poetry.lock", "Pipfile.lock", "requirements*.txt", "package-lock.json",
24
+ "pnpm-lock.yaml", "yarn.lock", "bun.lock*", "go.sum", "Cargo.lock", "gradle.lockfile"],
25
+ "setup_timeout": "10m",
26
+ "test_timeout": "10m",
27
+ },
28
+ "mine": {"since": "12 months ago", "max_tasks": 25, "max_candidates": 300, "max_files": 5, "max_lines": 200,
29
+ "verify_runs": 2, "min_statement_words": 8, "history_depth": 200},
30
+ "run": {"agents": [], "attempts": 1, "jobs": 3, "jobs_per_agent": 0, "timeout": "30m",
31
+ "max_cost": 25.0, "regression": True, "sandbox": "host"},
32
+ "docker": {"image": "", "mounts": []},
33
+ }
34
+ PLACEHOLDER = re.compile(r"\{(files|dirs|names)(?::([^}]*))?\}") # {names:,} joins with "," instead of " "
35
+ # Test config is reset like tests. Project manifests can hold legitimate changes, so they're only flagged.
36
+ TEST_CONFIG = ["conftest.py", "pytest.ini", "tox.ini", "jest.config.*", "jest.setup.*",
37
+ "vitest.config.*", "vitest.setup.*", "vitest.workspace.*", ".mocharc*"]
38
+ FLAGGED = ["pyproject.toml", "setup.cfg", "package.json", "go.mod", "Cargo.toml", "pom.xml", "build.gradle*"]
39
+
40
+ BUILTIN = {
41
+ "claude": {"cmd": "claude -p --output-format json --dangerously-skip-permissions", "parse": "claude-json"},
42
+ # Unverified: flags and output formats come from each CLI's docs and haven't been run by whichagent yet.
43
+ # `whichagent doctor` checks the flags against the installed CLI's --help.
44
+ "codex": {"cmd": "codex exec --json --sandbox workspace-write -", "parse": "codex-jsonl"},
45
+ "gemini": {"cmd": "gemini --output-format json --yolo", "parse": "gemini-json", "unverified": True},
46
+ "aider": {"cmd": "aider --message-file {prompt_file} --yes-always --no-auto-commits --no-gitignore"
47
+ " --no-check-update --no-show-release-notes --analytics-disable --no-pretty",
48
+ "stdin": False, "parse": "aider-text"},
49
+ "qwen": {"cmd": "qwen --output-format json --yolo", "parse": "gemini-json", "unverified": True},
50
+ "opencode": {"cmd": "opencode run --auto --format json", "parse": "opencode-json"},
51
+ "goose": {"cmd": "goose run --no-session --instructions {prompt_file}", "stdin": False, "unverified": True},
52
+ "amp": {"cmd": "amp --execute --dangerously-allow-all", "unverified": True},
53
+ "copilot": {"cmd": "copilot --allow-all-tools", "unverified": True},
54
+ "cursor": {"cmd": "cursor-agent --print --force --output-format json", "unverified": True},
55
+ }
56
+ AGENT_KEYS = {"name", "extends", "cmd", "args", "stdin", "parse", "timeout", "env", "overlay"}
57
+
58
+
59
+ def load(root, overrides=None):
60
+ """Config for the repo at root: defaults < detected stack < whichagent.toml < overrides ({section: {key: v}})."""
61
+ root = Path(root).resolve()
62
+ f = root / "whichagent.toml"
63
+ try:
64
+ data = tomllib.loads(f.read_text("utf-8")) if f.exists() else {}
65
+ except tomllib.TOMLDecodeError as e:
66
+ raise SystemExit(f"whichagent.toml: {e}")
67
+ overrides = overrides or {}
68
+ bad = [f"[{s}]" for s in data if s not in (*DEFAULTS, "agent", "prices")]
69
+ bad += [f"[{s}] {k}" for s in DEFAULTS for k in data.get(s, {}) if k not in DEFAULTS[s]]
70
+ bad += [f"[[agent]] {k}" for a in data.get("agent", []) for k in a if k not in AGENT_KEYS]
71
+ bad += ["[[agent]] without a name" for a in data.get("agent", []) if "name" not in a]
72
+ if bad:
73
+ raise SystemExit("whichagent.toml: unknown " + ", ".join(bad))
74
+ cfg = {s: {**d, **data.get(s, {}), **overrides.get(s, {})} for s, d in DEFAULTS.items()}
75
+ given = {**data.get("repo", {}), **overrides.get("repo", {})}
76
+ if not cfg["repo"]["test"] and (found := stacks.detect(root)):
77
+ cfg["repo"].update({k: v for k, v in found.items() if k not in given})
78
+ if cfg["repo"]["test"] and not PLACEHOLDER.search(cfg["repo"]["test"]):
79
+ raise SystemExit("The test command needs {files} (or {dirs} or {names}) where the task's tests go,"
80
+ " e.g. pytest -q {files}")
81
+ if cfg["run"]["sandbox"] not in ("host", "docker"):
82
+ raise SystemExit("[run] sandbox is host or docker")
83
+ defined = {a["name"]: a for a in data.get("agent", [])}
84
+ cfg["agents"] = {n: _agent(n, defined, cfg["run"]["timeout"]) for n in {**BUILTIN, **defined}}
85
+ cfg.update(root=root, prices=data.get("prices", {}), branch=git.branch(root))
86
+ return cfg
87
+
88
+
89
+ def _agent(name, defined, timeout, seen=()):
90
+ """An agent with `extends` resolved: built-in < parent < its own keys."""
91
+ if name in seen:
92
+ raise SystemExit(f"agent {name!r}: extends loop")
93
+ if name not in defined and name not in BUILTIN:
94
+ raise SystemExit(f"unknown agent {name!r}")
95
+ own = defined.get(name, {})
96
+ ext = own.get("extends")
97
+ base = _agent(ext, defined, timeout, seen + (name,)) if ext and ext != name else BUILTIN.get(name, {})
98
+ a = {"args": "", "stdin": True, "parse": "none", "env": {}, "overlay": {}, "timeout": timeout,
99
+ **base, **own, "name": name}
100
+ if "cmd" not in a:
101
+ raise SystemExit(f"agent {name!r}: needs cmd or extends")
102
+ return a
103
+
104
+
105
+ def selected(cfg):
106
+ """Agents to run: [run] agents, else your [[agent]] entries plus the built-ins found on PATH."""
107
+ names = cfg["run"]["agents"]
108
+ if not names and cfg["run"]["sandbox"] == "docker":
109
+ raise SystemExit("Docker mode: list the agents installed in the image in [run] agents or --agents")
110
+ names = names or [n for n, a in cfg["agents"].items() if n not in BUILTIN or shutil.which(argv(a)[0])]
111
+ missing = [n for n in names if n not in cfg["agents"]]
112
+ if missing:
113
+ raise SystemExit(f"unknown agent: {', '.join(missing)}")
114
+ return [cfg["agents"][n] for n in names]
115
+
116
+
117
+ def argv(agent):
118
+ return shlex.split(agent["cmd"]) + shlex.split(agent["args"])
119
+
120
+
121
+ def write_template(root, repo):
122
+ lines = ["# whichagent config: https://github.com/itzzdev09/whichagent/blob/main/ARCHITECTURE.md#12-configuration",
123
+ "[repo]"]
124
+ lines += [f"{k} = {json.dumps(repo[k])}" for k in ("install", "setup", "test", "suite", "report", "cache")
125
+ if repo.get(k)]
126
+ lines += ["", "[run]", "max_cost = 25.0 # USD, hard stop", ""]
127
+ (root / "whichagent.toml").write_text("\n".join(lines), "utf-8")
128
+
129
+
130
+ @functools.lru_cache(maxsize=None)
131
+ def _rx(pattern):
132
+ tokens = {"**/": "(?:.*/)?", "/**": "/.*", "**": ".*", "*": "[^/]*", "?": "[^/]"}
133
+ return re.compile("".join(tokens.get(t, re.escape(t)) for t in re.split(r"(\*\*/|/\*\*$|\*\*|\*|\?)", pattern)))
134
+
135
+
136
+ def match(path, patterns):
137
+ """gitignore-style glob: a pattern without '/' matches the file name at any depth."""
138
+ name = path.rsplit("/", 1)[-1]
139
+ return any(_rx(p).fullmatch(path if "/" in p else name) for p in patterns)
140
+
141
+
142
+ def is_test(cfg, path):
143
+ return match(path, cfg["repo"]["test_patterns"]) or match(path, TEST_CONFIG)
144
+
145
+
146
+ def secs(value):
147
+ m = re.fullmatch(r"(\d+(?:\.\d+)?)([smh]?)", str(value).strip())
148
+ if not m:
149
+ raise SystemExit(f"bad duration {value!r}: use e.g. 90s, 30m or 2h")
150
+ return float(m[1]) * {"": 1, "s": 1, "m": 60, "h": 3600}[m[2]]
151
+
152
+
153
+ def ask(question):
154
+ try:
155
+ return input(question + " [y/N] ").strip().lower() in ("y", "yes")
156
+ except EOFError:
157
+ return False