whichagent 0.1.0__tar.gz → 0.1.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {whichagent-0.1.0 → whichagent-0.1.1}/ARCHITECTURE.md +4 -4
  2. {whichagent-0.1.0 → whichagent-0.1.1}/PKG-INFO +6 -4
  3. {whichagent-0.1.0 → whichagent-0.1.1}/README.md +5 -3
  4. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/__init__.py +1 -1
  5. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/agents.py +43 -1
  6. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/config.py +7 -4
  7. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/run.py +9 -4
  8. {whichagent-0.1.0 → whichagent-0.1.1}/tests/test_pipeline.py +4 -0
  9. {whichagent-0.1.0 → whichagent-0.1.1}/tests/test_units.py +24 -0
  10. {whichagent-0.1.0 → whichagent-0.1.1}/.github/workflows/ci.yml +0 -0
  11. {whichagent-0.1.0 → whichagent-0.1.1}/.github/workflows/release.yml +0 -0
  12. {whichagent-0.1.0 → whichagent-0.1.1}/.gitignore +0 -0
  13. {whichagent-0.1.0 → whichagent-0.1.1}/LICENSE +0 -0
  14. {whichagent-0.1.0 → whichagent-0.1.1}/action.yml +0 -0
  15. {whichagent-0.1.0 → whichagent-0.1.1}/pyproject.toml +0 -0
  16. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/__main__.py +0 -0
  17. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/cli.py +0 -0
  18. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/git.py +0 -0
  19. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/grade.py +0 -0
  20. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/mine.py +0 -0
  21. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/report.py +0 -0
  22. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/sandbox.py +0 -0
  23. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/stacks.py +0 -0
  24. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/stats.py +0 -0
  25. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/store.py +0 -0
  26. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/testresults.py +0 -0
  27. {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/workspace.py +0 -0
  28. {whichagent-0.1.0 → whichagent-0.1.1}/tests/__init__.py +0 -0
  29. {whichagent-0.1.0 → whichagent-0.1.1}/tests/fake_agent.py +0 -0
  30. {whichagent-0.1.0 → whichagent-0.1.1}/tests/junit_runner.py +0 -0
  31. {whichagent-0.1.0 → whichagent-0.1.1}/tests/repos.py +0 -0
@@ -210,16 +210,16 @@ Built-in adapters. `whichagent doctor` checks each adapter's flags against the i
210
210
  |---|---|---|---|
211
211
  | Claude Code | `claude -p --output-format json --dangerously-skip-permissions` | stdin | `total_cost_usd` in the output |
212
212
  | Codex CLI | `codex exec --json --sandbox workspace-write -` (Codex's own sandbox stays on) | stdin | token usage events × prices |
213
- | Gemini CLI | `gemini --output-format json --yolo` | stdin | `stats` tokens × prices |
213
+ | Gemini CLI | `gemini --output-format json --yolo` with `GEMINI_CLI_TRUST_WORKSPACE=true` (untrusted folders ignore --yolo and ~/.gemini/.env) | stdin | `stats` tokens × prices |
214
214
  | Aider | `aider --message-file {prompt_file} --yes-always --no-auto-commits --no-gitignore` (plus quiet flags) | file | the "Cost: ... session" line; $0 for local models (ollama, lm_studio, llamafile) |
215
215
  | Qwen Code | `qwen --output-format json --yolo` | stdin | `stats` tokens × prices |
216
216
  | OpenCode | `opencode run --auto --format json` | stdin | `cost` on each step_finish event |
217
- | Goose | `goose run --no-session --instructions {prompt_file}` | file | none yet |
217
+ | Goose | `goose run --no-session --instructions {prompt_file} --output-format json` with `GOOSE_MODE=auto` | file | metadata tokens × prices ($0 for Ollama) |
218
218
  | Amp | `amp --execute --dangerously-allow-all` | stdin | none yet |
219
- | GitHub Copilot CLI | `copilot --allow-all-tools` | stdin | none yet |
219
+ | GitHub Copilot CLI | `copilot --allow-all-tools --no-ask-user --output-format json` | stdin | none: premium requests only (model and turns are parsed) |
220
220
  | Cursor CLI | `cursor-agent --print --force --output-format json` | stdin | none yet |
221
221
 
222
- Claude Code, Codex, OpenCode and Aider have been run end to end; every other adapter is marked unverified until it has. An agent's `env` values starting with `$` are read from your shell (`env = { GH_TOKEN = "$GH_TOKEN" }`), so credentials outside the allowlist never have to be written into the TOML.
222
+ Claude Code, Codex, Copilot, OpenCode and Aider have been run end to end; every other adapter is marked unverified until it has. An agent's `env` values starting with `$` are read from your shell (`env = { GH_TOKEN = "$GH_TOKEN" }`), so credentials outside the allowlist never have to be written into the TOML.
223
223
 
224
224
  ## 8. Grading
225
225
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: whichagent
3
- Version: 0.1.0
3
+ Version: 0.1.1
4
4
  Summary: Find out which coding agent actually works on your repo: a benchmark built from your own git history.
5
5
  Project-URL: Homepage, https://github.com/itzzdev09/whichagent
6
6
  Project-URL: Issues, https://github.com/itzzdev09/whichagent/issues
@@ -145,7 +145,9 @@ An adapter is a few lines of TOML, so adding your favorite agent makes a good fi
145
145
  | Qwen Code | `qwen --output-format json --yolo` | tokens x models.dev prices | unverified |
146
146
  | Aider | `aider --message-file ... --yes-always` | parsed from its output; $0 for local models | verified |
147
147
  | OpenCode | `opencode run --auto --format json` | reported by the CLI | verified |
148
- | Goose, Amp, Copilot CLI, Cursor CLI | headless mode, prompt on stdin or a file | time and pass/fail only | unverified |
148
+ | GitHub Copilot CLI | `copilot --allow-all-tools --no-ask-user --output-format json` | none: it reports premium requests, not tokens | verified |
149
+ | Goose | `goose run --no-session --instructions ... --output-format json` | tokens x prices; $0 on Ollama | unverified |
150
+ | Amp, Cursor CLI | headless mode, prompt on stdin | time and pass/fail only | unverified |
149
151
  | Anything else | any command; the prompt arrives on stdin or as `{prompt_file}` | optional | |
150
152
 
151
153
  Unverified adapters follow each CLI's docs but haven't been run by whichagent yet; `whichagent doctor` checks their flags on your machine. Several of these agents have free tiers or free models, which makes them cheap to compare. Agents get only an allowlist of environment variables; pass anything else with `env = { TOKEN = "$MY_TOKEN" }`, which reads it from your shell.
@@ -179,7 +181,7 @@ env = { OPENAI_BASE_URL = "http://127.0.0.1:11434/v1", OPENAI_API_KEY = "oll
179
181
  "qwen2.5-coder:7b" = { input = 0, output = 0 } # so the report shows $0 instead of "no cost"
180
182
  ```
181
183
 
182
- Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks. Gemini CLI and GitHub Copilot have free tiers when you sign in; free tiers are rate-limited, so try a few tasks first with `whichagent run --tasks ID1,ID2`. Model names starting with `ollama/`, `ollama_chat/`, `lm_studio/` or `llamafile/` are priced at $0 automatically; `[run] max_cost` only matters for paid agents.
184
+ Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks. GitHub Copilot's free plan works once you sign in (runs use premium requests from its monthly allowance). Gemini CLI's Google sign-in no longer serves individuals, so it needs a `GEMINI_API_KEY` from Google AI Studio, which has a free tier. Free tiers are rate-limited, so try a few tasks first with `whichagent run --tasks ID1,ID2`. Model names starting with `ollama/`, `ollama_chat/`, `lm_studio/` or `llamafile/` are priced at $0 automatically; `[run] max_cost` only matters for paid agents.
183
185
 
184
186
  ## Stacks
185
187
 
@@ -251,7 +253,7 @@ So that agents see the pull request's CLAUDE.md rather than the one in each old
251
253
 
252
254
  ## Status
253
255
 
254
- Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
256
+ Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, Copilot, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
255
257
 
256
258
  ## Contributing
257
259
 
@@ -127,7 +127,9 @@ An adapter is a few lines of TOML, so adding your favorite agent makes a good fi
127
127
  | Qwen Code | `qwen --output-format json --yolo` | tokens x models.dev prices | unverified |
128
128
  | Aider | `aider --message-file ... --yes-always` | parsed from its output; $0 for local models | verified |
129
129
  | OpenCode | `opencode run --auto --format json` | reported by the CLI | verified |
130
- | Goose, Amp, Copilot CLI, Cursor CLI | headless mode, prompt on stdin or a file | time and pass/fail only | unverified |
130
+ | GitHub Copilot CLI | `copilot --allow-all-tools --no-ask-user --output-format json` | none: it reports premium requests, not tokens | verified |
131
+ | Goose | `goose run --no-session --instructions ... --output-format json` | tokens x prices; $0 on Ollama | unverified |
132
+ | Amp, Cursor CLI | headless mode, prompt on stdin | time and pass/fail only | unverified |
131
133
  | Anything else | any command; the prompt arrives on stdin or as `{prompt_file}` | optional | |
132
134
 
133
135
  Unverified adapters follow each CLI's docs but haven't been run by whichagent yet; `whichagent doctor` checks their flags on your machine. Several of these agents have free tiers or free models, which makes them cheap to compare. Agents get only an allowlist of environment variables; pass anything else with `env = { TOKEN = "$MY_TOKEN" }`, which reads it from your shell.
@@ -161,7 +163,7 @@ env = { OPENAI_BASE_URL = "http://127.0.0.1:11434/v1", OPENAI_API_KEY = "oll
161
163
  "qwen2.5-coder:7b" = { input = 0, output = 0 } # so the report shows $0 instead of "no cost"
162
164
  ```
163
165
 
164
- Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks. Gemini CLI and GitHub Copilot have free tiers when you sign in; free tiers are rate-limited, so try a few tasks first with `whichagent run --tasks ID1,ID2`. Model names starting with `ollama/`, `ollama_chat/`, `lm_studio/` or `llamafile/` are priced at $0 automatically; `[run] max_cost` only matters for paid agents.
166
+ Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks. GitHub Copilot's free plan works once you sign in (runs use premium requests from its monthly allowance). Gemini CLI's Google sign-in no longer serves individuals, so it needs a `GEMINI_API_KEY` from Google AI Studio, which has a free tier. Free tiers are rate-limited, so try a few tasks first with `whichagent run --tasks ID1,ID2`. Model names starting with `ollama/`, `ollama_chat/`, `lm_studio/` or `llamafile/` are priced at $0 automatically; `[run] max_cost` only matters for paid agents.
165
167
 
166
168
  ## Stacks
167
169
 
@@ -233,7 +235,7 @@ So that agents see the pull request's CLAUDE.md rather than the one in each old
233
235
 
234
236
  ## Status
235
237
 
236
- Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
238
+ Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, Copilot, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
237
239
 
238
240
  ## Contributing
239
241
 
@@ -1,2 +1,2 @@
1
1
  """whichagent: benchmark coding agents on your repo's own git history."""
2
- __version__ = "0.1.0"
2
+ __version__ = "0.1.1"
@@ -127,8 +127,40 @@ def opencode(cfg, out, argv):
127
127
  "model": _flag(argv, "-m", "--model"), "error": error}
128
128
 
129
129
 
130
+ def copilot(cfg, out, argv):
131
+ """`copilot --output-format json` events: model and turns. It reports premium requests, not tokens or USD."""
132
+ model, turns, error = None, 0, False
133
+ for e in _events(out):
134
+ t = e.get("type")
135
+ if t == "model.call_start":
136
+ model = (e.get("data") or {}).get("model") or model
137
+ elif t == "assistant.turn_end":
138
+ turns += 1
139
+ elif t == "result":
140
+ error = e.get("exitCode", 0) != 0
141
+ return {"model": model, "turns": turns, "error": error}
142
+
143
+
144
+ def goose(cfg, out, argv):
145
+ """`goose run --output-format json`: one object after its banner, tokens and status under metadata.
146
+ The model is reported as provider/model, so local providers (ollama) are priced at $0."""
147
+ i = out.find("\n{")
148
+ d = _json(out[i + 1:] if i >= 0 else out)
149
+ if not isinstance(d, dict):
150
+ return {"error": True}
151
+ m = d.get("metadata") or {}
152
+ said = [x for x in d.get("messages") or [] if x.get("role") == "assistant"]
153
+ inf = ((said[-1].get("metadata") or {}).get("inference") or {}) if said else {}
154
+ provider = _flag(argv, "", "--provider") or inf.get("provider")
155
+ name = _flag(argv, "", "--model") or inf.get("requestedModel")
156
+ model = f"{provider}/{name}" if provider and name else name
157
+ tin, tout = m.get("input_tokens", 0), m.get("output_tokens", 0)
158
+ return {"cost_usd": cost(cfg, model, tin, tout, m.get("cache_read_input_tokens", 0)), "tokens_in": tin,
159
+ "tokens_out": tout, "turns": len(said), "model": model, "error": m.get("status", "completed") != "completed"}
160
+
161
+
130
162
  PARSERS = {"claude-json": claude, "codex-jsonl": codex, "gemini-json": gemini, "aider-text": aider,
131
- "opencode-json": opencode, "none": lambda cfg, out, argv: {}}
163
+ "opencode-json": opencode, "copilot-jsonl": copilot, "goose-json": goose, "none": lambda cfg, out, argv: {}}
132
164
 
133
165
 
134
166
  def _flag(argv, short, long):
@@ -149,6 +181,16 @@ def _codex_model():
149
181
 
150
182
 
151
183
  LOCAL = ("ollama", "ollama_chat", "lm_studio", "llamafile") # litellm prefixes for models on your machine
184
+ LOCAL_HOSTS = ("localhost", "127.0.0.1", "[::1]")
185
+ NO_COST = {"none", "copilot-jsonl"} # parsers that never report what a run cost
186
+
187
+
188
+ def free(argv, env):
189
+ """True when an agent runs a model on this machine (an ollama/lm_studio/llamafile model, a localhost endpoint)
190
+ or a model named free (OpenCode's -free, OpenRouter's :free): a run costs $0 before it reports anything."""
191
+ words = [w.split("=", 1)[-1] for w in argv] + [v for k, v in env.items() if "PROXY" not in k.upper()]
192
+ return any(w.split("/")[0] in LOCAL or w.endswith(("-free", ":free")) or any(h in w for h in LOCAL_HOSTS)
193
+ for w in words)
152
194
 
153
195
 
154
196
  def cost(cfg, model, tin, tout, cached=0):
@@ -42,15 +42,18 @@ BUILTIN = {
42
42
  # Unverified: flags and output formats come from each CLI's docs and haven't been run by whichagent yet.
43
43
  # `whichagent doctor` checks the flags against the installed CLI's --help.
44
44
  "codex": {"cmd": "codex exec --json --sandbox workspace-write -", "parse": "codex-jsonl"},
45
- "gemini": {"cmd": "gemini --output-format json --yolo", "parse": "gemini-json", "unverified": True},
45
+ # Gemini reads ~/.gemini/.env (GEMINI_API_KEY) and honors --yolo only in trusted folders
46
+ "gemini": {"cmd": "gemini --output-format json --yolo", "env": {"GEMINI_CLI_TRUST_WORKSPACE": "true"},
47
+ "parse": "gemini-json", "unverified": True},
46
48
  "aider": {"cmd": "aider --message-file {prompt_file} --yes-always --no-auto-commits --no-gitignore"
47
49
  " --no-check-update --no-show-release-notes --analytics-disable --no-pretty",
48
50
  "stdin": False, "parse": "aider-text"},
49
51
  "qwen": {"cmd": "qwen --output-format json --yolo", "parse": "gemini-json", "unverified": True},
50
52
  "opencode": {"cmd": "opencode run --auto --format json", "parse": "opencode-json"},
51
- "goose": {"cmd": "goose run --no-session --instructions {prompt_file}", "stdin": False, "unverified": True},
53
+ "goose": {"cmd": "goose run --no-session --instructions {prompt_file} --output-format json", "stdin": False,
54
+ "env": {"GOOSE_MODE": "auto"}, "parse": "goose-json", "unverified": True},
52
55
  "amp": {"cmd": "amp --execute --dangerously-allow-all", "unverified": True},
53
- "copilot": {"cmd": "copilot --allow-all-tools", "unverified": True},
56
+ "copilot": {"cmd": "copilot --allow-all-tools --no-ask-user --output-format json", "parse": "copilot-jsonl"},
54
57
  "cursor": {"cmd": "cursor-agent --print --force --output-format json", "unverified": True},
55
58
  }
56
59
  AGENT_KEYS = {"name", "extends", "cmd", "args", "stdin", "parse", "timeout", "env", "overlay"}
@@ -96,7 +99,7 @@ def _agent(name, defined, timeout, seen=()):
96
99
  ext = own.get("extends")
97
100
  base = _agent(ext, defined, timeout, seen + (name,)) if ext and ext != name else BUILTIN.get(name, {})
98
101
  a = {"args": "", "stdin": True, "parse": "none", "env": {}, "overlay": {}, "timeout": timeout,
99
- **base, **own, "name": name}
102
+ **base, **own, "name": name, "env": {**base.get("env", {}), **own.get("env", {})}}
100
103
  if "cmd" not in a:
101
104
  raise SystemExit(f"agent {name!r}: needs cmd or extends")
102
105
  return a
@@ -96,10 +96,15 @@ def run(cfg, agents, tasks, yes=False, keep=False, retry_errors=False, new_versi
96
96
  return []
97
97
  costs = {a["name"]: [x["cost_usd"] for x in old if x["agent"] == a["name"] and x["cost_usd"] is not None
98
98
  and x["status"] not in ERRORS] for a in agents} # a failed login costs $0 and says nothing
99
- lo = sum(statistics.median(costs[a["name"]]) if costs[a["name"]] else NO_HISTORY[0] for _, a, _ in todo)
100
- hi = sum(max(costs[a["name"]]) if costs[a["name"]] else NO_HISTORY[1] for _, a, _ in todo)
99
+ zero = {a["name"] for a in agents if ag.free(c.argv(a), a["env"]) or a["parse"] in ag.NO_COST}
100
+
101
+ def per_run(name): # (typical, worst) USD for one run: $0 for local and free models, else history, else a guess
102
+ xs = costs[name]
103
+ return (0.0, 0.0) if name in zero else (statistics.median(xs), max(xs)) if xs else NO_HISTORY
104
+ lo = sum(per_run(a["name"])[0] for _, a, _ in todo)
105
+ hi = sum(per_run(a["name"])[1] for _, a, _ in todo)
101
106
  log(f"{len(todo)} runs, estimated ${lo:.2f}-${hi:.2f} (stops before passing max_cost ${rc['max_cost']:.2f})")
102
- blind = [a["name"] for a in agents if a["parse"] == "none"]
107
+ blind = [a["name"] for a in agents if a["parse"] in ag.NO_COST]
103
108
  if blind:
104
109
  log(f"No cost data from {', '.join(blind)}: their spending isn't counted toward max_cost.")
105
110
  if not yes and not c.ask("Continue?"):
@@ -114,7 +119,7 @@ def run(cfg, agents, tasks, yes=False, keep=False, retry_errors=False, new_versi
114
119
  with slots[agent["name"]]:
115
120
  with lock:
116
121
  xs = costs[agent["name"]]
117
- est = sum(xs) / len(xs) if xs else NO_HISTORY[1]
122
+ est = 0.0 if agent["name"] in zero else sum(xs) / len(xs) if xs else NO_HISTORY[1]
118
123
  if sb.STOP.is_set() or budget["over"] or budget["spent"] + budget["inflight"] + est > rc["max_cost"]:
119
124
  budget["over"] = not sb.STOP.is_set()
120
125
  return
@@ -154,6 +154,10 @@ class Pipeline(unittest.TestCase):
154
154
  self.assertEqual(run.run(self.cfg, perfect, self.tasks, yes=True, new_version=True, log=quiet), [])
155
155
  cfg = config.load(self.repo, {"run": {"attempts": 5, "max_cost": 0.025, "jobs": 1}})
156
156
  self.assertEqual(len(run.run(cfg, perfect, self.tasks, yes=True, log=quiet)), 2) # $0.01 each
157
+ # an agent on a local model has no history, yet isn't held back by the $0.10-$1.00 guess for unknown agents
158
+ local = {**self.cfg["agents"]["noop"], "name": "noop-local", "args": "--model ollama/x"}
159
+ cfg = config.load(self.repo, {"run": {"max_cost": 0.5, "jobs": 1}})
160
+ self.assertEqual(len(run.run(cfg, [local], self.tasks, yes=True, log=quiet)), len(self.tasks))
157
161
 
158
162
  def test_cli_report_show_compare(self):
159
163
  out = io.StringIO()
@@ -68,6 +68,19 @@ class Units(unittest.TestCase):
68
68
  def test_durations(self):
69
69
  self.assertEqual([config.secs(v) for v in ("90", "90s", "30m", "2h")], [90, 90, 1800, 7200])
70
70
 
71
+ def test_free_agents(self):
72
+ self.assertTrue(agents.free(["aider", "--model", "ollama_chat/qwen2.5-coder:7b"], {}))
73
+ self.assertTrue(agents.free(["goose", "--provider", "ollama"], {}))
74
+ self.assertTrue(agents.free(["qwen"], {"OPENAI_BASE_URL": "http://127.0.0.1:11434/v1"}))
75
+ self.assertTrue(agents.free(["opencode", "run", "--model=opencode/deepseek-v4.1-flash-free"], {}))
76
+ self.assertFalse(agents.free(["claude", "-p"], {"HTTPS_PROXY": "http://localhost:3128"}))
77
+ self.assertFalse(agents.free(["codex", "exec", "-m", "gpt-x"], {}))
78
+
79
+ def test_extends_merges_env(self):
80
+ a = config._agent("g", {"g": {"name": "g", "extends": "gemini", "env": {"X": "1"}}}, 60)
81
+ self.assertEqual(a["env"], {"GEMINI_CLI_TRUST_WORKSPACE": "true", "X": "1"})
82
+ self.assertEqual(a["cmd"], config.BUILTIN["gemini"]["cmd"])
83
+
71
84
  def test_parsers(self):
72
85
  claude = json.dumps({"type": "result", "is_error": False, "num_turns": 7, "total_cost_usd": 0.12,
73
86
  "modelUsage": {"m-big": {"inputTokens": 10, "cacheReadInputTokens": 90,
@@ -100,6 +113,17 @@ class Units(unittest.TestCase):
100
113
  self.assertEqual(agents.parse(CFG, "opencode-json", oc, ["opencode", "run", "-m", "opencode/x-free"]),
101
114
  {"cost_usd": 0.5, "tokens_in": 2000, "tokens_out": 30, "turns": 2,
102
115
  "model": "opencode/x-free", "error": False})
116
+ cp = "\n".join(json.dumps(x) for x in [{"type": "model.call_start", "data": {"model": "m1"}},
117
+ {"type": "assistant.turn_end", "data": {}},
118
+ {"type": "assistant.turn_end", "data": {}},
119
+ {"type": "result", "exitCode": 1}])
120
+ said = {"role": "assistant", "metadata": {"inference": {"provider": "ollama", "requestedModel": "q:7b"}}}
121
+ goose = " __( O)> banner\n" + json.dumps({"messages": [{"role": "user"}, said, said], "metadata": {
122
+ "input_tokens": 300, "output_tokens": 20, "status": "completed"}}, indent=2)
123
+ self.assertEqual(agents.parse(CFG, "goose-json", goose, []),
124
+ {"cost_usd": 0.0, "tokens_in": 300, "tokens_out": 20, "turns": 2, "model": "ollama/q:7b",
125
+ "error": False})
126
+ self.assertEqual(agents.parse(CFG, "copilot-jsonl", cp, []), {"model": "m1", "turns": 2, "error": True})
103
127
 
104
128
 
105
129
  class TestResults(unittest.TestCase):
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes