whichagent 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {whichagent-0.1.0 → whichagent-0.1.1}/ARCHITECTURE.md +4 -4
- {whichagent-0.1.0 → whichagent-0.1.1}/PKG-INFO +6 -4
- {whichagent-0.1.0 → whichagent-0.1.1}/README.md +5 -3
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/__init__.py +1 -1
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/agents.py +43 -1
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/config.py +7 -4
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/run.py +9 -4
- {whichagent-0.1.0 → whichagent-0.1.1}/tests/test_pipeline.py +4 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/tests/test_units.py +24 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/.github/workflows/ci.yml +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/.github/workflows/release.yml +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/.gitignore +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/LICENSE +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/action.yml +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/pyproject.toml +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/__main__.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/cli.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/git.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/grade.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/mine.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/report.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/sandbox.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/stacks.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/stats.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/store.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/testresults.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/src/whichagent/workspace.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/tests/__init__.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/tests/fake_agent.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/tests/junit_runner.py +0 -0
- {whichagent-0.1.0 → whichagent-0.1.1}/tests/repos.py +0 -0
|
@@ -210,16 +210,16 @@ Built-in adapters. `whichagent doctor` checks each adapter's flags against the i
|
|
|
210
210
|
|---|---|---|---|
|
|
211
211
|
| Claude Code | `claude -p --output-format json --dangerously-skip-permissions` | stdin | `total_cost_usd` in the output |
|
|
212
212
|
| Codex CLI | `codex exec --json --sandbox workspace-write -` (Codex's own sandbox stays on) | stdin | token usage events × prices |
|
|
213
|
-
| Gemini CLI | `gemini --output-format json --yolo` | stdin | `stats` tokens × prices |
|
|
213
|
+
| Gemini CLI | `gemini --output-format json --yolo` with `GEMINI_CLI_TRUST_WORKSPACE=true` (untrusted folders ignore --yolo and ~/.gemini/.env) | stdin | `stats` tokens × prices |
|
|
214
214
|
| Aider | `aider --message-file {prompt_file} --yes-always --no-auto-commits --no-gitignore` (plus quiet flags) | file | the "Cost: ... session" line; $0 for local models (ollama, lm_studio, llamafile) |
|
|
215
215
|
| Qwen Code | `qwen --output-format json --yolo` | stdin | `stats` tokens × prices |
|
|
216
216
|
| OpenCode | `opencode run --auto --format json` | stdin | `cost` on each step_finish event |
|
|
217
|
-
| Goose | `goose run --no-session --instructions {prompt_file}` | file |
|
|
217
|
+
| Goose | `goose run --no-session --instructions {prompt_file} --output-format json` with `GOOSE_MODE=auto` | file | metadata tokens × prices ($0 for Ollama) |
|
|
218
218
|
| Amp | `amp --execute --dangerously-allow-all` | stdin | none yet |
|
|
219
|
-
| GitHub Copilot CLI | `copilot --allow-all-tools` | stdin | none
|
|
219
|
+
| GitHub Copilot CLI | `copilot --allow-all-tools --no-ask-user --output-format json` | stdin | none: premium requests only (model and turns are parsed) |
|
|
220
220
|
| Cursor CLI | `cursor-agent --print --force --output-format json` | stdin | none yet |
|
|
221
221
|
|
|
222
|
-
Claude Code, Codex, OpenCode and Aider have been run end to end; every other adapter is marked unverified until it has. An agent's `env` values starting with `$` are read from your shell (`env = { GH_TOKEN = "$GH_TOKEN" }`), so credentials outside the allowlist never have to be written into the TOML.
|
|
222
|
+
Claude Code, Codex, Copilot, OpenCode and Aider have been run end to end; every other adapter is marked unverified until it has. An agent's `env` values starting with `$` are read from your shell (`env = { GH_TOKEN = "$GH_TOKEN" }`), so credentials outside the allowlist never have to be written into the TOML.
|
|
223
223
|
|
|
224
224
|
## 8. Grading
|
|
225
225
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: whichagent
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Find out which coding agent actually works on your repo: a benchmark built from your own git history.
|
|
5
5
|
Project-URL: Homepage, https://github.com/itzzdev09/whichagent
|
|
6
6
|
Project-URL: Issues, https://github.com/itzzdev09/whichagent/issues
|
|
@@ -145,7 +145,9 @@ An adapter is a few lines of TOML, so adding your favorite agent makes a good fi
|
|
|
145
145
|
| Qwen Code | `qwen --output-format json --yolo` | tokens x models.dev prices | unverified |
|
|
146
146
|
| Aider | `aider --message-file ... --yes-always` | parsed from its output; $0 for local models | verified |
|
|
147
147
|
| OpenCode | `opencode run --auto --format json` | reported by the CLI | verified |
|
|
148
|
-
|
|
|
148
|
+
| GitHub Copilot CLI | `copilot --allow-all-tools --no-ask-user --output-format json` | none: it reports premium requests, not tokens | verified |
|
|
149
|
+
| Goose | `goose run --no-session --instructions ... --output-format json` | tokens x prices; $0 on Ollama | unverified |
|
|
150
|
+
| Amp, Cursor CLI | headless mode, prompt on stdin | time and pass/fail only | unverified |
|
|
149
151
|
| Anything else | any command; the prompt arrives on stdin or as `{prompt_file}` | optional | |
|
|
150
152
|
|
|
151
153
|
Unverified adapters follow each CLI's docs but haven't been run by whichagent yet; `whichagent doctor` checks their flags on your machine. Several of these agents have free tiers or free models, which makes them cheap to compare. Agents get only an allowlist of environment variables; pass anything else with `env = { TOKEN = "$MY_TOKEN" }`, which reads it from your shell.
|
|
@@ -179,7 +181,7 @@ env = { OPENAI_BASE_URL = "http://127.0.0.1:11434/v1", OPENAI_API_KEY = "oll
|
|
|
179
181
|
"qwen2.5-coder:7b" = { input = 0, output = 0 } # so the report shows $0 instead of "no cost"
|
|
180
182
|
```
|
|
181
183
|
|
|
182
|
-
Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks.
|
|
184
|
+
Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks. GitHub Copilot's free plan works once you sign in (runs use premium requests from its monthly allowance). Gemini CLI's Google sign-in no longer serves individuals, so it needs a `GEMINI_API_KEY` from Google AI Studio, which has a free tier. Free tiers are rate-limited, so try a few tasks first with `whichagent run --tasks ID1,ID2`. Model names starting with `ollama/`, `ollama_chat/`, `lm_studio/` or `llamafile/` are priced at $0 automatically; `[run] max_cost` only matters for paid agents.
|
|
183
185
|
|
|
184
186
|
## Stacks
|
|
185
187
|
|
|
@@ -251,7 +253,7 @@ So that agents see the pull request's CLAUDE.md rather than the one in each old
|
|
|
251
253
|
|
|
252
254
|
## Status
|
|
253
255
|
|
|
254
|
-
Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
|
|
256
|
+
Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, Copilot, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
|
|
255
257
|
|
|
256
258
|
## Contributing
|
|
257
259
|
|
|
@@ -127,7 +127,9 @@ An adapter is a few lines of TOML, so adding your favorite agent makes a good fi
|
|
|
127
127
|
| Qwen Code | `qwen --output-format json --yolo` | tokens x models.dev prices | unverified |
|
|
128
128
|
| Aider | `aider --message-file ... --yes-always` | parsed from its output; $0 for local models | verified |
|
|
129
129
|
| OpenCode | `opencode run --auto --format json` | reported by the CLI | verified |
|
|
130
|
-
|
|
|
130
|
+
| GitHub Copilot CLI | `copilot --allow-all-tools --no-ask-user --output-format json` | none: it reports premium requests, not tokens | verified |
|
|
131
|
+
| Goose | `goose run --no-session --instructions ... --output-format json` | tokens x prices; $0 on Ollama | unverified |
|
|
132
|
+
| Amp, Cursor CLI | headless mode, prompt on stdin | time and pass/fail only | unverified |
|
|
131
133
|
| Anything else | any command; the prompt arrives on stdin or as `{prompt_file}` | optional | |
|
|
132
134
|
|
|
133
135
|
Unverified adapters follow each CLI's docs but haven't been run by whichagent yet; `whichagent doctor` checks their flags on your machine. Several of these agents have free tiers or free models, which makes them cheap to compare. Agents get only an allowlist of environment variables; pass anything else with `env = { TOKEN = "$MY_TOKEN" }`, which reads it from your shell.
|
|
@@ -161,7 +163,7 @@ env = { OPENAI_BASE_URL = "http://127.0.0.1:11434/v1", OPENAI_API_KEY = "oll
|
|
|
161
163
|
"qwen2.5-coder:7b" = { input = 0, output = 0 } # so the report shows $0 instead of "no cost"
|
|
162
164
|
```
|
|
163
165
|
|
|
164
|
-
Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks.
|
|
166
|
+
Aider and OpenCode setups above have been run end to end. Small local models are weak agents: on commander.js, qwen2.5-coder:7b solved nothing through Aider and made no tool calls through Qwen Code, while OpenCode's free model solved 1 of 2 tasks. GitHub Copilot's free plan works once you sign in (runs use premium requests from its monthly allowance). Gemini CLI's Google sign-in no longer serves individuals, so it needs a `GEMINI_API_KEY` from Google AI Studio, which has a free tier. Free tiers are rate-limited, so try a few tasks first with `whichagent run --tasks ID1,ID2`. Model names starting with `ollama/`, `ollama_chat/`, `lm_studio/` or `llamafile/` are priced at $0 automatically; `[run] max_cost` only matters for paid agents.
|
|
165
167
|
|
|
166
168
|
## Stacks
|
|
167
169
|
|
|
@@ -233,7 +235,7 @@ So that agents see the pull request's CLAUDE.md rather than the one in each old
|
|
|
233
235
|
|
|
234
236
|
## Status
|
|
235
237
|
|
|
236
|
-
Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
|
|
238
|
+
Pre-release. Everything in [ARCHITECTURE.md](ARCHITECTURE.md) is built and covered by an offline test suite with fake agents; dogfooding on real repos is under way. Grading has been checked against real fixes: on 16 tasks mined from click and commander.js, the real fix is graded solved and no change is graded failed, every time. The Claude Code, Codex, Copilot, OpenCode and Aider adapters have been run end to end (OpenCode with a free model and Aider with a local Ollama model, both at $0); the other adapters, Docker mode and the GitHub Action haven't yet. Star or watch the repo to hear when it lands on PyPI.
|
|
237
239
|
|
|
238
240
|
## Contributing
|
|
239
241
|
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""whichagent: benchmark coding agents on your repo's own git history."""
|
|
2
|
-
__version__ = "0.1.
|
|
2
|
+
__version__ = "0.1.1"
|
|
@@ -127,8 +127,40 @@ def opencode(cfg, out, argv):
|
|
|
127
127
|
"model": _flag(argv, "-m", "--model"), "error": error}
|
|
128
128
|
|
|
129
129
|
|
|
130
|
+
def copilot(cfg, out, argv):
|
|
131
|
+
"""`copilot --output-format json` events: model and turns. It reports premium requests, not tokens or USD."""
|
|
132
|
+
model, turns, error = None, 0, False
|
|
133
|
+
for e in _events(out):
|
|
134
|
+
t = e.get("type")
|
|
135
|
+
if t == "model.call_start":
|
|
136
|
+
model = (e.get("data") or {}).get("model") or model
|
|
137
|
+
elif t == "assistant.turn_end":
|
|
138
|
+
turns += 1
|
|
139
|
+
elif t == "result":
|
|
140
|
+
error = e.get("exitCode", 0) != 0
|
|
141
|
+
return {"model": model, "turns": turns, "error": error}
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def goose(cfg, out, argv):
|
|
145
|
+
"""`goose run --output-format json`: one object after its banner, tokens and status under metadata.
|
|
146
|
+
The model is reported as provider/model, so local providers (ollama) are priced at $0."""
|
|
147
|
+
i = out.find("\n{")
|
|
148
|
+
d = _json(out[i + 1:] if i >= 0 else out)
|
|
149
|
+
if not isinstance(d, dict):
|
|
150
|
+
return {"error": True}
|
|
151
|
+
m = d.get("metadata") or {}
|
|
152
|
+
said = [x for x in d.get("messages") or [] if x.get("role") == "assistant"]
|
|
153
|
+
inf = ((said[-1].get("metadata") or {}).get("inference") or {}) if said else {}
|
|
154
|
+
provider = _flag(argv, "", "--provider") or inf.get("provider")
|
|
155
|
+
name = _flag(argv, "", "--model") or inf.get("requestedModel")
|
|
156
|
+
model = f"{provider}/{name}" if provider and name else name
|
|
157
|
+
tin, tout = m.get("input_tokens", 0), m.get("output_tokens", 0)
|
|
158
|
+
return {"cost_usd": cost(cfg, model, tin, tout, m.get("cache_read_input_tokens", 0)), "tokens_in": tin,
|
|
159
|
+
"tokens_out": tout, "turns": len(said), "model": model, "error": m.get("status", "completed") != "completed"}
|
|
160
|
+
|
|
161
|
+
|
|
130
162
|
PARSERS = {"claude-json": claude, "codex-jsonl": codex, "gemini-json": gemini, "aider-text": aider,
|
|
131
|
-
"opencode-json": opencode, "none": lambda cfg, out, argv: {}}
|
|
163
|
+
"opencode-json": opencode, "copilot-jsonl": copilot, "goose-json": goose, "none": lambda cfg, out, argv: {}}
|
|
132
164
|
|
|
133
165
|
|
|
134
166
|
def _flag(argv, short, long):
|
|
@@ -149,6 +181,16 @@ def _codex_model():
|
|
|
149
181
|
|
|
150
182
|
|
|
151
183
|
LOCAL = ("ollama", "ollama_chat", "lm_studio", "llamafile") # litellm prefixes for models on your machine
|
|
184
|
+
LOCAL_HOSTS = ("localhost", "127.0.0.1", "[::1]")
|
|
185
|
+
NO_COST = {"none", "copilot-jsonl"} # parsers that never report what a run cost
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def free(argv, env):
|
|
189
|
+
"""True when an agent runs a model on this machine (an ollama/lm_studio/llamafile model, a localhost endpoint)
|
|
190
|
+
or a model named free (OpenCode's -free, OpenRouter's :free): a run costs $0 before it reports anything."""
|
|
191
|
+
words = [w.split("=", 1)[-1] for w in argv] + [v for k, v in env.items() if "PROXY" not in k.upper()]
|
|
192
|
+
return any(w.split("/")[0] in LOCAL or w.endswith(("-free", ":free")) or any(h in w for h in LOCAL_HOSTS)
|
|
193
|
+
for w in words)
|
|
152
194
|
|
|
153
195
|
|
|
154
196
|
def cost(cfg, model, tin, tout, cached=0):
|
|
@@ -42,15 +42,18 @@ BUILTIN = {
|
|
|
42
42
|
# Unverified: flags and output formats come from each CLI's docs and haven't been run by whichagent yet.
|
|
43
43
|
# `whichagent doctor` checks the flags against the installed CLI's --help.
|
|
44
44
|
"codex": {"cmd": "codex exec --json --sandbox workspace-write -", "parse": "codex-jsonl"},
|
|
45
|
-
|
|
45
|
+
# Gemini reads ~/.gemini/.env (GEMINI_API_KEY) and honors --yolo only in trusted folders
|
|
46
|
+
"gemini": {"cmd": "gemini --output-format json --yolo", "env": {"GEMINI_CLI_TRUST_WORKSPACE": "true"},
|
|
47
|
+
"parse": "gemini-json", "unverified": True},
|
|
46
48
|
"aider": {"cmd": "aider --message-file {prompt_file} --yes-always --no-auto-commits --no-gitignore"
|
|
47
49
|
" --no-check-update --no-show-release-notes --analytics-disable --no-pretty",
|
|
48
50
|
"stdin": False, "parse": "aider-text"},
|
|
49
51
|
"qwen": {"cmd": "qwen --output-format json --yolo", "parse": "gemini-json", "unverified": True},
|
|
50
52
|
"opencode": {"cmd": "opencode run --auto --format json", "parse": "opencode-json"},
|
|
51
|
-
"goose": {"cmd": "goose run --no-session --instructions {prompt_file}", "stdin": False,
|
|
53
|
+
"goose": {"cmd": "goose run --no-session --instructions {prompt_file} --output-format json", "stdin": False,
|
|
54
|
+
"env": {"GOOSE_MODE": "auto"}, "parse": "goose-json", "unverified": True},
|
|
52
55
|
"amp": {"cmd": "amp --execute --dangerously-allow-all", "unverified": True},
|
|
53
|
-
"copilot": {"cmd": "copilot --allow-all-tools", "
|
|
56
|
+
"copilot": {"cmd": "copilot --allow-all-tools --no-ask-user --output-format json", "parse": "copilot-jsonl"},
|
|
54
57
|
"cursor": {"cmd": "cursor-agent --print --force --output-format json", "unverified": True},
|
|
55
58
|
}
|
|
56
59
|
AGENT_KEYS = {"name", "extends", "cmd", "args", "stdin", "parse", "timeout", "env", "overlay"}
|
|
@@ -96,7 +99,7 @@ def _agent(name, defined, timeout, seen=()):
|
|
|
96
99
|
ext = own.get("extends")
|
|
97
100
|
base = _agent(ext, defined, timeout, seen + (name,)) if ext and ext != name else BUILTIN.get(name, {})
|
|
98
101
|
a = {"args": "", "stdin": True, "parse": "none", "env": {}, "overlay": {}, "timeout": timeout,
|
|
99
|
-
**base, **own, "name": name}
|
|
102
|
+
**base, **own, "name": name, "env": {**base.get("env", {}), **own.get("env", {})}}
|
|
100
103
|
if "cmd" not in a:
|
|
101
104
|
raise SystemExit(f"agent {name!r}: needs cmd or extends")
|
|
102
105
|
return a
|
|
@@ -96,10 +96,15 @@ def run(cfg, agents, tasks, yes=False, keep=False, retry_errors=False, new_versi
|
|
|
96
96
|
return []
|
|
97
97
|
costs = {a["name"]: [x["cost_usd"] for x in old if x["agent"] == a["name"] and x["cost_usd"] is not None
|
|
98
98
|
and x["status"] not in ERRORS] for a in agents} # a failed login costs $0 and says nothing
|
|
99
|
-
|
|
100
|
-
|
|
99
|
+
zero = {a["name"] for a in agents if ag.free(c.argv(a), a["env"]) or a["parse"] in ag.NO_COST}
|
|
100
|
+
|
|
101
|
+
def per_run(name): # (typical, worst) USD for one run: $0 for local and free models, else history, else a guess
|
|
102
|
+
xs = costs[name]
|
|
103
|
+
return (0.0, 0.0) if name in zero else (statistics.median(xs), max(xs)) if xs else NO_HISTORY
|
|
104
|
+
lo = sum(per_run(a["name"])[0] for _, a, _ in todo)
|
|
105
|
+
hi = sum(per_run(a["name"])[1] for _, a, _ in todo)
|
|
101
106
|
log(f"{len(todo)} runs, estimated ${lo:.2f}-${hi:.2f} (stops before passing max_cost ${rc['max_cost']:.2f})")
|
|
102
|
-
blind = [a["name"] for a in agents if a["parse"]
|
|
107
|
+
blind = [a["name"] for a in agents if a["parse"] in ag.NO_COST]
|
|
103
108
|
if blind:
|
|
104
109
|
log(f"No cost data from {', '.join(blind)}: their spending isn't counted toward max_cost.")
|
|
105
110
|
if not yes and not c.ask("Continue?"):
|
|
@@ -114,7 +119,7 @@ def run(cfg, agents, tasks, yes=False, keep=False, retry_errors=False, new_versi
|
|
|
114
119
|
with slots[agent["name"]]:
|
|
115
120
|
with lock:
|
|
116
121
|
xs = costs[agent["name"]]
|
|
117
|
-
est = sum(xs) / len(xs) if xs else NO_HISTORY[1]
|
|
122
|
+
est = 0.0 if agent["name"] in zero else sum(xs) / len(xs) if xs else NO_HISTORY[1]
|
|
118
123
|
if sb.STOP.is_set() or budget["over"] or budget["spent"] + budget["inflight"] + est > rc["max_cost"]:
|
|
119
124
|
budget["over"] = not sb.STOP.is_set()
|
|
120
125
|
return
|
|
@@ -154,6 +154,10 @@ class Pipeline(unittest.TestCase):
|
|
|
154
154
|
self.assertEqual(run.run(self.cfg, perfect, self.tasks, yes=True, new_version=True, log=quiet), [])
|
|
155
155
|
cfg = config.load(self.repo, {"run": {"attempts": 5, "max_cost": 0.025, "jobs": 1}})
|
|
156
156
|
self.assertEqual(len(run.run(cfg, perfect, self.tasks, yes=True, log=quiet)), 2) # $0.01 each
|
|
157
|
+
# an agent on a local model has no history, yet isn't held back by the $0.10-$1.00 guess for unknown agents
|
|
158
|
+
local = {**self.cfg["agents"]["noop"], "name": "noop-local", "args": "--model ollama/x"}
|
|
159
|
+
cfg = config.load(self.repo, {"run": {"max_cost": 0.5, "jobs": 1}})
|
|
160
|
+
self.assertEqual(len(run.run(cfg, [local], self.tasks, yes=True, log=quiet)), len(self.tasks))
|
|
157
161
|
|
|
158
162
|
def test_cli_report_show_compare(self):
|
|
159
163
|
out = io.StringIO()
|
|
@@ -68,6 +68,19 @@ class Units(unittest.TestCase):
|
|
|
68
68
|
def test_durations(self):
|
|
69
69
|
self.assertEqual([config.secs(v) for v in ("90", "90s", "30m", "2h")], [90, 90, 1800, 7200])
|
|
70
70
|
|
|
71
|
+
def test_free_agents(self):
|
|
72
|
+
self.assertTrue(agents.free(["aider", "--model", "ollama_chat/qwen2.5-coder:7b"], {}))
|
|
73
|
+
self.assertTrue(agents.free(["goose", "--provider", "ollama"], {}))
|
|
74
|
+
self.assertTrue(agents.free(["qwen"], {"OPENAI_BASE_URL": "http://127.0.0.1:11434/v1"}))
|
|
75
|
+
self.assertTrue(agents.free(["opencode", "run", "--model=opencode/deepseek-v4.1-flash-free"], {}))
|
|
76
|
+
self.assertFalse(agents.free(["claude", "-p"], {"HTTPS_PROXY": "http://localhost:3128"}))
|
|
77
|
+
self.assertFalse(agents.free(["codex", "exec", "-m", "gpt-x"], {}))
|
|
78
|
+
|
|
79
|
+
def test_extends_merges_env(self):
|
|
80
|
+
a = config._agent("g", {"g": {"name": "g", "extends": "gemini", "env": {"X": "1"}}}, 60)
|
|
81
|
+
self.assertEqual(a["env"], {"GEMINI_CLI_TRUST_WORKSPACE": "true", "X": "1"})
|
|
82
|
+
self.assertEqual(a["cmd"], config.BUILTIN["gemini"]["cmd"])
|
|
83
|
+
|
|
71
84
|
def test_parsers(self):
|
|
72
85
|
claude = json.dumps({"type": "result", "is_error": False, "num_turns": 7, "total_cost_usd": 0.12,
|
|
73
86
|
"modelUsage": {"m-big": {"inputTokens": 10, "cacheReadInputTokens": 90,
|
|
@@ -100,6 +113,17 @@ class Units(unittest.TestCase):
|
|
|
100
113
|
self.assertEqual(agents.parse(CFG, "opencode-json", oc, ["opencode", "run", "-m", "opencode/x-free"]),
|
|
101
114
|
{"cost_usd": 0.5, "tokens_in": 2000, "tokens_out": 30, "turns": 2,
|
|
102
115
|
"model": "opencode/x-free", "error": False})
|
|
116
|
+
cp = "\n".join(json.dumps(x) for x in [{"type": "model.call_start", "data": {"model": "m1"}},
|
|
117
|
+
{"type": "assistant.turn_end", "data": {}},
|
|
118
|
+
{"type": "assistant.turn_end", "data": {}},
|
|
119
|
+
{"type": "result", "exitCode": 1}])
|
|
120
|
+
said = {"role": "assistant", "metadata": {"inference": {"provider": "ollama", "requestedModel": "q:7b"}}}
|
|
121
|
+
goose = " __( O)> banner\n" + json.dumps({"messages": [{"role": "user"}, said, said], "metadata": {
|
|
122
|
+
"input_tokens": 300, "output_tokens": 20, "status": "completed"}}, indent=2)
|
|
123
|
+
self.assertEqual(agents.parse(CFG, "goose-json", goose, []),
|
|
124
|
+
{"cost_usd": 0.0, "tokens_in": 300, "tokens_out": 20, "turns": 2, "model": "ollama/q:7b",
|
|
125
|
+
"error": False})
|
|
126
|
+
self.assertEqual(agents.parse(CFG, "copilot-jsonl", cp, []), {"model": "m1", "turns": 2, "error": True})
|
|
103
127
|
|
|
104
128
|
|
|
105
129
|
class TestResults(unittest.TestCase):
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|