whichagent 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- whichagent/__init__.py +2 -0
- whichagent/__main__.py +5 -0
- whichagent/agents.py +199 -0
- whichagent/cli.py +244 -0
- whichagent/config.py +157 -0
- whichagent/git.py +137 -0
- whichagent/grade.py +85 -0
- whichagent/mine.py +159 -0
- whichagent/report.py +312 -0
- whichagent/run.py +153 -0
- whichagent/sandbox.py +248 -0
- whichagent/stacks.py +55 -0
- whichagent/stats.py +78 -0
- whichagent/store.py +46 -0
- whichagent/testresults.py +91 -0
- whichagent/workspace.py +94 -0
- whichagent-0.1.0.dist-info/METADATA +262 -0
- whichagent-0.1.0.dist-info/RECORD +21 -0
- whichagent-0.1.0.dist-info/WHEEL +4 -0
- whichagent-0.1.0.dist-info/entry_points.txt +2 -0
- whichagent-0.1.0.dist-info/licenses/LICENSE +21 -0
whichagent/sandbox.py
ADDED
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"""Running commands: as a process tree on this machine, or in a Docker container that sees only the workspace.
|
|
2
|
+
|
|
3
|
+
Both executors have the same methods (shell, agent, env, which, path, close), so mining, running and grading
|
|
4
|
+
don't care where commands run.
|
|
5
|
+
"""
|
|
6
|
+
import functools
|
|
7
|
+
import hashlib
|
|
8
|
+
import itertools
|
|
9
|
+
import json
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import shutil
|
|
13
|
+
import signal
|
|
14
|
+
import subprocess
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
from pathlib import Path
|
|
18
|
+
|
|
19
|
+
STOP = threading.Event() # set on Ctrl-C: no new process starts
|
|
20
|
+
LIVE = set() # running processes, killed on Ctrl-C
|
|
21
|
+
CONTAINERS = set() # running containers, killed on Ctrl-C
|
|
22
|
+
_ids = itertools.count()
|
|
23
|
+
|
|
24
|
+
# Credentials and proxies: the only host variables a container gets. Everything else comes from its image.
|
|
25
|
+
CONTAINER_KEEP = set("""ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL CLAUDE_CODE_OAUTH_TOKEN
|
|
26
|
+
CLAUDE_CODE_USE_BEDROCK CLAUDE_CODE_USE_VERTEX AWS_PROFILE AWS_REGION AWS_DEFAULT_REGION OPENAI_API_KEY
|
|
27
|
+
OPENAI_BASE_URL GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_CLOUD_PROJECT GOOGLE_CLOUD_LOCATION GOOGLE_GENAI_USE_VERTEXAI
|
|
28
|
+
OPENROUTER_API_KEY DEEPSEEK_API_KEY MISTRAL_API_KEY GROQ_API_KEY XAI_API_KEY AZURE_OPENAI_API_KEY DASHSCOPE_API_KEY
|
|
29
|
+
HTTP_PROXY HTTPS_PROXY NO_PROXY ALL_PROXY""".split())
|
|
30
|
+
# On the host, agents also get what programs need to start: paths, home, temp, locale, the OS variables
|
|
31
|
+
# Windows needs, and the agents' config dirs. Nothing else from your shell.
|
|
32
|
+
ENV_KEEP = CONTAINER_KEEP | set("""PATH PATHEXT HOME USER LOGNAME SHELL TERM LANG LANGUAGE TZ TMPDIR TMP TEMP
|
|
33
|
+
USERPROFILE HOMEDRIVE HOMEPATH APPDATA LOCALAPPDATA PROGRAMDATA ALLUSERSPROFILE PUBLIC SYSTEMROOT SYSTEMDRIVE WINDIR
|
|
34
|
+
COMSPEC OS USERNAME USERDOMAIN COMPUTERNAME NUMBER_OF_PROCESSORS PROCESSOR_ARCHITECTURE PROGRAMFILES
|
|
35
|
+
PROGRAMFILES(X86) PROGRAMW6432 COMMONPROGRAMFILES COMMONPROGRAMFILES(X86) COMMONPROGRAMW6432 XDG_CONFIG_HOME
|
|
36
|
+
XDG_DATA_HOME XDG_CACHE_HOME XDG_STATE_HOME XDG_RUNTIME_DIR SSL_CERT_FILE SSL_CERT_DIR NODE_EXTRA_CA_CERTS
|
|
37
|
+
REQUESTS_CA_BUNDLE CLAUDE_CONFIG_DIR CLAUDE_CODE_GIT_BASH_PATH CODEX_HOME GOOGLE_APPLICATION_CREDENTIALS""".split())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def proc(cmd, cwd, timeout, out, err=None, env=None, stdin=None, shell=False, container=None):
|
|
41
|
+
"""Run cmd with output to files, killing its process tree (and container) at timeout.
|
|
42
|
+
|
|
43
|
+
Returns (exit code, or None on timeout; seconds)."""
|
|
44
|
+
if STOP.is_set():
|
|
45
|
+
raise RuntimeError("stopped")
|
|
46
|
+
start = time.monotonic()
|
|
47
|
+
with open(out, "wb") as o, open(err or os.devnull, "wb") as e, open(stdin or os.devnull, "rb") as i:
|
|
48
|
+
p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=shell, stdin=i, stdout=o,
|
|
49
|
+
stderr=e if err else subprocess.STDOUT, start_new_session=os.name != "nt")
|
|
50
|
+
LIVE.add(p)
|
|
51
|
+
if container:
|
|
52
|
+
CONTAINERS.add(container)
|
|
53
|
+
try:
|
|
54
|
+
if STOP.is_set():
|
|
55
|
+
_stop(p, container)
|
|
56
|
+
rc = p.wait(timeout)
|
|
57
|
+
except subprocess.TimeoutExpired:
|
|
58
|
+
rc = None
|
|
59
|
+
# ponytail: on Windows, children an agent leaves behind after a normal exit survive; Job Objects would fix it
|
|
60
|
+
if rc is None:
|
|
61
|
+
_stop(p, container)
|
|
62
|
+
elif os.name != "nt":
|
|
63
|
+
kill(p) # also stops what it left running in its process group
|
|
64
|
+
LIVE.discard(p)
|
|
65
|
+
CONTAINERS.discard(container)
|
|
66
|
+
return rc, time.monotonic() - start
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _stop(p, container):
|
|
70
|
+
if container: # killing the docker CLI alone leaves the container running
|
|
71
|
+
subprocess.run(["docker", "kill", container], capture_output=True)
|
|
72
|
+
kill(p)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def kill(p):
|
|
76
|
+
if os.name == "nt":
|
|
77
|
+
subprocess.run(["taskkill", "/F", "/T", "/PID", str(p.pid)], capture_output=True)
|
|
78
|
+
else:
|
|
79
|
+
try:
|
|
80
|
+
os.killpg(p.pid, signal.SIGKILL)
|
|
81
|
+
except OSError:
|
|
82
|
+
pass
|
|
83
|
+
p.wait()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def killall():
|
|
87
|
+
for name in CONTAINERS.copy():
|
|
88
|
+
subprocess.run(["docker", "kill", name], capture_output=True)
|
|
89
|
+
for p in LIVE.copy():
|
|
90
|
+
kill(p)
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _own(agent):
|
|
94
|
+
"""The agent's own env. A value like "$NAME" is read from your environment, so secrets stay out of the TOML."""
|
|
95
|
+
return {k: os.environ.get(v[1:], "") if isinstance(v, str) and v.startswith("$") else str(v)
|
|
96
|
+
for k, v in agent["env"].items()}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class Host:
|
|
100
|
+
"""Runs commands on this machine, as your user. Not a sandbox: see ARCHITECTURE.md section 14."""
|
|
101
|
+
|
|
102
|
+
def __init__(self, ws):
|
|
103
|
+
self.ws = Path(ws)
|
|
104
|
+
self.roots = [str(self.ws), self.ws.as_posix()] # stripped from test ids, so they match across workspaces
|
|
105
|
+
|
|
106
|
+
def path(self, p):
|
|
107
|
+
return str(p)
|
|
108
|
+
|
|
109
|
+
def which(self, binary):
|
|
110
|
+
return shutil.which(binary)
|
|
111
|
+
|
|
112
|
+
def env(self, agent):
|
|
113
|
+
keep = {k: v for k, v in os.environ.items() if k.upper() in ENV_KEEP or k.upper().startswith("LC_")}
|
|
114
|
+
return {**keep, **_own(agent)}
|
|
115
|
+
|
|
116
|
+
def shell(self, cmd, timeout, log):
|
|
117
|
+
return proc(cmd, self.ws, timeout, log, shell=True)
|
|
118
|
+
|
|
119
|
+
def agent(self, argv, timeout, out, err, env, stdin):
|
|
120
|
+
return proc(argv, self.ws, timeout, out, err=err, env=env, stdin=stdin)
|
|
121
|
+
|
|
122
|
+
def close(self):
|
|
123
|
+
pass
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class Docker:
|
|
127
|
+
"""Runs commands in a container that sees only this run's temp dir: the workspace, the prompt, the reports."""
|
|
128
|
+
|
|
129
|
+
def __init__(self, ws, image, mounts=()):
|
|
130
|
+
self.ws, self.image = Path(ws), image
|
|
131
|
+
self.mounts = [os.path.expanduser(m) for m in mounts]
|
|
132
|
+
self.cwd = f"/wa/{self.ws.name}"
|
|
133
|
+
self.roots = [self.cwd]
|
|
134
|
+
|
|
135
|
+
def path(self, p):
|
|
136
|
+
return "/wa/" + Path(p).relative_to(self.ws.parent).as_posix()
|
|
137
|
+
|
|
138
|
+
def which(self, binary):
|
|
139
|
+
return binary # it lives in the image; a missing one fails the run as an agent error
|
|
140
|
+
|
|
141
|
+
def env(self, agent):
|
|
142
|
+
return {**{k: v for k, v in os.environ.items() if k.upper() in CONTAINER_KEEP}, **_own(agent)}
|
|
143
|
+
|
|
144
|
+
def argv(self, name, env=(), stdin=False):
|
|
145
|
+
a = ["docker", "run", "--rm", "--name", name, "-v", f"{self.ws.parent}:/wa", "-w", self.cwd,
|
|
146
|
+
# the container runs as root and the mount is owned by you: without this, git refuses the workspace
|
|
147
|
+
"-e", "GIT_CONFIG_COUNT=1", "-e", "GIT_CONFIG_KEY_0=safe.directory", "-e", "GIT_CONFIG_VALUE_0=*"]
|
|
148
|
+
for m in self.mounts:
|
|
149
|
+
a += ["-v", m]
|
|
150
|
+
for k in env: # by name only: values travel in the docker CLI's environment, not on its command line
|
|
151
|
+
a += ["-e", k]
|
|
152
|
+
return a + (["-i"] if stdin else []) + [self.image]
|
|
153
|
+
|
|
154
|
+
def shell(self, cmd, timeout, log):
|
|
155
|
+
name = _name()
|
|
156
|
+
return proc(self.argv(name) + ["sh", "-c", cmd], self.ws, timeout, log, container=name)
|
|
157
|
+
|
|
158
|
+
def agent(self, argv, timeout, out, err, env, stdin):
|
|
159
|
+
name = _name()
|
|
160
|
+
return proc(self.argv(name, sorted(env), stdin is not None) + list(argv), self.ws, timeout, out, err=err,
|
|
161
|
+
env={**os.environ, **env}, stdin=stdin, container=name)
|
|
162
|
+
|
|
163
|
+
def close(self):
|
|
164
|
+
if hasattr(os, "getuid"): # files the container wrote as root go back to you, so the run dir can be deleted
|
|
165
|
+
subprocess.run(["docker", "run", "--rm", "-v", f"{self.ws.parent}:/wa", self.image, "chown", "-R",
|
|
166
|
+
f"{os.getuid()}:{os.getgid()}", "/wa"], capture_output=True)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _name():
|
|
170
|
+
return f"whichagent-{os.getpid()}-{next(_ids)}"
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
def executor(cfg, ws):
|
|
174
|
+
if cfg["run"]["sandbox"] == "docker":
|
|
175
|
+
return Docker(ws, image(cfg["root"], cfg["docker"]["image"]), cfg["docker"]["mounts"])
|
|
176
|
+
return Host(ws)
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def version(cfg, binary):
|
|
180
|
+
"""First line of `<binary> --version`, from the image in docker mode. Recorded on every result."""
|
|
181
|
+
if cfg["run"]["sandbox"] == "docker":
|
|
182
|
+
return _version(("docker", "run", "--rm", image(cfg["root"], cfg["docker"]["image"]), binary, "--version"))
|
|
183
|
+
exe = shutil.which(binary)
|
|
184
|
+
return _version((exe, "--version")) if exe else None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
@functools.cache
|
|
188
|
+
def _version(argv):
|
|
189
|
+
try:
|
|
190
|
+
r = subprocess.run(list(argv), capture_output=True, text=True, encoding="utf-8", errors="replace",
|
|
191
|
+
timeout=300)
|
|
192
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
193
|
+
return None
|
|
194
|
+
lines = (r.stdout or r.stderr).strip().splitlines()
|
|
195
|
+
return lines[0] if lines else None
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
@functools.cache
|
|
199
|
+
def image(root, configured=""):
|
|
200
|
+
"""The docker-mode image: [docker] image, else one built from .devcontainer/devcontainer.json or a Dockerfile."""
|
|
201
|
+
if configured:
|
|
202
|
+
return configured
|
|
203
|
+
dockerfile = context = None
|
|
204
|
+
for spec_path in (root / ".devcontainer" / "devcontainer.json", root / ".devcontainer.json"):
|
|
205
|
+
if spec_path.exists():
|
|
206
|
+
spec = json.loads(jsonc(spec_path.read_text("utf-8")))
|
|
207
|
+
if spec.get("image"):
|
|
208
|
+
return spec["image"]
|
|
209
|
+
build = spec.get("build") or {}
|
|
210
|
+
name = build.get("dockerfile") or spec.get("dockerFile")
|
|
211
|
+
if not name:
|
|
212
|
+
raise SystemExit(f"{spec_path}: docker mode needs an image or a dockerfile (compose isn't supported)")
|
|
213
|
+
dockerfile, context = spec_path.parent / name, spec_path.parent / build.get("context", ".")
|
|
214
|
+
break
|
|
215
|
+
if dockerfile is None and (root / "Dockerfile").exists():
|
|
216
|
+
dockerfile, context = root / "Dockerfile", root
|
|
217
|
+
if dockerfile is None:
|
|
218
|
+
raise SystemExit("Docker mode needs an image: set [docker] image, or add .devcontainer/devcontainer.json"
|
|
219
|
+
" or a Dockerfile")
|
|
220
|
+
tag = "whichagent-" + hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:12]
|
|
221
|
+
if subprocess.run(["docker", "image", "inspect", tag], capture_output=True).returncode:
|
|
222
|
+
if subprocess.run(["docker", "build", "-t", tag, "-f", str(dockerfile), str(context)]).returncode:
|
|
223
|
+
raise SystemExit(f"docker build of {dockerfile} failed")
|
|
224
|
+
return tag
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def jsonc(text):
|
|
228
|
+
"""JSON with comments and trailing commas (devcontainer.json) as plain JSON."""
|
|
229
|
+
out, i, string = [], 0, False
|
|
230
|
+
while i < len(text):
|
|
231
|
+
ch = text[i]
|
|
232
|
+
if string:
|
|
233
|
+
out.append(text[i:i + 2] if ch == "\\" else ch)
|
|
234
|
+
i += 2 if ch == "\\" else 1
|
|
235
|
+
string = ch != '"'
|
|
236
|
+
continue
|
|
237
|
+
if text.startswith("//", i):
|
|
238
|
+
end = text.find("\n", i)
|
|
239
|
+
i = len(text) if end < 0 else end
|
|
240
|
+
continue
|
|
241
|
+
if text.startswith("/*", i):
|
|
242
|
+
end = text.find("*/", i + 2)
|
|
243
|
+
i = len(text) if end < 0 else end + 2
|
|
244
|
+
continue
|
|
245
|
+
string = ch == '"'
|
|
246
|
+
out.append(ch)
|
|
247
|
+
i += 1
|
|
248
|
+
return re.sub(r",(\s*[}\]])", r"\1", "".join(out))
|
whichagent/stacks.py
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
"""Stack detection: [repo] install, test and suite commands for common toolchains, with per-test reports."""
|
|
2
|
+
import os
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def detect(root):
|
|
6
|
+
"""[repo] settings for the stack at root, or None when it isn't recognized."""
|
|
7
|
+
def has(name):
|
|
8
|
+
return (root / name).exists()
|
|
9
|
+
if has("uv.lock") or has("poetry.lock"):
|
|
10
|
+
tool = "uv" if has("uv.lock") else "poetry"
|
|
11
|
+
return {"install": "uv sync" if tool == "uv" else "poetry install",
|
|
12
|
+
"test": f"{tool} run pytest -q {{files}} --junitxml={{report}}",
|
|
13
|
+
"suite": f"{tool} run pytest -q --junitxml={{report}}"}
|
|
14
|
+
if node := _node(root):
|
|
15
|
+
return node
|
|
16
|
+
if has("go.mod"):
|
|
17
|
+
return {"install": "go mod download", "test": "go test -json {dirs} > {report}",
|
|
18
|
+
"suite": "go test -json ./... > {report}"}
|
|
19
|
+
if has("Cargo.toml"): # per-test results come from cargo's own output
|
|
20
|
+
return {"install": "cargo fetch", "test": "cargo test --no-fail-fast --test {names: --test }",
|
|
21
|
+
"suite": "cargo test --no-fail-fast"}
|
|
22
|
+
if has("pom.xml"):
|
|
23
|
+
mvn = "mvn -q -B"
|
|
24
|
+
return {"install": f"{mvn} -DskipTests test-compile",
|
|
25
|
+
"test": f"{mvn} test -Dtest={{names:,}} -Dsurefire.failIfNoSpecifiedTests=false"
|
|
26
|
+
" -Dsurefire.reportsDirectory={report}",
|
|
27
|
+
"suite": f"{mvn} test -Dsurefire.reportsDirectory={{report}}"}
|
|
28
|
+
if has("build.gradle") or has("build.gradle.kts"):
|
|
29
|
+
gradle = "gradlew" if os.name == "nt" else "./gradlew" # cmd.exe finds gradlew.bat in the current dir
|
|
30
|
+
return {"test": f"{gradle} test --tests {{names: --tests }}", "suite": f"{gradle} test",
|
|
31
|
+
"report": "build/test-results/test"}
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _node(root):
|
|
36
|
+
pkg = (root / "package.json").read_text("utf-8") if (root / "package.json").exists() else ""
|
|
37
|
+
if (root / "package-lock.json").exists():
|
|
38
|
+
install, run = "npm ci", "npx"
|
|
39
|
+
elif (root / "pnpm-lock.yaml").exists():
|
|
40
|
+
install, run = "pnpm install --frozen-lockfile", "pnpm exec"
|
|
41
|
+
elif (root / "yarn.lock").exists():
|
|
42
|
+
berry = (root / ".yarnrc.yml").exists()
|
|
43
|
+
install, run = f"yarn install {'--immutable' if berry else '--frozen-lockfile'}", "yarn"
|
|
44
|
+
else:
|
|
45
|
+
return None
|
|
46
|
+
if '"vitest"' in pkg:
|
|
47
|
+
test = f"{run} vitest run {{files}} --reporter=default --reporter=junit --outputFile.junit={{report}}"
|
|
48
|
+
elif '"jest"' in pkg:
|
|
49
|
+
test = f"{run} jest {{files}} --json --outputFile={{report}}"
|
|
50
|
+
elif "node --test" in pkg:
|
|
51
|
+
test = ("node --test --test-reporter=spec --test-reporter=junit --test-reporter-destination=stdout"
|
|
52
|
+
" --test-reporter-destination={report} {files}")
|
|
53
|
+
else:
|
|
54
|
+
return None
|
|
55
|
+
return {"install": install, "test": test, "suite": test.replace(" {files}", ""), "cache": True}
|
whichagent/stats.py
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""Solve rates with intervals, solved-every-time, cost per solved task and the paired sign test."""
|
|
2
|
+
import collections
|
|
3
|
+
import math
|
|
4
|
+
import random
|
|
5
|
+
import statistics
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def wilson(k, n, z=1.96):
|
|
9
|
+
"""95% Wilson interval for k successes in n: sane for small n and rates near 0% or 100%."""
|
|
10
|
+
if n == 0:
|
|
11
|
+
return 0.0, 1.0
|
|
12
|
+
p, d = k / n, 1 + z * z / n
|
|
13
|
+
center = (p + z * z / (2 * n)) / d
|
|
14
|
+
half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
|
|
15
|
+
return max(0.0, center - half), min(1.0, center + half)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def bootstrap(rates, reps=2000, seed=0):
|
|
19
|
+
"""95% interval for the mean per-task rate, resampling tasks: runs on one task aren't independent.
|
|
20
|
+
|
|
21
|
+
Never narrower than the task-level Wilson interval: on a few tasks that all score 100%, a plain
|
|
22
|
+
bootstrap would claim 100%-100%."""
|
|
23
|
+
rnd, n = random.Random(seed), len(rates)
|
|
24
|
+
means = sorted(sum(rnd.choices(rates, k=n)) / n for _ in range(reps))
|
|
25
|
+
lo, hi = wilson(sum(rates), n)
|
|
26
|
+
return min(lo, means[int(0.025 * reps)]), max(hi, means[int(0.975 * reps) - 1])
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def sign_test(wins, losses):
|
|
30
|
+
"""Exact two-sided sign test p-value (ties already dropped)."""
|
|
31
|
+
n = wins + losses
|
|
32
|
+
if n == 0:
|
|
33
|
+
return 1.0
|
|
34
|
+
return min(1.0, 2 * sum(math.comb(n, i) for i in range(min(wins, losses) + 1)) / 2 ** n)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def by_task(results):
|
|
38
|
+
"""{task: [solved?, ...]}. Setup errors aren't the agent's fault, so they don't count."""
|
|
39
|
+
out = {}
|
|
40
|
+
for r in results:
|
|
41
|
+
if r["status"] != "setup_error":
|
|
42
|
+
out.setdefault(r["task"], []).append(r["status"] == "solved")
|
|
43
|
+
return out
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def summary(results):
|
|
47
|
+
"""One agent's numbers, or None if it has no gradable runs."""
|
|
48
|
+
tasks = by_task(results)
|
|
49
|
+
if not tasks:
|
|
50
|
+
return None
|
|
51
|
+
runs = [r for r in results if r["status"] != "setup_error"]
|
|
52
|
+
rates = [sum(v) / len(v) for v in tasks.values()]
|
|
53
|
+
one_each = all(len(v) == 1 for v in tasks.values())
|
|
54
|
+
known = [r["cost_usd"] for r in runs if r["cost_usd"] is not None]
|
|
55
|
+
total = sum(known) if known else None
|
|
56
|
+
solved = sum(r["status"] == "solved" for r in runs)
|
|
57
|
+
secs = [r["secs"] for r in runs if r["secs"] is not None]
|
|
58
|
+
return {"tasks": len(tasks), "runs": len(runs), "solved": solved, "rate": sum(rates) / len(rates),
|
|
59
|
+
"ci": wilson(sum(rates), len(rates)) if one_each else bootstrap(rates),
|
|
60
|
+
"every": None if one_each else sum(all(v) for v in tasks.values()) / len(tasks),
|
|
61
|
+
"per_run": total / len(known) if known else None,
|
|
62
|
+
"per_solved": total / solved if known and solved else None,
|
|
63
|
+
"cost_complete": len(known) == len(runs),
|
|
64
|
+
"secs": statistics.median(secs) if secs else None,
|
|
65
|
+
"status": collections.Counter(r["status"] for r in results),
|
|
66
|
+
"tampered": sum(bool(r.get("tampered")) for r in runs),
|
|
67
|
+
"regressions": sum(r.get("regression") is True for r in runs),
|
|
68
|
+
"flagged": sum(bool(r.get("flagged")) for r in runs)} # edited pyproject.toml, package.json, ...
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def compare(ra, rb):
|
|
72
|
+
"""(wins, losses, ties, p) for agent A vs B on the tasks both ran: A wins a task with a higher solve rate."""
|
|
73
|
+
a, b = by_task(ra), by_task(rb)
|
|
74
|
+
shared = a.keys() & b.keys()
|
|
75
|
+
rate = {t: (sum(a[t]) / len(a[t]), sum(b[t]) / len(b[t])) for t in shared}
|
|
76
|
+
wins = sum(x > y for x, y in rate.values())
|
|
77
|
+
losses = sum(x < y for x, y in rate.values())
|
|
78
|
+
return wins, losses, len(shared) - wins - losses, sign_test(wins, losses)
|
whichagent/store.py
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
"""The .whichagent/ directory: tasks, results and run logs, kept as append-only JSONL so every command can resume."""
|
|
2
|
+
import json
|
|
3
|
+
import re
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def state(root):
|
|
8
|
+
""".whichagent/ in the repo, created with a .gitignore that keeps it out of git."""
|
|
9
|
+
d = Path(root) / ".whichagent"
|
|
10
|
+
if not d.exists():
|
|
11
|
+
d.mkdir()
|
|
12
|
+
(d / ".gitignore").write_text("*\n")
|
|
13
|
+
return d
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def read(path):
|
|
17
|
+
out = []
|
|
18
|
+
if Path(path).exists():
|
|
19
|
+
for line in Path(path).read_text("utf-8").splitlines():
|
|
20
|
+
try:
|
|
21
|
+
out.append(json.loads(line))
|
|
22
|
+
except ValueError:
|
|
23
|
+
pass # a line cut short by a crash
|
|
24
|
+
return out
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def append(path, obj):
|
|
28
|
+
with open(path, "a", encoding="utf-8") as f:
|
|
29
|
+
f.write(json.dumps(obj) + "\n")
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def latest(results, by_version=False):
|
|
33
|
+
"""The last result per run: a retried run replaces the earlier one. With by_version, runs of different
|
|
34
|
+
agent versions are kept apart instead."""
|
|
35
|
+
def key(r):
|
|
36
|
+
return r["task"], r["agent"], r["attempt"], r.get("agent_version") if by_version else None
|
|
37
|
+
return list({key(r): r for r in results}.values())
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def slug(name):
|
|
41
|
+
return re.sub(r"[^\w.-]+", "_", name).strip("_") or "x"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def run_dir(root, task_id, agent, version, attempt):
|
|
45
|
+
"""Prompt, logs, diff and test output of one run."""
|
|
46
|
+
return state(root) / "runs" / task_id / slug(f"{agent}@{version}" if version else agent) / str(attempt)
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Per-test results from test runners: JUnit XML (pytest, vitest, node, surefire, gradle), Jest JSON,
|
|
2
|
+
go test -json, and cargo test's own output."""
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import xml.etree.ElementTree as ET
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def parse(report, log="", roots=()):
|
|
10
|
+
"""{test id: passed} from report (a file, or a dir of XML files), else from the log; None when neither has any.
|
|
11
|
+
|
|
12
|
+
Skipped tests are left out. Paths under roots are made relative, so ids match across workspaces."""
|
|
13
|
+
out = {}
|
|
14
|
+
p = Path(report) if report else None
|
|
15
|
+
files = sorted(p.rglob("*.xml")) if p and p.is_dir() else [p] if p and p.is_file() else []
|
|
16
|
+
for f in files:
|
|
17
|
+
_merge(out, _report(f.read_text("utf-8", "replace")), roots)
|
|
18
|
+
if not files:
|
|
19
|
+
_merge(out, _cargo(log), roots)
|
|
20
|
+
return out or None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _merge(out, results, roots):
|
|
24
|
+
for key, ok in results.items():
|
|
25
|
+
for root in roots:
|
|
26
|
+
key = key.replace(root + "/", "").replace(root + "\\", "")
|
|
27
|
+
out[key] = out.get(key, True) and ok # a test reported twice passes only if it passed both times
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _report(text):
|
|
31
|
+
text = text.lstrip()
|
|
32
|
+
if text.startswith("<"):
|
|
33
|
+
return _junit(text)
|
|
34
|
+
try:
|
|
35
|
+
data = json.loads(text)
|
|
36
|
+
except ValueError:
|
|
37
|
+
return _go(text)
|
|
38
|
+
return _jest(data) if isinstance(data, dict) and "testResults" in data else _go(text)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _junit(text):
|
|
42
|
+
out = {}
|
|
43
|
+
try:
|
|
44
|
+
root = ET.fromstring(text)
|
|
45
|
+
except ET.ParseError:
|
|
46
|
+
return out
|
|
47
|
+
|
|
48
|
+
def walk(nodes, suites):
|
|
49
|
+
for n in nodes:
|
|
50
|
+
if n.tag in ("testsuites", "testsuite"):
|
|
51
|
+
walk(n, suites + [n.get("name", "")] if n.tag == "testsuite" else suites)
|
|
52
|
+
elif n.tag == "testcase" and n.find("skipped") is None:
|
|
53
|
+
key = "::".join(x for x in [*suites, n.get("file", ""), n.get("classname", ""), n.get("name", "")] if x)
|
|
54
|
+
out[key] = out.get(key, True) and n.find("failure") is None and n.find("error") is None
|
|
55
|
+
walk([root], [])
|
|
56
|
+
return out
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _jest(data):
|
|
60
|
+
out = {}
|
|
61
|
+
for f in data.get("testResults") or []:
|
|
62
|
+
cases = f.get("assertionResults") or []
|
|
63
|
+
for a in cases:
|
|
64
|
+
if a.get("status") in ("passed", "failed"):
|
|
65
|
+
key = f"{f.get('name', '')}::{a.get('fullName') or a.get('title')}"
|
|
66
|
+
out[key] = out.get(key, True) and a["status"] == "passed"
|
|
67
|
+
if not cases and f.get("status") == "failed":
|
|
68
|
+
out[f"{f.get('name', '')}::(file)"] = False # the file didn't even load
|
|
69
|
+
return out
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _go(text):
|
|
73
|
+
out = {}
|
|
74
|
+
for line in text.splitlines():
|
|
75
|
+
try:
|
|
76
|
+
e = json.loads(line)
|
|
77
|
+
except ValueError:
|
|
78
|
+
continue
|
|
79
|
+
if isinstance(e, dict) and e.get("Action") in ("pass", "fail"):
|
|
80
|
+
out[f"{e.get('Package', '')}::{e.get('Test') or '(package)'}"] = e["Action"] == "pass"
|
|
81
|
+
return out
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _cargo(log):
|
|
85
|
+
out, binary = {}, ""
|
|
86
|
+
for line in log.splitlines():
|
|
87
|
+
if m := re.match(r"\s*Running (.+?)(?: \(|$)", line):
|
|
88
|
+
binary = m[1]
|
|
89
|
+
elif m := re.match(r"test (\S+) \.\.\. (ok|FAILED)", line):
|
|
90
|
+
out[f"{binary}::{m[1]}"] = m[2] == "ok"
|
|
91
|
+
return out
|
whichagent/workspace.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""Ready-to-use workspaces: a leak-free clone at the base, dependencies (restored from the cache when the
|
|
2
|
+
lockfiles match), setup. Mining, running and doctor all use these, so tasks are validated where agents run."""
|
|
3
|
+
import hashlib
|
|
4
|
+
import os
|
|
5
|
+
import shutil
|
|
6
|
+
import tempfile
|
|
7
|
+
import threading
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from . import config as c, git as g, sandbox as sb, store
|
|
11
|
+
|
|
12
|
+
KEEP = 3 # dependency sets cached per repo, most recently used first; each can be hundreds of MB
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def create(cfg, task_id, base):
|
|
16
|
+
"""A fresh clone of the task's base in the temp dir, and the executor that runs commands in it."""
|
|
17
|
+
root = cfg["root"]
|
|
18
|
+
bare = g.base_clone(root, base, store.state(root) / "bases" / f"{task_id}.git", cfg["mine"]["history_depth"])
|
|
19
|
+
ws = g.workspace(bare, cfg["branch"], root.name)
|
|
20
|
+
return ws, sb.executor(cfg, ws)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def prepare(cfg, ws, box, logs):
|
|
24
|
+
"""Install dependencies (or restore them from the cache), then run setup. False when a step fails.
|
|
25
|
+
|
|
26
|
+
Output goes to install.log and setup.log in logs."""
|
|
27
|
+
rp, timeout = cfg["repo"], c.secs(cfg["repo"]["setup_timeout"])
|
|
28
|
+
if rp["install"]:
|
|
29
|
+
key = _key(cfg, ws)
|
|
30
|
+
if not (rp["cache"] and _restore(cfg, key, ws)):
|
|
31
|
+
if box.shell(rp["install"], timeout, logs / "install.log")[0] != 0:
|
|
32
|
+
return False
|
|
33
|
+
if rp["cache"]:
|
|
34
|
+
_save(cfg, key, ws)
|
|
35
|
+
if rp["setup"] and box.shell(rp["setup"], timeout, logs / "setup.log")[0] != 0:
|
|
36
|
+
return False
|
|
37
|
+
g.exclude_untracked(ws)
|
|
38
|
+
return True
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def remove(ws, box):
|
|
42
|
+
box.close()
|
|
43
|
+
g.rmtree(ws.parent)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def cache_dir(cfg):
|
|
47
|
+
"""In the temp dir, like workspaces: a dependency cache in a synced folder would sync gigabytes."""
|
|
48
|
+
root = cfg["root"]
|
|
49
|
+
tag = hashlib.sha256(str(root).encode()).hexdigest()[:8]
|
|
50
|
+
return Path(tempfile.gettempdir()) / "whichagent-cache" / f"{store.slug(root.name)}-{tag}"
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _key(cfg, ws):
|
|
54
|
+
"""The install command plus every lockfile's content: equal keys mean equal dependencies."""
|
|
55
|
+
h = hashlib.sha256(cfg["repo"]["install"].encode())
|
|
56
|
+
for p in sorted(g.git(ws, "ls-files").splitlines()):
|
|
57
|
+
if c.match(p, cfg["repo"]["skip_if_changed"]):
|
|
58
|
+
h.update(p.encode() + b"\0" + Path(ws, p).read_bytes())
|
|
59
|
+
return h.hexdigest()[:16]
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _save(cfg, key, ws):
|
|
63
|
+
"""Cache everything install created (node_modules at every level, and so on)."""
|
|
64
|
+
dest = cache_dir(cfg) / key
|
|
65
|
+
if dest.exists():
|
|
66
|
+
return
|
|
67
|
+
tmp = dest.with_name(f"{key}.tmp-{os.getpid()}-{threading.get_ident()}")
|
|
68
|
+
try:
|
|
69
|
+
tmp.mkdir(parents=True)
|
|
70
|
+
for p in g.git(ws, "ls-files", "--others", "--directory").splitlines():
|
|
71
|
+
src, to = ws / p.rstrip("/"), tmp / p.rstrip("/")
|
|
72
|
+
to.parent.mkdir(parents=True, exist_ok=True)
|
|
73
|
+
if src.is_dir():
|
|
74
|
+
shutil.copytree(src, to, symlinks=True)
|
|
75
|
+
elif src.exists():
|
|
76
|
+
shutil.copy2(src, to, follow_symlinks=False)
|
|
77
|
+
tmp.rename(dest)
|
|
78
|
+
except OSError:
|
|
79
|
+
g.rmtree(tmp) # another worker saved it first, or the disk is full: the cache is only a speedup
|
|
80
|
+
old = sorted((p for p in dest.parent.iterdir() if ".tmp-" not in p.name), key=lambda p: p.stat().st_mtime)
|
|
81
|
+
for p in old[:-KEEP]:
|
|
82
|
+
g.rmtree(p)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _restore(cfg, key, ws):
|
|
86
|
+
src = cache_dir(cfg) / key
|
|
87
|
+
if not src.is_dir():
|
|
88
|
+
return False
|
|
89
|
+
try:
|
|
90
|
+
os.utime(src) # most recently used
|
|
91
|
+
shutil.copytree(src, ws, symlinks=True, dirs_exist_ok=True)
|
|
92
|
+
return True
|
|
93
|
+
except OSError:
|
|
94
|
+
return False # evicted meanwhile: install instead
|