whichagent 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
whichagent/sandbox.py ADDED
@@ -0,0 +1,248 @@
1
+ """Running commands: as a process tree on this machine, or in a Docker container that sees only the workspace.
2
+
3
+ Both executors have the same methods (shell, agent, env, which, path, close), so mining, running and grading
4
+ don't care where commands run.
5
+ """
6
+ import functools
7
+ import hashlib
8
+ import itertools
9
+ import json
10
+ import os
11
+ import re
12
+ import shutil
13
+ import signal
14
+ import subprocess
15
+ import threading
16
+ import time
17
+ from pathlib import Path
18
+
19
+ STOP = threading.Event() # set on Ctrl-C: no new process starts
20
+ LIVE = set() # running processes, killed on Ctrl-C
21
+ CONTAINERS = set() # running containers, killed on Ctrl-C
22
+ _ids = itertools.count()
23
+
24
+ # Credentials and proxies: the only host variables a container gets. Everything else comes from its image.
25
+ CONTAINER_KEEP = set("""ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL CLAUDE_CODE_OAUTH_TOKEN
26
+ CLAUDE_CODE_USE_BEDROCK CLAUDE_CODE_USE_VERTEX AWS_PROFILE AWS_REGION AWS_DEFAULT_REGION OPENAI_API_KEY
27
+ OPENAI_BASE_URL GEMINI_API_KEY GOOGLE_API_KEY GOOGLE_CLOUD_PROJECT GOOGLE_CLOUD_LOCATION GOOGLE_GENAI_USE_VERTEXAI
28
+ OPENROUTER_API_KEY DEEPSEEK_API_KEY MISTRAL_API_KEY GROQ_API_KEY XAI_API_KEY AZURE_OPENAI_API_KEY DASHSCOPE_API_KEY
29
+ HTTP_PROXY HTTPS_PROXY NO_PROXY ALL_PROXY""".split())
30
+ # On the host, agents also get what programs need to start: paths, home, temp, locale, the OS variables
31
+ # Windows needs, and the agents' config dirs. Nothing else from your shell.
32
+ ENV_KEEP = CONTAINER_KEEP | set("""PATH PATHEXT HOME USER LOGNAME SHELL TERM LANG LANGUAGE TZ TMPDIR TMP TEMP
33
+ USERPROFILE HOMEDRIVE HOMEPATH APPDATA LOCALAPPDATA PROGRAMDATA ALLUSERSPROFILE PUBLIC SYSTEMROOT SYSTEMDRIVE WINDIR
34
+ COMSPEC OS USERNAME USERDOMAIN COMPUTERNAME NUMBER_OF_PROCESSORS PROCESSOR_ARCHITECTURE PROGRAMFILES
35
+ PROGRAMFILES(X86) PROGRAMW6432 COMMONPROGRAMFILES COMMONPROGRAMFILES(X86) COMMONPROGRAMW6432 XDG_CONFIG_HOME
36
+ XDG_DATA_HOME XDG_CACHE_HOME XDG_STATE_HOME XDG_RUNTIME_DIR SSL_CERT_FILE SSL_CERT_DIR NODE_EXTRA_CA_CERTS
37
+ REQUESTS_CA_BUNDLE CLAUDE_CONFIG_DIR CLAUDE_CODE_GIT_BASH_PATH CODEX_HOME GOOGLE_APPLICATION_CREDENTIALS""".split())
38
+
39
+
40
+ def proc(cmd, cwd, timeout, out, err=None, env=None, stdin=None, shell=False, container=None):
41
+ """Run cmd with output to files, killing its process tree (and container) at timeout.
42
+
43
+ Returns (exit code, or None on timeout; seconds)."""
44
+ if STOP.is_set():
45
+ raise RuntimeError("stopped")
46
+ start = time.monotonic()
47
+ with open(out, "wb") as o, open(err or os.devnull, "wb") as e, open(stdin or os.devnull, "rb") as i:
48
+ p = subprocess.Popen(cmd, cwd=cwd, env=env, shell=shell, stdin=i, stdout=o,
49
+ stderr=e if err else subprocess.STDOUT, start_new_session=os.name != "nt")
50
+ LIVE.add(p)
51
+ if container:
52
+ CONTAINERS.add(container)
53
+ try:
54
+ if STOP.is_set():
55
+ _stop(p, container)
56
+ rc = p.wait(timeout)
57
+ except subprocess.TimeoutExpired:
58
+ rc = None
59
+ # ponytail: on Windows, children an agent leaves behind after a normal exit survive; Job Objects would fix it
60
+ if rc is None:
61
+ _stop(p, container)
62
+ elif os.name != "nt":
63
+ kill(p) # also stops what it left running in its process group
64
+ LIVE.discard(p)
65
+ CONTAINERS.discard(container)
66
+ return rc, time.monotonic() - start
67
+
68
+
69
+ def _stop(p, container):
70
+ if container: # killing the docker CLI alone leaves the container running
71
+ subprocess.run(["docker", "kill", container], capture_output=True)
72
+ kill(p)
73
+
74
+
75
+ def kill(p):
76
+ if os.name == "nt":
77
+ subprocess.run(["taskkill", "/F", "/T", "/PID", str(p.pid)], capture_output=True)
78
+ else:
79
+ try:
80
+ os.killpg(p.pid, signal.SIGKILL)
81
+ except OSError:
82
+ pass
83
+ p.wait()
84
+
85
+
86
+ def killall():
87
+ for name in CONTAINERS.copy():
88
+ subprocess.run(["docker", "kill", name], capture_output=True)
89
+ for p in LIVE.copy():
90
+ kill(p)
91
+
92
+
93
+ def _own(agent):
94
+ """The agent's own env. A value like "$NAME" is read from your environment, so secrets stay out of the TOML."""
95
+ return {k: os.environ.get(v[1:], "") if isinstance(v, str) and v.startswith("$") else str(v)
96
+ for k, v in agent["env"].items()}
97
+
98
+
99
+ class Host:
100
+ """Runs commands on this machine, as your user. Not a sandbox: see ARCHITECTURE.md section 14."""
101
+
102
+ def __init__(self, ws):
103
+ self.ws = Path(ws)
104
+ self.roots = [str(self.ws), self.ws.as_posix()] # stripped from test ids, so they match across workspaces
105
+
106
+ def path(self, p):
107
+ return str(p)
108
+
109
+ def which(self, binary):
110
+ return shutil.which(binary)
111
+
112
+ def env(self, agent):
113
+ keep = {k: v for k, v in os.environ.items() if k.upper() in ENV_KEEP or k.upper().startswith("LC_")}
114
+ return {**keep, **_own(agent)}
115
+
116
+ def shell(self, cmd, timeout, log):
117
+ return proc(cmd, self.ws, timeout, log, shell=True)
118
+
119
+ def agent(self, argv, timeout, out, err, env, stdin):
120
+ return proc(argv, self.ws, timeout, out, err=err, env=env, stdin=stdin)
121
+
122
+ def close(self):
123
+ pass
124
+
125
+
126
+ class Docker:
127
+ """Runs commands in a container that sees only this run's temp dir: the workspace, the prompt, the reports."""
128
+
129
+ def __init__(self, ws, image, mounts=()):
130
+ self.ws, self.image = Path(ws), image
131
+ self.mounts = [os.path.expanduser(m) for m in mounts]
132
+ self.cwd = f"/wa/{self.ws.name}"
133
+ self.roots = [self.cwd]
134
+
135
+ def path(self, p):
136
+ return "/wa/" + Path(p).relative_to(self.ws.parent).as_posix()
137
+
138
+ def which(self, binary):
139
+ return binary # it lives in the image; a missing one fails the run as an agent error
140
+
141
+ def env(self, agent):
142
+ return {**{k: v for k, v in os.environ.items() if k.upper() in CONTAINER_KEEP}, **_own(agent)}
143
+
144
+ def argv(self, name, env=(), stdin=False):
145
+ a = ["docker", "run", "--rm", "--name", name, "-v", f"{self.ws.parent}:/wa", "-w", self.cwd,
146
+ # the container runs as root and the mount is owned by you: without this, git refuses the workspace
147
+ "-e", "GIT_CONFIG_COUNT=1", "-e", "GIT_CONFIG_KEY_0=safe.directory", "-e", "GIT_CONFIG_VALUE_0=*"]
148
+ for m in self.mounts:
149
+ a += ["-v", m]
150
+ for k in env: # by name only: values travel in the docker CLI's environment, not on its command line
151
+ a += ["-e", k]
152
+ return a + (["-i"] if stdin else []) + [self.image]
153
+
154
+ def shell(self, cmd, timeout, log):
155
+ name = _name()
156
+ return proc(self.argv(name) + ["sh", "-c", cmd], self.ws, timeout, log, container=name)
157
+
158
+ def agent(self, argv, timeout, out, err, env, stdin):
159
+ name = _name()
160
+ return proc(self.argv(name, sorted(env), stdin is not None) + list(argv), self.ws, timeout, out, err=err,
161
+ env={**os.environ, **env}, stdin=stdin, container=name)
162
+
163
+ def close(self):
164
+ if hasattr(os, "getuid"): # files the container wrote as root go back to you, so the run dir can be deleted
165
+ subprocess.run(["docker", "run", "--rm", "-v", f"{self.ws.parent}:/wa", self.image, "chown", "-R",
166
+ f"{os.getuid()}:{os.getgid()}", "/wa"], capture_output=True)
167
+
168
+
169
+ def _name():
170
+ return f"whichagent-{os.getpid()}-{next(_ids)}"
171
+
172
+
173
+ def executor(cfg, ws):
174
+ if cfg["run"]["sandbox"] == "docker":
175
+ return Docker(ws, image(cfg["root"], cfg["docker"]["image"]), cfg["docker"]["mounts"])
176
+ return Host(ws)
177
+
178
+
179
+ def version(cfg, binary):
180
+ """First line of `<binary> --version`, from the image in docker mode. Recorded on every result."""
181
+ if cfg["run"]["sandbox"] == "docker":
182
+ return _version(("docker", "run", "--rm", image(cfg["root"], cfg["docker"]["image"]), binary, "--version"))
183
+ exe = shutil.which(binary)
184
+ return _version((exe, "--version")) if exe else None
185
+
186
+
187
+ @functools.cache
188
+ def _version(argv):
189
+ try:
190
+ r = subprocess.run(list(argv), capture_output=True, text=True, encoding="utf-8", errors="replace",
191
+ timeout=300)
192
+ except (OSError, subprocess.TimeoutExpired):
193
+ return None
194
+ lines = (r.stdout or r.stderr).strip().splitlines()
195
+ return lines[0] if lines else None
196
+
197
+
198
+ @functools.cache
199
+ def image(root, configured=""):
200
+ """The docker-mode image: [docker] image, else one built from .devcontainer/devcontainer.json or a Dockerfile."""
201
+ if configured:
202
+ return configured
203
+ dockerfile = context = None
204
+ for spec_path in (root / ".devcontainer" / "devcontainer.json", root / ".devcontainer.json"):
205
+ if spec_path.exists():
206
+ spec = json.loads(jsonc(spec_path.read_text("utf-8")))
207
+ if spec.get("image"):
208
+ return spec["image"]
209
+ build = spec.get("build") or {}
210
+ name = build.get("dockerfile") or spec.get("dockerFile")
211
+ if not name:
212
+ raise SystemExit(f"{spec_path}: docker mode needs an image or a dockerfile (compose isn't supported)")
213
+ dockerfile, context = spec_path.parent / name, spec_path.parent / build.get("context", ".")
214
+ break
215
+ if dockerfile is None and (root / "Dockerfile").exists():
216
+ dockerfile, context = root / "Dockerfile", root
217
+ if dockerfile is None:
218
+ raise SystemExit("Docker mode needs an image: set [docker] image, or add .devcontainer/devcontainer.json"
219
+ " or a Dockerfile")
220
+ tag = "whichagent-" + hashlib.sha256(dockerfile.read_bytes()).hexdigest()[:12]
221
+ if subprocess.run(["docker", "image", "inspect", tag], capture_output=True).returncode:
222
+ if subprocess.run(["docker", "build", "-t", tag, "-f", str(dockerfile), str(context)]).returncode:
223
+ raise SystemExit(f"docker build of {dockerfile} failed")
224
+ return tag
225
+
226
+
227
+ def jsonc(text):
228
+ """JSON with comments and trailing commas (devcontainer.json) as plain JSON."""
229
+ out, i, string = [], 0, False
230
+ while i < len(text):
231
+ ch = text[i]
232
+ if string:
233
+ out.append(text[i:i + 2] if ch == "\\" else ch)
234
+ i += 2 if ch == "\\" else 1
235
+ string = ch != '"'
236
+ continue
237
+ if text.startswith("//", i):
238
+ end = text.find("\n", i)
239
+ i = len(text) if end < 0 else end
240
+ continue
241
+ if text.startswith("/*", i):
242
+ end = text.find("*/", i + 2)
243
+ i = len(text) if end < 0 else end + 2
244
+ continue
245
+ string = ch == '"'
246
+ out.append(ch)
247
+ i += 1
248
+ return re.sub(r",(\s*[}\]])", r"\1", "".join(out))
whichagent/stacks.py ADDED
@@ -0,0 +1,55 @@
1
+ """Stack detection: [repo] install, test and suite commands for common toolchains, with per-test reports."""
2
+ import os
3
+
4
+
5
+ def detect(root):
6
+ """[repo] settings for the stack at root, or None when it isn't recognized."""
7
+ def has(name):
8
+ return (root / name).exists()
9
+ if has("uv.lock") or has("poetry.lock"):
10
+ tool = "uv" if has("uv.lock") else "poetry"
11
+ return {"install": "uv sync" if tool == "uv" else "poetry install",
12
+ "test": f"{tool} run pytest -q {{files}} --junitxml={{report}}",
13
+ "suite": f"{tool} run pytest -q --junitxml={{report}}"}
14
+ if node := _node(root):
15
+ return node
16
+ if has("go.mod"):
17
+ return {"install": "go mod download", "test": "go test -json {dirs} > {report}",
18
+ "suite": "go test -json ./... > {report}"}
19
+ if has("Cargo.toml"): # per-test results come from cargo's own output
20
+ return {"install": "cargo fetch", "test": "cargo test --no-fail-fast --test {names: --test }",
21
+ "suite": "cargo test --no-fail-fast"}
22
+ if has("pom.xml"):
23
+ mvn = "mvn -q -B"
24
+ return {"install": f"{mvn} -DskipTests test-compile",
25
+ "test": f"{mvn} test -Dtest={{names:,}} -Dsurefire.failIfNoSpecifiedTests=false"
26
+ " -Dsurefire.reportsDirectory={report}",
27
+ "suite": f"{mvn} test -Dsurefire.reportsDirectory={{report}}"}
28
+ if has("build.gradle") or has("build.gradle.kts"):
29
+ gradle = "gradlew" if os.name == "nt" else "./gradlew" # cmd.exe finds gradlew.bat in the current dir
30
+ return {"test": f"{gradle} test --tests {{names: --tests }}", "suite": f"{gradle} test",
31
+ "report": "build/test-results/test"}
32
+ return None
33
+
34
+
35
+ def _node(root):
36
+ pkg = (root / "package.json").read_text("utf-8") if (root / "package.json").exists() else ""
37
+ if (root / "package-lock.json").exists():
38
+ install, run = "npm ci", "npx"
39
+ elif (root / "pnpm-lock.yaml").exists():
40
+ install, run = "pnpm install --frozen-lockfile", "pnpm exec"
41
+ elif (root / "yarn.lock").exists():
42
+ berry = (root / ".yarnrc.yml").exists()
43
+ install, run = f"yarn install {'--immutable' if berry else '--frozen-lockfile'}", "yarn"
44
+ else:
45
+ return None
46
+ if '"vitest"' in pkg:
47
+ test = f"{run} vitest run {{files}} --reporter=default --reporter=junit --outputFile.junit={{report}}"
48
+ elif '"jest"' in pkg:
49
+ test = f"{run} jest {{files}} --json --outputFile={{report}}"
50
+ elif "node --test" in pkg:
51
+ test = ("node --test --test-reporter=spec --test-reporter=junit --test-reporter-destination=stdout"
52
+ " --test-reporter-destination={report} {files}")
53
+ else:
54
+ return None
55
+ return {"install": install, "test": test, "suite": test.replace(" {files}", ""), "cache": True}
whichagent/stats.py ADDED
@@ -0,0 +1,78 @@
1
+ """Solve rates with intervals, solved-every-time, cost per solved task and the paired sign test."""
2
+ import collections
3
+ import math
4
+ import random
5
+ import statistics
6
+
7
+
8
+ def wilson(k, n, z=1.96):
9
+ """95% Wilson interval for k successes in n: sane for small n and rates near 0% or 100%."""
10
+ if n == 0:
11
+ return 0.0, 1.0
12
+ p, d = k / n, 1 + z * z / n
13
+ center = (p + z * z / (2 * n)) / d
14
+ half = z * math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d
15
+ return max(0.0, center - half), min(1.0, center + half)
16
+
17
+
18
+ def bootstrap(rates, reps=2000, seed=0):
19
+ """95% interval for the mean per-task rate, resampling tasks: runs on one task aren't independent.
20
+
21
+ Never narrower than the task-level Wilson interval: on a few tasks that all score 100%, a plain
22
+ bootstrap would claim 100%-100%."""
23
+ rnd, n = random.Random(seed), len(rates)
24
+ means = sorted(sum(rnd.choices(rates, k=n)) / n for _ in range(reps))
25
+ lo, hi = wilson(sum(rates), n)
26
+ return min(lo, means[int(0.025 * reps)]), max(hi, means[int(0.975 * reps) - 1])
27
+
28
+
29
+ def sign_test(wins, losses):
30
+ """Exact two-sided sign test p-value (ties already dropped)."""
31
+ n = wins + losses
32
+ if n == 0:
33
+ return 1.0
34
+ return min(1.0, 2 * sum(math.comb(n, i) for i in range(min(wins, losses) + 1)) / 2 ** n)
35
+
36
+
37
+ def by_task(results):
38
+ """{task: [solved?, ...]}. Setup errors aren't the agent's fault, so they don't count."""
39
+ out = {}
40
+ for r in results:
41
+ if r["status"] != "setup_error":
42
+ out.setdefault(r["task"], []).append(r["status"] == "solved")
43
+ return out
44
+
45
+
46
+ def summary(results):
47
+ """One agent's numbers, or None if it has no gradable runs."""
48
+ tasks = by_task(results)
49
+ if not tasks:
50
+ return None
51
+ runs = [r for r in results if r["status"] != "setup_error"]
52
+ rates = [sum(v) / len(v) for v in tasks.values()]
53
+ one_each = all(len(v) == 1 for v in tasks.values())
54
+ known = [r["cost_usd"] for r in runs if r["cost_usd"] is not None]
55
+ total = sum(known) if known else None
56
+ solved = sum(r["status"] == "solved" for r in runs)
57
+ secs = [r["secs"] for r in runs if r["secs"] is not None]
58
+ return {"tasks": len(tasks), "runs": len(runs), "solved": solved, "rate": sum(rates) / len(rates),
59
+ "ci": wilson(sum(rates), len(rates)) if one_each else bootstrap(rates),
60
+ "every": None if one_each else sum(all(v) for v in tasks.values()) / len(tasks),
61
+ "per_run": total / len(known) if known else None,
62
+ "per_solved": total / solved if known and solved else None,
63
+ "cost_complete": len(known) == len(runs),
64
+ "secs": statistics.median(secs) if secs else None,
65
+ "status": collections.Counter(r["status"] for r in results),
66
+ "tampered": sum(bool(r.get("tampered")) for r in runs),
67
+ "regressions": sum(r.get("regression") is True for r in runs),
68
+ "flagged": sum(bool(r.get("flagged")) for r in runs)} # edited pyproject.toml, package.json, ...
69
+
70
+
71
+ def compare(ra, rb):
72
+ """(wins, losses, ties, p) for agent A vs B on the tasks both ran: A wins a task with a higher solve rate."""
73
+ a, b = by_task(ra), by_task(rb)
74
+ shared = a.keys() & b.keys()
75
+ rate = {t: (sum(a[t]) / len(a[t]), sum(b[t]) / len(b[t])) for t in shared}
76
+ wins = sum(x > y for x, y in rate.values())
77
+ losses = sum(x < y for x, y in rate.values())
78
+ return wins, losses, len(shared) - wins - losses, sign_test(wins, losses)
whichagent/store.py ADDED
@@ -0,0 +1,46 @@
1
+ """The .whichagent/ directory: tasks, results and run logs, kept as append-only JSONL so every command can resume."""
2
+ import json
3
+ import re
4
+ from pathlib import Path
5
+
6
+
7
+ def state(root):
8
+ """.whichagent/ in the repo, created with a .gitignore that keeps it out of git."""
9
+ d = Path(root) / ".whichagent"
10
+ if not d.exists():
11
+ d.mkdir()
12
+ (d / ".gitignore").write_text("*\n")
13
+ return d
14
+
15
+
16
+ def read(path):
17
+ out = []
18
+ if Path(path).exists():
19
+ for line in Path(path).read_text("utf-8").splitlines():
20
+ try:
21
+ out.append(json.loads(line))
22
+ except ValueError:
23
+ pass # a line cut short by a crash
24
+ return out
25
+
26
+
27
+ def append(path, obj):
28
+ with open(path, "a", encoding="utf-8") as f:
29
+ f.write(json.dumps(obj) + "\n")
30
+
31
+
32
+ def latest(results, by_version=False):
33
+ """The last result per run: a retried run replaces the earlier one. With by_version, runs of different
34
+ agent versions are kept apart instead."""
35
+ def key(r):
36
+ return r["task"], r["agent"], r["attempt"], r.get("agent_version") if by_version else None
37
+ return list({key(r): r for r in results}.values())
38
+
39
+
40
+ def slug(name):
41
+ return re.sub(r"[^\w.-]+", "_", name).strip("_") or "x"
42
+
43
+
44
+ def run_dir(root, task_id, agent, version, attempt):
45
+ """Prompt, logs, diff and test output of one run."""
46
+ return state(root) / "runs" / task_id / slug(f"{agent}@{version}" if version else agent) / str(attempt)
@@ -0,0 +1,91 @@
1
+ """Per-test results from test runners: JUnit XML (pytest, vitest, node, surefire, gradle), Jest JSON,
2
+ go test -json, and cargo test's own output."""
3
+ import json
4
+ import re
5
+ import xml.etree.ElementTree as ET
6
+ from pathlib import Path
7
+
8
+
9
+ def parse(report, log="", roots=()):
10
+ """{test id: passed} from report (a file, or a dir of XML files), else from the log; None when neither has any.
11
+
12
+ Skipped tests are left out. Paths under roots are made relative, so ids match across workspaces."""
13
+ out = {}
14
+ p = Path(report) if report else None
15
+ files = sorted(p.rglob("*.xml")) if p and p.is_dir() else [p] if p and p.is_file() else []
16
+ for f in files:
17
+ _merge(out, _report(f.read_text("utf-8", "replace")), roots)
18
+ if not files:
19
+ _merge(out, _cargo(log), roots)
20
+ return out or None
21
+
22
+
23
+ def _merge(out, results, roots):
24
+ for key, ok in results.items():
25
+ for root in roots:
26
+ key = key.replace(root + "/", "").replace(root + "\\", "")
27
+ out[key] = out.get(key, True) and ok # a test reported twice passes only if it passed both times
28
+
29
+
30
+ def _report(text):
31
+ text = text.lstrip()
32
+ if text.startswith("<"):
33
+ return _junit(text)
34
+ try:
35
+ data = json.loads(text)
36
+ except ValueError:
37
+ return _go(text)
38
+ return _jest(data) if isinstance(data, dict) and "testResults" in data else _go(text)
39
+
40
+
41
+ def _junit(text):
42
+ out = {}
43
+ try:
44
+ root = ET.fromstring(text)
45
+ except ET.ParseError:
46
+ return out
47
+
48
+ def walk(nodes, suites):
49
+ for n in nodes:
50
+ if n.tag in ("testsuites", "testsuite"):
51
+ walk(n, suites + [n.get("name", "")] if n.tag == "testsuite" else suites)
52
+ elif n.tag == "testcase" and n.find("skipped") is None:
53
+ key = "::".join(x for x in [*suites, n.get("file", ""), n.get("classname", ""), n.get("name", "")] if x)
54
+ out[key] = out.get(key, True) and n.find("failure") is None and n.find("error") is None
55
+ walk([root], [])
56
+ return out
57
+
58
+
59
+ def _jest(data):
60
+ out = {}
61
+ for f in data.get("testResults") or []:
62
+ cases = f.get("assertionResults") or []
63
+ for a in cases:
64
+ if a.get("status") in ("passed", "failed"):
65
+ key = f"{f.get('name', '')}::{a.get('fullName') or a.get('title')}"
66
+ out[key] = out.get(key, True) and a["status"] == "passed"
67
+ if not cases and f.get("status") == "failed":
68
+ out[f"{f.get('name', '')}::(file)"] = False # the file didn't even load
69
+ return out
70
+
71
+
72
+ def _go(text):
73
+ out = {}
74
+ for line in text.splitlines():
75
+ try:
76
+ e = json.loads(line)
77
+ except ValueError:
78
+ continue
79
+ if isinstance(e, dict) and e.get("Action") in ("pass", "fail"):
80
+ out[f"{e.get('Package', '')}::{e.get('Test') or '(package)'}"] = e["Action"] == "pass"
81
+ return out
82
+
83
+
84
+ def _cargo(log):
85
+ out, binary = {}, ""
86
+ for line in log.splitlines():
87
+ if m := re.match(r"\s*Running (.+?)(?: \(|$)", line):
88
+ binary = m[1]
89
+ elif m := re.match(r"test (\S+) \.\.\. (ok|FAILED)", line):
90
+ out[f"{binary}::{m[1]}"] = m[2] == "ok"
91
+ return out
@@ -0,0 +1,94 @@
1
+ """Ready-to-use workspaces: a leak-free clone at the base, dependencies (restored from the cache when the
2
+ lockfiles match), setup. Mining, running and doctor all use these, so tasks are validated where agents run."""
3
+ import hashlib
4
+ import os
5
+ import shutil
6
+ import tempfile
7
+ import threading
8
+ from pathlib import Path
9
+
10
+ from . import config as c, git as g, sandbox as sb, store
11
+
12
+ KEEP = 3 # dependency sets cached per repo, most recently used first; each can be hundreds of MB
13
+
14
+
15
+ def create(cfg, task_id, base):
16
+ """A fresh clone of the task's base in the temp dir, and the executor that runs commands in it."""
17
+ root = cfg["root"]
18
+ bare = g.base_clone(root, base, store.state(root) / "bases" / f"{task_id}.git", cfg["mine"]["history_depth"])
19
+ ws = g.workspace(bare, cfg["branch"], root.name)
20
+ return ws, sb.executor(cfg, ws)
21
+
22
+
23
+ def prepare(cfg, ws, box, logs):
24
+ """Install dependencies (or restore them from the cache), then run setup. False when a step fails.
25
+
26
+ Output goes to install.log and setup.log in logs."""
27
+ rp, timeout = cfg["repo"], c.secs(cfg["repo"]["setup_timeout"])
28
+ if rp["install"]:
29
+ key = _key(cfg, ws)
30
+ if not (rp["cache"] and _restore(cfg, key, ws)):
31
+ if box.shell(rp["install"], timeout, logs / "install.log")[0] != 0:
32
+ return False
33
+ if rp["cache"]:
34
+ _save(cfg, key, ws)
35
+ if rp["setup"] and box.shell(rp["setup"], timeout, logs / "setup.log")[0] != 0:
36
+ return False
37
+ g.exclude_untracked(ws)
38
+ return True
39
+
40
+
41
+ def remove(ws, box):
42
+ box.close()
43
+ g.rmtree(ws.parent)
44
+
45
+
46
+ def cache_dir(cfg):
47
+ """In the temp dir, like workspaces: a dependency cache in a synced folder would sync gigabytes."""
48
+ root = cfg["root"]
49
+ tag = hashlib.sha256(str(root).encode()).hexdigest()[:8]
50
+ return Path(tempfile.gettempdir()) / "whichagent-cache" / f"{store.slug(root.name)}-{tag}"
51
+
52
+
53
+ def _key(cfg, ws):
54
+ """The install command plus every lockfile's content: equal keys mean equal dependencies."""
55
+ h = hashlib.sha256(cfg["repo"]["install"].encode())
56
+ for p in sorted(g.git(ws, "ls-files").splitlines()):
57
+ if c.match(p, cfg["repo"]["skip_if_changed"]):
58
+ h.update(p.encode() + b"\0" + Path(ws, p).read_bytes())
59
+ return h.hexdigest()[:16]
60
+
61
+
62
+ def _save(cfg, key, ws):
63
+ """Cache everything install created (node_modules at every level, and so on)."""
64
+ dest = cache_dir(cfg) / key
65
+ if dest.exists():
66
+ return
67
+ tmp = dest.with_name(f"{key}.tmp-{os.getpid()}-{threading.get_ident()}")
68
+ try:
69
+ tmp.mkdir(parents=True)
70
+ for p in g.git(ws, "ls-files", "--others", "--directory").splitlines():
71
+ src, to = ws / p.rstrip("/"), tmp / p.rstrip("/")
72
+ to.parent.mkdir(parents=True, exist_ok=True)
73
+ if src.is_dir():
74
+ shutil.copytree(src, to, symlinks=True)
75
+ elif src.exists():
76
+ shutil.copy2(src, to, follow_symlinks=False)
77
+ tmp.rename(dest)
78
+ except OSError:
79
+ g.rmtree(tmp) # another worker saved it first, or the disk is full: the cache is only a speedup
80
+ old = sorted((p for p in dest.parent.iterdir() if ".tmp-" not in p.name), key=lambda p: p.stat().st_mtime)
81
+ for p in old[:-KEEP]:
82
+ g.rmtree(p)
83
+
84
+
85
+ def _restore(cfg, key, ws):
86
+ src = cache_dir(cfg) / key
87
+ if not src.is_dir():
88
+ return False
89
+ try:
90
+ os.utime(src) # most recently used
91
+ shutil.copytree(src, ws, symlinks=True, dirs_exist_ok=True)
92
+ return True
93
+ except OSError:
94
+ return False # evicted meanwhile: install instead