roller-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- roller/__init__.py +31 -0
- roller/__main__.py +6 -0
- roller/actf.py +431 -0
- roller/artifacts.py +124 -0
- roller/assets/deepseek/journal.mjs +44 -0
- roller/assets/deepseek/roller.patch.yml +15 -0
- roller/assets/deepseek/runner.py +149 -0
- roller/assets/hermes/runner.py +322 -0
- roller/assets/openclaw_capture.py +32 -0
- roller/assets/openclaw_drain.mjs +21 -0
- roller/assets/opencode/journal.mjs +42 -0
- roller/assets/opencode/runner.py +203 -0
- roller/assets/pi/runner.mjs +160 -0
- roller/assets/pi/runner.py +93 -0
- roller/benches/__init__.py +73 -0
- roller/benches/base.py +306 -0
- roller/benches/grading.py +553 -0
- roller/benches/pinchbench.py +976 -0
- roller/benches/qwenclaw/__init__.py +49 -0
- roller/benches/qwenclaw/adapter.py +167 -0
- roller/benches/qwenclaw/grader_runtime.py +395 -0
- roller/benches/qwenclaw/report.py +49 -0
- roller/benches/qwenclaw/task_parser.py +181 -0
- roller/benches/qwenclaw/workspace.py +202 -0
- roller/benches/report.py +166 -0
- roller/benches/skillsbench.py +708 -0
- roller/benches/transcripts.py +102 -0
- roller/benches/wildclaw.py +569 -0
- roller/bridge_capture.py +73 -0
- roller/bundles.py +545 -0
- roller/capture.py +385 -0
- roller/capture_protocol.py +301 -0
- roller/capture_runtime.py +48 -0
- roller/capture_states.py +37 -0
- roller/claude_capture.py +185 -0
- roller/claude_code_actf.py +594 -0
- roller/cli.py +1732 -0
- roller/codex_actf.py +584 -0
- roller/codex_capture.py +99 -0
- roller/codex_tools.py +137 -0
- roller/collector.py +818 -0
- roller/compaction_sft.py +672 -0
- roller/conversation.py +432 -0
- roller/deepseek_actf.py +135 -0
- roller/harness_registry.py +61 -0
- roller/harnesses/__init__.py +29 -0
- roller/harnesses/base.py +64 -0
- roller/harnesses/bridge.py +448 -0
- roller/harnesses/claude_code.py +839 -0
- roller/harnesses/codex.py +808 -0
- roller/harnesses/config_overrides.py +100 -0
- roller/harnesses/container.py +206 -0
- roller/harnesses/deepseek.py +104 -0
- roller/harnesses/docker.py +72 -0
- roller/harnesses/gateway_ports.py +155 -0
- roller/harnesses/hermes.py +119 -0
- roller/harnesses/openclaw.py +1605 -0
- roller/harnesses/opencode.py +193 -0
- roller/harnesses/pi.py +111 -0
- roller/hermes_actf.py +147 -0
- roller/image_build.py +206 -0
- roller/integrity.py +332 -0
- roller/model_protocols.py +201 -0
- roller/models.py +169 -0
- roller/openclaw_capture.py +140 -0
- roller/openclaw_delivery.py +211 -0
- roller/openclaw_usage.py +59 -0
- roller/opencode_actf.py +305 -0
- roller/opencode_capture.py +96 -0
- roller/pi_actf.py +172 -0
- roller/response_metadata.py +89 -0
- roller/run_lock.py +297 -0
- roller/sft.py +471 -0
- roller/storage.py +53 -0
- roller/tasks.py +689 -0
- roller/thinking.py +50 -0
- roller/thinking_capabilities.py +152 -0
- roller/trajectory_audit.py +386 -0
- roller/usage.py +168 -0
- roller/version.py +1 -0
- roller/viewer/__init__.py +3 -0
- roller/viewer/app.py +722 -0
- roller/viewer/static/app.js +752 -0
- roller/viewer/static/index.html +105 -0
- roller/viewer/static/style.css +213 -0
- roller/wire_trace.py +650 -0
- roller_agent-0.2.0.dist-info/METADATA +397 -0
- roller_agent-0.2.0.dist-info/RECORD +92 -0
- roller_agent-0.2.0.dist-info/WHEEL +5 -0
- roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
- roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
- roller_agent-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,708 @@
|
|
|
1
|
+
"""SkillsBench adapter with task environments and skill ablation variants."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
from collections.abc import Iterable, Sequence
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
import yaml
|
|
15
|
+
|
|
16
|
+
from roller.benches.base import (
|
|
17
|
+
BenchAdapter,
|
|
18
|
+
BenchBinding,
|
|
19
|
+
BenchFormatError,
|
|
20
|
+
GradeSpec,
|
|
21
|
+
validate_timeout_multiplier,
|
|
22
|
+
)
|
|
23
|
+
from roller.benches.report import REPORT_SCHEMA, sample_score
|
|
24
|
+
|
|
25
|
+
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)", re.DOTALL)
|
|
26
|
+
WORKDIR_RE = re.compile(r"^\s*WORKDIR\s+(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
|
|
27
|
+
ENV_RE = re.compile(r"^\s*ENV\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$", re.MULTILINE)
|
|
28
|
+
SKILL_COPY_RE = re.compile(r"^\s*COPY\s+(?:_deps/)?skills(?:/)?\s+.*$", re.MULTILINE)
|
|
29
|
+
HEREDOC_RE = re.compile(r"<<-?['\"]?([A-Za-z_][A-Za-z0-9_]*)['\"]?\s*$")
|
|
30
|
+
PYTHON_STDIN_RE = re.compile(r"\b(python3?\s+-)\s+<<-?['\"]?[A-Za-z_][A-Za-z0-9_]*['\"]?\s*$")
|
|
31
|
+
CAT_REDIRECT_RE = re.compile(r"\bcat\s+>\s+([^\s]+)\s+<<-?['\"]?[A-Za-z_][A-Za-z0-9_]*['\"]?\s*$")
|
|
32
|
+
FROM_RE = re.compile(r"^(\s*FROM\s+(?:--platform=\S+\s+)?)(\S+)(.*)$", re.IGNORECASE)
|
|
33
|
+
VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
|
|
34
|
+
RESOURCE_KEYS = {
|
|
35
|
+
"network_mode",
|
|
36
|
+
"build_timeout_sec",
|
|
37
|
+
"os",
|
|
38
|
+
"cpus",
|
|
39
|
+
"memory_mb",
|
|
40
|
+
"storage_mb",
|
|
41
|
+
"gpus",
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
_VERIFIER_RUNNER = r'''
|
|
45
|
+
import os
|
|
46
|
+
from pathlib import Path
|
|
47
|
+
import subprocess
|
|
48
|
+
import tempfile
|
|
49
|
+
import time
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
_RETRYABLE_INFRASTRUCTURE_ERRORS = {
|
|
53
|
+
"verifier dependency download failed",
|
|
54
|
+
"verifier DNS failure",
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _infrastructure_error(output):
|
|
59
|
+
lowered = output.lower()
|
|
60
|
+
tests_completed = (
|
|
61
|
+
"short test summary info" in lowered
|
|
62
|
+
or (
|
|
63
|
+
"test session starts" in lowered
|
|
64
|
+
and any(marker in lowered for marker in (" passed in ", " failed in ", " error in "))
|
|
65
|
+
)
|
|
66
|
+
)
|
|
67
|
+
markers = (
|
|
68
|
+
("uvx: command not found", "uvx is unavailable", False),
|
|
69
|
+
("uv: command not found", "uv is unavailable", False),
|
|
70
|
+
("failed to download", "verifier dependency download failed", True),
|
|
71
|
+
("temporary failure in name resolution", "verifier DNS failure", True),
|
|
72
|
+
("could not resolve host", "verifier DNS failure", True),
|
|
73
|
+
("executable doesn't exist at", "Playwright browser is unavailable", False),
|
|
74
|
+
("please run the following command to download new browsers", "Playwright browser is unavailable", False),
|
|
75
|
+
("browser executable not found", "Playwright browser is unavailable", False),
|
|
76
|
+
("err_access_denied", "local verifier request was intercepted by a proxy", False),
|
|
77
|
+
)
|
|
78
|
+
for marker, message, ignore_after_tests in markers:
|
|
79
|
+
if marker in lowered:
|
|
80
|
+
if ignore_after_tests and tests_completed:
|
|
81
|
+
continue
|
|
82
|
+
return message
|
|
83
|
+
return None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def roller_grade(workspace_path, task_workspace, verifier_env):
|
|
87
|
+
environment = dict(os.environ)
|
|
88
|
+
environment.update({str(key): str(value) for key, value in verifier_env.items()})
|
|
89
|
+
reward_path = Path("/logs/verifier/reward.txt")
|
|
90
|
+
reward_path.parent.mkdir(parents=True, exist_ok=True)
|
|
91
|
+
# Official harnesses provide these mount points. Roller stages the verifier
|
|
92
|
+
# into the live task container, so create their empty equivalents here.
|
|
93
|
+
Path("/tests").mkdir(parents=True, exist_ok=True)
|
|
94
|
+
# v1.1 verifier scripts repeatedly bootstrap uv from the network and then
|
|
95
|
+
# source this file. The task overlay already ships pinned uv/uvx binaries;
|
|
96
|
+
# this fallback lets verification proceed when the redundant download is
|
|
97
|
+
# interrupted by a cluster proxy.
|
|
98
|
+
local_bin = Path.home() / ".local" / "bin"
|
|
99
|
+
local_bin.mkdir(parents=True, exist_ok=True)
|
|
100
|
+
env_path = local_bin / "env"
|
|
101
|
+
if not env_path.exists():
|
|
102
|
+
env_path.write_text('export PATH="$HOME/.local/bin:$PATH"\n', encoding="utf-8")
|
|
103
|
+
result = None
|
|
104
|
+
output = ""
|
|
105
|
+
verifier_attempts = 0
|
|
106
|
+
for verifier_attempts in range(1, 4):
|
|
107
|
+
reward_path.unlink(missing_ok=True)
|
|
108
|
+
# Some upstream verifiers leave a background service alive after their
|
|
109
|
+
# shell exits. A PIPE would remain open in that descendant and make
|
|
110
|
+
# subprocess.run() wait until the outer timeout despite completed tests.
|
|
111
|
+
# A regular file lets us observe the verifier process itself, while the
|
|
112
|
+
# disposable task container still cleans up any leftover service.
|
|
113
|
+
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log:
|
|
114
|
+
result = subprocess.run(
|
|
115
|
+
["/bin/bash", "/verifier/test.sh"],
|
|
116
|
+
cwd=task_workspace,
|
|
117
|
+
env=environment,
|
|
118
|
+
stdout=log,
|
|
119
|
+
stderr=subprocess.STDOUT,
|
|
120
|
+
text=True,
|
|
121
|
+
timeout=__VERIFIER_TIMEOUT__,
|
|
122
|
+
)
|
|
123
|
+
log.flush()
|
|
124
|
+
output = Path(log.name).read_text(encoding="utf-8", errors="replace")
|
|
125
|
+
infrastructure_error = _infrastructure_error(output)
|
|
126
|
+
if (
|
|
127
|
+
infrastructure_error not in _RETRYABLE_INFRASTRUCTURE_ERRORS
|
|
128
|
+
or verifier_attempts == 3
|
|
129
|
+
):
|
|
130
|
+
break
|
|
131
|
+
time.sleep(5 * verifier_attempts)
|
|
132
|
+
assert result is not None
|
|
133
|
+
if not reward_path.is_file():
|
|
134
|
+
raise RuntimeError(
|
|
135
|
+
"SkillsBench verifier produced no /logs/verifier/reward.txt: "
|
|
136
|
+
+ output[-2000:]
|
|
137
|
+
)
|
|
138
|
+
try:
|
|
139
|
+
reward = float(reward_path.read_text(encoding="utf-8").strip())
|
|
140
|
+
except ValueError as exc:
|
|
141
|
+
raise RuntimeError("SkillsBench verifier reward is not numeric") from exc
|
|
142
|
+
payload = {
|
|
143
|
+
"overall_score": max(0.0, min(1.0, reward)),
|
|
144
|
+
"passed": reward > 0,
|
|
145
|
+
"score_eligible": True,
|
|
146
|
+
"verifier_exit_code": result.returncode,
|
|
147
|
+
"verifier_attempts": verifier_attempts,
|
|
148
|
+
"verifier_output": output[-8000:],
|
|
149
|
+
}
|
|
150
|
+
infrastructure_error = _infrastructure_error(output)
|
|
151
|
+
if infrastructure_error:
|
|
152
|
+
payload["score_eligible"] = False
|
|
153
|
+
payload["error"] = infrastructure_error
|
|
154
|
+
return payload
|
|
155
|
+
'''
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
_INFRASTRUCTURE_OUTPUT_MARKERS = (
|
|
159
|
+
"uvx: command not found",
|
|
160
|
+
"uv: command not found",
|
|
161
|
+
"failed to download",
|
|
162
|
+
"temporary failure in name resolution",
|
|
163
|
+
"could not resolve host",
|
|
164
|
+
"executable doesn't exist at",
|
|
165
|
+
"please run the following command to download new browsers",
|
|
166
|
+
"browser executable not found",
|
|
167
|
+
"err_access_denied",
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _score_error(row: dict[str, Any], scores: dict[str, Any] | None) -> str | None:
|
|
172
|
+
error = row.get("bench_error")
|
|
173
|
+
if isinstance(error, str) and error:
|
|
174
|
+
return error
|
|
175
|
+
if not isinstance(scores, dict):
|
|
176
|
+
return None
|
|
177
|
+
error = scores.get("error")
|
|
178
|
+
if isinstance(error, str) and error:
|
|
179
|
+
return error
|
|
180
|
+
output = scores.get("verifier_output")
|
|
181
|
+
if isinstance(output, str):
|
|
182
|
+
lowered = output.lower()
|
|
183
|
+
tests_completed = (
|
|
184
|
+
"short test summary info" in lowered
|
|
185
|
+
or (
|
|
186
|
+
"test session starts" in lowered
|
|
187
|
+
and any(
|
|
188
|
+
marker in lowered
|
|
189
|
+
for marker in (" passed in ", " failed in ", " error in ")
|
|
190
|
+
)
|
|
191
|
+
)
|
|
192
|
+
)
|
|
193
|
+
for marker in _INFRASTRUCTURE_OUTPUT_MARKERS:
|
|
194
|
+
if marker in lowered:
|
|
195
|
+
if marker in {
|
|
196
|
+
"failed to download",
|
|
197
|
+
"temporary failure in name resolution",
|
|
198
|
+
"could not resolve host",
|
|
199
|
+
} and tests_completed:
|
|
200
|
+
continue
|
|
201
|
+
return f"verifier infrastructure failure: {marker}"
|
|
202
|
+
return None
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _task_metadata(path: Path) -> tuple[dict[str, Any], str]:
|
|
206
|
+
match = FRONTMATTER_RE.match(path.read_text(encoding="utf-8"))
|
|
207
|
+
if match is None:
|
|
208
|
+
raise BenchFormatError(f"{path}: missing YAML frontmatter")
|
|
209
|
+
try:
|
|
210
|
+
metadata = yaml.safe_load(match.group(1)) or {}
|
|
211
|
+
except yaml.YAMLError as exc:
|
|
212
|
+
raise BenchFormatError(f"{path}: invalid YAML frontmatter: {exc}") from exc
|
|
213
|
+
if not isinstance(metadata, dict):
|
|
214
|
+
raise BenchFormatError(f"{path}: frontmatter must be an object")
|
|
215
|
+
prompt = match.group(2).strip()
|
|
216
|
+
if not prompt:
|
|
217
|
+
raise BenchFormatError(f"{path}: task prompt is empty")
|
|
218
|
+
return metadata, prompt
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def _portable_heredocs(value: str) -> str:
|
|
222
|
+
"""Rewrite Dockerfile heredocs for engines whose parser predates BuildKit.
|
|
223
|
+
|
|
224
|
+
SkillsBench currently uses Python-stdin, file-writing, and direct RUN
|
|
225
|
+
heredocs. Their shell-equivalent forms work in both Docker and Podman 4.x.
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
lines = value.splitlines()
|
|
229
|
+
output: list[str] = []
|
|
230
|
+
index = 0
|
|
231
|
+
while index < len(lines):
|
|
232
|
+
line = lines[index]
|
|
233
|
+
marker = HEREDOC_RE.search(line)
|
|
234
|
+
if marker is None:
|
|
235
|
+
output.append(line)
|
|
236
|
+
index += 1
|
|
237
|
+
continue
|
|
238
|
+
delimiter = marker.group(1)
|
|
239
|
+
end = index + 1
|
|
240
|
+
while end < len(lines) and lines[end].strip() != delimiter:
|
|
241
|
+
end += 1
|
|
242
|
+
if end == len(lines):
|
|
243
|
+
raise BenchFormatError(f"unterminated Dockerfile heredoc {delimiter!r}")
|
|
244
|
+
body = "\n".join(lines[index + 1 : end]) + "\n"
|
|
245
|
+
if ".from_pretrained(" in body:
|
|
246
|
+
original = base64.b64encode(body.encode("utf-8")).decode("ascii")
|
|
247
|
+
backup = (
|
|
248
|
+
"\n import seisbench\n"
|
|
249
|
+
" seisbench.use_backup_repository()"
|
|
250
|
+
if "seisbench.models" in body
|
|
251
|
+
else ""
|
|
252
|
+
)
|
|
253
|
+
body = f'''import base64
|
|
254
|
+
import time
|
|
255
|
+
|
|
256
|
+
source = base64.b64decode({original!r}).decode("utf-8")
|
|
257
|
+
last_error = None
|
|
258
|
+
for attempt in range(6):
|
|
259
|
+
try:
|
|
260
|
+
exec(compile(source, "<skillsbench-preload>", "exec"), globals())
|
|
261
|
+
break
|
|
262
|
+
except Exception as exc:
|
|
263
|
+
last_error = exc
|
|
264
|
+
if attempt == 0:{backup or chr(10) + " pass"}
|
|
265
|
+
if attempt == 5:
|
|
266
|
+
raise
|
|
267
|
+
time.sleep(20 * (attempt + 1))
|
|
268
|
+
else:
|
|
269
|
+
raise RuntimeError("SkillsBench preload failed") from last_error
|
|
270
|
+
'''
|
|
271
|
+
encoded = base64.b64encode(body.encode("utf-8")).decode("ascii")
|
|
272
|
+
# SeisBench 0.10 imports pkg_resources, which was removed from recent
|
|
273
|
+
# setuptools releases selected by an otherwise-unpinned upstream pip
|
|
274
|
+
# install. Pin the last compatible series immediately before its cache
|
|
275
|
+
# warm-up script so current clean builds remain reproducible.
|
|
276
|
+
compatibility = (
|
|
277
|
+
"pip install --no-cache-dir 'setuptools<81' && "
|
|
278
|
+
if "import seisbench" in body
|
|
279
|
+
else ""
|
|
280
|
+
)
|
|
281
|
+
python_match = PYTHON_STDIN_RE.search(line)
|
|
282
|
+
cat_match = CAT_REDIRECT_RE.search(line)
|
|
283
|
+
if python_match is not None:
|
|
284
|
+
replacement = (
|
|
285
|
+
f"{compatibility}printf %s {encoded} | base64 -d | "
|
|
286
|
+
f"{python_match.group(1)}"
|
|
287
|
+
)
|
|
288
|
+
output.append(line[: python_match.start()] + replacement)
|
|
289
|
+
elif cat_match is not None:
|
|
290
|
+
replacement = f"printf %s {encoded} | base64 -d > {cat_match.group(1)}"
|
|
291
|
+
output.append(line[: cat_match.start()] + replacement)
|
|
292
|
+
elif line.strip().upper().startswith("RUN <<"):
|
|
293
|
+
shell = "/bin/bash" if "pipefail" in body else "/bin/sh"
|
|
294
|
+
output.append(f"RUN printf %s {encoded} | base64 -d | {shell}")
|
|
295
|
+
else:
|
|
296
|
+
raise BenchFormatError(
|
|
297
|
+
f"unsupported Dockerfile heredoc syntax: {line.strip()}"
|
|
298
|
+
)
|
|
299
|
+
index = end + 1
|
|
300
|
+
return "\n".join(output)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def _qualified_from_images(value: str) -> str:
|
|
304
|
+
"""Apply Docker Hub's implicit namespace rule for strict Podman hosts."""
|
|
305
|
+
|
|
306
|
+
output: list[str] = []
|
|
307
|
+
for line in value.splitlines():
|
|
308
|
+
match = FROM_RE.match(line)
|
|
309
|
+
if match is None:
|
|
310
|
+
output.append(line)
|
|
311
|
+
continue
|
|
312
|
+
image = match.group(2)
|
|
313
|
+
first_component = image.split("/", 1)[0]
|
|
314
|
+
if (
|
|
315
|
+
"/" in image
|
|
316
|
+
and not image.startswith("$")
|
|
317
|
+
and "." not in first_component
|
|
318
|
+
and ":" not in first_component
|
|
319
|
+
and first_component != "localhost"
|
|
320
|
+
):
|
|
321
|
+
image = "docker.io/" + image
|
|
322
|
+
output.append(match.group(1) + image + match.group(3))
|
|
323
|
+
return "\n".join(output)
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
def _portable_tar_extraction(value: str) -> str:
|
|
327
|
+
"""Prevent archive ownership metadata from breaking rootless builds."""
|
|
328
|
+
|
|
329
|
+
output: list[str] = []
|
|
330
|
+
for line in value.splitlines():
|
|
331
|
+
extraction = re.search(
|
|
332
|
+
r"(?:^\s*|[;&|]\s*|\bRUN\s+)tar\s+"
|
|
333
|
+
r"(?:--[\w-]+(?:=[^\s]+)?\s+)*-[A-Za-z]*x[A-Za-z]*\b",
|
|
334
|
+
line,
|
|
335
|
+
re.IGNORECASE,
|
|
336
|
+
)
|
|
337
|
+
if extraction is not None:
|
|
338
|
+
if "--no-same-owner" not in line:
|
|
339
|
+
line = re.sub(
|
|
340
|
+
r"\btar\s+",
|
|
341
|
+
"tar --no-same-owner --no-same-permissions ",
|
|
342
|
+
line,
|
|
343
|
+
count=1,
|
|
344
|
+
flags=re.IGNORECASE,
|
|
345
|
+
)
|
|
346
|
+
output.append(line)
|
|
347
|
+
return "\n".join(output)
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def _sanitized_dockerfile(path: Path) -> str:
|
|
351
|
+
value = _portable_heredocs(path.read_text(encoding="utf-8"))
|
|
352
|
+
value = _qualified_from_images(value)
|
|
353
|
+
value = _portable_tar_extraction(value)
|
|
354
|
+
# v1.1 predates the upstream PyCBC compatibility pin. Retain the release
|
|
355
|
+
# task while applying the same dependency-only fix now present on main.
|
|
356
|
+
if "pycbc==2.8.5" in value and "astropy==7.2.1" not in value:
|
|
357
|
+
value = value.replace(
|
|
358
|
+
' "setuptools<81" \\\n',
|
|
359
|
+
' "setuptools<81" \\\n "astropy==7.2.1" \\\n',
|
|
360
|
+
1,
|
|
361
|
+
)
|
|
362
|
+
# Coursier's interactive setup installs a newer copy of itself from GitHub
|
|
363
|
+
# before installing Scala. That extra download is unnecessary and is less
|
|
364
|
+
# reliable behind evaluation-cluster proxies; the pinned launcher already
|
|
365
|
+
# present in the image can install the requested Scala binaries directly.
|
|
366
|
+
if "coursier/releases/download/" in value:
|
|
367
|
+
value = value.replace("curl -fL ", "curl -fL --retry 5 --retry-all-errors --retry-delay 5 ")
|
|
368
|
+
value = value.replace("./cs setup --yes && ./cs install ", "./cs install ")
|
|
369
|
+
# A no-skill run must not inherit task skills accidentally. Runtime staging
|
|
370
|
+
# injects them only for the with-skill condition.
|
|
371
|
+
value = SKILL_COPY_RE.sub("# Roller: task skills are injected at runtime", value)
|
|
372
|
+
return value.rstrip() + "\n"
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _workspace_from_dockerfile(source: str) -> str:
|
|
376
|
+
environment = {name: value.strip('"\'') for name, value in ENV_RE.findall(source)}
|
|
377
|
+
matches = WORKDIR_RE.findall(source)
|
|
378
|
+
value = matches[-1].strip().strip('"\'') if matches else "/root"
|
|
379
|
+
value = VAR_RE.sub(lambda match: environment.get(match.group(1), match.group(0)), value)
|
|
380
|
+
return value if value.startswith("/") else "/root"
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def _environment_digest(root: Path, dockerfile: str) -> str:
|
|
384
|
+
digest = hashlib.sha256()
|
|
385
|
+
# Bump this salt whenever the harness overlay contract changes so a stale
|
|
386
|
+
# locally tagged task image cannot silently bypass the new layer.
|
|
387
|
+
digest.update(b"roller-skillsbench-opencode-v2\0")
|
|
388
|
+
digest.update(dockerfile.encode("utf-8"))
|
|
389
|
+
for path in sorted(root.rglob("*")):
|
|
390
|
+
relative = path.relative_to(root)
|
|
391
|
+
if "skills" in relative.parts or not path.is_file():
|
|
392
|
+
continue
|
|
393
|
+
digest.update(relative.as_posix().encode("utf-8") + b"\0")
|
|
394
|
+
with path.open("rb") as stream:
|
|
395
|
+
while chunk := stream.read(1024 * 1024):
|
|
396
|
+
digest.update(chunk)
|
|
397
|
+
return digest.hexdigest()
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def _image_component(value: str) -> str:
|
|
401
|
+
return re.sub(r"[^a-z0-9_.-]+", "-", value.lower()).strip("-.") or "task"
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def _task_set(root: Path) -> list[dict[str, Any]]:
|
|
405
|
+
path = root / "integrations" / "agentbeats" / "task_sets" / "skillsbench-v1.1.json"
|
|
406
|
+
if path.is_file():
|
|
407
|
+
try:
|
|
408
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
409
|
+
except json.JSONDecodeError as exc:
|
|
410
|
+
raise BenchFormatError(f"invalid SkillsBench task set {path}: {exc}") from exc
|
|
411
|
+
tasks = payload.get("tasks") if isinstance(payload, dict) else None
|
|
412
|
+
if isinstance(tasks, list) and all(isinstance(item, dict) for item in tasks):
|
|
413
|
+
return tasks
|
|
414
|
+
return [{"task_id": path.name} for path in sorted((root / "tasks").iterdir()) if path.is_dir()]
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
class SkillsBenchAdapter(BenchAdapter):
|
|
418
|
+
name = "skillsbench"
|
|
419
|
+
description = "SkillsBench — task-specific environments, with-skill/no-skill ablation"
|
|
420
|
+
variants = ("with-skill", "no-skill")
|
|
421
|
+
default_variant = "with-skill"
|
|
422
|
+
|
|
423
|
+
@staticmethod
|
|
424
|
+
def _root(root: Path) -> Path:
|
|
425
|
+
root = root.expanduser().resolve()
|
|
426
|
+
if not (root / "tasks").is_dir():
|
|
427
|
+
raise FileNotFoundError(f"SkillsBench tasks/ not found under: {root}")
|
|
428
|
+
return root
|
|
429
|
+
|
|
430
|
+
def load(
|
|
431
|
+
self,
|
|
432
|
+
root: Path,
|
|
433
|
+
*,
|
|
434
|
+
categories: Sequence[str] | None = None,
|
|
435
|
+
task_ids: Sequence[str] | None = None,
|
|
436
|
+
variant: str | None = None,
|
|
437
|
+
timeout_multiplier: float = 1.0,
|
|
438
|
+
) -> list[dict[str, Any]]:
|
|
439
|
+
root = self._root(root)
|
|
440
|
+
aliases = {
|
|
441
|
+
"with-skills": "with-skill",
|
|
442
|
+
"without-skills": "no-skill",
|
|
443
|
+
"without-skill": "no-skill",
|
|
444
|
+
}
|
|
445
|
+
variant = aliases.get(variant or "", variant) or self.default_variant
|
|
446
|
+
if variant not in self.variants:
|
|
447
|
+
raise BenchFormatError(
|
|
448
|
+
f"{self.name}: variant must be one of {', '.join(self.variants)}"
|
|
449
|
+
)
|
|
450
|
+
entries = _task_set(root)
|
|
451
|
+
known = {
|
|
452
|
+
item.get("task_id") for item in entries if isinstance(item.get("task_id"), str)
|
|
453
|
+
}
|
|
454
|
+
requested = set(task_ids or ())
|
|
455
|
+
missing = requested - known
|
|
456
|
+
if missing:
|
|
457
|
+
raise BenchFormatError(
|
|
458
|
+
f"{self.name}: unknown task id(s): {', '.join(sorted(missing))}"
|
|
459
|
+
)
|
|
460
|
+
category_filter = set(categories or ())
|
|
461
|
+
rows: list[dict[str, Any]] = []
|
|
462
|
+
for entry in entries:
|
|
463
|
+
task_id = entry.get("task_id")
|
|
464
|
+
if not isinstance(task_id, str):
|
|
465
|
+
continue
|
|
466
|
+
if requested and task_id not in requested:
|
|
467
|
+
continue
|
|
468
|
+
task_dir = root / "tasks" / task_id
|
|
469
|
+
metadata, prompt = _task_metadata(task_dir / "task.md")
|
|
470
|
+
taxonomy = metadata.get("metadata") or {}
|
|
471
|
+
if not isinstance(taxonomy, dict):
|
|
472
|
+
raise BenchFormatError(f"{task_id}: metadata.metadata must be an object")
|
|
473
|
+
category = str(entry.get("category") or taxonomy.get("category") or "")
|
|
474
|
+
if category_filter and category not in category_filter:
|
|
475
|
+
continue
|
|
476
|
+
rows.append(
|
|
477
|
+
self._row(
|
|
478
|
+
task_dir,
|
|
479
|
+
task_id,
|
|
480
|
+
metadata,
|
|
481
|
+
prompt,
|
|
482
|
+
category,
|
|
483
|
+
variant,
|
|
484
|
+
validate_timeout_multiplier(timeout_multiplier),
|
|
485
|
+
)
|
|
486
|
+
)
|
|
487
|
+
return rows
|
|
488
|
+
|
|
489
|
+
def _row(
|
|
490
|
+
self,
|
|
491
|
+
task_dir: Path,
|
|
492
|
+
task_id: str,
|
|
493
|
+
metadata: dict[str, Any],
|
|
494
|
+
prompt: str,
|
|
495
|
+
category: str,
|
|
496
|
+
variant: str,
|
|
497
|
+
timeout_multiplier: float,
|
|
498
|
+
) -> dict[str, Any]:
|
|
499
|
+
environment_dir = task_dir / "environment"
|
|
500
|
+
dockerfile_path = environment_dir / "Dockerfile"
|
|
501
|
+
verifier_dir = task_dir / "verifier"
|
|
502
|
+
if not dockerfile_path.is_file() or not (verifier_dir / "test.sh").is_file():
|
|
503
|
+
raise BenchFormatError(f"{task_id}: environment Dockerfile or verifier/test.sh missing")
|
|
504
|
+
dockerfile = _sanitized_dockerfile(dockerfile_path)
|
|
505
|
+
digest = _environment_digest(environment_dir, dockerfile)
|
|
506
|
+
component = _image_component(task_id)[:80]
|
|
507
|
+
short = digest[:16]
|
|
508
|
+
base_image = f"roller-skillsbench-env-{component}:{short}"
|
|
509
|
+
image = f"roller-skillsbench-opencode-{component}:{short}"
|
|
510
|
+
agent = metadata.get("agent") or {}
|
|
511
|
+
verifier = metadata.get("verifier") or {}
|
|
512
|
+
# v1.1 calls this block ``environment``. Current development versions
|
|
513
|
+
# renamed it to ``sandbox``; accepting both keeps the adapter useful
|
|
514
|
+
# while allowing official runs to stay pinned to the v1.1 release.
|
|
515
|
+
sandbox = metadata.get("sandbox") or metadata.get("environment") or {}
|
|
516
|
+
if not isinstance(agent, dict) or not isinstance(verifier, dict):
|
|
517
|
+
raise BenchFormatError(f"{task_id}: agent and verifier must be objects")
|
|
518
|
+
if not isinstance(sandbox, dict):
|
|
519
|
+
raise BenchFormatError(f"{task_id}: sandbox must be an object")
|
|
520
|
+
timeout_value = agent.get("timeout_sec", 900)
|
|
521
|
+
verifier_timeout_value = verifier.get("timeout_sec", 900)
|
|
522
|
+
try:
|
|
523
|
+
timeout = max(1, math.ceil(float(timeout_value) * timeout_multiplier))
|
|
524
|
+
verifier_timeout = max(
|
|
525
|
+
1, math.ceil(float(verifier_timeout_value) * timeout_multiplier)
|
|
526
|
+
)
|
|
527
|
+
except (TypeError, ValueError) as exc:
|
|
528
|
+
raise BenchFormatError(f"{task_id}: invalid agent/verifier timeout") from exc
|
|
529
|
+
build_timeout_value = sandbox.get("build_timeout_sec", 3600)
|
|
530
|
+
try:
|
|
531
|
+
build_timeout = max(1, math.ceil(float(build_timeout_value)))
|
|
532
|
+
except (TypeError, ValueError) as exc:
|
|
533
|
+
raise BenchFormatError(f"{task_id}: invalid sandbox.build_timeout_sec") from exc
|
|
534
|
+
build_args = {
|
|
535
|
+
str(key): value
|
|
536
|
+
for key, value in sandbox.items()
|
|
537
|
+
if key not in RESOURCE_KEYS and isinstance(value, (str, int, float, bool))
|
|
538
|
+
}
|
|
539
|
+
workdir = _workspace_from_dockerfile(dockerfile)
|
|
540
|
+
skills_dir = environment_dir / "skills"
|
|
541
|
+
files: dict[str, Any] = {"output_dir": ".roller_outputs"}
|
|
542
|
+
setup: list[str] = []
|
|
543
|
+
skill_targets = (
|
|
544
|
+
"/roller/opencode-home/config/opencode/skills",
|
|
545
|
+
"/root/.config/opencode/skills",
|
|
546
|
+
"/root/.opencode/skill",
|
|
547
|
+
"/root/.agents/skills",
|
|
548
|
+
"/root/.claude/skills",
|
|
549
|
+
"/root/.codex/skills",
|
|
550
|
+
"/root/.gemini/skills",
|
|
551
|
+
"/skills",
|
|
552
|
+
)
|
|
553
|
+
cleanup = "rm -rf " + " ".join(skill_targets)
|
|
554
|
+
if variant == "with-skill":
|
|
555
|
+
if not skills_dir.is_dir():
|
|
556
|
+
raise BenchFormatError(f"{task_id}: environment/skills is missing")
|
|
557
|
+
files["context"] = [
|
|
558
|
+
{
|
|
559
|
+
"role": "skill",
|
|
560
|
+
"source": str(skills_dir.resolve()),
|
|
561
|
+
"target": "skills",
|
|
562
|
+
"capture": "hash",
|
|
563
|
+
}
|
|
564
|
+
]
|
|
565
|
+
copies = " && ".join(
|
|
566
|
+
f"mkdir -p {target} && cp -a /roller/task/context/skills/. {target}/"
|
|
567
|
+
for target in skill_targets
|
|
568
|
+
)
|
|
569
|
+
setup.append(f"{cleanup} && {copies}")
|
|
570
|
+
else:
|
|
571
|
+
setup.append(cleanup)
|
|
572
|
+
verifier_env = verifier.get("env") or {}
|
|
573
|
+
if not isinstance(verifier_env, dict):
|
|
574
|
+
raise BenchFormatError(f"{task_id}: verifier.env must be an object")
|
|
575
|
+
grade = GradeSpec(
|
|
576
|
+
code=_VERIFIER_RUNNER.replace("__VERIFIER_TIMEOUT__", str(verifier_timeout)),
|
|
577
|
+
entrypoint="roller_grade",
|
|
578
|
+
kwargs={"task_workspace": workdir, "verifier_env": verifier_env},
|
|
579
|
+
timeout_seconds=verifier_timeout + 30,
|
|
580
|
+
needs_transcript=False,
|
|
581
|
+
workspace_path="/",
|
|
582
|
+
stage=((str(verifier_dir.resolve()), "verifier"),),
|
|
583
|
+
)
|
|
584
|
+
binding = BenchBinding(
|
|
585
|
+
self.name,
|
|
586
|
+
grade=grade,
|
|
587
|
+
category=category,
|
|
588
|
+
metadata={
|
|
589
|
+
"variant": variant,
|
|
590
|
+
"difficulty": (metadata.get("metadata") or {}).get("difficulty"),
|
|
591
|
+
"environment_digest": digest,
|
|
592
|
+
"taxonomy": dict(metadata.get("metadata") or {}),
|
|
593
|
+
"resource_request": {
|
|
594
|
+
key: sandbox[key] for key in RESOURCE_KEYS if key in sandbox
|
|
595
|
+
},
|
|
596
|
+
"oracle_declared": "oracle" in metadata,
|
|
597
|
+
},
|
|
598
|
+
)
|
|
599
|
+
return {
|
|
600
|
+
"schema_version": "roller.task/v1",
|
|
601
|
+
"id": task_id,
|
|
602
|
+
"harness": "opencode",
|
|
603
|
+
"prompt": prompt,
|
|
604
|
+
"timeout_seconds": timeout,
|
|
605
|
+
"files": files,
|
|
606
|
+
"runtime": {
|
|
607
|
+
"image": image,
|
|
608
|
+
"image_build": {
|
|
609
|
+
"base_image": base_image,
|
|
610
|
+
"context": str(environment_dir.resolve()),
|
|
611
|
+
"dockerfile": dockerfile,
|
|
612
|
+
"harness": "opencode",
|
|
613
|
+
"args": build_args,
|
|
614
|
+
"timeout_seconds": build_timeout,
|
|
615
|
+
},
|
|
616
|
+
"workspace_path": workdir,
|
|
617
|
+
"setup": setup,
|
|
618
|
+
},
|
|
619
|
+
"bench": binding.as_dict(),
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
def aggregate(self, samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
|
|
623
|
+
"""Report SkillsBench's score-eligible pass rate and mean reward."""
|
|
624
|
+
|
|
625
|
+
rows = list(samples)
|
|
626
|
+
task_rows: list[dict[str, Any]] = []
|
|
627
|
+
category_values: dict[str, list[tuple[float, bool]]] = {}
|
|
628
|
+
rewards: list[float] = []
|
|
629
|
+
passed = 0
|
|
630
|
+
for row in rows:
|
|
631
|
+
scores = row.get("bench_scores")
|
|
632
|
+
score_key = row.get("bench_score_key") or "overall_score"
|
|
633
|
+
value = sample_score(scores, score_key) if isinstance(scores, dict) else None
|
|
634
|
+
error = _score_error(row, scores if isinstance(scores, dict) else None)
|
|
635
|
+
declared_eligible = (
|
|
636
|
+
scores.get("score_eligible", True)
|
|
637
|
+
if isinstance(scores, dict)
|
|
638
|
+
else False
|
|
639
|
+
)
|
|
640
|
+
status = row.get("status")
|
|
641
|
+
eligible = bool(
|
|
642
|
+
status == "ok"
|
|
643
|
+
and value is not None
|
|
644
|
+
and not error
|
|
645
|
+
and declared_eligible is not False
|
|
646
|
+
)
|
|
647
|
+
is_passed = bool(
|
|
648
|
+
eligible
|
|
649
|
+
and (
|
|
650
|
+
scores.get("passed")
|
|
651
|
+
if isinstance(scores, dict) and isinstance(scores.get("passed"), bool)
|
|
652
|
+
else value > 0
|
|
653
|
+
)
|
|
654
|
+
)
|
|
655
|
+
category = row.get("bench_category")
|
|
656
|
+
if eligible:
|
|
657
|
+
assert value is not None
|
|
658
|
+
rewards.append(value)
|
|
659
|
+
passed += int(is_passed)
|
|
660
|
+
if isinstance(category, str) and category:
|
|
661
|
+
category_values.setdefault(category, []).append((value, is_passed))
|
|
662
|
+
elif not error:
|
|
663
|
+
error = str(row.get("error") or status or "not score eligible")
|
|
664
|
+
task_rows.append(
|
|
665
|
+
{
|
|
666
|
+
"task_id": row.get("task_id"),
|
|
667
|
+
"model": row.get("model"),
|
|
668
|
+
"category": category,
|
|
669
|
+
"status": status,
|
|
670
|
+
"score": value if eligible else None,
|
|
671
|
+
"score_eligible": eligible,
|
|
672
|
+
"passed": is_passed,
|
|
673
|
+
"error": error,
|
|
674
|
+
}
|
|
675
|
+
)
|
|
676
|
+
eligible_count = len(rewards)
|
|
677
|
+
mean_reward = sum(rewards) / eligible_count if eligible_count else None
|
|
678
|
+
pass_rate = passed / eligible_count if eligible_count else None
|
|
679
|
+
return {
|
|
680
|
+
"schema_version": REPORT_SCHEMA,
|
|
681
|
+
"metric": "pass_rate",
|
|
682
|
+
"task_count": len(rows),
|
|
683
|
+
"scored_count": eligible_count,
|
|
684
|
+
"score_eligible_count": eligible_count,
|
|
685
|
+
"unscored_count": len(rows) - eligible_count,
|
|
686
|
+
"infra_failed_count": len(rows) - eligible_count,
|
|
687
|
+
"passed_count": passed,
|
|
688
|
+
"pass_rate": round(pass_rate, 6) if pass_rate is not None else None,
|
|
689
|
+
"mean_reward": round(mean_reward, 6) if mean_reward is not None else None,
|
|
690
|
+
"average": round(mean_reward, 6) if mean_reward is not None else None,
|
|
691
|
+
"scored_average": round(mean_reward, 6) if mean_reward is not None else None,
|
|
692
|
+
"categories": {
|
|
693
|
+
name: {
|
|
694
|
+
"task_count": len(values),
|
|
695
|
+
"pass_rate": round(
|
|
696
|
+
sum(int(item[1]) for item in values) / len(values), 6
|
|
697
|
+
),
|
|
698
|
+
"average": round(
|
|
699
|
+
sum(item[0] for item in values) / len(values), 6
|
|
700
|
+
),
|
|
701
|
+
}
|
|
702
|
+
for name, values in sorted(category_values.items())
|
|
703
|
+
},
|
|
704
|
+
"tasks": task_rows,
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
__all__ = ["SkillsBenchAdapter"]
|