roller-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. roller/__init__.py +31 -0
  2. roller/__main__.py +6 -0
  3. roller/actf.py +431 -0
  4. roller/artifacts.py +124 -0
  5. roller/assets/deepseek/journal.mjs +44 -0
  6. roller/assets/deepseek/roller.patch.yml +15 -0
  7. roller/assets/deepseek/runner.py +149 -0
  8. roller/assets/hermes/runner.py +322 -0
  9. roller/assets/openclaw_capture.py +32 -0
  10. roller/assets/openclaw_drain.mjs +21 -0
  11. roller/assets/opencode/journal.mjs +42 -0
  12. roller/assets/opencode/runner.py +203 -0
  13. roller/assets/pi/runner.mjs +160 -0
  14. roller/assets/pi/runner.py +93 -0
  15. roller/benches/__init__.py +73 -0
  16. roller/benches/base.py +306 -0
  17. roller/benches/grading.py +553 -0
  18. roller/benches/pinchbench.py +976 -0
  19. roller/benches/qwenclaw/__init__.py +49 -0
  20. roller/benches/qwenclaw/adapter.py +167 -0
  21. roller/benches/qwenclaw/grader_runtime.py +395 -0
  22. roller/benches/qwenclaw/report.py +49 -0
  23. roller/benches/qwenclaw/task_parser.py +181 -0
  24. roller/benches/qwenclaw/workspace.py +202 -0
  25. roller/benches/report.py +166 -0
  26. roller/benches/skillsbench.py +708 -0
  27. roller/benches/transcripts.py +102 -0
  28. roller/benches/wildclaw.py +569 -0
  29. roller/bridge_capture.py +73 -0
  30. roller/bundles.py +545 -0
  31. roller/capture.py +385 -0
  32. roller/capture_protocol.py +301 -0
  33. roller/capture_runtime.py +48 -0
  34. roller/capture_states.py +37 -0
  35. roller/claude_capture.py +185 -0
  36. roller/claude_code_actf.py +594 -0
  37. roller/cli.py +1732 -0
  38. roller/codex_actf.py +584 -0
  39. roller/codex_capture.py +99 -0
  40. roller/codex_tools.py +137 -0
  41. roller/collector.py +818 -0
  42. roller/compaction_sft.py +672 -0
  43. roller/conversation.py +432 -0
  44. roller/deepseek_actf.py +135 -0
  45. roller/harness_registry.py +61 -0
  46. roller/harnesses/__init__.py +29 -0
  47. roller/harnesses/base.py +64 -0
  48. roller/harnesses/bridge.py +448 -0
  49. roller/harnesses/claude_code.py +839 -0
  50. roller/harnesses/codex.py +808 -0
  51. roller/harnesses/config_overrides.py +100 -0
  52. roller/harnesses/container.py +206 -0
  53. roller/harnesses/deepseek.py +104 -0
  54. roller/harnesses/docker.py +72 -0
  55. roller/harnesses/gateway_ports.py +155 -0
  56. roller/harnesses/hermes.py +119 -0
  57. roller/harnesses/openclaw.py +1605 -0
  58. roller/harnesses/opencode.py +193 -0
  59. roller/harnesses/pi.py +111 -0
  60. roller/hermes_actf.py +147 -0
  61. roller/image_build.py +206 -0
  62. roller/integrity.py +332 -0
  63. roller/model_protocols.py +201 -0
  64. roller/models.py +169 -0
  65. roller/openclaw_capture.py +140 -0
  66. roller/openclaw_delivery.py +211 -0
  67. roller/openclaw_usage.py +59 -0
  68. roller/opencode_actf.py +305 -0
  69. roller/opencode_capture.py +96 -0
  70. roller/pi_actf.py +172 -0
  71. roller/response_metadata.py +89 -0
  72. roller/run_lock.py +297 -0
  73. roller/sft.py +471 -0
  74. roller/storage.py +53 -0
  75. roller/tasks.py +689 -0
  76. roller/thinking.py +50 -0
  77. roller/thinking_capabilities.py +152 -0
  78. roller/trajectory_audit.py +386 -0
  79. roller/usage.py +168 -0
  80. roller/version.py +1 -0
  81. roller/viewer/__init__.py +3 -0
  82. roller/viewer/app.py +722 -0
  83. roller/viewer/static/app.js +752 -0
  84. roller/viewer/static/index.html +105 -0
  85. roller/viewer/static/style.css +213 -0
  86. roller/wire_trace.py +650 -0
  87. roller_agent-0.2.0.dist-info/METADATA +397 -0
  88. roller_agent-0.2.0.dist-info/RECORD +92 -0
  89. roller_agent-0.2.0.dist-info/WHEEL +5 -0
  90. roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
  91. roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
  92. roller_agent-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,708 @@
1
+ """SkillsBench adapter with task environments and skill ablation variants."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import base64
6
+ import hashlib
7
+ import json
8
+ import math
9
+ import re
10
+ from collections.abc import Iterable, Sequence
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ import yaml
15
+
16
+ from roller.benches.base import (
17
+ BenchAdapter,
18
+ BenchBinding,
19
+ BenchFormatError,
20
+ GradeSpec,
21
+ validate_timeout_multiplier,
22
+ )
23
+ from roller.benches.report import REPORT_SCHEMA, sample_score
24
+
25
+ FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)", re.DOTALL)
26
+ WORKDIR_RE = re.compile(r"^\s*WORKDIR\s+(.+?)\s*$", re.MULTILINE | re.IGNORECASE)
27
+ ENV_RE = re.compile(r"^\s*ENV\s+([A-Za-z_][A-Za-z0-9_]*)=(\S+)\s*$", re.MULTILINE)
28
+ SKILL_COPY_RE = re.compile(r"^\s*COPY\s+(?:_deps/)?skills(?:/)?\s+.*$", re.MULTILINE)
29
+ HEREDOC_RE = re.compile(r"<<-?['\"]?([A-Za-z_][A-Za-z0-9_]*)['\"]?\s*$")
30
+ PYTHON_STDIN_RE = re.compile(r"\b(python3?\s+-)\s+<<-?['\"]?[A-Za-z_][A-Za-z0-9_]*['\"]?\s*$")
31
+ CAT_REDIRECT_RE = re.compile(r"\bcat\s+>\s+([^\s]+)\s+<<-?['\"]?[A-Za-z_][A-Za-z0-9_]*['\"]?\s*$")
32
+ FROM_RE = re.compile(r"^(\s*FROM\s+(?:--platform=\S+\s+)?)(\S+)(.*)$", re.IGNORECASE)
33
+ VAR_RE = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")
34
+ RESOURCE_KEYS = {
35
+ "network_mode",
36
+ "build_timeout_sec",
37
+ "os",
38
+ "cpus",
39
+ "memory_mb",
40
+ "storage_mb",
41
+ "gpus",
42
+ }
43
+
44
+ _VERIFIER_RUNNER = r'''
45
+ import os
46
+ from pathlib import Path
47
+ import subprocess
48
+ import tempfile
49
+ import time
50
+
51
+
52
+ _RETRYABLE_INFRASTRUCTURE_ERRORS = {
53
+ "verifier dependency download failed",
54
+ "verifier DNS failure",
55
+ }
56
+
57
+
58
+ def _infrastructure_error(output):
59
+ lowered = output.lower()
60
+ tests_completed = (
61
+ "short test summary info" in lowered
62
+ or (
63
+ "test session starts" in lowered
64
+ and any(marker in lowered for marker in (" passed in ", " failed in ", " error in "))
65
+ )
66
+ )
67
+ markers = (
68
+ ("uvx: command not found", "uvx is unavailable", False),
69
+ ("uv: command not found", "uv is unavailable", False),
70
+ ("failed to download", "verifier dependency download failed", True),
71
+ ("temporary failure in name resolution", "verifier DNS failure", True),
72
+ ("could not resolve host", "verifier DNS failure", True),
73
+ ("executable doesn't exist at", "Playwright browser is unavailable", False),
74
+ ("please run the following command to download new browsers", "Playwright browser is unavailable", False),
75
+ ("browser executable not found", "Playwright browser is unavailable", False),
76
+ ("err_access_denied", "local verifier request was intercepted by a proxy", False),
77
+ )
78
+ for marker, message, ignore_after_tests in markers:
79
+ if marker in lowered:
80
+ if ignore_after_tests and tests_completed:
81
+ continue
82
+ return message
83
+ return None
84
+
85
+
86
+ def roller_grade(workspace_path, task_workspace, verifier_env):
87
+ environment = dict(os.environ)
88
+ environment.update({str(key): str(value) for key, value in verifier_env.items()})
89
+ reward_path = Path("/logs/verifier/reward.txt")
90
+ reward_path.parent.mkdir(parents=True, exist_ok=True)
91
+ # Official harnesses provide these mount points. Roller stages the verifier
92
+ # into the live task container, so create their empty equivalents here.
93
+ Path("/tests").mkdir(parents=True, exist_ok=True)
94
+ # v1.1 verifier scripts repeatedly bootstrap uv from the network and then
95
+ # source this file. The task overlay already ships pinned uv/uvx binaries;
96
+ # this fallback lets verification proceed when the redundant download is
97
+ # interrupted by a cluster proxy.
98
+ local_bin = Path.home() / ".local" / "bin"
99
+ local_bin.mkdir(parents=True, exist_ok=True)
100
+ env_path = local_bin / "env"
101
+ if not env_path.exists():
102
+ env_path.write_text('export PATH="$HOME/.local/bin:$PATH"\n', encoding="utf-8")
103
+ result = None
104
+ output = ""
105
+ verifier_attempts = 0
106
+ for verifier_attempts in range(1, 4):
107
+ reward_path.unlink(missing_ok=True)
108
+ # Some upstream verifiers leave a background service alive after their
109
+ # shell exits. A PIPE would remain open in that descendant and make
110
+ # subprocess.run() wait until the outer timeout despite completed tests.
111
+ # A regular file lets us observe the verifier process itself, while the
112
+ # disposable task container still cleans up any leftover service.
113
+ with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8") as log:
114
+ result = subprocess.run(
115
+ ["/bin/bash", "/verifier/test.sh"],
116
+ cwd=task_workspace,
117
+ env=environment,
118
+ stdout=log,
119
+ stderr=subprocess.STDOUT,
120
+ text=True,
121
+ timeout=__VERIFIER_TIMEOUT__,
122
+ )
123
+ log.flush()
124
+ output = Path(log.name).read_text(encoding="utf-8", errors="replace")
125
+ infrastructure_error = _infrastructure_error(output)
126
+ if (
127
+ infrastructure_error not in _RETRYABLE_INFRASTRUCTURE_ERRORS
128
+ or verifier_attempts == 3
129
+ ):
130
+ break
131
+ time.sleep(5 * verifier_attempts)
132
+ assert result is not None
133
+ if not reward_path.is_file():
134
+ raise RuntimeError(
135
+ "SkillsBench verifier produced no /logs/verifier/reward.txt: "
136
+ + output[-2000:]
137
+ )
138
+ try:
139
+ reward = float(reward_path.read_text(encoding="utf-8").strip())
140
+ except ValueError as exc:
141
+ raise RuntimeError("SkillsBench verifier reward is not numeric") from exc
142
+ payload = {
143
+ "overall_score": max(0.0, min(1.0, reward)),
144
+ "passed": reward > 0,
145
+ "score_eligible": True,
146
+ "verifier_exit_code": result.returncode,
147
+ "verifier_attempts": verifier_attempts,
148
+ "verifier_output": output[-8000:],
149
+ }
150
+ infrastructure_error = _infrastructure_error(output)
151
+ if infrastructure_error:
152
+ payload["score_eligible"] = False
153
+ payload["error"] = infrastructure_error
154
+ return payload
155
+ '''
156
+
157
+
158
+ _INFRASTRUCTURE_OUTPUT_MARKERS = (
159
+ "uvx: command not found",
160
+ "uv: command not found",
161
+ "failed to download",
162
+ "temporary failure in name resolution",
163
+ "could not resolve host",
164
+ "executable doesn't exist at",
165
+ "please run the following command to download new browsers",
166
+ "browser executable not found",
167
+ "err_access_denied",
168
+ )
169
+
170
+
171
+ def _score_error(row: dict[str, Any], scores: dict[str, Any] | None) -> str | None:
172
+ error = row.get("bench_error")
173
+ if isinstance(error, str) and error:
174
+ return error
175
+ if not isinstance(scores, dict):
176
+ return None
177
+ error = scores.get("error")
178
+ if isinstance(error, str) and error:
179
+ return error
180
+ output = scores.get("verifier_output")
181
+ if isinstance(output, str):
182
+ lowered = output.lower()
183
+ tests_completed = (
184
+ "short test summary info" in lowered
185
+ or (
186
+ "test session starts" in lowered
187
+ and any(
188
+ marker in lowered
189
+ for marker in (" passed in ", " failed in ", " error in ")
190
+ )
191
+ )
192
+ )
193
+ for marker in _INFRASTRUCTURE_OUTPUT_MARKERS:
194
+ if marker in lowered:
195
+ if marker in {
196
+ "failed to download",
197
+ "temporary failure in name resolution",
198
+ "could not resolve host",
199
+ } and tests_completed:
200
+ continue
201
+ return f"verifier infrastructure failure: {marker}"
202
+ return None
203
+
204
+
205
+ def _task_metadata(path: Path) -> tuple[dict[str, Any], str]:
206
+ match = FRONTMATTER_RE.match(path.read_text(encoding="utf-8"))
207
+ if match is None:
208
+ raise BenchFormatError(f"{path}: missing YAML frontmatter")
209
+ try:
210
+ metadata = yaml.safe_load(match.group(1)) or {}
211
+ except yaml.YAMLError as exc:
212
+ raise BenchFormatError(f"{path}: invalid YAML frontmatter: {exc}") from exc
213
+ if not isinstance(metadata, dict):
214
+ raise BenchFormatError(f"{path}: frontmatter must be an object")
215
+ prompt = match.group(2).strip()
216
+ if not prompt:
217
+ raise BenchFormatError(f"{path}: task prompt is empty")
218
+ return metadata, prompt
219
+
220
+
221
+ def _portable_heredocs(value: str) -> str:
222
+ """Rewrite Dockerfile heredocs for engines whose parser predates BuildKit.
223
+
224
+ SkillsBench currently uses Python-stdin, file-writing, and direct RUN
225
+ heredocs. Their shell-equivalent forms work in both Docker and Podman 4.x.
226
+ """
227
+
228
+ lines = value.splitlines()
229
+ output: list[str] = []
230
+ index = 0
231
+ while index < len(lines):
232
+ line = lines[index]
233
+ marker = HEREDOC_RE.search(line)
234
+ if marker is None:
235
+ output.append(line)
236
+ index += 1
237
+ continue
238
+ delimiter = marker.group(1)
239
+ end = index + 1
240
+ while end < len(lines) and lines[end].strip() != delimiter:
241
+ end += 1
242
+ if end == len(lines):
243
+ raise BenchFormatError(f"unterminated Dockerfile heredoc {delimiter!r}")
244
+ body = "\n".join(lines[index + 1 : end]) + "\n"
245
+ if ".from_pretrained(" in body:
246
+ original = base64.b64encode(body.encode("utf-8")).decode("ascii")
247
+ backup = (
248
+ "\n import seisbench\n"
249
+ " seisbench.use_backup_repository()"
250
+ if "seisbench.models" in body
251
+ else ""
252
+ )
253
+ body = f'''import base64
254
+ import time
255
+
256
+ source = base64.b64decode({original!r}).decode("utf-8")
257
+ last_error = None
258
+ for attempt in range(6):
259
+ try:
260
+ exec(compile(source, "<skillsbench-preload>", "exec"), globals())
261
+ break
262
+ except Exception as exc:
263
+ last_error = exc
264
+ if attempt == 0:{backup or chr(10) + " pass"}
265
+ if attempt == 5:
266
+ raise
267
+ time.sleep(20 * (attempt + 1))
268
+ else:
269
+ raise RuntimeError("SkillsBench preload failed") from last_error
270
+ '''
271
+ encoded = base64.b64encode(body.encode("utf-8")).decode("ascii")
272
+ # SeisBench 0.10 imports pkg_resources, which was removed from recent
273
+ # setuptools releases selected by an otherwise-unpinned upstream pip
274
+ # install. Pin the last compatible series immediately before its cache
275
+ # warm-up script so current clean builds remain reproducible.
276
+ compatibility = (
277
+ "pip install --no-cache-dir 'setuptools<81' && "
278
+ if "import seisbench" in body
279
+ else ""
280
+ )
281
+ python_match = PYTHON_STDIN_RE.search(line)
282
+ cat_match = CAT_REDIRECT_RE.search(line)
283
+ if python_match is not None:
284
+ replacement = (
285
+ f"{compatibility}printf %s {encoded} | base64 -d | "
286
+ f"{python_match.group(1)}"
287
+ )
288
+ output.append(line[: python_match.start()] + replacement)
289
+ elif cat_match is not None:
290
+ replacement = f"printf %s {encoded} | base64 -d > {cat_match.group(1)}"
291
+ output.append(line[: cat_match.start()] + replacement)
292
+ elif line.strip().upper().startswith("RUN <<"):
293
+ shell = "/bin/bash" if "pipefail" in body else "/bin/sh"
294
+ output.append(f"RUN printf %s {encoded} | base64 -d | {shell}")
295
+ else:
296
+ raise BenchFormatError(
297
+ f"unsupported Dockerfile heredoc syntax: {line.strip()}"
298
+ )
299
+ index = end + 1
300
+ return "\n".join(output)
301
+
302
+
303
+ def _qualified_from_images(value: str) -> str:
304
+ """Apply Docker Hub's implicit namespace rule for strict Podman hosts."""
305
+
306
+ output: list[str] = []
307
+ for line in value.splitlines():
308
+ match = FROM_RE.match(line)
309
+ if match is None:
310
+ output.append(line)
311
+ continue
312
+ image = match.group(2)
313
+ first_component = image.split("/", 1)[0]
314
+ if (
315
+ "/" in image
316
+ and not image.startswith("$")
317
+ and "." not in first_component
318
+ and ":" not in first_component
319
+ and first_component != "localhost"
320
+ ):
321
+ image = "docker.io/" + image
322
+ output.append(match.group(1) + image + match.group(3))
323
+ return "\n".join(output)
324
+
325
+
326
+ def _portable_tar_extraction(value: str) -> str:
327
+ """Prevent archive ownership metadata from breaking rootless builds."""
328
+
329
+ output: list[str] = []
330
+ for line in value.splitlines():
331
+ extraction = re.search(
332
+ r"(?:^\s*|[;&|]\s*|\bRUN\s+)tar\s+"
333
+ r"(?:--[\w-]+(?:=[^\s]+)?\s+)*-[A-Za-z]*x[A-Za-z]*\b",
334
+ line,
335
+ re.IGNORECASE,
336
+ )
337
+ if extraction is not None:
338
+ if "--no-same-owner" not in line:
339
+ line = re.sub(
340
+ r"\btar\s+",
341
+ "tar --no-same-owner --no-same-permissions ",
342
+ line,
343
+ count=1,
344
+ flags=re.IGNORECASE,
345
+ )
346
+ output.append(line)
347
+ return "\n".join(output)
348
+
349
+
350
+ def _sanitized_dockerfile(path: Path) -> str:
351
+ value = _portable_heredocs(path.read_text(encoding="utf-8"))
352
+ value = _qualified_from_images(value)
353
+ value = _portable_tar_extraction(value)
354
+ # v1.1 predates the upstream PyCBC compatibility pin. Retain the release
355
+ # task while applying the same dependency-only fix now present on main.
356
+ if "pycbc==2.8.5" in value and "astropy==7.2.1" not in value:
357
+ value = value.replace(
358
+ ' "setuptools<81" \\\n',
359
+ ' "setuptools<81" \\\n "astropy==7.2.1" \\\n',
360
+ 1,
361
+ )
362
+ # Coursier's interactive setup installs a newer copy of itself from GitHub
363
+ # before installing Scala. That extra download is unnecessary and is less
364
+ # reliable behind evaluation-cluster proxies; the pinned launcher already
365
+ # present in the image can install the requested Scala binaries directly.
366
+ if "coursier/releases/download/" in value:
367
+ value = value.replace("curl -fL ", "curl -fL --retry 5 --retry-all-errors --retry-delay 5 ")
368
+ value = value.replace("./cs setup --yes && ./cs install ", "./cs install ")
369
+ # A no-skill run must not inherit task skills accidentally. Runtime staging
370
+ # injects them only for the with-skill condition.
371
+ value = SKILL_COPY_RE.sub("# Roller: task skills are injected at runtime", value)
372
+ return value.rstrip() + "\n"
373
+
374
+
375
+ def _workspace_from_dockerfile(source: str) -> str:
376
+ environment = {name: value.strip('"\'') for name, value in ENV_RE.findall(source)}
377
+ matches = WORKDIR_RE.findall(source)
378
+ value = matches[-1].strip().strip('"\'') if matches else "/root"
379
+ value = VAR_RE.sub(lambda match: environment.get(match.group(1), match.group(0)), value)
380
+ return value if value.startswith("/") else "/root"
381
+
382
+
383
+ def _environment_digest(root: Path, dockerfile: str) -> str:
384
+ digest = hashlib.sha256()
385
+ # Bump this salt whenever the harness overlay contract changes so a stale
386
+ # locally tagged task image cannot silently bypass the new layer.
387
+ digest.update(b"roller-skillsbench-opencode-v2\0")
388
+ digest.update(dockerfile.encode("utf-8"))
389
+ for path in sorted(root.rglob("*")):
390
+ relative = path.relative_to(root)
391
+ if "skills" in relative.parts or not path.is_file():
392
+ continue
393
+ digest.update(relative.as_posix().encode("utf-8") + b"\0")
394
+ with path.open("rb") as stream:
395
+ while chunk := stream.read(1024 * 1024):
396
+ digest.update(chunk)
397
+ return digest.hexdigest()
398
+
399
+
400
+ def _image_component(value: str) -> str:
401
+ return re.sub(r"[^a-z0-9_.-]+", "-", value.lower()).strip("-.") or "task"
402
+
403
+
404
+ def _task_set(root: Path) -> list[dict[str, Any]]:
405
+ path = root / "integrations" / "agentbeats" / "task_sets" / "skillsbench-v1.1.json"
406
+ if path.is_file():
407
+ try:
408
+ payload = json.loads(path.read_text(encoding="utf-8"))
409
+ except json.JSONDecodeError as exc:
410
+ raise BenchFormatError(f"invalid SkillsBench task set {path}: {exc}") from exc
411
+ tasks = payload.get("tasks") if isinstance(payload, dict) else None
412
+ if isinstance(tasks, list) and all(isinstance(item, dict) for item in tasks):
413
+ return tasks
414
+ return [{"task_id": path.name} for path in sorted((root / "tasks").iterdir()) if path.is_dir()]
415
+
416
+
417
+ class SkillsBenchAdapter(BenchAdapter):
418
+ name = "skillsbench"
419
+ description = "SkillsBench — task-specific environments, with-skill/no-skill ablation"
420
+ variants = ("with-skill", "no-skill")
421
+ default_variant = "with-skill"
422
+
423
+ @staticmethod
424
+ def _root(root: Path) -> Path:
425
+ root = root.expanduser().resolve()
426
+ if not (root / "tasks").is_dir():
427
+ raise FileNotFoundError(f"SkillsBench tasks/ not found under: {root}")
428
+ return root
429
+
430
+ def load(
431
+ self,
432
+ root: Path,
433
+ *,
434
+ categories: Sequence[str] | None = None,
435
+ task_ids: Sequence[str] | None = None,
436
+ variant: str | None = None,
437
+ timeout_multiplier: float = 1.0,
438
+ ) -> list[dict[str, Any]]:
439
+ root = self._root(root)
440
+ aliases = {
441
+ "with-skills": "with-skill",
442
+ "without-skills": "no-skill",
443
+ "without-skill": "no-skill",
444
+ }
445
+ variant = aliases.get(variant or "", variant) or self.default_variant
446
+ if variant not in self.variants:
447
+ raise BenchFormatError(
448
+ f"{self.name}: variant must be one of {', '.join(self.variants)}"
449
+ )
450
+ entries = _task_set(root)
451
+ known = {
452
+ item.get("task_id") for item in entries if isinstance(item.get("task_id"), str)
453
+ }
454
+ requested = set(task_ids or ())
455
+ missing = requested - known
456
+ if missing:
457
+ raise BenchFormatError(
458
+ f"{self.name}: unknown task id(s): {', '.join(sorted(missing))}"
459
+ )
460
+ category_filter = set(categories or ())
461
+ rows: list[dict[str, Any]] = []
462
+ for entry in entries:
463
+ task_id = entry.get("task_id")
464
+ if not isinstance(task_id, str):
465
+ continue
466
+ if requested and task_id not in requested:
467
+ continue
468
+ task_dir = root / "tasks" / task_id
469
+ metadata, prompt = _task_metadata(task_dir / "task.md")
470
+ taxonomy = metadata.get("metadata") or {}
471
+ if not isinstance(taxonomy, dict):
472
+ raise BenchFormatError(f"{task_id}: metadata.metadata must be an object")
473
+ category = str(entry.get("category") or taxonomy.get("category") or "")
474
+ if category_filter and category not in category_filter:
475
+ continue
476
+ rows.append(
477
+ self._row(
478
+ task_dir,
479
+ task_id,
480
+ metadata,
481
+ prompt,
482
+ category,
483
+ variant,
484
+ validate_timeout_multiplier(timeout_multiplier),
485
+ )
486
+ )
487
+ return rows
488
+
489
+ def _row(
490
+ self,
491
+ task_dir: Path,
492
+ task_id: str,
493
+ metadata: dict[str, Any],
494
+ prompt: str,
495
+ category: str,
496
+ variant: str,
497
+ timeout_multiplier: float,
498
+ ) -> dict[str, Any]:
499
+ environment_dir = task_dir / "environment"
500
+ dockerfile_path = environment_dir / "Dockerfile"
501
+ verifier_dir = task_dir / "verifier"
502
+ if not dockerfile_path.is_file() or not (verifier_dir / "test.sh").is_file():
503
+ raise BenchFormatError(f"{task_id}: environment Dockerfile or verifier/test.sh missing")
504
+ dockerfile = _sanitized_dockerfile(dockerfile_path)
505
+ digest = _environment_digest(environment_dir, dockerfile)
506
+ component = _image_component(task_id)[:80]
507
+ short = digest[:16]
508
+ base_image = f"roller-skillsbench-env-{component}:{short}"
509
+ image = f"roller-skillsbench-opencode-{component}:{short}"
510
+ agent = metadata.get("agent") or {}
511
+ verifier = metadata.get("verifier") or {}
512
+ # v1.1 calls this block ``environment``. Current development versions
513
+ # renamed it to ``sandbox``; accepting both keeps the adapter useful
514
+ # while allowing official runs to stay pinned to the v1.1 release.
515
+ sandbox = metadata.get("sandbox") or metadata.get("environment") or {}
516
+ if not isinstance(agent, dict) or not isinstance(verifier, dict):
517
+ raise BenchFormatError(f"{task_id}: agent and verifier must be objects")
518
+ if not isinstance(sandbox, dict):
519
+ raise BenchFormatError(f"{task_id}: sandbox must be an object")
520
+ timeout_value = agent.get("timeout_sec", 900)
521
+ verifier_timeout_value = verifier.get("timeout_sec", 900)
522
+ try:
523
+ timeout = max(1, math.ceil(float(timeout_value) * timeout_multiplier))
524
+ verifier_timeout = max(
525
+ 1, math.ceil(float(verifier_timeout_value) * timeout_multiplier)
526
+ )
527
+ except (TypeError, ValueError) as exc:
528
+ raise BenchFormatError(f"{task_id}: invalid agent/verifier timeout") from exc
529
+ build_timeout_value = sandbox.get("build_timeout_sec", 3600)
530
+ try:
531
+ build_timeout = max(1, math.ceil(float(build_timeout_value)))
532
+ except (TypeError, ValueError) as exc:
533
+ raise BenchFormatError(f"{task_id}: invalid sandbox.build_timeout_sec") from exc
534
+ build_args = {
535
+ str(key): value
536
+ for key, value in sandbox.items()
537
+ if key not in RESOURCE_KEYS and isinstance(value, (str, int, float, bool))
538
+ }
539
+ workdir = _workspace_from_dockerfile(dockerfile)
540
+ skills_dir = environment_dir / "skills"
541
+ files: dict[str, Any] = {"output_dir": ".roller_outputs"}
542
+ setup: list[str] = []
543
+ skill_targets = (
544
+ "/roller/opencode-home/config/opencode/skills",
545
+ "/root/.config/opencode/skills",
546
+ "/root/.opencode/skill",
547
+ "/root/.agents/skills",
548
+ "/root/.claude/skills",
549
+ "/root/.codex/skills",
550
+ "/root/.gemini/skills",
551
+ "/skills",
552
+ )
553
+ cleanup = "rm -rf " + " ".join(skill_targets)
554
+ if variant == "with-skill":
555
+ if not skills_dir.is_dir():
556
+ raise BenchFormatError(f"{task_id}: environment/skills is missing")
557
+ files["context"] = [
558
+ {
559
+ "role": "skill",
560
+ "source": str(skills_dir.resolve()),
561
+ "target": "skills",
562
+ "capture": "hash",
563
+ }
564
+ ]
565
+ copies = " && ".join(
566
+ f"mkdir -p {target} && cp -a /roller/task/context/skills/. {target}/"
567
+ for target in skill_targets
568
+ )
569
+ setup.append(f"{cleanup} && {copies}")
570
+ else:
571
+ setup.append(cleanup)
572
+ verifier_env = verifier.get("env") or {}
573
+ if not isinstance(verifier_env, dict):
574
+ raise BenchFormatError(f"{task_id}: verifier.env must be an object")
575
+ grade = GradeSpec(
576
+ code=_VERIFIER_RUNNER.replace("__VERIFIER_TIMEOUT__", str(verifier_timeout)),
577
+ entrypoint="roller_grade",
578
+ kwargs={"task_workspace": workdir, "verifier_env": verifier_env},
579
+ timeout_seconds=verifier_timeout + 30,
580
+ needs_transcript=False,
581
+ workspace_path="/",
582
+ stage=((str(verifier_dir.resolve()), "verifier"),),
583
+ )
584
+ binding = BenchBinding(
585
+ self.name,
586
+ grade=grade,
587
+ category=category,
588
+ metadata={
589
+ "variant": variant,
590
+ "difficulty": (metadata.get("metadata") or {}).get("difficulty"),
591
+ "environment_digest": digest,
592
+ "taxonomy": dict(metadata.get("metadata") or {}),
593
+ "resource_request": {
594
+ key: sandbox[key] for key in RESOURCE_KEYS if key in sandbox
595
+ },
596
+ "oracle_declared": "oracle" in metadata,
597
+ },
598
+ )
599
+ return {
600
+ "schema_version": "roller.task/v1",
601
+ "id": task_id,
602
+ "harness": "opencode",
603
+ "prompt": prompt,
604
+ "timeout_seconds": timeout,
605
+ "files": files,
606
+ "runtime": {
607
+ "image": image,
608
+ "image_build": {
609
+ "base_image": base_image,
610
+ "context": str(environment_dir.resolve()),
611
+ "dockerfile": dockerfile,
612
+ "harness": "opencode",
613
+ "args": build_args,
614
+ "timeout_seconds": build_timeout,
615
+ },
616
+ "workspace_path": workdir,
617
+ "setup": setup,
618
+ },
619
+ "bench": binding.as_dict(),
620
+ }
621
+
622
+ def aggregate(self, samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
623
+ """Report SkillsBench's score-eligible pass rate and mean reward."""
624
+
625
+ rows = list(samples)
626
+ task_rows: list[dict[str, Any]] = []
627
+ category_values: dict[str, list[tuple[float, bool]]] = {}
628
+ rewards: list[float] = []
629
+ passed = 0
630
+ for row in rows:
631
+ scores = row.get("bench_scores")
632
+ score_key = row.get("bench_score_key") or "overall_score"
633
+ value = sample_score(scores, score_key) if isinstance(scores, dict) else None
634
+ error = _score_error(row, scores if isinstance(scores, dict) else None)
635
+ declared_eligible = (
636
+ scores.get("score_eligible", True)
637
+ if isinstance(scores, dict)
638
+ else False
639
+ )
640
+ status = row.get("status")
641
+ eligible = bool(
642
+ status == "ok"
643
+ and value is not None
644
+ and not error
645
+ and declared_eligible is not False
646
+ )
647
+ is_passed = bool(
648
+ eligible
649
+ and (
650
+ scores.get("passed")
651
+ if isinstance(scores, dict) and isinstance(scores.get("passed"), bool)
652
+ else value > 0
653
+ )
654
+ )
655
+ category = row.get("bench_category")
656
+ if eligible:
657
+ assert value is not None
658
+ rewards.append(value)
659
+ passed += int(is_passed)
660
+ if isinstance(category, str) and category:
661
+ category_values.setdefault(category, []).append((value, is_passed))
662
+ elif not error:
663
+ error = str(row.get("error") or status or "not score eligible")
664
+ task_rows.append(
665
+ {
666
+ "task_id": row.get("task_id"),
667
+ "model": row.get("model"),
668
+ "category": category,
669
+ "status": status,
670
+ "score": value if eligible else None,
671
+ "score_eligible": eligible,
672
+ "passed": is_passed,
673
+ "error": error,
674
+ }
675
+ )
676
+ eligible_count = len(rewards)
677
+ mean_reward = sum(rewards) / eligible_count if eligible_count else None
678
+ pass_rate = passed / eligible_count if eligible_count else None
679
+ return {
680
+ "schema_version": REPORT_SCHEMA,
681
+ "metric": "pass_rate",
682
+ "task_count": len(rows),
683
+ "scored_count": eligible_count,
684
+ "score_eligible_count": eligible_count,
685
+ "unscored_count": len(rows) - eligible_count,
686
+ "infra_failed_count": len(rows) - eligible_count,
687
+ "passed_count": passed,
688
+ "pass_rate": round(pass_rate, 6) if pass_rate is not None else None,
689
+ "mean_reward": round(mean_reward, 6) if mean_reward is not None else None,
690
+ "average": round(mean_reward, 6) if mean_reward is not None else None,
691
+ "scored_average": round(mean_reward, 6) if mean_reward is not None else None,
692
+ "categories": {
693
+ name: {
694
+ "task_count": len(values),
695
+ "pass_rate": round(
696
+ sum(int(item[1]) for item in values) / len(values), 6
697
+ ),
698
+ "average": round(
699
+ sum(item[0] for item in values) / len(values), 6
700
+ ),
701
+ }
702
+ for name, values in sorted(category_values.items())
703
+ },
704
+ "tasks": task_rows,
705
+ }
706
+
707
+
708
+ __all__ = ["SkillsBenchAdapter"]