roller-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- roller/__init__.py +31 -0
- roller/__main__.py +6 -0
- roller/actf.py +431 -0
- roller/artifacts.py +124 -0
- roller/assets/deepseek/journal.mjs +44 -0
- roller/assets/deepseek/roller.patch.yml +15 -0
- roller/assets/deepseek/runner.py +149 -0
- roller/assets/hermes/runner.py +322 -0
- roller/assets/openclaw_capture.py +32 -0
- roller/assets/openclaw_drain.mjs +21 -0
- roller/assets/opencode/journal.mjs +42 -0
- roller/assets/opencode/runner.py +203 -0
- roller/assets/pi/runner.mjs +160 -0
- roller/assets/pi/runner.py +93 -0
- roller/benches/__init__.py +73 -0
- roller/benches/base.py +306 -0
- roller/benches/grading.py +553 -0
- roller/benches/pinchbench.py +976 -0
- roller/benches/qwenclaw/__init__.py +49 -0
- roller/benches/qwenclaw/adapter.py +167 -0
- roller/benches/qwenclaw/grader_runtime.py +395 -0
- roller/benches/qwenclaw/report.py +49 -0
- roller/benches/qwenclaw/task_parser.py +181 -0
- roller/benches/qwenclaw/workspace.py +202 -0
- roller/benches/report.py +166 -0
- roller/benches/skillsbench.py +708 -0
- roller/benches/transcripts.py +102 -0
- roller/benches/wildclaw.py +569 -0
- roller/bridge_capture.py +73 -0
- roller/bundles.py +545 -0
- roller/capture.py +385 -0
- roller/capture_protocol.py +301 -0
- roller/capture_runtime.py +48 -0
- roller/capture_states.py +37 -0
- roller/claude_capture.py +185 -0
- roller/claude_code_actf.py +594 -0
- roller/cli.py +1732 -0
- roller/codex_actf.py +584 -0
- roller/codex_capture.py +99 -0
- roller/codex_tools.py +137 -0
- roller/collector.py +818 -0
- roller/compaction_sft.py +672 -0
- roller/conversation.py +432 -0
- roller/deepseek_actf.py +135 -0
- roller/harness_registry.py +61 -0
- roller/harnesses/__init__.py +29 -0
- roller/harnesses/base.py +64 -0
- roller/harnesses/bridge.py +448 -0
- roller/harnesses/claude_code.py +839 -0
- roller/harnesses/codex.py +808 -0
- roller/harnesses/config_overrides.py +100 -0
- roller/harnesses/container.py +206 -0
- roller/harnesses/deepseek.py +104 -0
- roller/harnesses/docker.py +72 -0
- roller/harnesses/gateway_ports.py +155 -0
- roller/harnesses/hermes.py +119 -0
- roller/harnesses/openclaw.py +1605 -0
- roller/harnesses/opencode.py +193 -0
- roller/harnesses/pi.py +111 -0
- roller/hermes_actf.py +147 -0
- roller/image_build.py +206 -0
- roller/integrity.py +332 -0
- roller/model_protocols.py +201 -0
- roller/models.py +169 -0
- roller/openclaw_capture.py +140 -0
- roller/openclaw_delivery.py +211 -0
- roller/openclaw_usage.py +59 -0
- roller/opencode_actf.py +305 -0
- roller/opencode_capture.py +96 -0
- roller/pi_actf.py +172 -0
- roller/response_metadata.py +89 -0
- roller/run_lock.py +297 -0
- roller/sft.py +471 -0
- roller/storage.py +53 -0
- roller/tasks.py +689 -0
- roller/thinking.py +50 -0
- roller/thinking_capabilities.py +152 -0
- roller/trajectory_audit.py +386 -0
- roller/usage.py +168 -0
- roller/version.py +1 -0
- roller/viewer/__init__.py +3 -0
- roller/viewer/app.py +722 -0
- roller/viewer/static/app.js +752 -0
- roller/viewer/static/index.html +105 -0
- roller/viewer/static/style.css +213 -0
- roller/wire_trace.py +650 -0
- roller_agent-0.2.0.dist-info/METADATA +397 -0
- roller_agent-0.2.0.dist-info/RECORD +92 -0
- roller_agent-0.2.0.dist-info/WHEEL +5 -0
- roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
- roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
- roller_agent-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
"""QwenClawBench Markdown task parsing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import re
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import yaml
|
|
11
|
+
|
|
12
|
+
from roller.benches.base import BenchFormatError
|
|
13
|
+
|
|
14
|
+
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)$", re.DOTALL)
|
|
15
|
+
SECTION_RE = re.compile(r"^##\s+(.+)$")
|
|
16
|
+
PYTHON_FENCE_RE = re.compile(r"```python\s*\n(.*?)\n\s*```", re.DOTALL)
|
|
17
|
+
TASK_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
|
|
18
|
+
DEFAULT_TIMEOUT_SECONDS = 1800
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(frozen=True)
|
|
22
|
+
class QwenClawTask:
|
|
23
|
+
task_id: str
|
|
24
|
+
name: str
|
|
25
|
+
category: str
|
|
26
|
+
subcategory: str
|
|
27
|
+
grading_type: str
|
|
28
|
+
grading_weights: dict[str, float]
|
|
29
|
+
timeout_seconds: int
|
|
30
|
+
input_modality: str
|
|
31
|
+
external_dependency: str
|
|
32
|
+
workspace_files: tuple[dict[str, Any], ...]
|
|
33
|
+
prompt: str
|
|
34
|
+
expected_behavior: str
|
|
35
|
+
grading_criteria: tuple[str, ...]
|
|
36
|
+
automated_code: str
|
|
37
|
+
llm_judge_rubric: str
|
|
38
|
+
source_path: Path
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def split_sections(body: str) -> dict[str, str]:
|
|
42
|
+
sections: dict[str, str] = {}
|
|
43
|
+
current: str | None = None
|
|
44
|
+
lines: list[str] = []
|
|
45
|
+
for line in body.splitlines():
|
|
46
|
+
match = SECTION_RE.match(line)
|
|
47
|
+
if match:
|
|
48
|
+
if current is not None:
|
|
49
|
+
sections[current] = "\n".join(lines).strip()
|
|
50
|
+
current = match.group(1).strip()
|
|
51
|
+
lines = []
|
|
52
|
+
elif current is not None:
|
|
53
|
+
lines.append(line)
|
|
54
|
+
if current is not None:
|
|
55
|
+
sections[current] = "\n".join(lines).strip()
|
|
56
|
+
return sections
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _grading_criteria(raw: str) -> tuple[str, ...]:
|
|
60
|
+
criteria: list[str] = []
|
|
61
|
+
for line in raw.splitlines():
|
|
62
|
+
match = re.match(r"^-\s+\[[ xX]\]\s+(.+)$", line.strip())
|
|
63
|
+
if match:
|
|
64
|
+
criteria.append(match.group(1).strip())
|
|
65
|
+
return tuple(criteria)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _python_code(raw: str, label: str) -> str:
|
|
69
|
+
match = PYTHON_FENCE_RE.search(raw)
|
|
70
|
+
if not match:
|
|
71
|
+
raise BenchFormatError(f"{label}: missing fenced Python grader")
|
|
72
|
+
code = match.group(1).strip()
|
|
73
|
+
if not code:
|
|
74
|
+
raise BenchFormatError(f"{label}: automated grader is empty")
|
|
75
|
+
try:
|
|
76
|
+
compile(code, label, "exec")
|
|
77
|
+
except SyntaxError as exc:
|
|
78
|
+
raise BenchFormatError(f"{label}: invalid automated grader: {exc}") from exc
|
|
79
|
+
return code
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def parse_task_markdown(
|
|
83
|
+
text: str, label: str, source_path: Path | None = None
|
|
84
|
+
) -> QwenClawTask:
|
|
85
|
+
match = FRONTMATTER_RE.match(text)
|
|
86
|
+
if not match:
|
|
87
|
+
raise BenchFormatError(f"{label}: missing YAML frontmatter")
|
|
88
|
+
frontmatter = re.sub(
|
|
89
|
+
r"^source_query:\s*.*?(?=\n\w+:\s|\Z)",
|
|
90
|
+
"",
|
|
91
|
+
match.group(1),
|
|
92
|
+
flags=re.MULTILINE | re.DOTALL,
|
|
93
|
+
)
|
|
94
|
+
frontmatter = re.sub(
|
|
95
|
+
r"^source_line_number:.*$", "", frontmatter, flags=re.MULTILINE
|
|
96
|
+
)
|
|
97
|
+
try:
|
|
98
|
+
metadata = yaml.safe_load(frontmatter)
|
|
99
|
+
except yaml.YAMLError as exc:
|
|
100
|
+
raise BenchFormatError(f"{label}: invalid YAML frontmatter: {exc}") from exc
|
|
101
|
+
if not isinstance(metadata, dict):
|
|
102
|
+
raise BenchFormatError(f"{label}: frontmatter must be an object")
|
|
103
|
+
sections = split_sections(match.group(2))
|
|
104
|
+
|
|
105
|
+
task_id = metadata.get("id")
|
|
106
|
+
if not isinstance(task_id, str) or not task_id.strip():
|
|
107
|
+
raise BenchFormatError(f"{label}: id must be a non-empty string")
|
|
108
|
+
task_id = task_id.strip()
|
|
109
|
+
if not TASK_ID_RE.fullmatch(task_id):
|
|
110
|
+
raise BenchFormatError(f"{label}: unsafe task id {task_id!r}")
|
|
111
|
+
prompt = sections.get("Prompt", "").strip()
|
|
112
|
+
if not prompt:
|
|
113
|
+
raise BenchFormatError(f"{label}: missing ## Prompt")
|
|
114
|
+
grading_type = metadata.get("grading_type", "automated")
|
|
115
|
+
if grading_type not in {"automated", "llm_judge", "hybrid"}:
|
|
116
|
+
raise BenchFormatError(f"{label}: unsupported grading_type {grading_type!r}")
|
|
117
|
+
raw_weights = metadata.get("grading_weights") or {
|
|
118
|
+
"automated": 0.5,
|
|
119
|
+
"llm_judge": 0.5,
|
|
120
|
+
}
|
|
121
|
+
if not isinstance(raw_weights, dict):
|
|
122
|
+
raise BenchFormatError(f"{label}: grading_weights must be an object")
|
|
123
|
+
try:
|
|
124
|
+
weights = {
|
|
125
|
+
"automated": float(raw_weights.get("automated", 0.5)),
|
|
126
|
+
"llm_judge": float(raw_weights.get("llm_judge", 0.5)),
|
|
127
|
+
}
|
|
128
|
+
except (TypeError, ValueError) as exc:
|
|
129
|
+
raise BenchFormatError(f"{label}: grading weights must be numeric") from exc
|
|
130
|
+
if min(weights.values()) < 0 or sum(weights.values()) <= 0:
|
|
131
|
+
raise BenchFormatError(
|
|
132
|
+
f"{label}: grading weights must be non-negative and non-zero"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
raw_timeout = metadata.get("timeout_seconds", DEFAULT_TIMEOUT_SECONDS)
|
|
136
|
+
if isinstance(raw_timeout, bool):
|
|
137
|
+
raise BenchFormatError(f"{label}: timeout_seconds must be an integer")
|
|
138
|
+
try:
|
|
139
|
+
timeout_seconds = int(raw_timeout)
|
|
140
|
+
except (TypeError, ValueError) as exc:
|
|
141
|
+
raise BenchFormatError(f"{label}: timeout_seconds must be an integer") from exc
|
|
142
|
+
if timeout_seconds <= 0:
|
|
143
|
+
raise BenchFormatError(f"{label}: timeout_seconds must be positive")
|
|
144
|
+
|
|
145
|
+
workspace_files = metadata.get("workspace_files", [])
|
|
146
|
+
if not isinstance(workspace_files, list) or not all(
|
|
147
|
+
isinstance(item, dict) for item in workspace_files
|
|
148
|
+
):
|
|
149
|
+
raise BenchFormatError(f"{label}: workspace_files must be a list of objects")
|
|
150
|
+
automated_code = ""
|
|
151
|
+
if grading_type in {"automated", "hybrid"}:
|
|
152
|
+
automated_code = _python_code(
|
|
153
|
+
sections.get("Automated Checks", ""), f"{label} ## Automated Checks"
|
|
154
|
+
)
|
|
155
|
+
rubric = sections.get("LLM Judge Rubric", "").strip()
|
|
156
|
+
if grading_type in {"llm_judge", "hybrid"} and not rubric:
|
|
157
|
+
rubric = "\n".join(
|
|
158
|
+
f"- {item}"
|
|
159
|
+
for item in _grading_criteria(sections.get("Grading Criteria", ""))
|
|
160
|
+
)
|
|
161
|
+
if grading_type in {"llm_judge", "hybrid"} and not rubric:
|
|
162
|
+
raise BenchFormatError(f"{label}: LLM judge task has no rubric")
|
|
163
|
+
|
|
164
|
+
return QwenClawTask(
|
|
165
|
+
task_id=task_id,
|
|
166
|
+
name=str(metadata.get("name") or "").strip(),
|
|
167
|
+
category=str(metadata.get("category") or "").strip(),
|
|
168
|
+
subcategory=str(metadata.get("subcategory") or "").strip(),
|
|
169
|
+
grading_type=grading_type,
|
|
170
|
+
grading_weights=weights,
|
|
171
|
+
timeout_seconds=timeout_seconds,
|
|
172
|
+
input_modality=str(metadata.get("input_modality") or "").strip(),
|
|
173
|
+
external_dependency=str(metadata.get("external_dependency") or "").strip(),
|
|
174
|
+
workspace_files=tuple(workspace_files),
|
|
175
|
+
prompt=prompt,
|
|
176
|
+
expected_behavior=sections.get("Expected Behavior", "").strip(),
|
|
177
|
+
grading_criteria=_grading_criteria(sections.get("Grading Criteria", "")),
|
|
178
|
+
automated_code=automated_code,
|
|
179
|
+
llm_judge_rubric=rubric,
|
|
180
|
+
source_path=source_path or Path(label),
|
|
181
|
+
)
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""QwenClawBench workspace materialization and path compatibility."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import shlex
|
|
9
|
+
import shutil
|
|
10
|
+
import stat
|
|
11
|
+
import tempfile
|
|
12
|
+
from pathlib import Path, PurePosixPath
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from roller.benches.base import BenchFormatError
|
|
16
|
+
from roller.benches.qwenclaw.task_parser import QwenClawTask
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _relative_path(value: Any, label: str) -> str:
|
|
20
|
+
if not isinstance(value, str) or not value.strip():
|
|
21
|
+
raise BenchFormatError(f"{label}: must be a non-empty relative path")
|
|
22
|
+
raw = value.strip().replace("\\", "/")
|
|
23
|
+
path = PurePosixPath(raw)
|
|
24
|
+
if (
|
|
25
|
+
path.is_absolute()
|
|
26
|
+
or not path.parts
|
|
27
|
+
or any(part in {"", ".", ".."} for part in path.parts)
|
|
28
|
+
):
|
|
29
|
+
raise BenchFormatError(f"{label}: unsafe path {value!r}")
|
|
30
|
+
return path.as_posix()
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _source_path(assets_root: Path, task_id: str, source: str) -> Path:
|
|
34
|
+
relative = _relative_path(source, f"{task_id}.workspace_files.source")
|
|
35
|
+
candidates = (assets_root / task_id / relative, assets_root / relative)
|
|
36
|
+
for candidate in candidates:
|
|
37
|
+
if not candidate.exists():
|
|
38
|
+
continue
|
|
39
|
+
cursor = assets_root
|
|
40
|
+
for part in candidate.relative_to(assets_root).parts:
|
|
41
|
+
cursor = cursor / part
|
|
42
|
+
if cursor.is_symlink():
|
|
43
|
+
raise BenchFormatError(
|
|
44
|
+
f"{task_id}: workspace source is a symlink: {cursor}"
|
|
45
|
+
)
|
|
46
|
+
resolved = candidate.resolve()
|
|
47
|
+
if not resolved.is_relative_to(assets_root.resolve()):
|
|
48
|
+
raise BenchFormatError(
|
|
49
|
+
f"{task_id}: workspace source escapes assets: {source}"
|
|
50
|
+
)
|
|
51
|
+
return candidate
|
|
52
|
+
searched = ", ".join(str(path) for path in candidates)
|
|
53
|
+
raise BenchFormatError(
|
|
54
|
+
f"{task_id}: missing workspace source {source!r}; searched {searched}"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _hash_source(digest: Any, source: Path, logical: str) -> None:
|
|
59
|
+
if source.is_dir():
|
|
60
|
+
for path in sorted(source.rglob("*")):
|
|
61
|
+
if path.is_symlink():
|
|
62
|
+
raise BenchFormatError(f"workspace source contains a symlink: {path}")
|
|
63
|
+
relative = path.relative_to(source).as_posix()
|
|
64
|
+
mode = path.stat().st_mode
|
|
65
|
+
if path.is_dir():
|
|
66
|
+
digest.update(f"D:{logical}/{relative}\0".encode())
|
|
67
|
+
elif path.is_file() and stat.S_ISREG(mode):
|
|
68
|
+
digest.update(f"F:{logical}/{relative}\0".encode())
|
|
69
|
+
with path.open("rb") as handle:
|
|
70
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
71
|
+
digest.update(chunk)
|
|
72
|
+
else:
|
|
73
|
+
raise BenchFormatError(
|
|
74
|
+
f"workspace source is not a regular file: {path}"
|
|
75
|
+
)
|
|
76
|
+
return
|
|
77
|
+
mode = source.stat().st_mode
|
|
78
|
+
if not stat.S_ISREG(mode):
|
|
79
|
+
raise BenchFormatError(f"workspace source is not a regular file: {source}")
|
|
80
|
+
digest.update(f"F:{logical}\0".encode())
|
|
81
|
+
with source.open("rb") as handle:
|
|
82
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
83
|
+
digest.update(chunk)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _cache_root() -> Path:
|
|
87
|
+
configured = (os.environ.get("QWENCLAW_CACHE_DIR") or "").strip()
|
|
88
|
+
if configured:
|
|
89
|
+
return Path(configured).expanduser().resolve()
|
|
90
|
+
xdg = (os.environ.get("XDG_CACHE_HOME") or "").strip()
|
|
91
|
+
base = Path(xdg).expanduser() if xdg else Path.home() / ".cache"
|
|
92
|
+
return (base / "roller" / "benches" / "qwenclawbench").resolve()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def materialize_workspace(
|
|
96
|
+
task: QwenClawTask, assets_root: Path, cache_root: Path | None = None
|
|
97
|
+
) -> Path:
|
|
98
|
+
"""Build the exact source-to-destination workspace expected upstream."""
|
|
99
|
+
|
|
100
|
+
entries: list[tuple[str, Path | None, str | None]] = []
|
|
101
|
+
destinations: list[str] = []
|
|
102
|
+
digest = hashlib.sha256()
|
|
103
|
+
digest.update(task.source_path.read_bytes())
|
|
104
|
+
for index, item in enumerate(task.workspace_files):
|
|
105
|
+
destination_value = item.get("dest", item.get("path"))
|
|
106
|
+
destination = _relative_path(
|
|
107
|
+
destination_value, f"{task.task_id}.workspace_files[{index}].dest"
|
|
108
|
+
)
|
|
109
|
+
for existing in destinations:
|
|
110
|
+
if (
|
|
111
|
+
destination == existing
|
|
112
|
+
or destination.startswith(existing + "/")
|
|
113
|
+
or existing.startswith(destination + "/")
|
|
114
|
+
):
|
|
115
|
+
raise BenchFormatError(
|
|
116
|
+
f"{task.task_id}: colliding workspace destinations "
|
|
117
|
+
f"{existing!r} and {destination!r}"
|
|
118
|
+
)
|
|
119
|
+
destinations.append(destination)
|
|
120
|
+
if "content" in item:
|
|
121
|
+
content = item["content"]
|
|
122
|
+
if not isinstance(content, str):
|
|
123
|
+
raise BenchFormatError(
|
|
124
|
+
f"{task.task_id}.workspace_files[{index}].content must be a string"
|
|
125
|
+
)
|
|
126
|
+
digest.update(f"C:{destination}\0{content}".encode())
|
|
127
|
+
entries.append((destination, None, content))
|
|
128
|
+
continue
|
|
129
|
+
source_value = item.get("source")
|
|
130
|
+
if not isinstance(source_value, str):
|
|
131
|
+
raise BenchFormatError(
|
|
132
|
+
f"{task.task_id}.workspace_files[{index}].source must be a string"
|
|
133
|
+
)
|
|
134
|
+
source = _source_path(assets_root, task.task_id, source_value)
|
|
135
|
+
digest.update(f"M:{source_value}->{destination}\0".encode())
|
|
136
|
+
_hash_source(digest, source, destination)
|
|
137
|
+
entries.append((destination, source, None))
|
|
138
|
+
|
|
139
|
+
fingerprint = digest.hexdigest()
|
|
140
|
+
task_cache = (cache_root or _cache_root()) / task.task_id
|
|
141
|
+
target = task_cache / fingerprint
|
|
142
|
+
workspace = target / "workspace"
|
|
143
|
+
if (target / "manifest.json").is_file() and workspace.is_dir():
|
|
144
|
+
return workspace
|
|
145
|
+
|
|
146
|
+
task_cache.mkdir(parents=True, exist_ok=True)
|
|
147
|
+
temporary = Path(tempfile.mkdtemp(prefix=f".{fingerprint}.", dir=task_cache))
|
|
148
|
+
temporary_workspace = temporary / "workspace"
|
|
149
|
+
temporary_workspace.mkdir()
|
|
150
|
+
try:
|
|
151
|
+
for destination, source, content in entries:
|
|
152
|
+
output = temporary_workspace / destination
|
|
153
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
154
|
+
if content is not None:
|
|
155
|
+
output.write_text(content, encoding="utf-8")
|
|
156
|
+
elif source is not None and source.is_dir():
|
|
157
|
+
shutil.copytree(source, output, copy_function=shutil.copy2)
|
|
158
|
+
elif source is not None:
|
|
159
|
+
shutil.copy2(source, output)
|
|
160
|
+
(temporary / "manifest.json").write_text(
|
|
161
|
+
json.dumps(
|
|
162
|
+
{
|
|
163
|
+
"schema_version": "roller.qwenclaw_workspace/v1",
|
|
164
|
+
"task_id": task.task_id,
|
|
165
|
+
"fingerprint": fingerprint,
|
|
166
|
+
"source": str(task.source_path),
|
|
167
|
+
},
|
|
168
|
+
ensure_ascii=False,
|
|
169
|
+
indent=2,
|
|
170
|
+
)
|
|
171
|
+
+ "\n",
|
|
172
|
+
encoding="utf-8",
|
|
173
|
+
)
|
|
174
|
+
try:
|
|
175
|
+
temporary.rename(target)
|
|
176
|
+
except FileExistsError:
|
|
177
|
+
shutil.rmtree(temporary)
|
|
178
|
+
except Exception:
|
|
179
|
+
shutil.rmtree(temporary, ignore_errors=True)
|
|
180
|
+
raise
|
|
181
|
+
return workspace
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def workspace_compatibility_script(workspace: str) -> str:
|
|
185
|
+
"""Make ``workspace/...`` and workspace-relative paths refer to one tree."""
|
|
186
|
+
|
|
187
|
+
root = workspace.rstrip("/")
|
|
188
|
+
alias = f"{root}/workspace"
|
|
189
|
+
quoted_root = shlex.quote(root)
|
|
190
|
+
quoted_alias = shlex.quote(alias)
|
|
191
|
+
return (
|
|
192
|
+
"set -eu; "
|
|
193
|
+
f"if [ -L {quoted_alias} ]; then "
|
|
194
|
+
f'test "$(readlink -f {quoted_alias})" = "$(readlink -f {quoted_root})"; '
|
|
195
|
+
f"elif [ -d {quoted_alias} ]; then "
|
|
196
|
+
f"cp -a {quoted_alias}/. {quoted_root}/; "
|
|
197
|
+
f"rm -rf {quoted_alias}; "
|
|
198
|
+
f"ln -s . {quoted_alias}; "
|
|
199
|
+
f"elif [ -e {quoted_alias} ]; then "
|
|
200
|
+
"echo 'workspace compatibility path is not a directory' >&2; exit 1; "
|
|
201
|
+
f"else ln -s . {quoted_alias}; fi"
|
|
202
|
+
)
|
roller/benches/report.py
ADDED
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
from typing import Any, Iterable
|
|
6
|
+
|
|
7
|
+
SCORE_FILE = "score.json"
|
|
8
|
+
REPORT_SCHEMA = "roller.bench_report/v1"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _numeric(scores: dict[str, Any]) -> dict[str, float]:
|
|
12
|
+
return {
|
|
13
|
+
key: float(value)
|
|
14
|
+
for key, value in scores.items()
|
|
15
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool)
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def sample_score(scores: dict[str, Any], score_key: str = "overall_score") -> float | None:
|
|
20
|
+
"""Reduce one grader payload to a single number.
|
|
21
|
+
|
|
22
|
+
Follows WildClawBench's rule: prefer the declared aggregate key, otherwise
|
|
23
|
+
average whatever numeric fields the grader returned.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
numeric = _numeric(scores)
|
|
27
|
+
if not numeric:
|
|
28
|
+
return None
|
|
29
|
+
if score_key in numeric:
|
|
30
|
+
return numeric[score_key]
|
|
31
|
+
return sum(numeric.values()) / len(numeric)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def read_scores(sample_dir: Path) -> dict[str, Any] | None:
|
|
35
|
+
path = sample_dir / SCORE_FILE
|
|
36
|
+
if not path.is_file():
|
|
37
|
+
return None
|
|
38
|
+
try:
|
|
39
|
+
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
40
|
+
except (OSError, json.JSONDecodeError):
|
|
41
|
+
return None
|
|
42
|
+
return payload if isinstance(payload, dict) else None
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def mean_aggregate(samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
|
|
46
|
+
"""Average scored samples over *all* samples, counting failures as zero.
|
|
47
|
+
|
|
48
|
+
An agent that crashed or timed out earned no score; dropping it from the
|
|
49
|
+
denominator would silently reward failure. Unscored samples are reported
|
|
50
|
+
separately so a broken grader is not mistaken for a weak model.
|
|
51
|
+
"""
|
|
52
|
+
|
|
53
|
+
rows = list(samples)
|
|
54
|
+
per_task: list[dict[str, Any]] = []
|
|
55
|
+
per_category: dict[str, list[float]] = {}
|
|
56
|
+
total = 0.0
|
|
57
|
+
scored = 0
|
|
58
|
+
for row in rows:
|
|
59
|
+
scores = row.get("bench_scores")
|
|
60
|
+
score_key = row.get("bench_score_key") or "overall_score"
|
|
61
|
+
grader_error = row.get("bench_error")
|
|
62
|
+
if not grader_error and isinstance(scores, dict):
|
|
63
|
+
grader_error = scores.get("error")
|
|
64
|
+
# A synthetic zero returned alongside a grader/infrastructure error is
|
|
65
|
+
# not a model score. Keep it unscored so broken grading cannot depress
|
|
66
|
+
# benchmark results while still surfacing the failure in the report.
|
|
67
|
+
value = (
|
|
68
|
+
sample_score(scores, score_key)
|
|
69
|
+
if isinstance(scores, dict) and not grader_error
|
|
70
|
+
else None
|
|
71
|
+
)
|
|
72
|
+
category = row.get("bench_category")
|
|
73
|
+
if value is not None:
|
|
74
|
+
total += value
|
|
75
|
+
scored += 1
|
|
76
|
+
if isinstance(category, str) and category:
|
|
77
|
+
per_category.setdefault(category, []).append(value)
|
|
78
|
+
per_task.append(
|
|
79
|
+
{
|
|
80
|
+
"task_id": row.get("task_id"),
|
|
81
|
+
"model": row.get("model"),
|
|
82
|
+
"category": category,
|
|
83
|
+
"status": row.get("status"),
|
|
84
|
+
"score": value,
|
|
85
|
+
"error": grader_error,
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
task_count = len(rows)
|
|
89
|
+
return {
|
|
90
|
+
"schema_version": REPORT_SCHEMA,
|
|
91
|
+
"task_count": task_count,
|
|
92
|
+
"scored_count": scored,
|
|
93
|
+
"unscored_count": task_count - scored,
|
|
94
|
+
"average": round(total / task_count, 6) if task_count else None,
|
|
95
|
+
"scored_average": round(total / scored, 6) if scored else None,
|
|
96
|
+
"categories": {
|
|
97
|
+
name: {
|
|
98
|
+
"task_count": len(values),
|
|
99
|
+
"average": round(sum(values) / len(values), 6),
|
|
100
|
+
}
|
|
101
|
+
for name, values in sorted(per_category.items())
|
|
102
|
+
},
|
|
103
|
+
"tasks": per_task,
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def format_report(bench_name: str, report: dict[str, Any]) -> str:
|
|
108
|
+
lines = [
|
|
109
|
+
"",
|
|
110
|
+
"=" * 62,
|
|
111
|
+
f" Bench report — {bench_name}",
|
|
112
|
+
"=" * 62,
|
|
113
|
+
]
|
|
114
|
+
average = report.get("average")
|
|
115
|
+
if average is None:
|
|
116
|
+
lines.append(" no samples")
|
|
117
|
+
else:
|
|
118
|
+
filled = max(0, min(10, int(round(average * 10))))
|
|
119
|
+
bar = "█" * filled + "░" * (10 - filled)
|
|
120
|
+
lines.append(
|
|
121
|
+
f" scored {report['scored_count']}/{report['task_count']}"
|
|
122
|
+
f" (unscored {report['unscored_count']})"
|
|
123
|
+
)
|
|
124
|
+
pass_rate = report.get("pass_rate")
|
|
125
|
+
if isinstance(pass_rate, (int, float)):
|
|
126
|
+
pass_bar_count = max(0, min(10, int(round(pass_rate * 10))))
|
|
127
|
+
pass_bar = "█" * pass_bar_count + "░" * (10 - pass_bar_count)
|
|
128
|
+
lines.append(f" pass rate {pass_bar} {pass_rate:.4f}")
|
|
129
|
+
lines.append(f" mean reward {bar} {average:.4f}")
|
|
130
|
+
else:
|
|
131
|
+
lines.append(f" average {bar} {average:.4f}")
|
|
132
|
+
scored_average = report.get("scored_average")
|
|
133
|
+
if (
|
|
134
|
+
scored_average is not None
|
|
135
|
+
and report["unscored_count"]
|
|
136
|
+
and not isinstance(pass_rate, (int, float))
|
|
137
|
+
):
|
|
138
|
+
lines.append(f" average over scored samples only: {scored_average:.4f}")
|
|
139
|
+
categories = report.get("categories") or {}
|
|
140
|
+
if categories:
|
|
141
|
+
lines.append("")
|
|
142
|
+
for name, row in categories.items():
|
|
143
|
+
if isinstance(row.get("pass_rate"), (int, float)):
|
|
144
|
+
lines.append(
|
|
145
|
+
f" pass {row['pass_rate']:.4f} reward {row['average']:.4f}"
|
|
146
|
+
f" {name} ({row['task_count']} trials)"
|
|
147
|
+
)
|
|
148
|
+
else:
|
|
149
|
+
lines.append(
|
|
150
|
+
f" {row['average']:.4f} {name} ({row['task_count']} tasks)"
|
|
151
|
+
)
|
|
152
|
+
failures = [
|
|
153
|
+
row
|
|
154
|
+
for row in report.get("tasks", [])
|
|
155
|
+
if row.get("score") is None
|
|
156
|
+
]
|
|
157
|
+
if failures:
|
|
158
|
+
lines.append("")
|
|
159
|
+
lines.append(f" unscored samples ({len(failures)}):")
|
|
160
|
+
for row in failures[:20]:
|
|
161
|
+
detail = row.get("error") or row.get("status") or "unknown"
|
|
162
|
+
lines.append(f" - {row.get('task_id')}: {detail}")
|
|
163
|
+
if len(failures) > 20:
|
|
164
|
+
lines.append(f" ... and {len(failures) - 20} more")
|
|
165
|
+
lines.append("=" * 62)
|
|
166
|
+
return "\n".join(lines)
|