roller-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. roller/__init__.py +31 -0
  2. roller/__main__.py +6 -0
  3. roller/actf.py +431 -0
  4. roller/artifacts.py +124 -0
  5. roller/assets/deepseek/journal.mjs +44 -0
  6. roller/assets/deepseek/roller.patch.yml +15 -0
  7. roller/assets/deepseek/runner.py +149 -0
  8. roller/assets/hermes/runner.py +322 -0
  9. roller/assets/openclaw_capture.py +32 -0
  10. roller/assets/openclaw_drain.mjs +21 -0
  11. roller/assets/opencode/journal.mjs +42 -0
  12. roller/assets/opencode/runner.py +203 -0
  13. roller/assets/pi/runner.mjs +160 -0
  14. roller/assets/pi/runner.py +93 -0
  15. roller/benches/__init__.py +73 -0
  16. roller/benches/base.py +306 -0
  17. roller/benches/grading.py +553 -0
  18. roller/benches/pinchbench.py +976 -0
  19. roller/benches/qwenclaw/__init__.py +49 -0
  20. roller/benches/qwenclaw/adapter.py +167 -0
  21. roller/benches/qwenclaw/grader_runtime.py +395 -0
  22. roller/benches/qwenclaw/report.py +49 -0
  23. roller/benches/qwenclaw/task_parser.py +181 -0
  24. roller/benches/qwenclaw/workspace.py +202 -0
  25. roller/benches/report.py +166 -0
  26. roller/benches/skillsbench.py +708 -0
  27. roller/benches/transcripts.py +102 -0
  28. roller/benches/wildclaw.py +569 -0
  29. roller/bridge_capture.py +73 -0
  30. roller/bundles.py +545 -0
  31. roller/capture.py +385 -0
  32. roller/capture_protocol.py +301 -0
  33. roller/capture_runtime.py +48 -0
  34. roller/capture_states.py +37 -0
  35. roller/claude_capture.py +185 -0
  36. roller/claude_code_actf.py +594 -0
  37. roller/cli.py +1732 -0
  38. roller/codex_actf.py +584 -0
  39. roller/codex_capture.py +99 -0
  40. roller/codex_tools.py +137 -0
  41. roller/collector.py +818 -0
  42. roller/compaction_sft.py +672 -0
  43. roller/conversation.py +432 -0
  44. roller/deepseek_actf.py +135 -0
  45. roller/harness_registry.py +61 -0
  46. roller/harnesses/__init__.py +29 -0
  47. roller/harnesses/base.py +64 -0
  48. roller/harnesses/bridge.py +448 -0
  49. roller/harnesses/claude_code.py +839 -0
  50. roller/harnesses/codex.py +808 -0
  51. roller/harnesses/config_overrides.py +100 -0
  52. roller/harnesses/container.py +206 -0
  53. roller/harnesses/deepseek.py +104 -0
  54. roller/harnesses/docker.py +72 -0
  55. roller/harnesses/gateway_ports.py +155 -0
  56. roller/harnesses/hermes.py +119 -0
  57. roller/harnesses/openclaw.py +1605 -0
  58. roller/harnesses/opencode.py +193 -0
  59. roller/harnesses/pi.py +111 -0
  60. roller/hermes_actf.py +147 -0
  61. roller/image_build.py +206 -0
  62. roller/integrity.py +332 -0
  63. roller/model_protocols.py +201 -0
  64. roller/models.py +169 -0
  65. roller/openclaw_capture.py +140 -0
  66. roller/openclaw_delivery.py +211 -0
  67. roller/openclaw_usage.py +59 -0
  68. roller/opencode_actf.py +305 -0
  69. roller/opencode_capture.py +96 -0
  70. roller/pi_actf.py +172 -0
  71. roller/response_metadata.py +89 -0
  72. roller/run_lock.py +297 -0
  73. roller/sft.py +471 -0
  74. roller/storage.py +53 -0
  75. roller/tasks.py +689 -0
  76. roller/thinking.py +50 -0
  77. roller/thinking_capabilities.py +152 -0
  78. roller/trajectory_audit.py +386 -0
  79. roller/usage.py +168 -0
  80. roller/version.py +1 -0
  81. roller/viewer/__init__.py +3 -0
  82. roller/viewer/app.py +722 -0
  83. roller/viewer/static/app.js +752 -0
  84. roller/viewer/static/index.html +105 -0
  85. roller/viewer/static/style.css +213 -0
  86. roller/wire_trace.py +650 -0
  87. roller_agent-0.2.0.dist-info/METADATA +397 -0
  88. roller_agent-0.2.0.dist-info/RECORD +92 -0
  89. roller_agent-0.2.0.dist-info/WHEEL +5 -0
  90. roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
  91. roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
  92. roller_agent-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,181 @@
1
+ """QwenClawBench Markdown task parsing."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import re
6
+ from dataclasses import dataclass
7
+ from pathlib import Path
8
+ from typing import Any
9
+
10
+ import yaml
11
+
12
+ from roller.benches.base import BenchFormatError
13
+
14
+ FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)$", re.DOTALL)
15
+ SECTION_RE = re.compile(r"^##\s+(.+)$")
16
+ PYTHON_FENCE_RE = re.compile(r"```python\s*\n(.*?)\n\s*```", re.DOTALL)
17
+ TASK_ID_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
18
+ DEFAULT_TIMEOUT_SECONDS = 1800
19
+
20
+
21
+ @dataclass(frozen=True)
22
+ class QwenClawTask:
23
+ task_id: str
24
+ name: str
25
+ category: str
26
+ subcategory: str
27
+ grading_type: str
28
+ grading_weights: dict[str, float]
29
+ timeout_seconds: int
30
+ input_modality: str
31
+ external_dependency: str
32
+ workspace_files: tuple[dict[str, Any], ...]
33
+ prompt: str
34
+ expected_behavior: str
35
+ grading_criteria: tuple[str, ...]
36
+ automated_code: str
37
+ llm_judge_rubric: str
38
+ source_path: Path
39
+
40
+
41
+ def split_sections(body: str) -> dict[str, str]:
42
+ sections: dict[str, str] = {}
43
+ current: str | None = None
44
+ lines: list[str] = []
45
+ for line in body.splitlines():
46
+ match = SECTION_RE.match(line)
47
+ if match:
48
+ if current is not None:
49
+ sections[current] = "\n".join(lines).strip()
50
+ current = match.group(1).strip()
51
+ lines = []
52
+ elif current is not None:
53
+ lines.append(line)
54
+ if current is not None:
55
+ sections[current] = "\n".join(lines).strip()
56
+ return sections
57
+
58
+
59
+ def _grading_criteria(raw: str) -> tuple[str, ...]:
60
+ criteria: list[str] = []
61
+ for line in raw.splitlines():
62
+ match = re.match(r"^-\s+\[[ xX]\]\s+(.+)$", line.strip())
63
+ if match:
64
+ criteria.append(match.group(1).strip())
65
+ return tuple(criteria)
66
+
67
+
68
+ def _python_code(raw: str, label: str) -> str:
69
+ match = PYTHON_FENCE_RE.search(raw)
70
+ if not match:
71
+ raise BenchFormatError(f"{label}: missing fenced Python grader")
72
+ code = match.group(1).strip()
73
+ if not code:
74
+ raise BenchFormatError(f"{label}: automated grader is empty")
75
+ try:
76
+ compile(code, label, "exec")
77
+ except SyntaxError as exc:
78
+ raise BenchFormatError(f"{label}: invalid automated grader: {exc}") from exc
79
+ return code
80
+
81
+
82
+ def parse_task_markdown(
83
+ text: str, label: str, source_path: Path | None = None
84
+ ) -> QwenClawTask:
85
+ match = FRONTMATTER_RE.match(text)
86
+ if not match:
87
+ raise BenchFormatError(f"{label}: missing YAML frontmatter")
88
+ frontmatter = re.sub(
89
+ r"^source_query:\s*.*?(?=\n\w+:\s|\Z)",
90
+ "",
91
+ match.group(1),
92
+ flags=re.MULTILINE | re.DOTALL,
93
+ )
94
+ frontmatter = re.sub(
95
+ r"^source_line_number:.*$", "", frontmatter, flags=re.MULTILINE
96
+ )
97
+ try:
98
+ metadata = yaml.safe_load(frontmatter)
99
+ except yaml.YAMLError as exc:
100
+ raise BenchFormatError(f"{label}: invalid YAML frontmatter: {exc}") from exc
101
+ if not isinstance(metadata, dict):
102
+ raise BenchFormatError(f"{label}: frontmatter must be an object")
103
+ sections = split_sections(match.group(2))
104
+
105
+ task_id = metadata.get("id")
106
+ if not isinstance(task_id, str) or not task_id.strip():
107
+ raise BenchFormatError(f"{label}: id must be a non-empty string")
108
+ task_id = task_id.strip()
109
+ if not TASK_ID_RE.fullmatch(task_id):
110
+ raise BenchFormatError(f"{label}: unsafe task id {task_id!r}")
111
+ prompt = sections.get("Prompt", "").strip()
112
+ if not prompt:
113
+ raise BenchFormatError(f"{label}: missing ## Prompt")
114
+ grading_type = metadata.get("grading_type", "automated")
115
+ if grading_type not in {"automated", "llm_judge", "hybrid"}:
116
+ raise BenchFormatError(f"{label}: unsupported grading_type {grading_type!r}")
117
+ raw_weights = metadata.get("grading_weights") or {
118
+ "automated": 0.5,
119
+ "llm_judge": 0.5,
120
+ }
121
+ if not isinstance(raw_weights, dict):
122
+ raise BenchFormatError(f"{label}: grading_weights must be an object")
123
+ try:
124
+ weights = {
125
+ "automated": float(raw_weights.get("automated", 0.5)),
126
+ "llm_judge": float(raw_weights.get("llm_judge", 0.5)),
127
+ }
128
+ except (TypeError, ValueError) as exc:
129
+ raise BenchFormatError(f"{label}: grading weights must be numeric") from exc
130
+ if min(weights.values()) < 0 or sum(weights.values()) <= 0:
131
+ raise BenchFormatError(
132
+ f"{label}: grading weights must be non-negative and non-zero"
133
+ )
134
+
135
+ raw_timeout = metadata.get("timeout_seconds", DEFAULT_TIMEOUT_SECONDS)
136
+ if isinstance(raw_timeout, bool):
137
+ raise BenchFormatError(f"{label}: timeout_seconds must be an integer")
138
+ try:
139
+ timeout_seconds = int(raw_timeout)
140
+ except (TypeError, ValueError) as exc:
141
+ raise BenchFormatError(f"{label}: timeout_seconds must be an integer") from exc
142
+ if timeout_seconds <= 0:
143
+ raise BenchFormatError(f"{label}: timeout_seconds must be positive")
144
+
145
+ workspace_files = metadata.get("workspace_files", [])
146
+ if not isinstance(workspace_files, list) or not all(
147
+ isinstance(item, dict) for item in workspace_files
148
+ ):
149
+ raise BenchFormatError(f"{label}: workspace_files must be a list of objects")
150
+ automated_code = ""
151
+ if grading_type in {"automated", "hybrid"}:
152
+ automated_code = _python_code(
153
+ sections.get("Automated Checks", ""), f"{label} ## Automated Checks"
154
+ )
155
+ rubric = sections.get("LLM Judge Rubric", "").strip()
156
+ if grading_type in {"llm_judge", "hybrid"} and not rubric:
157
+ rubric = "\n".join(
158
+ f"- {item}"
159
+ for item in _grading_criteria(sections.get("Grading Criteria", ""))
160
+ )
161
+ if grading_type in {"llm_judge", "hybrid"} and not rubric:
162
+ raise BenchFormatError(f"{label}: LLM judge task has no rubric")
163
+
164
+ return QwenClawTask(
165
+ task_id=task_id,
166
+ name=str(metadata.get("name") or "").strip(),
167
+ category=str(metadata.get("category") or "").strip(),
168
+ subcategory=str(metadata.get("subcategory") or "").strip(),
169
+ grading_type=grading_type,
170
+ grading_weights=weights,
171
+ timeout_seconds=timeout_seconds,
172
+ input_modality=str(metadata.get("input_modality") or "").strip(),
173
+ external_dependency=str(metadata.get("external_dependency") or "").strip(),
174
+ workspace_files=tuple(workspace_files),
175
+ prompt=prompt,
176
+ expected_behavior=sections.get("Expected Behavior", "").strip(),
177
+ grading_criteria=_grading_criteria(sections.get("Grading Criteria", "")),
178
+ automated_code=automated_code,
179
+ llm_judge_rubric=rubric,
180
+ source_path=source_path or Path(label),
181
+ )
@@ -0,0 +1,202 @@
1
+ """QwenClawBench workspace materialization and path compatibility."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import os
8
+ import shlex
9
+ import shutil
10
+ import stat
11
+ import tempfile
12
+ from pathlib import Path, PurePosixPath
13
+ from typing import Any
14
+
15
+ from roller.benches.base import BenchFormatError
16
+ from roller.benches.qwenclaw.task_parser import QwenClawTask
17
+
18
+
19
+ def _relative_path(value: Any, label: str) -> str:
20
+ if not isinstance(value, str) or not value.strip():
21
+ raise BenchFormatError(f"{label}: must be a non-empty relative path")
22
+ raw = value.strip().replace("\\", "/")
23
+ path = PurePosixPath(raw)
24
+ if (
25
+ path.is_absolute()
26
+ or not path.parts
27
+ or any(part in {"", ".", ".."} for part in path.parts)
28
+ ):
29
+ raise BenchFormatError(f"{label}: unsafe path {value!r}")
30
+ return path.as_posix()
31
+
32
+
33
+ def _source_path(assets_root: Path, task_id: str, source: str) -> Path:
34
+ relative = _relative_path(source, f"{task_id}.workspace_files.source")
35
+ candidates = (assets_root / task_id / relative, assets_root / relative)
36
+ for candidate in candidates:
37
+ if not candidate.exists():
38
+ continue
39
+ cursor = assets_root
40
+ for part in candidate.relative_to(assets_root).parts:
41
+ cursor = cursor / part
42
+ if cursor.is_symlink():
43
+ raise BenchFormatError(
44
+ f"{task_id}: workspace source is a symlink: {cursor}"
45
+ )
46
+ resolved = candidate.resolve()
47
+ if not resolved.is_relative_to(assets_root.resolve()):
48
+ raise BenchFormatError(
49
+ f"{task_id}: workspace source escapes assets: {source}"
50
+ )
51
+ return candidate
52
+ searched = ", ".join(str(path) for path in candidates)
53
+ raise BenchFormatError(
54
+ f"{task_id}: missing workspace source {source!r}; searched {searched}"
55
+ )
56
+
57
+
58
+ def _hash_source(digest: Any, source: Path, logical: str) -> None:
59
+ if source.is_dir():
60
+ for path in sorted(source.rglob("*")):
61
+ if path.is_symlink():
62
+ raise BenchFormatError(f"workspace source contains a symlink: {path}")
63
+ relative = path.relative_to(source).as_posix()
64
+ mode = path.stat().st_mode
65
+ if path.is_dir():
66
+ digest.update(f"D:{logical}/{relative}\0".encode())
67
+ elif path.is_file() and stat.S_ISREG(mode):
68
+ digest.update(f"F:{logical}/{relative}\0".encode())
69
+ with path.open("rb") as handle:
70
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
71
+ digest.update(chunk)
72
+ else:
73
+ raise BenchFormatError(
74
+ f"workspace source is not a regular file: {path}"
75
+ )
76
+ return
77
+ mode = source.stat().st_mode
78
+ if not stat.S_ISREG(mode):
79
+ raise BenchFormatError(f"workspace source is not a regular file: {source}")
80
+ digest.update(f"F:{logical}\0".encode())
81
+ with source.open("rb") as handle:
82
+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):
83
+ digest.update(chunk)
84
+
85
+
86
+ def _cache_root() -> Path:
87
+ configured = (os.environ.get("QWENCLAW_CACHE_DIR") or "").strip()
88
+ if configured:
89
+ return Path(configured).expanduser().resolve()
90
+ xdg = (os.environ.get("XDG_CACHE_HOME") or "").strip()
91
+ base = Path(xdg).expanduser() if xdg else Path.home() / ".cache"
92
+ return (base / "roller" / "benches" / "qwenclawbench").resolve()
93
+
94
+
95
+ def materialize_workspace(
96
+ task: QwenClawTask, assets_root: Path, cache_root: Path | None = None
97
+ ) -> Path:
98
+ """Build the exact source-to-destination workspace expected upstream."""
99
+
100
+ entries: list[tuple[str, Path | None, str | None]] = []
101
+ destinations: list[str] = []
102
+ digest = hashlib.sha256()
103
+ digest.update(task.source_path.read_bytes())
104
+ for index, item in enumerate(task.workspace_files):
105
+ destination_value = item.get("dest", item.get("path"))
106
+ destination = _relative_path(
107
+ destination_value, f"{task.task_id}.workspace_files[{index}].dest"
108
+ )
109
+ for existing in destinations:
110
+ if (
111
+ destination == existing
112
+ or destination.startswith(existing + "/")
113
+ or existing.startswith(destination + "/")
114
+ ):
115
+ raise BenchFormatError(
116
+ f"{task.task_id}: colliding workspace destinations "
117
+ f"{existing!r} and {destination!r}"
118
+ )
119
+ destinations.append(destination)
120
+ if "content" in item:
121
+ content = item["content"]
122
+ if not isinstance(content, str):
123
+ raise BenchFormatError(
124
+ f"{task.task_id}.workspace_files[{index}].content must be a string"
125
+ )
126
+ digest.update(f"C:{destination}\0{content}".encode())
127
+ entries.append((destination, None, content))
128
+ continue
129
+ source_value = item.get("source")
130
+ if not isinstance(source_value, str):
131
+ raise BenchFormatError(
132
+ f"{task.task_id}.workspace_files[{index}].source must be a string"
133
+ )
134
+ source = _source_path(assets_root, task.task_id, source_value)
135
+ digest.update(f"M:{source_value}->{destination}\0".encode())
136
+ _hash_source(digest, source, destination)
137
+ entries.append((destination, source, None))
138
+
139
+ fingerprint = digest.hexdigest()
140
+ task_cache = (cache_root or _cache_root()) / task.task_id
141
+ target = task_cache / fingerprint
142
+ workspace = target / "workspace"
143
+ if (target / "manifest.json").is_file() and workspace.is_dir():
144
+ return workspace
145
+
146
+ task_cache.mkdir(parents=True, exist_ok=True)
147
+ temporary = Path(tempfile.mkdtemp(prefix=f".{fingerprint}.", dir=task_cache))
148
+ temporary_workspace = temporary / "workspace"
149
+ temporary_workspace.mkdir()
150
+ try:
151
+ for destination, source, content in entries:
152
+ output = temporary_workspace / destination
153
+ output.parent.mkdir(parents=True, exist_ok=True)
154
+ if content is not None:
155
+ output.write_text(content, encoding="utf-8")
156
+ elif source is not None and source.is_dir():
157
+ shutil.copytree(source, output, copy_function=shutil.copy2)
158
+ elif source is not None:
159
+ shutil.copy2(source, output)
160
+ (temporary / "manifest.json").write_text(
161
+ json.dumps(
162
+ {
163
+ "schema_version": "roller.qwenclaw_workspace/v1",
164
+ "task_id": task.task_id,
165
+ "fingerprint": fingerprint,
166
+ "source": str(task.source_path),
167
+ },
168
+ ensure_ascii=False,
169
+ indent=2,
170
+ )
171
+ + "\n",
172
+ encoding="utf-8",
173
+ )
174
+ try:
175
+ temporary.rename(target)
176
+ except FileExistsError:
177
+ shutil.rmtree(temporary)
178
+ except Exception:
179
+ shutil.rmtree(temporary, ignore_errors=True)
180
+ raise
181
+ return workspace
182
+
183
+
184
+ def workspace_compatibility_script(workspace: str) -> str:
185
+ """Make ``workspace/...`` and workspace-relative paths refer to one tree."""
186
+
187
+ root = workspace.rstrip("/")
188
+ alias = f"{root}/workspace"
189
+ quoted_root = shlex.quote(root)
190
+ quoted_alias = shlex.quote(alias)
191
+ return (
192
+ "set -eu; "
193
+ f"if [ -L {quoted_alias} ]; then "
194
+ f'test "$(readlink -f {quoted_alias})" = "$(readlink -f {quoted_root})"; '
195
+ f"elif [ -d {quoted_alias} ]; then "
196
+ f"cp -a {quoted_alias}/. {quoted_root}/; "
197
+ f"rm -rf {quoted_alias}; "
198
+ f"ln -s . {quoted_alias}; "
199
+ f"elif [ -e {quoted_alias} ]; then "
200
+ "echo 'workspace compatibility path is not a directory' >&2; exit 1; "
201
+ f"else ln -s . {quoted_alias}; fi"
202
+ )
@@ -0,0 +1,166 @@
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ from pathlib import Path
5
+ from typing import Any, Iterable
6
+
7
+ SCORE_FILE = "score.json"
8
+ REPORT_SCHEMA = "roller.bench_report/v1"
9
+
10
+
11
+ def _numeric(scores: dict[str, Any]) -> dict[str, float]:
12
+ return {
13
+ key: float(value)
14
+ for key, value in scores.items()
15
+ if isinstance(value, (int, float)) and not isinstance(value, bool)
16
+ }
17
+
18
+
19
+ def sample_score(scores: dict[str, Any], score_key: str = "overall_score") -> float | None:
20
+ """Reduce one grader payload to a single number.
21
+
22
+ Follows WildClawBench's rule: prefer the declared aggregate key, otherwise
23
+ average whatever numeric fields the grader returned.
24
+ """
25
+
26
+ numeric = _numeric(scores)
27
+ if not numeric:
28
+ return None
29
+ if score_key in numeric:
30
+ return numeric[score_key]
31
+ return sum(numeric.values()) / len(numeric)
32
+
33
+
34
+ def read_scores(sample_dir: Path) -> dict[str, Any] | None:
35
+ path = sample_dir / SCORE_FILE
36
+ if not path.is_file():
37
+ return None
38
+ try:
39
+ payload = json.loads(path.read_text(encoding="utf-8"))
40
+ except (OSError, json.JSONDecodeError):
41
+ return None
42
+ return payload if isinstance(payload, dict) else None
43
+
44
+
45
+ def mean_aggregate(samples: Iterable[dict[str, Any]]) -> dict[str, Any]:
46
+ """Average scored samples over *all* samples, counting failures as zero.
47
+
48
+ An agent that crashed or timed out earned no score; dropping it from the
49
+ denominator would silently reward failure. Unscored samples are reported
50
+ separately so a broken grader is not mistaken for a weak model.
51
+ """
52
+
53
+ rows = list(samples)
54
+ per_task: list[dict[str, Any]] = []
55
+ per_category: dict[str, list[float]] = {}
56
+ total = 0.0
57
+ scored = 0
58
+ for row in rows:
59
+ scores = row.get("bench_scores")
60
+ score_key = row.get("bench_score_key") or "overall_score"
61
+ grader_error = row.get("bench_error")
62
+ if not grader_error and isinstance(scores, dict):
63
+ grader_error = scores.get("error")
64
+ # A synthetic zero returned alongside a grader/infrastructure error is
65
+ # not a model score. Keep it unscored so broken grading cannot depress
66
+ # benchmark results while still surfacing the failure in the report.
67
+ value = (
68
+ sample_score(scores, score_key)
69
+ if isinstance(scores, dict) and not grader_error
70
+ else None
71
+ )
72
+ category = row.get("bench_category")
73
+ if value is not None:
74
+ total += value
75
+ scored += 1
76
+ if isinstance(category, str) and category:
77
+ per_category.setdefault(category, []).append(value)
78
+ per_task.append(
79
+ {
80
+ "task_id": row.get("task_id"),
81
+ "model": row.get("model"),
82
+ "category": category,
83
+ "status": row.get("status"),
84
+ "score": value,
85
+ "error": grader_error,
86
+ }
87
+ )
88
+ task_count = len(rows)
89
+ return {
90
+ "schema_version": REPORT_SCHEMA,
91
+ "task_count": task_count,
92
+ "scored_count": scored,
93
+ "unscored_count": task_count - scored,
94
+ "average": round(total / task_count, 6) if task_count else None,
95
+ "scored_average": round(total / scored, 6) if scored else None,
96
+ "categories": {
97
+ name: {
98
+ "task_count": len(values),
99
+ "average": round(sum(values) / len(values), 6),
100
+ }
101
+ for name, values in sorted(per_category.items())
102
+ },
103
+ "tasks": per_task,
104
+ }
105
+
106
+
107
+ def format_report(bench_name: str, report: dict[str, Any]) -> str:
108
+ lines = [
109
+ "",
110
+ "=" * 62,
111
+ f" Bench report — {bench_name}",
112
+ "=" * 62,
113
+ ]
114
+ average = report.get("average")
115
+ if average is None:
116
+ lines.append(" no samples")
117
+ else:
118
+ filled = max(0, min(10, int(round(average * 10))))
119
+ bar = "█" * filled + "░" * (10 - filled)
120
+ lines.append(
121
+ f" scored {report['scored_count']}/{report['task_count']}"
122
+ f" (unscored {report['unscored_count']})"
123
+ )
124
+ pass_rate = report.get("pass_rate")
125
+ if isinstance(pass_rate, (int, float)):
126
+ pass_bar_count = max(0, min(10, int(round(pass_rate * 10))))
127
+ pass_bar = "█" * pass_bar_count + "░" * (10 - pass_bar_count)
128
+ lines.append(f" pass rate {pass_bar} {pass_rate:.4f}")
129
+ lines.append(f" mean reward {bar} {average:.4f}")
130
+ else:
131
+ lines.append(f" average {bar} {average:.4f}")
132
+ scored_average = report.get("scored_average")
133
+ if (
134
+ scored_average is not None
135
+ and report["unscored_count"]
136
+ and not isinstance(pass_rate, (int, float))
137
+ ):
138
+ lines.append(f" average over scored samples only: {scored_average:.4f}")
139
+ categories = report.get("categories") or {}
140
+ if categories:
141
+ lines.append("")
142
+ for name, row in categories.items():
143
+ if isinstance(row.get("pass_rate"), (int, float)):
144
+ lines.append(
145
+ f" pass {row['pass_rate']:.4f} reward {row['average']:.4f}"
146
+ f" {name} ({row['task_count']} trials)"
147
+ )
148
+ else:
149
+ lines.append(
150
+ f" {row['average']:.4f} {name} ({row['task_count']} tasks)"
151
+ )
152
+ failures = [
153
+ row
154
+ for row in report.get("tasks", [])
155
+ if row.get("score") is None
156
+ ]
157
+ if failures:
158
+ lines.append("")
159
+ lines.append(f" unscored samples ({len(failures)}):")
160
+ for row in failures[:20]:
161
+ detail = row.get("error") or row.get("status") or "unknown"
162
+ lines.append(f" - {row.get('task_id')}: {detail}")
163
+ if len(failures) > 20:
164
+ lines.append(f" ... and {len(failures) - 20} more")
165
+ lines.append("=" * 62)
166
+ return "\n".join(lines)