roller-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- roller/__init__.py +31 -0
- roller/__main__.py +6 -0
- roller/actf.py +431 -0
- roller/artifacts.py +124 -0
- roller/assets/deepseek/journal.mjs +44 -0
- roller/assets/deepseek/roller.patch.yml +15 -0
- roller/assets/deepseek/runner.py +149 -0
- roller/assets/hermes/runner.py +322 -0
- roller/assets/openclaw_capture.py +32 -0
- roller/assets/openclaw_drain.mjs +21 -0
- roller/assets/opencode/journal.mjs +42 -0
- roller/assets/opencode/runner.py +203 -0
- roller/assets/pi/runner.mjs +160 -0
- roller/assets/pi/runner.py +93 -0
- roller/benches/__init__.py +73 -0
- roller/benches/base.py +306 -0
- roller/benches/grading.py +553 -0
- roller/benches/pinchbench.py +976 -0
- roller/benches/qwenclaw/__init__.py +49 -0
- roller/benches/qwenclaw/adapter.py +167 -0
- roller/benches/qwenclaw/grader_runtime.py +395 -0
- roller/benches/qwenclaw/report.py +49 -0
- roller/benches/qwenclaw/task_parser.py +181 -0
- roller/benches/qwenclaw/workspace.py +202 -0
- roller/benches/report.py +166 -0
- roller/benches/skillsbench.py +708 -0
- roller/benches/transcripts.py +102 -0
- roller/benches/wildclaw.py +569 -0
- roller/bridge_capture.py +73 -0
- roller/bundles.py +545 -0
- roller/capture.py +385 -0
- roller/capture_protocol.py +301 -0
- roller/capture_runtime.py +48 -0
- roller/capture_states.py +37 -0
- roller/claude_capture.py +185 -0
- roller/claude_code_actf.py +594 -0
- roller/cli.py +1732 -0
- roller/codex_actf.py +584 -0
- roller/codex_capture.py +99 -0
- roller/codex_tools.py +137 -0
- roller/collector.py +818 -0
- roller/compaction_sft.py +672 -0
- roller/conversation.py +432 -0
- roller/deepseek_actf.py +135 -0
- roller/harness_registry.py +61 -0
- roller/harnesses/__init__.py +29 -0
- roller/harnesses/base.py +64 -0
- roller/harnesses/bridge.py +448 -0
- roller/harnesses/claude_code.py +839 -0
- roller/harnesses/codex.py +808 -0
- roller/harnesses/config_overrides.py +100 -0
- roller/harnesses/container.py +206 -0
- roller/harnesses/deepseek.py +104 -0
- roller/harnesses/docker.py +72 -0
- roller/harnesses/gateway_ports.py +155 -0
- roller/harnesses/hermes.py +119 -0
- roller/harnesses/openclaw.py +1605 -0
- roller/harnesses/opencode.py +193 -0
- roller/harnesses/pi.py +111 -0
- roller/hermes_actf.py +147 -0
- roller/image_build.py +206 -0
- roller/integrity.py +332 -0
- roller/model_protocols.py +201 -0
- roller/models.py +169 -0
- roller/openclaw_capture.py +140 -0
- roller/openclaw_delivery.py +211 -0
- roller/openclaw_usage.py +59 -0
- roller/opencode_actf.py +305 -0
- roller/opencode_capture.py +96 -0
- roller/pi_actf.py +172 -0
- roller/response_metadata.py +89 -0
- roller/run_lock.py +297 -0
- roller/sft.py +471 -0
- roller/storage.py +53 -0
- roller/tasks.py +689 -0
- roller/thinking.py +50 -0
- roller/thinking_capabilities.py +152 -0
- roller/trajectory_audit.py +386 -0
- roller/usage.py +168 -0
- roller/version.py +1 -0
- roller/viewer/__init__.py +3 -0
- roller/viewer/app.py +722 -0
- roller/viewer/static/app.js +752 -0
- roller/viewer/static/index.html +105 -0
- roller/viewer/static/style.css +213 -0
- roller/wire_trace.py +650 -0
- roller_agent-0.2.0.dist-info/METADATA +397 -0
- roller_agent-0.2.0.dist-info/RECORD +92 -0
- roller_agent-0.2.0.dist-info/WHEEL +5 -0
- roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
- roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
- roller_agent-0.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,976 @@
|
|
|
1
|
+
"""PinchBench adapter, including automated, LLM-judge and hybrid grading."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import re
|
|
7
|
+
from collections.abc import Sequence
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import yaml
|
|
12
|
+
|
|
13
|
+
from roller.benches.base import (
|
|
14
|
+
BenchAdapter,
|
|
15
|
+
BenchBinding,
|
|
16
|
+
BenchFormatError,
|
|
17
|
+
GradeSpec,
|
|
18
|
+
validate_timeout_multiplier,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)", re.DOTALL)
|
|
22
|
+
SECTION_RE = re.compile(r"^##\s+(.+?)\s*$")
|
|
23
|
+
PYTHON_FENCE_RE = re.compile(r"```python\s*\n(.*?)\n```", re.DOTALL | re.IGNORECASE)
|
|
24
|
+
CODE_FENCE_RE = re.compile(r"```\s*\n(.*?)\n```", re.DOTALL)
|
|
25
|
+
GRADING_TYPES = {"automated", "llm_judge", "hybrid"}
|
|
26
|
+
JUDGE_ENV = ()
|
|
27
|
+
JUDGE_OPTIONAL_ENV = (
|
|
28
|
+
"PINCHBENCH_JUDGE_BASE_URL",
|
|
29
|
+
"PINCHBENCH_JUDGE_API_KEY",
|
|
30
|
+
"PINCHBENCH_JUDGE_MODEL",
|
|
31
|
+
"PINCHBENCH_JUDGE_TIMEOUT_SECONDS",
|
|
32
|
+
"PINCHBENCH_JUDGE_MAX_COMPLETION_TOKENS",
|
|
33
|
+
"PINCHBENCH_JUDGE_MAX_TOKENS",
|
|
34
|
+
"PINCHBENCH_JUDGE_MAX_TOKENS_FIELD",
|
|
35
|
+
"PINCHBENCH_JUDGE_RETRIES",
|
|
36
|
+
"PINCHBENCH_JUDGE_STREAM",
|
|
37
|
+
"PINCHBENCH_JUDGE_NO_PROXY",
|
|
38
|
+
"OPENROUTER_API_KEY",
|
|
39
|
+
"OPENROUTER_BASE_URL",
|
|
40
|
+
"HTTP_PROXY",
|
|
41
|
+
"HTTPS_PROXY",
|
|
42
|
+
"NO_PROXY",
|
|
43
|
+
"http_proxy",
|
|
44
|
+
"https_proxy",
|
|
45
|
+
"no_proxy",
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
_PINCH_FWS_SETUP = r'''set -euo pipefail
|
|
49
|
+
tools=/tmp/roller-pinch-tools
|
|
50
|
+
mkdir -p "$tools/bin" "$HOME/.config/gws/cache"
|
|
51
|
+
|
|
52
|
+
download_checked() {
|
|
53
|
+
url=$1
|
|
54
|
+
destination=$2
|
|
55
|
+
checksum=$3
|
|
56
|
+
if ! echo "$checksum $destination" | sha256sum -c - >/dev/null 2>&1; then
|
|
57
|
+
rm -f "$destination"
|
|
58
|
+
curl --fail --location --silent --show-error \
|
|
59
|
+
--retry 5 --retry-all-errors --connect-timeout 30 --max-time 600 \
|
|
60
|
+
"$url" -o "$destination"
|
|
61
|
+
echo "$checksum $destination" | sha256sum -c - >/dev/null
|
|
62
|
+
fi
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if [ ! -x "$tools/node_modules/.bin/fws" ]; then
|
|
66
|
+
npm install --prefix "$tools" --no-audit --no-fund @juppytt/fws@0.4.4
|
|
67
|
+
fi
|
|
68
|
+
mkdir -p "$tools/node_modules/@juppytt/fws/node_modules/.bin"
|
|
69
|
+
ln -sf "$tools/node_modules/.bin/tsx" \
|
|
70
|
+
"$tools/node_modules/@juppytt/fws/node_modules/.bin/tsx"
|
|
71
|
+
|
|
72
|
+
download_checked \
|
|
73
|
+
https://github.com/googleworkspace/cli/releases/download/v0.22.3/google-workspace-cli-x86_64-unknown-linux-gnu.tar.gz \
|
|
74
|
+
/tmp/roller-pinch-gws.tar.gz \
|
|
75
|
+
b951ef847b38dd41a23d31a3aeaf3cb650421aac607bbfe63727bf4f4213ce44
|
|
76
|
+
if [ ! -x "$tools/google-workspace-cli-x86_64-unknown-linux-gnu/gws" ]; then
|
|
77
|
+
tar -xzf /tmp/roller-pinch-gws.tar.gz -C "$tools"
|
|
78
|
+
fi
|
|
79
|
+
ln -sf "$tools/google-workspace-cli-x86_64-unknown-linux-gnu/gws" \
|
|
80
|
+
"$tools/bin/gws.real"
|
|
81
|
+
|
|
82
|
+
download_checked \
|
|
83
|
+
https://github.com/cli/cli/releases/download/v2.79.0/gh_2.79.0_linux_amd64.tar.gz \
|
|
84
|
+
/tmp/roller-pinch-gh.tar.gz \
|
|
85
|
+
e7af0c72a607c0528fda1989f7c8e3be85e67d321889002af0e2938ad9c8fb68
|
|
86
|
+
if [ ! -x "$tools/gh_2.79.0_linux_amd64/bin/gh" ]; then
|
|
87
|
+
tar -xzf /tmp/roller-pinch-gh.tar.gz -C "$tools"
|
|
88
|
+
fi
|
|
89
|
+
ln -sf "$tools/gh_2.79.0_linux_amd64/bin/gh" "$tools/bin/gh.real"
|
|
90
|
+
|
|
91
|
+
for discovery in gmail:v1 calendar:v3 drive:v3 tasks:v1 sheets:v4 people:v1; do
|
|
92
|
+
api=${discovery%%:*}
|
|
93
|
+
version=${discovery#*:}
|
|
94
|
+
destination="$HOME/.config/gws/cache/${api}_${version}.json"
|
|
95
|
+
if ! python3 -m json.tool "$destination" >/dev/null 2>&1; then
|
|
96
|
+
rm -f "$destination"
|
|
97
|
+
curl --fail --location --silent --show-error \
|
|
98
|
+
--retry 5 --retry-all-errors --connect-timeout 30 --max-time 600 \
|
|
99
|
+
"https://www.googleapis.com/discovery/v1/apis/${api}/${version}/rest" \
|
|
100
|
+
-o "$destination"
|
|
101
|
+
python3 -m json.tool "$destination" >/dev/null
|
|
102
|
+
fi
|
|
103
|
+
done
|
|
104
|
+
|
|
105
|
+
export PATH="$tools/bin:$tools/node_modules/.bin:$PATH"
|
|
106
|
+
: "${ROLLER_GATEWAY_PORT:?Roller did not provide the OpenClaw gateway port}"
|
|
107
|
+
fws_port=$((ROLLER_GATEWAY_PORT + 1))
|
|
108
|
+
if [ "$fws_port" -gt 65534 ]; then
|
|
109
|
+
fws_port=$((ROLLER_GATEWAY_PORT - 2))
|
|
110
|
+
fi
|
|
111
|
+
fws_output=$(fws server start -p "$fws_port")
|
|
112
|
+
printf '%s\n' "$fws_output"
|
|
113
|
+
printf '%s\n' "$fws_output" | sed -n '/^[[:space:]]*export /p' \
|
|
114
|
+
> /roller/task/fws-environment
|
|
115
|
+
test -s /roller/task/fws-environment
|
|
116
|
+
|
|
117
|
+
printf '%s\n' \
|
|
118
|
+
'#!/bin/bash' \
|
|
119
|
+
'set -a' \
|
|
120
|
+
'. /roller/task/fws-environment' \
|
|
121
|
+
'set +a' \
|
|
122
|
+
'exec /tmp/roller-pinch-tools/bin/gh.real "$@"' > "$tools/bin/gh"
|
|
123
|
+
printf '%s\n' \
|
|
124
|
+
'#!/bin/bash' \
|
|
125
|
+
'set -a' \
|
|
126
|
+
'. /roller/task/fws-environment' \
|
|
127
|
+
'set +a' \
|
|
128
|
+
'exec /tmp/roller-pinch-tools/bin/gws.real "$@"' > "$tools/bin/gws"
|
|
129
|
+
chmod +x "$tools/bin/gh" "$tools/bin/gws"
|
|
130
|
+
printf '%s\n' \
|
|
131
|
+
'export PATH=/tmp/roller-pinch-tools/bin:/tmp/roller-pinch-tools/node_modules/.bin:$PATH' \
|
|
132
|
+
> /roller/task/environment
|
|
133
|
+
|
|
134
|
+
fws_ready=false
|
|
135
|
+
for attempt in $(seq 1 20); do
|
|
136
|
+
if GH_REPO=testuser/my-project "$tools/bin/gh" api /user >/dev/null 2>&1; then
|
|
137
|
+
fws_ready=true
|
|
138
|
+
break
|
|
139
|
+
fi
|
|
140
|
+
sleep 0.5
|
|
141
|
+
done
|
|
142
|
+
$fws_ready || { echo 'FWS mock service failed its readiness probe' >&2; exit 1; }'''
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _pinch_runtime_setup(task: dict[str, Any]) -> list[str]:
|
|
146
|
+
"""Provision the mock workspace services declared by upstream PinchBench."""
|
|
147
|
+
|
|
148
|
+
prerequisites = task["prerequisites"]
|
|
149
|
+
needs_fws = task["category"].casefold() in {"gws", "github"} or any(
|
|
150
|
+
"fws" in prerequisite.casefold() for prerequisite in prerequisites
|
|
151
|
+
)
|
|
152
|
+
return [_PINCH_FWS_SETUP] if needs_fws else []
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _split_markdown(text: str, label: str) -> tuple[dict[str, Any], dict[str, str]]:
|
|
156
|
+
match = FRONTMATTER_RE.match(text)
|
|
157
|
+
if match is None:
|
|
158
|
+
raise BenchFormatError(f"{label}: missing YAML frontmatter")
|
|
159
|
+
try:
|
|
160
|
+
metadata = yaml.safe_load(match.group(1)) or {}
|
|
161
|
+
except yaml.YAMLError as exc:
|
|
162
|
+
raise BenchFormatError(f"{label}: invalid YAML frontmatter: {exc}") from exc
|
|
163
|
+
if not isinstance(metadata, dict):
|
|
164
|
+
raise BenchFormatError(f"{label}: frontmatter must be an object")
|
|
165
|
+
sections: dict[str, str] = {}
|
|
166
|
+
current: str | None = None
|
|
167
|
+
lines: list[str] = []
|
|
168
|
+
for line in match.group(2).splitlines():
|
|
169
|
+
header = SECTION_RE.match(line)
|
|
170
|
+
if header:
|
|
171
|
+
if current is not None:
|
|
172
|
+
sections[current] = "\n".join(lines).strip()
|
|
173
|
+
current, lines = header.group(1), []
|
|
174
|
+
elif current is not None:
|
|
175
|
+
lines.append(line)
|
|
176
|
+
if current is not None:
|
|
177
|
+
sections[current] = "\n".join(lines).strip()
|
|
178
|
+
return metadata, sections
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _strip_python_fence(value: str) -> str:
|
|
182
|
+
stripped = value.strip()
|
|
183
|
+
match = PYTHON_FENCE_RE.search(stripped) or CODE_FENCE_RE.search(stripped)
|
|
184
|
+
return match.group(1).strip() if match else stripped
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def parse_task_markdown(text: str, label: str) -> dict[str, Any]:
|
|
188
|
+
metadata, sections = _split_markdown(text, label)
|
|
189
|
+
task_id = metadata.get("id")
|
|
190
|
+
if not isinstance(task_id, str) or not task_id.strip():
|
|
191
|
+
raise BenchFormatError(f"{label}: id must be a non-empty string")
|
|
192
|
+
grading_type = metadata.get("grading_type", "automated")
|
|
193
|
+
if grading_type not in GRADING_TYPES:
|
|
194
|
+
raise BenchFormatError(
|
|
195
|
+
f"{label}: grading_type must be one of {sorted(GRADING_TYPES)}"
|
|
196
|
+
)
|
|
197
|
+
timeout = metadata.get("timeout_seconds", 120)
|
|
198
|
+
if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0:
|
|
199
|
+
raise BenchFormatError(f"{label}: timeout_seconds must be a positive integer")
|
|
200
|
+
prompt = sections.get("Prompt", "").strip()
|
|
201
|
+
sessions = metadata.get("sessions")
|
|
202
|
+
parsed_sessions: list[dict[str, Any]] = []
|
|
203
|
+
if sessions is not None:
|
|
204
|
+
if not isinstance(sessions, list) or not sessions:
|
|
205
|
+
raise BenchFormatError(f"{label}: sessions must be a non-empty list")
|
|
206
|
+
for index, session in enumerate(sessions):
|
|
207
|
+
if not isinstance(session, dict):
|
|
208
|
+
raise BenchFormatError(f"{label}: sessions[{index}] must be an object")
|
|
209
|
+
session_prompt = session.get("prompt")
|
|
210
|
+
reset = session.get("new_session", False)
|
|
211
|
+
if not isinstance(session_prompt, str) or not session_prompt.strip():
|
|
212
|
+
raise BenchFormatError(
|
|
213
|
+
f"{label}: sessions[{index}].prompt must be non-empty"
|
|
214
|
+
)
|
|
215
|
+
if not isinstance(reset, bool) or (index == 0 and reset):
|
|
216
|
+
raise BenchFormatError(
|
|
217
|
+
f"{label}: sessions[{index}].new_session must be false for the first "
|
|
218
|
+
"session and boolean elsewhere"
|
|
219
|
+
)
|
|
220
|
+
parsed_sessions.append(
|
|
221
|
+
{
|
|
222
|
+
"role": "user",
|
|
223
|
+
"content": session_prompt.strip(),
|
|
224
|
+
"new_session": reset,
|
|
225
|
+
}
|
|
226
|
+
)
|
|
227
|
+
elif not prompt:
|
|
228
|
+
raise BenchFormatError(f"{label}: missing ## Prompt")
|
|
229
|
+
workspace_files = metadata.get("workspace_files", [])
|
|
230
|
+
if not isinstance(workspace_files, list):
|
|
231
|
+
raise BenchFormatError(f"{label}: workspace_files must be a list")
|
|
232
|
+
prerequisites = metadata.get("prerequisites", [])
|
|
233
|
+
if not isinstance(prerequisites, list) or not all(
|
|
234
|
+
isinstance(item, str) and item.strip() for item in prerequisites
|
|
235
|
+
):
|
|
236
|
+
raise BenchFormatError(f"{label}: prerequisites must be a string list")
|
|
237
|
+
return {
|
|
238
|
+
"id": task_id.strip(),
|
|
239
|
+
"name": str(metadata.get("name") or task_id).strip(),
|
|
240
|
+
"category": str(metadata.get("category") or "").strip(),
|
|
241
|
+
"grading_type": grading_type,
|
|
242
|
+
"timeout_seconds": timeout,
|
|
243
|
+
"workspace_files": workspace_files,
|
|
244
|
+
"prompt": prompt,
|
|
245
|
+
"sessions": parsed_sessions,
|
|
246
|
+
"expected_behavior": sections.get("Expected Behavior", "").strip(),
|
|
247
|
+
"grading_criteria": [
|
|
248
|
+
match.group(1)
|
|
249
|
+
for line in sections.get("Grading Criteria", "").splitlines()
|
|
250
|
+
if (match := re.match(r"^-\s+\[[ x]\]\s+(.+)$", line.strip()))
|
|
251
|
+
],
|
|
252
|
+
"automated_checks": _strip_python_fence(
|
|
253
|
+
sections.get("Automated Checks", "")
|
|
254
|
+
),
|
|
255
|
+
"llm_judge_rubric": sections.get("LLM Judge Rubric", "").strip(),
|
|
256
|
+
"grading_weights": metadata.get("grading_weights") or {},
|
|
257
|
+
"prerequisites": [item.strip() for item in prerequisites],
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
_PINCH_WRAPPER = r'''
|
|
262
|
+
import json as _pinch_json
|
|
263
|
+
import os as _pinch_os
|
|
264
|
+
import re as _pinch_re
|
|
265
|
+
from pathlib import Path as _PinchPath
|
|
266
|
+
import time as _pinch_time
|
|
267
|
+
from urllib import error as _pinch_error
|
|
268
|
+
from urllib import request as _pinch_request
|
|
269
|
+
|
|
270
|
+
_PINCH_TASK_PROMPT = __TASK_PROMPT__
|
|
271
|
+
_PINCH_EXPECTED = __EXPECTED__
|
|
272
|
+
_PINCH_CRITERIA = __CRITERIA__
|
|
273
|
+
_PINCH_RUBRIC = __RUBRIC__
|
|
274
|
+
_PINCH_GRADING_TYPE = __GRADING_TYPE__
|
|
275
|
+
_PINCH_WEIGHTS = __WEIGHTS__
|
|
276
|
+
_PINCH_AUTOMATED = globals().get("grade")
|
|
277
|
+
_PINCH_OFFICIAL_JUDGE_MODEL = "anthropic/claude-haiku-4.5"
|
|
278
|
+
_PINCH_JUDGE_SYSTEM_MSG = (
|
|
279
|
+
"You are a strict grading function. "
|
|
280
|
+
"Respond with ONLY a JSON object, no prose, no markdown fences, no extra text."
|
|
281
|
+
)
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def _pinch_average(values):
|
|
285
|
+
numbers = [float(value) for value in values.values()
|
|
286
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool)]
|
|
287
|
+
return sum(numbers) / len(numbers) if numbers else 0.0
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _pinch_normalize_transcript(transcript):
|
|
291
|
+
"""Present OpenClaw messages in the list-of-parts shape Pinch expects."""
|
|
292
|
+
normalized = []
|
|
293
|
+
for event in transcript:
|
|
294
|
+
if not isinstance(event, dict):
|
|
295
|
+
continue
|
|
296
|
+
event = dict(event)
|
|
297
|
+
message = event.get("message")
|
|
298
|
+
if isinstance(message, dict):
|
|
299
|
+
message = dict(message)
|
|
300
|
+
content = message.get("content", [])
|
|
301
|
+
if isinstance(content, str):
|
|
302
|
+
content = [{"type": "text", "text": content}]
|
|
303
|
+
elif isinstance(content, dict):
|
|
304
|
+
content = [content]
|
|
305
|
+
elif isinstance(content, list):
|
|
306
|
+
content = [
|
|
307
|
+
item
|
|
308
|
+
if isinstance(item, dict)
|
|
309
|
+
else {"type": "text", "text": str(item)}
|
|
310
|
+
for item in content
|
|
311
|
+
]
|
|
312
|
+
else:
|
|
313
|
+
content = []
|
|
314
|
+
message["content"] = content
|
|
315
|
+
event["message"] = message
|
|
316
|
+
normalized.append(event)
|
|
317
|
+
return normalized
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _pinch_transcript_text(transcript):
|
|
321
|
+
# Equivalent to upstream PinchBench's _summarize_transcript.
|
|
322
|
+
pieces = []
|
|
323
|
+
for event in transcript:
|
|
324
|
+
if not isinstance(event, dict) or event.get("type") != "message":
|
|
325
|
+
continue
|
|
326
|
+
message = event.get("message", {})
|
|
327
|
+
role = message.get("role")
|
|
328
|
+
content = message.get("content", [])
|
|
329
|
+
if role == "assistant":
|
|
330
|
+
for item in content:
|
|
331
|
+
if item.get("type") == "toolCall":
|
|
332
|
+
args = item.get("arguments", {})
|
|
333
|
+
truncated_args = {}
|
|
334
|
+
for key, value in args.items():
|
|
335
|
+
if isinstance(value, str) and len(value) > 200:
|
|
336
|
+
truncated_args[key] = value[:200] + "...[truncated]"
|
|
337
|
+
else:
|
|
338
|
+
truncated_args[key] = value
|
|
339
|
+
pieces.append(
|
|
340
|
+
f"Tool: {item.get('name')}({_pinch_json.dumps(truncated_args)})"
|
|
341
|
+
)
|
|
342
|
+
elif item.get("type") == "text":
|
|
343
|
+
value = item.get("text", "").strip()
|
|
344
|
+
if value:
|
|
345
|
+
pieces.append(f"Assistant: {value[:2000]}")
|
|
346
|
+
elif role == "toolResult":
|
|
347
|
+
if content:
|
|
348
|
+
pieces.append(f"Result: {str(content[0])[:200]}")
|
|
349
|
+
elif role == "user":
|
|
350
|
+
if content:
|
|
351
|
+
pieces.append(f"User: {content[0]}")
|
|
352
|
+
return "\n".join(pieces)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def _pinch_workspace_text(workspace_path):
|
|
356
|
+
root = _PinchPath(workspace_path)
|
|
357
|
+
if not root.is_dir():
|
|
358
|
+
return ""
|
|
359
|
+
skip_names = {
|
|
360
|
+
"BOOTSTRAP.md", "SOUL.md", "USER.md", "IDENTITY.md",
|
|
361
|
+
"HEARTBEAT.md", "TOOLS.md", "AGENTS.md",
|
|
362
|
+
}
|
|
363
|
+
skip_dirs = {".git", ".openclaw", "__pycache__", "node_modules", "skills"}
|
|
364
|
+
pieces = []
|
|
365
|
+
for path in sorted(root.rglob("*")):
|
|
366
|
+
if not path.is_file():
|
|
367
|
+
continue
|
|
368
|
+
relative = path.relative_to(root)
|
|
369
|
+
if any(part.startswith(".") or part in skip_dirs for part in relative.parts):
|
|
370
|
+
continue
|
|
371
|
+
if path.name in skip_names:
|
|
372
|
+
continue
|
|
373
|
+
try:
|
|
374
|
+
value = path.read_text(encoding="utf-8")
|
|
375
|
+
except (OSError, UnicodeDecodeError):
|
|
376
|
+
continue
|
|
377
|
+
pieces.append(f"### File: {relative}\n{value}")
|
|
378
|
+
return "\n\n".join(pieces)
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _pinch_parse_json(text):
|
|
382
|
+
text = text.strip()
|
|
383
|
+
if not text:
|
|
384
|
+
return {}
|
|
385
|
+
try:
|
|
386
|
+
value = _pinch_json.loads(text)
|
|
387
|
+
if isinstance(value, dict):
|
|
388
|
+
return value
|
|
389
|
+
except Exception:
|
|
390
|
+
pass
|
|
391
|
+
fenced = _pinch_re.search(r"```(?:json)?\s*(.*?)\s*```", text, _pinch_re.S)
|
|
392
|
+
if fenced:
|
|
393
|
+
try:
|
|
394
|
+
value = _pinch_json.loads(fenced.group(1))
|
|
395
|
+
if isinstance(value, dict):
|
|
396
|
+
return value
|
|
397
|
+
except Exception:
|
|
398
|
+
pass
|
|
399
|
+
candidates = []
|
|
400
|
+
depth = 0
|
|
401
|
+
current = []
|
|
402
|
+
for char in text:
|
|
403
|
+
if char == "{":
|
|
404
|
+
if depth == 0:
|
|
405
|
+
current = []
|
|
406
|
+
depth += 1
|
|
407
|
+
if depth > 0:
|
|
408
|
+
current.append(char)
|
|
409
|
+
if char == "}":
|
|
410
|
+
depth -= 1
|
|
411
|
+
if depth == 0 and current:
|
|
412
|
+
candidates.append("".join(current))
|
|
413
|
+
for candidate in reversed(candidates):
|
|
414
|
+
try:
|
|
415
|
+
value = _pinch_json.loads(candidate)
|
|
416
|
+
if isinstance(value, dict) and "scores" in value:
|
|
417
|
+
return value
|
|
418
|
+
except Exception:
|
|
419
|
+
continue
|
|
420
|
+
for candidate in reversed(candidates):
|
|
421
|
+
try:
|
|
422
|
+
value = _pinch_json.loads(candidate)
|
|
423
|
+
if isinstance(value, dict):
|
|
424
|
+
return value
|
|
425
|
+
except Exception:
|
|
426
|
+
continue
|
|
427
|
+
score_match = _pinch_re.search(
|
|
428
|
+
r"(?:total|overall|final)\s*(?:score)?[:\s]*(0\.\d+|1\.0+)",
|
|
429
|
+
text,
|
|
430
|
+
_pinch_re.I,
|
|
431
|
+
)
|
|
432
|
+
if score_match:
|
|
433
|
+
total = float(score_match.group(1))
|
|
434
|
+
if 0.0 <= total <= 1.0:
|
|
435
|
+
return {"scores": {}, "total": total, "notes": "Score extracted from prose"}
|
|
436
|
+
return {}
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def _pinch_score(value):
|
|
440
|
+
if isinstance(value, bool):
|
|
441
|
+
return None
|
|
442
|
+
if isinstance(value, (int, float)):
|
|
443
|
+
return float(value)
|
|
444
|
+
if isinstance(value, str):
|
|
445
|
+
try:
|
|
446
|
+
return float(value.strip())
|
|
447
|
+
except ValueError:
|
|
448
|
+
return None
|
|
449
|
+
if isinstance(value, dict):
|
|
450
|
+
for key in ("score", "value", "weighted_score"):
|
|
451
|
+
if key in value:
|
|
452
|
+
return _pinch_score(value[key])
|
|
453
|
+
return None
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _pinch_named_scores(parsed):
|
|
457
|
+
scores = {}
|
|
458
|
+
for key in ("scores", "criteria_scores"):
|
|
459
|
+
values = parsed.get(key)
|
|
460
|
+
if isinstance(values, dict):
|
|
461
|
+
for name, value in values.items():
|
|
462
|
+
coerced = _pinch_score(value)
|
|
463
|
+
if coerced is not None:
|
|
464
|
+
scores[str(name)] = coerced
|
|
465
|
+
criteria = parsed.get("criterion_scores")
|
|
466
|
+
if isinstance(criteria, dict):
|
|
467
|
+
for name, value in criteria.items():
|
|
468
|
+
coerced = _pinch_score(value)
|
|
469
|
+
if coerced is not None:
|
|
470
|
+
scores[str(name)] = coerced
|
|
471
|
+
elif isinstance(criteria, list):
|
|
472
|
+
for index, item in enumerate(criteria, start=1):
|
|
473
|
+
if isinstance(item, dict):
|
|
474
|
+
name = (
|
|
475
|
+
item.get("name") or item.get("criterion")
|
|
476
|
+
or item.get("label") or f"criterion_{index}"
|
|
477
|
+
)
|
|
478
|
+
coerced = _pinch_score(item)
|
|
479
|
+
else:
|
|
480
|
+
name = f"criterion_{index}"
|
|
481
|
+
coerced = _pinch_score(item)
|
|
482
|
+
if coerced is not None:
|
|
483
|
+
scores[str(name)] = coerced
|
|
484
|
+
for key, value in parsed.items():
|
|
485
|
+
if _pinch_re.fullmatch(r"criterion\d+", str(key), _pinch_re.I):
|
|
486
|
+
coerced = _pinch_score(value)
|
|
487
|
+
if coerced is not None:
|
|
488
|
+
scores[str(key)] = coerced
|
|
489
|
+
return scores
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
def _pinch_normalize(parsed):
|
|
493
|
+
scores = _pinch_named_scores(parsed)
|
|
494
|
+
total = None
|
|
495
|
+
for key in ("total", "score", "overall_score", "completionScore", "total_score"):
|
|
496
|
+
if key in parsed:
|
|
497
|
+
total = _pinch_score(parsed[key])
|
|
498
|
+
if total is not None:
|
|
499
|
+
break
|
|
500
|
+
if total is None and isinstance(parsed.get("overall"), dict):
|
|
501
|
+
total = _pinch_score(parsed["overall"])
|
|
502
|
+
if total is None and scores:
|
|
503
|
+
total = _pinch_average(scores)
|
|
504
|
+
values = [value for value in scores.values() if isinstance(value, (int, float))]
|
|
505
|
+
if (
|
|
506
|
+
values and total is not None and total > 1.0
|
|
507
|
+
and all(0.0 <= float(value) <= 1.0 for value in values)
|
|
508
|
+
):
|
|
509
|
+
total = sum(values) / len(values)
|
|
510
|
+
notes = ""
|
|
511
|
+
for key in ("notes", "justification", "reasoning"):
|
|
512
|
+
if key in parsed:
|
|
513
|
+
notes = str(parsed[key])
|
|
514
|
+
break
|
|
515
|
+
return {"scores": scores, "total": total, "notes": notes}
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _pinch_env_int(name, default, minimum):
|
|
519
|
+
raw = _pinch_os.environ.get(name, "").strip()
|
|
520
|
+
if not raw:
|
|
521
|
+
return default
|
|
522
|
+
try:
|
|
523
|
+
value = int(raw)
|
|
524
|
+
except ValueError as exc:
|
|
525
|
+
raise RuntimeError(f"{name} must be an integer") from exc
|
|
526
|
+
if value < minimum:
|
|
527
|
+
raise RuntimeError(f"{name} must be at least {minimum}")
|
|
528
|
+
return value
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def _pinch_env_bool(name, default):
|
|
532
|
+
raw = _pinch_os.environ.get(name, "").strip().lower()
|
|
533
|
+
if not raw:
|
|
534
|
+
return default
|
|
535
|
+
if raw in {"1", "true", "yes", "on"}:
|
|
536
|
+
return True
|
|
537
|
+
if raw in {"0", "false", "no", "off"}:
|
|
538
|
+
return False
|
|
539
|
+
raise RuntimeError(f"{name} must be a boolean")
|
|
540
|
+
|
|
541
|
+
|
|
542
|
+
def _pinch_env_first(*names, default=""):
|
|
543
|
+
for name in names:
|
|
544
|
+
value = _pinch_os.environ.get(name, "").strip()
|
|
545
|
+
if value:
|
|
546
|
+
return value
|
|
547
|
+
return default
|
|
548
|
+
|
|
549
|
+
|
|
550
|
+
def _pinch_read_response(response, stream):
|
|
551
|
+
if not stream:
|
|
552
|
+
return _pinch_json.loads(response.read().decode("utf-8"))
|
|
553
|
+
raw_chunks = []
|
|
554
|
+
content_chunks = []
|
|
555
|
+
reasoning_chunks = []
|
|
556
|
+
finish_reason = None
|
|
557
|
+
saw_sse = False
|
|
558
|
+
for raw_line in response:
|
|
559
|
+
raw_chunks.append(raw_line)
|
|
560
|
+
line = raw_line.decode("utf-8", errors="replace").strip()
|
|
561
|
+
if not line.startswith("data:"):
|
|
562
|
+
continue
|
|
563
|
+
saw_sse = True
|
|
564
|
+
data = line[5:].strip()
|
|
565
|
+
if not data or data == "[DONE]":
|
|
566
|
+
continue
|
|
567
|
+
event = _pinch_json.loads(data)
|
|
568
|
+
if event.get("error"):
|
|
569
|
+
raise RuntimeError(f"PinchBench judge stream failed: {event['error']}")
|
|
570
|
+
choices = event.get("choices") or []
|
|
571
|
+
if not choices:
|
|
572
|
+
continue
|
|
573
|
+
choice = choices[0]
|
|
574
|
+
delta = choice.get("delta") or choice.get("message") or {}
|
|
575
|
+
content = delta.get("content")
|
|
576
|
+
if isinstance(content, str):
|
|
577
|
+
content_chunks.append(content)
|
|
578
|
+
elif isinstance(content, list):
|
|
579
|
+
for item in content:
|
|
580
|
+
if isinstance(item, dict) and isinstance(item.get("text"), str):
|
|
581
|
+
content_chunks.append(item["text"])
|
|
582
|
+
reasoning = delta.get("reasoning_content")
|
|
583
|
+
if isinstance(reasoning, str):
|
|
584
|
+
reasoning_chunks.append(reasoning)
|
|
585
|
+
if choice.get("finish_reason") is not None:
|
|
586
|
+
finish_reason = choice["finish_reason"]
|
|
587
|
+
if not saw_sse:
|
|
588
|
+
return _pinch_json.loads(b"".join(raw_chunks).decode("utf-8"))
|
|
589
|
+
return {
|
|
590
|
+
"choices": [{
|
|
591
|
+
"message": {
|
|
592
|
+
"content": "".join(content_chunks),
|
|
593
|
+
"reasoning_content": "".join(reasoning_chunks),
|
|
594
|
+
},
|
|
595
|
+
"finish_reason": finish_reason,
|
|
596
|
+
}]
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _pinch_judge(transcript, workspace_path):
|
|
601
|
+
endpoint = _pinch_env_first(
|
|
602
|
+
"PINCHBENCH_JUDGE_BASE_URL",
|
|
603
|
+
"OPENROUTER_BASE_URL",
|
|
604
|
+
default="https://openrouter.ai/api/v1",
|
|
605
|
+
).rstrip("/")
|
|
606
|
+
api_key = _pinch_env_first("PINCHBENCH_JUDGE_API_KEY", "OPENROUTER_API_KEY")
|
|
607
|
+
model = _pinch_env_first(
|
|
608
|
+
"PINCHBENCH_JUDGE_MODEL", default=_PINCH_OFFICIAL_JUDGE_MODEL
|
|
609
|
+
)
|
|
610
|
+
if not api_key:
|
|
611
|
+
raise RuntimeError(
|
|
612
|
+
"LLM grading requires PINCHBENCH_JUDGE_API_KEY or OPENROUTER_API_KEY"
|
|
613
|
+
)
|
|
614
|
+
if endpoint.endswith("/v1"):
|
|
615
|
+
endpoint += "/chat/completions"
|
|
616
|
+
elif not endpoint.endswith("/chat/completions"):
|
|
617
|
+
endpoint += "/v1/chat/completions"
|
|
618
|
+
rubric = _PINCH_RUBRIC or "\n".join(
|
|
619
|
+
f"- {criterion}" for criterion in _PINCH_CRITERIA
|
|
620
|
+
)
|
|
621
|
+
request_timeout = _pinch_env_int("PINCHBENCH_JUDGE_TIMEOUT_SECONDS", 300, 1)
|
|
622
|
+
max_tokens = _pinch_env_int(
|
|
623
|
+
"PINCHBENCH_JUDGE_MAX_COMPLETION_TOKENS",
|
|
624
|
+
_pinch_env_int("PINCHBENCH_JUDGE_MAX_TOKENS", 2048, 1),
|
|
625
|
+
1,
|
|
626
|
+
)
|
|
627
|
+
max_tokens_field = _pinch_env_first(
|
|
628
|
+
"PINCHBENCH_JUDGE_MAX_TOKENS_FIELD", default="max_completion_tokens"
|
|
629
|
+
)
|
|
630
|
+
if max_tokens_field not in {"max_completion_tokens", "max_tokens"}:
|
|
631
|
+
raise RuntimeError(
|
|
632
|
+
"PINCHBENCH_JUDGE_MAX_TOKENS_FIELD must be max_completion_tokens or max_tokens"
|
|
633
|
+
)
|
|
634
|
+
# Upstream PinchBench performs one direct judge request. Keep retries
|
|
635
|
+
# opt-in so the default remains score-comparable while overloaded custom
|
|
636
|
+
# gateways can still request a more forgiving transport policy.
|
|
637
|
+
retries = _pinch_env_int("PINCHBENCH_JUDGE_RETRIES", 0, 0)
|
|
638
|
+
stream = _pinch_env_bool("PINCHBENCH_JUDGE_STREAM", False)
|
|
639
|
+
workspace = _pinch_workspace_text(workspace_path)
|
|
640
|
+
workspace_section = ""
|
|
641
|
+
if workspace.strip():
|
|
642
|
+
workspace_section = f"## Workspace Files Created by Agent\n{workspace}\n\n"
|
|
643
|
+
prompt = (
|
|
644
|
+
"You are a grading function. Your ONLY job is to output a single JSON object.\n\n"
|
|
645
|
+
"CRITICAL RULES FOR YOU, THE GRADER (not the agent being graded):\n"
|
|
646
|
+
"- Do NOT use any tools (no Read, Write, exec, or any other tool calls)\n"
|
|
647
|
+
"- Do NOT create files or run commands\n"
|
|
648
|
+
"- Do NOT write any prose, explanation, or commentary outside the JSON\n"
|
|
649
|
+
"- Respond with ONLY a JSON object — nothing else\n\n"
|
|
650
|
+
"IMPORTANT: The agent being graded may have used tools (read, write, exec, apply_patch, "
|
|
651
|
+
"todowrite, etc.) during task execution. This is normal and expected. Do NOT treat the "
|
|
652
|
+
"agent's tool usage as a rule violation — the rules above apply only to you, the grader.\n\n"
|
|
653
|
+
"Be a strict evaluator. Reserve 1.0 for genuinely excellent performance. "
|
|
654
|
+
"An average acceptable completion should score around 0.6-0.7. "
|
|
655
|
+
"Deduct points for unnecessary steps, verbose output, and inefficient tool usage.\n\n"
|
|
656
|
+
f"## Task\n{_PINCH_TASK_PROMPT}\n\n"
|
|
657
|
+
f"## Expected Behavior\n{_PINCH_EXPECTED}\n\n"
|
|
658
|
+
f"## Agent Transcript (summarized)\n{_pinch_transcript_text(transcript)}\n\n"
|
|
659
|
+
f"{workspace_section}"
|
|
660
|
+
f"## Grading Rubric\n{rubric}\n\n"
|
|
661
|
+
"Score each criterion from 0.0 to 1.0.\n"
|
|
662
|
+
'The "total" field must also be between 0.0 and 1.0, and it must be the '
|
|
663
|
+
"arithmetic mean of the criterion scores, not their sum.\n\n"
|
|
664
|
+
"Respond with ONLY this JSON structure (no markdown, no code fences, no extra text):\n"
|
|
665
|
+
'{"scores": {"criterion_name": 0.0}, "total": 0.0, "notes": "brief justification"}'
|
|
666
|
+
)
|
|
667
|
+
payload_dict = {
|
|
668
|
+
"model": model,
|
|
669
|
+
"messages": [
|
|
670
|
+
{"role": "system", "content": _PINCH_JUDGE_SYSTEM_MSG},
|
|
671
|
+
{"role": "user", "content": prompt},
|
|
672
|
+
],
|
|
673
|
+
"temperature": 0.0,
|
|
674
|
+
max_tokens_field: max_tokens,
|
|
675
|
+
}
|
|
676
|
+
if stream:
|
|
677
|
+
payload_dict["stream"] = True
|
|
678
|
+
payload = _pinch_json.dumps(payload_dict).encode("utf-8")
|
|
679
|
+
no_proxy = _pinch_env_bool("PINCHBENCH_JUDGE_NO_PROXY", False)
|
|
680
|
+
opener = _pinch_request.build_opener(
|
|
681
|
+
_pinch_request.ProxyHandler({}) if no_proxy else _pinch_request.ProxyHandler()
|
|
682
|
+
)
|
|
683
|
+
parsed = {}
|
|
684
|
+
last_error = None
|
|
685
|
+
attempts_used = 0
|
|
686
|
+
content_source = None
|
|
687
|
+
finish_reason = None
|
|
688
|
+
for attempt in range(retries + 1):
|
|
689
|
+
attempts_used = attempt + 1
|
|
690
|
+
req = _pinch_request.Request(
|
|
691
|
+
endpoint,
|
|
692
|
+
data=payload,
|
|
693
|
+
headers={"Authorization": "Bearer " + api_key, "Content-Type": "application/json"},
|
|
694
|
+
method="POST",
|
|
695
|
+
)
|
|
696
|
+
try:
|
|
697
|
+
with opener.open(req, timeout=request_timeout) as response:
|
|
698
|
+
body = _pinch_read_response(response, stream)
|
|
699
|
+
choices = body.get("choices", [])
|
|
700
|
+
message = choices[0].get("message", {}) if choices else {}
|
|
701
|
+
text = message.get("content", "")
|
|
702
|
+
content_source = "content"
|
|
703
|
+
if not isinstance(text, str) or not text.strip():
|
|
704
|
+
text = message.get("reasoning_content", "")
|
|
705
|
+
content_source = "reasoning_content"
|
|
706
|
+
finish_reason = choices[0].get("finish_reason") if choices else None
|
|
707
|
+
parsed = _pinch_parse_json(text if isinstance(text, str) else "")
|
|
708
|
+
if _pinch_normalize(parsed)["total"] is not None:
|
|
709
|
+
break
|
|
710
|
+
last_error = ValueError("response contained no parseable score")
|
|
711
|
+
except _pinch_error.HTTPError as exc:
|
|
712
|
+
last_error = exc
|
|
713
|
+
except (_pinch_error.URLError, TimeoutError, OSError, ValueError) as exc:
|
|
714
|
+
last_error = exc
|
|
715
|
+
if attempt < retries:
|
|
716
|
+
_pinch_time.sleep(min(2 ** attempt, 8))
|
|
717
|
+
failure_notes = (
|
|
718
|
+
f"LLM judge failed: no parseable score after {retries + 1} "
|
|
719
|
+
f"attempt(s): {last_error}"
|
|
720
|
+
)
|
|
721
|
+
normalized = _pinch_normalize(parsed)
|
|
722
|
+
scores = normalized["scores"]
|
|
723
|
+
total = normalized["total"]
|
|
724
|
+
notes = normalized["notes"]
|
|
725
|
+
if total is None:
|
|
726
|
+
total = 0.0
|
|
727
|
+
notes = failure_notes if not parsed else "LLM judge failed: response parsed but no score extracted"
|
|
728
|
+
provenance = {
|
|
729
|
+
"judge_model": model,
|
|
730
|
+
"judge_protocol": "openai_chat_completions",
|
|
731
|
+
"judge_temperature": 0.0,
|
|
732
|
+
"judge_max_tokens_field": max_tokens_field,
|
|
733
|
+
"judge_max_tokens": max_tokens,
|
|
734
|
+
"judge_stream": stream,
|
|
735
|
+
"judge_attempts": attempts_used,
|
|
736
|
+
"judge_content_source": content_source,
|
|
737
|
+
"judge_finish_reason": finish_reason,
|
|
738
|
+
}
|
|
739
|
+
return (
|
|
740
|
+
scores,
|
|
741
|
+
total,
|
|
742
|
+
notes,
|
|
743
|
+
provenance,
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
|
|
747
|
+
def roller_grade(transcript, workspace_path):
|
|
748
|
+
transcript = _pinch_normalize_transcript(transcript)
|
|
749
|
+
automated_scores = {}
|
|
750
|
+
automated_total = 0.0
|
|
751
|
+
if _PINCH_GRADING_TYPE in ("automated", "hybrid"):
|
|
752
|
+
if not callable(_PINCH_AUTOMATED):
|
|
753
|
+
raise RuntimeError("PinchBench task has no automated grade function")
|
|
754
|
+
private_key = _PinchPath(workspace_path) / ".roller-private" / "image_classification_answer_key.json"
|
|
755
|
+
if private_key.is_file():
|
|
756
|
+
globals()["_PINCHBENCH_PRIVATE_IMAGE_KEY_PATH"] = str(private_key)
|
|
757
|
+
automated_scores = _PINCH_AUTOMATED(transcript, workspace_path)
|
|
758
|
+
if not isinstance(automated_scores, dict):
|
|
759
|
+
raise TypeError("PinchBench automated grader must return a dict")
|
|
760
|
+
automated_total = _pinch_average(automated_scores)
|
|
761
|
+
judge_scores, judge_total, notes, judge_provenance = {}, 0.0, "", {}
|
|
762
|
+
if _PINCH_GRADING_TYPE in ("llm_judge", "hybrid"):
|
|
763
|
+
judge_scores, judge_total, notes, judge_provenance = _pinch_judge(
|
|
764
|
+
transcript, workspace_path
|
|
765
|
+
)
|
|
766
|
+
if _PINCH_GRADING_TYPE == "automated":
|
|
767
|
+
result, total = dict(automated_scores), automated_total
|
|
768
|
+
elif _PINCH_GRADING_TYPE == "llm_judge":
|
|
769
|
+
result, total = {"llm_judge." + key: value for key, value in judge_scores.items()}, judge_total
|
|
770
|
+
else:
|
|
771
|
+
auto_weight = float(_PINCH_WEIGHTS.get("automated", 0.5))
|
|
772
|
+
judge_weight = float(_PINCH_WEIGHTS.get("llm_judge", 0.5))
|
|
773
|
+
denominator = auto_weight + judge_weight
|
|
774
|
+
if denominator <= 0:
|
|
775
|
+
auto_weight = judge_weight = 0.5
|
|
776
|
+
denominator = 1.0
|
|
777
|
+
total = (automated_total * auto_weight + judge_total * judge_weight) / denominator
|
|
778
|
+
result = {"automated." + key: value for key, value in automated_scores.items()}
|
|
779
|
+
result.update({"llm_judge." + key: value for key, value in judge_scores.items()})
|
|
780
|
+
result["overall_score"] = total
|
|
781
|
+
result.update(judge_provenance)
|
|
782
|
+
if notes:
|
|
783
|
+
result["judge_notes"] = notes
|
|
784
|
+
return result
|
|
785
|
+
'''
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def _grader_source(task: dict[str, Any]) -> str:
|
|
789
|
+
auto = task["automated_checks"]
|
|
790
|
+
if task["grading_type"] in {"automated", "hybrid"} and not auto:
|
|
791
|
+
raise BenchFormatError(
|
|
792
|
+
f"{task['id']}: {task['grading_type']} task has no ## Automated Checks"
|
|
793
|
+
)
|
|
794
|
+
source = (auto.rstrip() + "\n\n") if auto else ""
|
|
795
|
+
replacements = {
|
|
796
|
+
"__TASK_PROMPT__": repr(task["prompt"] or "\n\n".join(
|
|
797
|
+
item["content"] for item in task["sessions"]
|
|
798
|
+
)),
|
|
799
|
+
"__EXPECTED__": repr(task["expected_behavior"]),
|
|
800
|
+
"__CRITERIA__": repr(task["grading_criteria"]),
|
|
801
|
+
"__RUBRIC__": repr(task["llm_judge_rubric"]),
|
|
802
|
+
"__GRADING_TYPE__": repr(task["grading_type"]),
|
|
803
|
+
"__WEIGHTS__": repr(task["grading_weights"]),
|
|
804
|
+
}
|
|
805
|
+
wrapper = _PINCH_WRAPPER
|
|
806
|
+
for marker, value in replacements.items():
|
|
807
|
+
wrapper = wrapper.replace(marker, value)
|
|
808
|
+
return source + wrapper.strip() + "\n"
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
class PinchBenchAdapter(BenchAdapter):
|
|
812
|
+
name = "pinchbench"
|
|
813
|
+
description = "PinchBench — real-world OpenClaw tasks with automated and LLM grading"
|
|
814
|
+
variants = ("all", "automated-only", "core")
|
|
815
|
+
default_variant = "all"
|
|
816
|
+
|
|
817
|
+
@staticmethod
|
|
818
|
+
def _tasks_root(root: Path) -> Path:
|
|
819
|
+
root = root.expanduser().resolve()
|
|
820
|
+
tasks = root / "tasks" if (root / "tasks" / "manifest.yaml").is_file() else root
|
|
821
|
+
if not (tasks / "manifest.yaml").is_file():
|
|
822
|
+
raise FileNotFoundError(f"PinchBench manifest not found under: {root}")
|
|
823
|
+
return tasks
|
|
824
|
+
|
|
825
|
+
def load(
|
|
826
|
+
self,
|
|
827
|
+
root: Path,
|
|
828
|
+
*,
|
|
829
|
+
categories: Sequence[str] | None = None,
|
|
830
|
+
task_ids: Sequence[str] | None = None,
|
|
831
|
+
variant: str | None = None,
|
|
832
|
+
timeout_multiplier: float = 1.0,
|
|
833
|
+
) -> list[dict[str, Any]]:
|
|
834
|
+
tasks_root = self._tasks_root(root)
|
|
835
|
+
timeout_multiplier = validate_timeout_multiplier(timeout_multiplier)
|
|
836
|
+
variant = variant or self.default_variant
|
|
837
|
+
if variant not in self.variants:
|
|
838
|
+
raise BenchFormatError(
|
|
839
|
+
f"{self.name}: variant must be one of {', '.join(self.variants)}"
|
|
840
|
+
)
|
|
841
|
+
manifest = yaml.safe_load((tasks_root / "manifest.yaml").read_text()) or {}
|
|
842
|
+
if not isinstance(manifest, dict) or not isinstance(manifest.get("categories"), dict):
|
|
843
|
+
raise BenchFormatError("pinchbench manifest: categories must be an object")
|
|
844
|
+
ordered: list[tuple[str, str]] = []
|
|
845
|
+
category_by_id: dict[str, str] = {}
|
|
846
|
+
for category, identifiers in manifest["categories"].items():
|
|
847
|
+
if not isinstance(identifiers, list):
|
|
848
|
+
raise BenchFormatError(
|
|
849
|
+
f"pinchbench manifest: category {category!r} must be a list"
|
|
850
|
+
)
|
|
851
|
+
for task_id in identifiers:
|
|
852
|
+
if not isinstance(task_id, str):
|
|
853
|
+
raise BenchFormatError("pinchbench manifest: task ids must be strings")
|
|
854
|
+
category_by_id[task_id] = str(category)
|
|
855
|
+
ordered.append((task_id, str(category)))
|
|
856
|
+
first = manifest.get("run_first", [])
|
|
857
|
+
if isinstance(first, list):
|
|
858
|
+
first_ids = [item for item in first if isinstance(item, str)]
|
|
859
|
+
ordered.sort(key=lambda item: first_ids.index(item[0]) if item[0] in first_ids else len(first_ids))
|
|
860
|
+
requested = set(task_ids or ())
|
|
861
|
+
missing = requested - set(category_by_id)
|
|
862
|
+
if missing:
|
|
863
|
+
raise BenchFormatError(
|
|
864
|
+
f"{self.name}: unknown task id(s): {', '.join(sorted(missing))}"
|
|
865
|
+
)
|
|
866
|
+
core = set(manifest.get("core", []))
|
|
867
|
+
category_filter = set(categories or ())
|
|
868
|
+
rows: list[dict[str, Any]] = []
|
|
869
|
+
for task_id, manifest_category in ordered:
|
|
870
|
+
if requested and task_id not in requested:
|
|
871
|
+
continue
|
|
872
|
+
if category_filter and manifest_category not in category_filter:
|
|
873
|
+
continue
|
|
874
|
+
task_path = tasks_root / f"{task_id}.md"
|
|
875
|
+
if not task_path.is_file():
|
|
876
|
+
raise BenchFormatError(f"pinchbench manifest references missing {task_path}")
|
|
877
|
+
task = parse_task_markdown(task_path.read_text(encoding="utf-8"), task_path.name)
|
|
878
|
+
# The manifest is upstream's declared source of truth. A few task
|
|
879
|
+
# frontmatters lag category moves, so use the manifest rather than
|
|
880
|
+
# making an otherwise runnable checkout fail export.
|
|
881
|
+
if variant == "automated-only" and task["grading_type"] != "automated":
|
|
882
|
+
continue
|
|
883
|
+
if variant == "core" and task_id not in core:
|
|
884
|
+
continue
|
|
885
|
+
rows.append(
|
|
886
|
+
self._row(
|
|
887
|
+
tasks_root.parent,
|
|
888
|
+
task,
|
|
889
|
+
manifest_category,
|
|
890
|
+
variant,
|
|
891
|
+
timeout_multiplier,
|
|
892
|
+
)
|
|
893
|
+
)
|
|
894
|
+
return rows
|
|
895
|
+
|
|
896
|
+
def _row(
|
|
897
|
+
self,
|
|
898
|
+
root: Path,
|
|
899
|
+
task: dict[str, Any],
|
|
900
|
+
category: str,
|
|
901
|
+
variant: str,
|
|
902
|
+
timeout_multiplier: float,
|
|
903
|
+
) -> dict[str, Any]:
|
|
904
|
+
inputs: list[dict[str, Any]] = []
|
|
905
|
+
for index, item in enumerate(task["workspace_files"]):
|
|
906
|
+
if not isinstance(item, dict):
|
|
907
|
+
raise BenchFormatError(
|
|
908
|
+
f"{task['id']}: workspace_files[{index}] must be an object"
|
|
909
|
+
)
|
|
910
|
+
if "content" in item:
|
|
911
|
+
target = item.get("path") or item.get("dest")
|
|
912
|
+
if not isinstance(target, str) or not isinstance(item["content"], str):
|
|
913
|
+
raise BenchFormatError(
|
|
914
|
+
f"{task['id']}: inline workspace file needs string path/content"
|
|
915
|
+
)
|
|
916
|
+
inputs.append({"target": target, "content": item["content"]})
|
|
917
|
+
else:
|
|
918
|
+
source, target = item.get("source"), item.get("dest")
|
|
919
|
+
if not isinstance(source, str) or not isinstance(target, str):
|
|
920
|
+
raise BenchFormatError(
|
|
921
|
+
f"{task['id']}: workspace file needs string source/dest"
|
|
922
|
+
)
|
|
923
|
+
inputs.append({"source": source, "target": target})
|
|
924
|
+
|
|
925
|
+
stage: list[tuple[str, str]] = []
|
|
926
|
+
private_key = root / "assets" / "image_classification_answer_key.json"
|
|
927
|
+
if task["id"] == "task_image_identification" and private_key.is_file():
|
|
928
|
+
stage.append(
|
|
929
|
+
(str(private_key.resolve()), ".roller-private/image_classification_answer_key.json")
|
|
930
|
+
)
|
|
931
|
+
judge_required = task["grading_type"] in {"llm_judge", "hybrid"}
|
|
932
|
+
grade = GradeSpec(
|
|
933
|
+
code=_grader_source(task),
|
|
934
|
+
entrypoint="roller_grade",
|
|
935
|
+
env_names=JUDGE_ENV if judge_required else (),
|
|
936
|
+
optional_env_names=JUDGE_OPTIONAL_ENV if judge_required else (),
|
|
937
|
+
timeout_seconds=3600 if judge_required else 120,
|
|
938
|
+
stage=tuple(stage),
|
|
939
|
+
)
|
|
940
|
+
binding = BenchBinding(
|
|
941
|
+
self.name,
|
|
942
|
+
grade=grade,
|
|
943
|
+
category=category,
|
|
944
|
+
metadata={
|
|
945
|
+
"name": task["name"],
|
|
946
|
+
"grading_type": task["grading_type"],
|
|
947
|
+
"variant": variant,
|
|
948
|
+
"prerequisites": task["prerequisites"],
|
|
949
|
+
},
|
|
950
|
+
)
|
|
951
|
+
row: dict[str, Any] = {
|
|
952
|
+
"schema_version": "roller.task/v1",
|
|
953
|
+
"id": task["id"],
|
|
954
|
+
"harness": "openclaw",
|
|
955
|
+
"timeout_seconds": max(
|
|
956
|
+
1, math.ceil(task["timeout_seconds"] * timeout_multiplier)
|
|
957
|
+
),
|
|
958
|
+
"file_root": str((root / "assets").resolve()),
|
|
959
|
+
"files": {
|
|
960
|
+
"inputs": inputs,
|
|
961
|
+
"input_mode": "copy",
|
|
962
|
+
"output_dir": "outputs",
|
|
963
|
+
},
|
|
964
|
+
"bench": binding.as_dict(),
|
|
965
|
+
}
|
|
966
|
+
setup = _pinch_runtime_setup(task)
|
|
967
|
+
if setup:
|
|
968
|
+
row["runtime"] = {"setup": setup}
|
|
969
|
+
if task["sessions"]:
|
|
970
|
+
row["messages"] = task["sessions"]
|
|
971
|
+
else:
|
|
972
|
+
row["prompt"] = task["prompt"]
|
|
973
|
+
return row
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
__all__ = ["PinchBenchAdapter", "parse_task_markdown"]
|