roller-agent 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. roller/__init__.py +31 -0
  2. roller/__main__.py +6 -0
  3. roller/actf.py +431 -0
  4. roller/artifacts.py +124 -0
  5. roller/assets/deepseek/journal.mjs +44 -0
  6. roller/assets/deepseek/roller.patch.yml +15 -0
  7. roller/assets/deepseek/runner.py +149 -0
  8. roller/assets/hermes/runner.py +322 -0
  9. roller/assets/openclaw_capture.py +32 -0
  10. roller/assets/openclaw_drain.mjs +21 -0
  11. roller/assets/opencode/journal.mjs +42 -0
  12. roller/assets/opencode/runner.py +203 -0
  13. roller/assets/pi/runner.mjs +160 -0
  14. roller/assets/pi/runner.py +93 -0
  15. roller/benches/__init__.py +73 -0
  16. roller/benches/base.py +306 -0
  17. roller/benches/grading.py +553 -0
  18. roller/benches/pinchbench.py +976 -0
  19. roller/benches/qwenclaw/__init__.py +49 -0
  20. roller/benches/qwenclaw/adapter.py +167 -0
  21. roller/benches/qwenclaw/grader_runtime.py +395 -0
  22. roller/benches/qwenclaw/report.py +49 -0
  23. roller/benches/qwenclaw/task_parser.py +181 -0
  24. roller/benches/qwenclaw/workspace.py +202 -0
  25. roller/benches/report.py +166 -0
  26. roller/benches/skillsbench.py +708 -0
  27. roller/benches/transcripts.py +102 -0
  28. roller/benches/wildclaw.py +569 -0
  29. roller/bridge_capture.py +73 -0
  30. roller/bundles.py +545 -0
  31. roller/capture.py +385 -0
  32. roller/capture_protocol.py +301 -0
  33. roller/capture_runtime.py +48 -0
  34. roller/capture_states.py +37 -0
  35. roller/claude_capture.py +185 -0
  36. roller/claude_code_actf.py +594 -0
  37. roller/cli.py +1732 -0
  38. roller/codex_actf.py +584 -0
  39. roller/codex_capture.py +99 -0
  40. roller/codex_tools.py +137 -0
  41. roller/collector.py +818 -0
  42. roller/compaction_sft.py +672 -0
  43. roller/conversation.py +432 -0
  44. roller/deepseek_actf.py +135 -0
  45. roller/harness_registry.py +61 -0
  46. roller/harnesses/__init__.py +29 -0
  47. roller/harnesses/base.py +64 -0
  48. roller/harnesses/bridge.py +448 -0
  49. roller/harnesses/claude_code.py +839 -0
  50. roller/harnesses/codex.py +808 -0
  51. roller/harnesses/config_overrides.py +100 -0
  52. roller/harnesses/container.py +206 -0
  53. roller/harnesses/deepseek.py +104 -0
  54. roller/harnesses/docker.py +72 -0
  55. roller/harnesses/gateway_ports.py +155 -0
  56. roller/harnesses/hermes.py +119 -0
  57. roller/harnesses/openclaw.py +1605 -0
  58. roller/harnesses/opencode.py +193 -0
  59. roller/harnesses/pi.py +111 -0
  60. roller/hermes_actf.py +147 -0
  61. roller/image_build.py +206 -0
  62. roller/integrity.py +332 -0
  63. roller/model_protocols.py +201 -0
  64. roller/models.py +169 -0
  65. roller/openclaw_capture.py +140 -0
  66. roller/openclaw_delivery.py +211 -0
  67. roller/openclaw_usage.py +59 -0
  68. roller/opencode_actf.py +305 -0
  69. roller/opencode_capture.py +96 -0
  70. roller/pi_actf.py +172 -0
  71. roller/response_metadata.py +89 -0
  72. roller/run_lock.py +297 -0
  73. roller/sft.py +471 -0
  74. roller/storage.py +53 -0
  75. roller/tasks.py +689 -0
  76. roller/thinking.py +50 -0
  77. roller/thinking_capabilities.py +152 -0
  78. roller/trajectory_audit.py +386 -0
  79. roller/usage.py +168 -0
  80. roller/version.py +1 -0
  81. roller/viewer/__init__.py +3 -0
  82. roller/viewer/app.py +722 -0
  83. roller/viewer/static/app.js +752 -0
  84. roller/viewer/static/index.html +105 -0
  85. roller/viewer/static/style.css +213 -0
  86. roller/wire_trace.py +650 -0
  87. roller_agent-0.2.0.dist-info/METADATA +397 -0
  88. roller_agent-0.2.0.dist-info/RECORD +92 -0
  89. roller_agent-0.2.0.dist-info/WHEEL +5 -0
  90. roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
  91. roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
  92. roller_agent-0.2.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,976 @@
1
+ """PinchBench adapter, including automated, LLM-judge and hybrid grading."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import re
7
+ from collections.abc import Sequence
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ import yaml
12
+
13
+ from roller.benches.base import (
14
+ BenchAdapter,
15
+ BenchBinding,
16
+ BenchFormatError,
17
+ GradeSpec,
18
+ validate_timeout_multiplier,
19
+ )
20
+
21
+ FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---\s*\n(.*)", re.DOTALL)
22
+ SECTION_RE = re.compile(r"^##\s+(.+?)\s*$")
23
+ PYTHON_FENCE_RE = re.compile(r"```python\s*\n(.*?)\n```", re.DOTALL | re.IGNORECASE)
24
+ CODE_FENCE_RE = re.compile(r"```\s*\n(.*?)\n```", re.DOTALL)
25
+ GRADING_TYPES = {"automated", "llm_judge", "hybrid"}
26
+ JUDGE_ENV = ()
27
+ JUDGE_OPTIONAL_ENV = (
28
+ "PINCHBENCH_JUDGE_BASE_URL",
29
+ "PINCHBENCH_JUDGE_API_KEY",
30
+ "PINCHBENCH_JUDGE_MODEL",
31
+ "PINCHBENCH_JUDGE_TIMEOUT_SECONDS",
32
+ "PINCHBENCH_JUDGE_MAX_COMPLETION_TOKENS",
33
+ "PINCHBENCH_JUDGE_MAX_TOKENS",
34
+ "PINCHBENCH_JUDGE_MAX_TOKENS_FIELD",
35
+ "PINCHBENCH_JUDGE_RETRIES",
36
+ "PINCHBENCH_JUDGE_STREAM",
37
+ "PINCHBENCH_JUDGE_NO_PROXY",
38
+ "OPENROUTER_API_KEY",
39
+ "OPENROUTER_BASE_URL",
40
+ "HTTP_PROXY",
41
+ "HTTPS_PROXY",
42
+ "NO_PROXY",
43
+ "http_proxy",
44
+ "https_proxy",
45
+ "no_proxy",
46
+ )
47
+
48
+ _PINCH_FWS_SETUP = r'''set -euo pipefail
49
+ tools=/tmp/roller-pinch-tools
50
+ mkdir -p "$tools/bin" "$HOME/.config/gws/cache"
51
+
52
+ download_checked() {
53
+ url=$1
54
+ destination=$2
55
+ checksum=$3
56
+ if ! echo "$checksum $destination" | sha256sum -c - >/dev/null 2>&1; then
57
+ rm -f "$destination"
58
+ curl --fail --location --silent --show-error \
59
+ --retry 5 --retry-all-errors --connect-timeout 30 --max-time 600 \
60
+ "$url" -o "$destination"
61
+ echo "$checksum $destination" | sha256sum -c - >/dev/null
62
+ fi
63
+ }
64
+
65
+ if [ ! -x "$tools/node_modules/.bin/fws" ]; then
66
+ npm install --prefix "$tools" --no-audit --no-fund @juppytt/fws@0.4.4
67
+ fi
68
+ mkdir -p "$tools/node_modules/@juppytt/fws/node_modules/.bin"
69
+ ln -sf "$tools/node_modules/.bin/tsx" \
70
+ "$tools/node_modules/@juppytt/fws/node_modules/.bin/tsx"
71
+
72
+ download_checked \
73
+ https://github.com/googleworkspace/cli/releases/download/v0.22.3/google-workspace-cli-x86_64-unknown-linux-gnu.tar.gz \
74
+ /tmp/roller-pinch-gws.tar.gz \
75
+ b951ef847b38dd41a23d31a3aeaf3cb650421aac607bbfe63727bf4f4213ce44
76
+ if [ ! -x "$tools/google-workspace-cli-x86_64-unknown-linux-gnu/gws" ]; then
77
+ tar -xzf /tmp/roller-pinch-gws.tar.gz -C "$tools"
78
+ fi
79
+ ln -sf "$tools/google-workspace-cli-x86_64-unknown-linux-gnu/gws" \
80
+ "$tools/bin/gws.real"
81
+
82
+ download_checked \
83
+ https://github.com/cli/cli/releases/download/v2.79.0/gh_2.79.0_linux_amd64.tar.gz \
84
+ /tmp/roller-pinch-gh.tar.gz \
85
+ e7af0c72a607c0528fda1989f7c8e3be85e67d321889002af0e2938ad9c8fb68
86
+ if [ ! -x "$tools/gh_2.79.0_linux_amd64/bin/gh" ]; then
87
+ tar -xzf /tmp/roller-pinch-gh.tar.gz -C "$tools"
88
+ fi
89
+ ln -sf "$tools/gh_2.79.0_linux_amd64/bin/gh" "$tools/bin/gh.real"
90
+
91
+ for discovery in gmail:v1 calendar:v3 drive:v3 tasks:v1 sheets:v4 people:v1; do
92
+ api=${discovery%%:*}
93
+ version=${discovery#*:}
94
+ destination="$HOME/.config/gws/cache/${api}_${version}.json"
95
+ if ! python3 -m json.tool "$destination" >/dev/null 2>&1; then
96
+ rm -f "$destination"
97
+ curl --fail --location --silent --show-error \
98
+ --retry 5 --retry-all-errors --connect-timeout 30 --max-time 600 \
99
+ "https://www.googleapis.com/discovery/v1/apis/${api}/${version}/rest" \
100
+ -o "$destination"
101
+ python3 -m json.tool "$destination" >/dev/null
102
+ fi
103
+ done
104
+
105
+ export PATH="$tools/bin:$tools/node_modules/.bin:$PATH"
106
+ : "${ROLLER_GATEWAY_PORT:?Roller did not provide the OpenClaw gateway port}"
107
+ fws_port=$((ROLLER_GATEWAY_PORT + 1))
108
+ if [ "$fws_port" -gt 65534 ]; then
109
+ fws_port=$((ROLLER_GATEWAY_PORT - 2))
110
+ fi
111
+ fws_output=$(fws server start -p "$fws_port")
112
+ printf '%s\n' "$fws_output"
113
+ printf '%s\n' "$fws_output" | sed -n '/^[[:space:]]*export /p' \
114
+ > /roller/task/fws-environment
115
+ test -s /roller/task/fws-environment
116
+
117
+ printf '%s\n' \
118
+ '#!/bin/bash' \
119
+ 'set -a' \
120
+ '. /roller/task/fws-environment' \
121
+ 'set +a' \
122
+ 'exec /tmp/roller-pinch-tools/bin/gh.real "$@"' > "$tools/bin/gh"
123
+ printf '%s\n' \
124
+ '#!/bin/bash' \
125
+ 'set -a' \
126
+ '. /roller/task/fws-environment' \
127
+ 'set +a' \
128
+ 'exec /tmp/roller-pinch-tools/bin/gws.real "$@"' > "$tools/bin/gws"
129
+ chmod +x "$tools/bin/gh" "$tools/bin/gws"
130
+ printf '%s\n' \
131
+ 'export PATH=/tmp/roller-pinch-tools/bin:/tmp/roller-pinch-tools/node_modules/.bin:$PATH' \
132
+ > /roller/task/environment
133
+
134
+ fws_ready=false
135
+ for attempt in $(seq 1 20); do
136
+ if GH_REPO=testuser/my-project "$tools/bin/gh" api /user >/dev/null 2>&1; then
137
+ fws_ready=true
138
+ break
139
+ fi
140
+ sleep 0.5
141
+ done
142
+ $fws_ready || { echo 'FWS mock service failed its readiness probe' >&2; exit 1; }'''
143
+
144
+
145
+ def _pinch_runtime_setup(task: dict[str, Any]) -> list[str]:
146
+ """Provision the mock workspace services declared by upstream PinchBench."""
147
+
148
+ prerequisites = task["prerequisites"]
149
+ needs_fws = task["category"].casefold() in {"gws", "github"} or any(
150
+ "fws" in prerequisite.casefold() for prerequisite in prerequisites
151
+ )
152
+ return [_PINCH_FWS_SETUP] if needs_fws else []
153
+
154
+
155
+ def _split_markdown(text: str, label: str) -> tuple[dict[str, Any], dict[str, str]]:
156
+ match = FRONTMATTER_RE.match(text)
157
+ if match is None:
158
+ raise BenchFormatError(f"{label}: missing YAML frontmatter")
159
+ try:
160
+ metadata = yaml.safe_load(match.group(1)) or {}
161
+ except yaml.YAMLError as exc:
162
+ raise BenchFormatError(f"{label}: invalid YAML frontmatter: {exc}") from exc
163
+ if not isinstance(metadata, dict):
164
+ raise BenchFormatError(f"{label}: frontmatter must be an object")
165
+ sections: dict[str, str] = {}
166
+ current: str | None = None
167
+ lines: list[str] = []
168
+ for line in match.group(2).splitlines():
169
+ header = SECTION_RE.match(line)
170
+ if header:
171
+ if current is not None:
172
+ sections[current] = "\n".join(lines).strip()
173
+ current, lines = header.group(1), []
174
+ elif current is not None:
175
+ lines.append(line)
176
+ if current is not None:
177
+ sections[current] = "\n".join(lines).strip()
178
+ return metadata, sections
179
+
180
+
181
+ def _strip_python_fence(value: str) -> str:
182
+ stripped = value.strip()
183
+ match = PYTHON_FENCE_RE.search(stripped) or CODE_FENCE_RE.search(stripped)
184
+ return match.group(1).strip() if match else stripped
185
+
186
+
187
+ def parse_task_markdown(text: str, label: str) -> dict[str, Any]:
188
+ metadata, sections = _split_markdown(text, label)
189
+ task_id = metadata.get("id")
190
+ if not isinstance(task_id, str) or not task_id.strip():
191
+ raise BenchFormatError(f"{label}: id must be a non-empty string")
192
+ grading_type = metadata.get("grading_type", "automated")
193
+ if grading_type not in GRADING_TYPES:
194
+ raise BenchFormatError(
195
+ f"{label}: grading_type must be one of {sorted(GRADING_TYPES)}"
196
+ )
197
+ timeout = metadata.get("timeout_seconds", 120)
198
+ if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0:
199
+ raise BenchFormatError(f"{label}: timeout_seconds must be a positive integer")
200
+ prompt = sections.get("Prompt", "").strip()
201
+ sessions = metadata.get("sessions")
202
+ parsed_sessions: list[dict[str, Any]] = []
203
+ if sessions is not None:
204
+ if not isinstance(sessions, list) or not sessions:
205
+ raise BenchFormatError(f"{label}: sessions must be a non-empty list")
206
+ for index, session in enumerate(sessions):
207
+ if not isinstance(session, dict):
208
+ raise BenchFormatError(f"{label}: sessions[{index}] must be an object")
209
+ session_prompt = session.get("prompt")
210
+ reset = session.get("new_session", False)
211
+ if not isinstance(session_prompt, str) or not session_prompt.strip():
212
+ raise BenchFormatError(
213
+ f"{label}: sessions[{index}].prompt must be non-empty"
214
+ )
215
+ if not isinstance(reset, bool) or (index == 0 and reset):
216
+ raise BenchFormatError(
217
+ f"{label}: sessions[{index}].new_session must be false for the first "
218
+ "session and boolean elsewhere"
219
+ )
220
+ parsed_sessions.append(
221
+ {
222
+ "role": "user",
223
+ "content": session_prompt.strip(),
224
+ "new_session": reset,
225
+ }
226
+ )
227
+ elif not prompt:
228
+ raise BenchFormatError(f"{label}: missing ## Prompt")
229
+ workspace_files = metadata.get("workspace_files", [])
230
+ if not isinstance(workspace_files, list):
231
+ raise BenchFormatError(f"{label}: workspace_files must be a list")
232
+ prerequisites = metadata.get("prerequisites", [])
233
+ if not isinstance(prerequisites, list) or not all(
234
+ isinstance(item, str) and item.strip() for item in prerequisites
235
+ ):
236
+ raise BenchFormatError(f"{label}: prerequisites must be a string list")
237
+ return {
238
+ "id": task_id.strip(),
239
+ "name": str(metadata.get("name") or task_id).strip(),
240
+ "category": str(metadata.get("category") or "").strip(),
241
+ "grading_type": grading_type,
242
+ "timeout_seconds": timeout,
243
+ "workspace_files": workspace_files,
244
+ "prompt": prompt,
245
+ "sessions": parsed_sessions,
246
+ "expected_behavior": sections.get("Expected Behavior", "").strip(),
247
+ "grading_criteria": [
248
+ match.group(1)
249
+ for line in sections.get("Grading Criteria", "").splitlines()
250
+ if (match := re.match(r"^-\s+\[[ x]\]\s+(.+)$", line.strip()))
251
+ ],
252
+ "automated_checks": _strip_python_fence(
253
+ sections.get("Automated Checks", "")
254
+ ),
255
+ "llm_judge_rubric": sections.get("LLM Judge Rubric", "").strip(),
256
+ "grading_weights": metadata.get("grading_weights") or {},
257
+ "prerequisites": [item.strip() for item in prerequisites],
258
+ }
259
+
260
+
261
+ _PINCH_WRAPPER = r'''
262
+ import json as _pinch_json
263
+ import os as _pinch_os
264
+ import re as _pinch_re
265
+ from pathlib import Path as _PinchPath
266
+ import time as _pinch_time
267
+ from urllib import error as _pinch_error
268
+ from urllib import request as _pinch_request
269
+
270
+ _PINCH_TASK_PROMPT = __TASK_PROMPT__
271
+ _PINCH_EXPECTED = __EXPECTED__
272
+ _PINCH_CRITERIA = __CRITERIA__
273
+ _PINCH_RUBRIC = __RUBRIC__
274
+ _PINCH_GRADING_TYPE = __GRADING_TYPE__
275
+ _PINCH_WEIGHTS = __WEIGHTS__
276
+ _PINCH_AUTOMATED = globals().get("grade")
277
+ _PINCH_OFFICIAL_JUDGE_MODEL = "anthropic/claude-haiku-4.5"
278
+ _PINCH_JUDGE_SYSTEM_MSG = (
279
+ "You are a strict grading function. "
280
+ "Respond with ONLY a JSON object, no prose, no markdown fences, no extra text."
281
+ )
282
+
283
+
284
+ def _pinch_average(values):
285
+ numbers = [float(value) for value in values.values()
286
+ if isinstance(value, (int, float)) and not isinstance(value, bool)]
287
+ return sum(numbers) / len(numbers) if numbers else 0.0
288
+
289
+
290
+ def _pinch_normalize_transcript(transcript):
291
+ """Present OpenClaw messages in the list-of-parts shape Pinch expects."""
292
+ normalized = []
293
+ for event in transcript:
294
+ if not isinstance(event, dict):
295
+ continue
296
+ event = dict(event)
297
+ message = event.get("message")
298
+ if isinstance(message, dict):
299
+ message = dict(message)
300
+ content = message.get("content", [])
301
+ if isinstance(content, str):
302
+ content = [{"type": "text", "text": content}]
303
+ elif isinstance(content, dict):
304
+ content = [content]
305
+ elif isinstance(content, list):
306
+ content = [
307
+ item
308
+ if isinstance(item, dict)
309
+ else {"type": "text", "text": str(item)}
310
+ for item in content
311
+ ]
312
+ else:
313
+ content = []
314
+ message["content"] = content
315
+ event["message"] = message
316
+ normalized.append(event)
317
+ return normalized
318
+
319
+
320
+ def _pinch_transcript_text(transcript):
321
+ # Equivalent to upstream PinchBench's _summarize_transcript.
322
+ pieces = []
323
+ for event in transcript:
324
+ if not isinstance(event, dict) or event.get("type") != "message":
325
+ continue
326
+ message = event.get("message", {})
327
+ role = message.get("role")
328
+ content = message.get("content", [])
329
+ if role == "assistant":
330
+ for item in content:
331
+ if item.get("type") == "toolCall":
332
+ args = item.get("arguments", {})
333
+ truncated_args = {}
334
+ for key, value in args.items():
335
+ if isinstance(value, str) and len(value) > 200:
336
+ truncated_args[key] = value[:200] + "...[truncated]"
337
+ else:
338
+ truncated_args[key] = value
339
+ pieces.append(
340
+ f"Tool: {item.get('name')}({_pinch_json.dumps(truncated_args)})"
341
+ )
342
+ elif item.get("type") == "text":
343
+ value = item.get("text", "").strip()
344
+ if value:
345
+ pieces.append(f"Assistant: {value[:2000]}")
346
+ elif role == "toolResult":
347
+ if content:
348
+ pieces.append(f"Result: {str(content[0])[:200]}")
349
+ elif role == "user":
350
+ if content:
351
+ pieces.append(f"User: {content[0]}")
352
+ return "\n".join(pieces)
353
+
354
+
355
+ def _pinch_workspace_text(workspace_path):
356
+ root = _PinchPath(workspace_path)
357
+ if not root.is_dir():
358
+ return ""
359
+ skip_names = {
360
+ "BOOTSTRAP.md", "SOUL.md", "USER.md", "IDENTITY.md",
361
+ "HEARTBEAT.md", "TOOLS.md", "AGENTS.md",
362
+ }
363
+ skip_dirs = {".git", ".openclaw", "__pycache__", "node_modules", "skills"}
364
+ pieces = []
365
+ for path in sorted(root.rglob("*")):
366
+ if not path.is_file():
367
+ continue
368
+ relative = path.relative_to(root)
369
+ if any(part.startswith(".") or part in skip_dirs for part in relative.parts):
370
+ continue
371
+ if path.name in skip_names:
372
+ continue
373
+ try:
374
+ value = path.read_text(encoding="utf-8")
375
+ except (OSError, UnicodeDecodeError):
376
+ continue
377
+ pieces.append(f"### File: {relative}\n{value}")
378
+ return "\n\n".join(pieces)
379
+
380
+
381
+ def _pinch_parse_json(text):
382
+ text = text.strip()
383
+ if not text:
384
+ return {}
385
+ try:
386
+ value = _pinch_json.loads(text)
387
+ if isinstance(value, dict):
388
+ return value
389
+ except Exception:
390
+ pass
391
+ fenced = _pinch_re.search(r"```(?:json)?\s*(.*?)\s*```", text, _pinch_re.S)
392
+ if fenced:
393
+ try:
394
+ value = _pinch_json.loads(fenced.group(1))
395
+ if isinstance(value, dict):
396
+ return value
397
+ except Exception:
398
+ pass
399
+ candidates = []
400
+ depth = 0
401
+ current = []
402
+ for char in text:
403
+ if char == "{":
404
+ if depth == 0:
405
+ current = []
406
+ depth += 1
407
+ if depth > 0:
408
+ current.append(char)
409
+ if char == "}":
410
+ depth -= 1
411
+ if depth == 0 and current:
412
+ candidates.append("".join(current))
413
+ for candidate in reversed(candidates):
414
+ try:
415
+ value = _pinch_json.loads(candidate)
416
+ if isinstance(value, dict) and "scores" in value:
417
+ return value
418
+ except Exception:
419
+ continue
420
+ for candidate in reversed(candidates):
421
+ try:
422
+ value = _pinch_json.loads(candidate)
423
+ if isinstance(value, dict):
424
+ return value
425
+ except Exception:
426
+ continue
427
+ score_match = _pinch_re.search(
428
+ r"(?:total|overall|final)\s*(?:score)?[:\s]*(0\.\d+|1\.0+)",
429
+ text,
430
+ _pinch_re.I,
431
+ )
432
+ if score_match:
433
+ total = float(score_match.group(1))
434
+ if 0.0 <= total <= 1.0:
435
+ return {"scores": {}, "total": total, "notes": "Score extracted from prose"}
436
+ return {}
437
+
438
+
439
+ def _pinch_score(value):
440
+ if isinstance(value, bool):
441
+ return None
442
+ if isinstance(value, (int, float)):
443
+ return float(value)
444
+ if isinstance(value, str):
445
+ try:
446
+ return float(value.strip())
447
+ except ValueError:
448
+ return None
449
+ if isinstance(value, dict):
450
+ for key in ("score", "value", "weighted_score"):
451
+ if key in value:
452
+ return _pinch_score(value[key])
453
+ return None
454
+
455
+
456
+ def _pinch_named_scores(parsed):
457
+ scores = {}
458
+ for key in ("scores", "criteria_scores"):
459
+ values = parsed.get(key)
460
+ if isinstance(values, dict):
461
+ for name, value in values.items():
462
+ coerced = _pinch_score(value)
463
+ if coerced is not None:
464
+ scores[str(name)] = coerced
465
+ criteria = parsed.get("criterion_scores")
466
+ if isinstance(criteria, dict):
467
+ for name, value in criteria.items():
468
+ coerced = _pinch_score(value)
469
+ if coerced is not None:
470
+ scores[str(name)] = coerced
471
+ elif isinstance(criteria, list):
472
+ for index, item in enumerate(criteria, start=1):
473
+ if isinstance(item, dict):
474
+ name = (
475
+ item.get("name") or item.get("criterion")
476
+ or item.get("label") or f"criterion_{index}"
477
+ )
478
+ coerced = _pinch_score(item)
479
+ else:
480
+ name = f"criterion_{index}"
481
+ coerced = _pinch_score(item)
482
+ if coerced is not None:
483
+ scores[str(name)] = coerced
484
+ for key, value in parsed.items():
485
+ if _pinch_re.fullmatch(r"criterion\d+", str(key), _pinch_re.I):
486
+ coerced = _pinch_score(value)
487
+ if coerced is not None:
488
+ scores[str(key)] = coerced
489
+ return scores
490
+
491
+
492
+ def _pinch_normalize(parsed):
493
+ scores = _pinch_named_scores(parsed)
494
+ total = None
495
+ for key in ("total", "score", "overall_score", "completionScore", "total_score"):
496
+ if key in parsed:
497
+ total = _pinch_score(parsed[key])
498
+ if total is not None:
499
+ break
500
+ if total is None and isinstance(parsed.get("overall"), dict):
501
+ total = _pinch_score(parsed["overall"])
502
+ if total is None and scores:
503
+ total = _pinch_average(scores)
504
+ values = [value for value in scores.values() if isinstance(value, (int, float))]
505
+ if (
506
+ values and total is not None and total > 1.0
507
+ and all(0.0 <= float(value) <= 1.0 for value in values)
508
+ ):
509
+ total = sum(values) / len(values)
510
+ notes = ""
511
+ for key in ("notes", "justification", "reasoning"):
512
+ if key in parsed:
513
+ notes = str(parsed[key])
514
+ break
515
+ return {"scores": scores, "total": total, "notes": notes}
516
+
517
+
518
+ def _pinch_env_int(name, default, minimum):
519
+ raw = _pinch_os.environ.get(name, "").strip()
520
+ if not raw:
521
+ return default
522
+ try:
523
+ value = int(raw)
524
+ except ValueError as exc:
525
+ raise RuntimeError(f"{name} must be an integer") from exc
526
+ if value < minimum:
527
+ raise RuntimeError(f"{name} must be at least {minimum}")
528
+ return value
529
+
530
+
531
+ def _pinch_env_bool(name, default):
532
+ raw = _pinch_os.environ.get(name, "").strip().lower()
533
+ if not raw:
534
+ return default
535
+ if raw in {"1", "true", "yes", "on"}:
536
+ return True
537
+ if raw in {"0", "false", "no", "off"}:
538
+ return False
539
+ raise RuntimeError(f"{name} must be a boolean")
540
+
541
+
542
+ def _pinch_env_first(*names, default=""):
543
+ for name in names:
544
+ value = _pinch_os.environ.get(name, "").strip()
545
+ if value:
546
+ return value
547
+ return default
548
+
549
+
550
+ def _pinch_read_response(response, stream):
551
+ if not stream:
552
+ return _pinch_json.loads(response.read().decode("utf-8"))
553
+ raw_chunks = []
554
+ content_chunks = []
555
+ reasoning_chunks = []
556
+ finish_reason = None
557
+ saw_sse = False
558
+ for raw_line in response:
559
+ raw_chunks.append(raw_line)
560
+ line = raw_line.decode("utf-8", errors="replace").strip()
561
+ if not line.startswith("data:"):
562
+ continue
563
+ saw_sse = True
564
+ data = line[5:].strip()
565
+ if not data or data == "[DONE]":
566
+ continue
567
+ event = _pinch_json.loads(data)
568
+ if event.get("error"):
569
+ raise RuntimeError(f"PinchBench judge stream failed: {event['error']}")
570
+ choices = event.get("choices") or []
571
+ if not choices:
572
+ continue
573
+ choice = choices[0]
574
+ delta = choice.get("delta") or choice.get("message") or {}
575
+ content = delta.get("content")
576
+ if isinstance(content, str):
577
+ content_chunks.append(content)
578
+ elif isinstance(content, list):
579
+ for item in content:
580
+ if isinstance(item, dict) and isinstance(item.get("text"), str):
581
+ content_chunks.append(item["text"])
582
+ reasoning = delta.get("reasoning_content")
583
+ if isinstance(reasoning, str):
584
+ reasoning_chunks.append(reasoning)
585
+ if choice.get("finish_reason") is not None:
586
+ finish_reason = choice["finish_reason"]
587
+ if not saw_sse:
588
+ return _pinch_json.loads(b"".join(raw_chunks).decode("utf-8"))
589
+ return {
590
+ "choices": [{
591
+ "message": {
592
+ "content": "".join(content_chunks),
593
+ "reasoning_content": "".join(reasoning_chunks),
594
+ },
595
+ "finish_reason": finish_reason,
596
+ }]
597
+ }
598
+
599
+
600
+ def _pinch_judge(transcript, workspace_path):
601
+ endpoint = _pinch_env_first(
602
+ "PINCHBENCH_JUDGE_BASE_URL",
603
+ "OPENROUTER_BASE_URL",
604
+ default="https://openrouter.ai/api/v1",
605
+ ).rstrip("/")
606
+ api_key = _pinch_env_first("PINCHBENCH_JUDGE_API_KEY", "OPENROUTER_API_KEY")
607
+ model = _pinch_env_first(
608
+ "PINCHBENCH_JUDGE_MODEL", default=_PINCH_OFFICIAL_JUDGE_MODEL
609
+ )
610
+ if not api_key:
611
+ raise RuntimeError(
612
+ "LLM grading requires PINCHBENCH_JUDGE_API_KEY or OPENROUTER_API_KEY"
613
+ )
614
+ if endpoint.endswith("/v1"):
615
+ endpoint += "/chat/completions"
616
+ elif not endpoint.endswith("/chat/completions"):
617
+ endpoint += "/v1/chat/completions"
618
+ rubric = _PINCH_RUBRIC or "\n".join(
619
+ f"- {criterion}" for criterion in _PINCH_CRITERIA
620
+ )
621
+ request_timeout = _pinch_env_int("PINCHBENCH_JUDGE_TIMEOUT_SECONDS", 300, 1)
622
+ max_tokens = _pinch_env_int(
623
+ "PINCHBENCH_JUDGE_MAX_COMPLETION_TOKENS",
624
+ _pinch_env_int("PINCHBENCH_JUDGE_MAX_TOKENS", 2048, 1),
625
+ 1,
626
+ )
627
+ max_tokens_field = _pinch_env_first(
628
+ "PINCHBENCH_JUDGE_MAX_TOKENS_FIELD", default="max_completion_tokens"
629
+ )
630
+ if max_tokens_field not in {"max_completion_tokens", "max_tokens"}:
631
+ raise RuntimeError(
632
+ "PINCHBENCH_JUDGE_MAX_TOKENS_FIELD must be max_completion_tokens or max_tokens"
633
+ )
634
+ # Upstream PinchBench performs one direct judge request. Keep retries
635
+ # opt-in so the default remains score-comparable while overloaded custom
636
+ # gateways can still request a more forgiving transport policy.
637
+ retries = _pinch_env_int("PINCHBENCH_JUDGE_RETRIES", 0, 0)
638
+ stream = _pinch_env_bool("PINCHBENCH_JUDGE_STREAM", False)
639
+ workspace = _pinch_workspace_text(workspace_path)
640
+ workspace_section = ""
641
+ if workspace.strip():
642
+ workspace_section = f"## Workspace Files Created by Agent\n{workspace}\n\n"
643
+ prompt = (
644
+ "You are a grading function. Your ONLY job is to output a single JSON object.\n\n"
645
+ "CRITICAL RULES FOR YOU, THE GRADER (not the agent being graded):\n"
646
+ "- Do NOT use any tools (no Read, Write, exec, or any other tool calls)\n"
647
+ "- Do NOT create files or run commands\n"
648
+ "- Do NOT write any prose, explanation, or commentary outside the JSON\n"
649
+ "- Respond with ONLY a JSON object — nothing else\n\n"
650
+ "IMPORTANT: The agent being graded may have used tools (read, write, exec, apply_patch, "
651
+ "todowrite, etc.) during task execution. This is normal and expected. Do NOT treat the "
652
+ "agent's tool usage as a rule violation — the rules above apply only to you, the grader.\n\n"
653
+ "Be a strict evaluator. Reserve 1.0 for genuinely excellent performance. "
654
+ "An average acceptable completion should score around 0.6-0.7. "
655
+ "Deduct points for unnecessary steps, verbose output, and inefficient tool usage.\n\n"
656
+ f"## Task\n{_PINCH_TASK_PROMPT}\n\n"
657
+ f"## Expected Behavior\n{_PINCH_EXPECTED}\n\n"
658
+ f"## Agent Transcript (summarized)\n{_pinch_transcript_text(transcript)}\n\n"
659
+ f"{workspace_section}"
660
+ f"## Grading Rubric\n{rubric}\n\n"
661
+ "Score each criterion from 0.0 to 1.0.\n"
662
+ 'The "total" field must also be between 0.0 and 1.0, and it must be the '
663
+ "arithmetic mean of the criterion scores, not their sum.\n\n"
664
+ "Respond with ONLY this JSON structure (no markdown, no code fences, no extra text):\n"
665
+ '{"scores": {"criterion_name": 0.0}, "total": 0.0, "notes": "brief justification"}'
666
+ )
667
+ payload_dict = {
668
+ "model": model,
669
+ "messages": [
670
+ {"role": "system", "content": _PINCH_JUDGE_SYSTEM_MSG},
671
+ {"role": "user", "content": prompt},
672
+ ],
673
+ "temperature": 0.0,
674
+ max_tokens_field: max_tokens,
675
+ }
676
+ if stream:
677
+ payload_dict["stream"] = True
678
+ payload = _pinch_json.dumps(payload_dict).encode("utf-8")
679
+ no_proxy = _pinch_env_bool("PINCHBENCH_JUDGE_NO_PROXY", False)
680
+ opener = _pinch_request.build_opener(
681
+ _pinch_request.ProxyHandler({}) if no_proxy else _pinch_request.ProxyHandler()
682
+ )
683
+ parsed = {}
684
+ last_error = None
685
+ attempts_used = 0
686
+ content_source = None
687
+ finish_reason = None
688
+ for attempt in range(retries + 1):
689
+ attempts_used = attempt + 1
690
+ req = _pinch_request.Request(
691
+ endpoint,
692
+ data=payload,
693
+ headers={"Authorization": "Bearer " + api_key, "Content-Type": "application/json"},
694
+ method="POST",
695
+ )
696
+ try:
697
+ with opener.open(req, timeout=request_timeout) as response:
698
+ body = _pinch_read_response(response, stream)
699
+ choices = body.get("choices", [])
700
+ message = choices[0].get("message", {}) if choices else {}
701
+ text = message.get("content", "")
702
+ content_source = "content"
703
+ if not isinstance(text, str) or not text.strip():
704
+ text = message.get("reasoning_content", "")
705
+ content_source = "reasoning_content"
706
+ finish_reason = choices[0].get("finish_reason") if choices else None
707
+ parsed = _pinch_parse_json(text if isinstance(text, str) else "")
708
+ if _pinch_normalize(parsed)["total"] is not None:
709
+ break
710
+ last_error = ValueError("response contained no parseable score")
711
+ except _pinch_error.HTTPError as exc:
712
+ last_error = exc
713
+ except (_pinch_error.URLError, TimeoutError, OSError, ValueError) as exc:
714
+ last_error = exc
715
+ if attempt < retries:
716
+ _pinch_time.sleep(min(2 ** attempt, 8))
717
+ failure_notes = (
718
+ f"LLM judge failed: no parseable score after {retries + 1} "
719
+ f"attempt(s): {last_error}"
720
+ )
721
+ normalized = _pinch_normalize(parsed)
722
+ scores = normalized["scores"]
723
+ total = normalized["total"]
724
+ notes = normalized["notes"]
725
+ if total is None:
726
+ total = 0.0
727
+ notes = failure_notes if not parsed else "LLM judge failed: response parsed but no score extracted"
728
+ provenance = {
729
+ "judge_model": model,
730
+ "judge_protocol": "openai_chat_completions",
731
+ "judge_temperature": 0.0,
732
+ "judge_max_tokens_field": max_tokens_field,
733
+ "judge_max_tokens": max_tokens,
734
+ "judge_stream": stream,
735
+ "judge_attempts": attempts_used,
736
+ "judge_content_source": content_source,
737
+ "judge_finish_reason": finish_reason,
738
+ }
739
+ return (
740
+ scores,
741
+ total,
742
+ notes,
743
+ provenance,
744
+ )
745
+
746
+
747
+ def roller_grade(transcript, workspace_path):
748
+ transcript = _pinch_normalize_transcript(transcript)
749
+ automated_scores = {}
750
+ automated_total = 0.0
751
+ if _PINCH_GRADING_TYPE in ("automated", "hybrid"):
752
+ if not callable(_PINCH_AUTOMATED):
753
+ raise RuntimeError("PinchBench task has no automated grade function")
754
+ private_key = _PinchPath(workspace_path) / ".roller-private" / "image_classification_answer_key.json"
755
+ if private_key.is_file():
756
+ globals()["_PINCHBENCH_PRIVATE_IMAGE_KEY_PATH"] = str(private_key)
757
+ automated_scores = _PINCH_AUTOMATED(transcript, workspace_path)
758
+ if not isinstance(automated_scores, dict):
759
+ raise TypeError("PinchBench automated grader must return a dict")
760
+ automated_total = _pinch_average(automated_scores)
761
+ judge_scores, judge_total, notes, judge_provenance = {}, 0.0, "", {}
762
+ if _PINCH_GRADING_TYPE in ("llm_judge", "hybrid"):
763
+ judge_scores, judge_total, notes, judge_provenance = _pinch_judge(
764
+ transcript, workspace_path
765
+ )
766
+ if _PINCH_GRADING_TYPE == "automated":
767
+ result, total = dict(automated_scores), automated_total
768
+ elif _PINCH_GRADING_TYPE == "llm_judge":
769
+ result, total = {"llm_judge." + key: value for key, value in judge_scores.items()}, judge_total
770
+ else:
771
+ auto_weight = float(_PINCH_WEIGHTS.get("automated", 0.5))
772
+ judge_weight = float(_PINCH_WEIGHTS.get("llm_judge", 0.5))
773
+ denominator = auto_weight + judge_weight
774
+ if denominator <= 0:
775
+ auto_weight = judge_weight = 0.5
776
+ denominator = 1.0
777
+ total = (automated_total * auto_weight + judge_total * judge_weight) / denominator
778
+ result = {"automated." + key: value for key, value in automated_scores.items()}
779
+ result.update({"llm_judge." + key: value for key, value in judge_scores.items()})
780
+ result["overall_score"] = total
781
+ result.update(judge_provenance)
782
+ if notes:
783
+ result["judge_notes"] = notes
784
+ return result
785
+ '''
786
+
787
+
788
+ def _grader_source(task: dict[str, Any]) -> str:
789
+ auto = task["automated_checks"]
790
+ if task["grading_type"] in {"automated", "hybrid"} and not auto:
791
+ raise BenchFormatError(
792
+ f"{task['id']}: {task['grading_type']} task has no ## Automated Checks"
793
+ )
794
+ source = (auto.rstrip() + "\n\n") if auto else ""
795
+ replacements = {
796
+ "__TASK_PROMPT__": repr(task["prompt"] or "\n\n".join(
797
+ item["content"] for item in task["sessions"]
798
+ )),
799
+ "__EXPECTED__": repr(task["expected_behavior"]),
800
+ "__CRITERIA__": repr(task["grading_criteria"]),
801
+ "__RUBRIC__": repr(task["llm_judge_rubric"]),
802
+ "__GRADING_TYPE__": repr(task["grading_type"]),
803
+ "__WEIGHTS__": repr(task["grading_weights"]),
804
+ }
805
+ wrapper = _PINCH_WRAPPER
806
+ for marker, value in replacements.items():
807
+ wrapper = wrapper.replace(marker, value)
808
+ return source + wrapper.strip() + "\n"
809
+
810
+
811
+ class PinchBenchAdapter(BenchAdapter):
812
+ name = "pinchbench"
813
+ description = "PinchBench — real-world OpenClaw tasks with automated and LLM grading"
814
+ variants = ("all", "automated-only", "core")
815
+ default_variant = "all"
816
+
817
+ @staticmethod
818
+ def _tasks_root(root: Path) -> Path:
819
+ root = root.expanduser().resolve()
820
+ tasks = root / "tasks" if (root / "tasks" / "manifest.yaml").is_file() else root
821
+ if not (tasks / "manifest.yaml").is_file():
822
+ raise FileNotFoundError(f"PinchBench manifest not found under: {root}")
823
+ return tasks
824
+
825
+ def load(
826
+ self,
827
+ root: Path,
828
+ *,
829
+ categories: Sequence[str] | None = None,
830
+ task_ids: Sequence[str] | None = None,
831
+ variant: str | None = None,
832
+ timeout_multiplier: float = 1.0,
833
+ ) -> list[dict[str, Any]]:
834
+ tasks_root = self._tasks_root(root)
835
+ timeout_multiplier = validate_timeout_multiplier(timeout_multiplier)
836
+ variant = variant or self.default_variant
837
+ if variant not in self.variants:
838
+ raise BenchFormatError(
839
+ f"{self.name}: variant must be one of {', '.join(self.variants)}"
840
+ )
841
+ manifest = yaml.safe_load((tasks_root / "manifest.yaml").read_text()) or {}
842
+ if not isinstance(manifest, dict) or not isinstance(manifest.get("categories"), dict):
843
+ raise BenchFormatError("pinchbench manifest: categories must be an object")
844
+ ordered: list[tuple[str, str]] = []
845
+ category_by_id: dict[str, str] = {}
846
+ for category, identifiers in manifest["categories"].items():
847
+ if not isinstance(identifiers, list):
848
+ raise BenchFormatError(
849
+ f"pinchbench manifest: category {category!r} must be a list"
850
+ )
851
+ for task_id in identifiers:
852
+ if not isinstance(task_id, str):
853
+ raise BenchFormatError("pinchbench manifest: task ids must be strings")
854
+ category_by_id[task_id] = str(category)
855
+ ordered.append((task_id, str(category)))
856
+ first = manifest.get("run_first", [])
857
+ if isinstance(first, list):
858
+ first_ids = [item for item in first if isinstance(item, str)]
859
+ ordered.sort(key=lambda item: first_ids.index(item[0]) if item[0] in first_ids else len(first_ids))
860
+ requested = set(task_ids or ())
861
+ missing = requested - set(category_by_id)
862
+ if missing:
863
+ raise BenchFormatError(
864
+ f"{self.name}: unknown task id(s): {', '.join(sorted(missing))}"
865
+ )
866
+ core = set(manifest.get("core", []))
867
+ category_filter = set(categories or ())
868
+ rows: list[dict[str, Any]] = []
869
+ for task_id, manifest_category in ordered:
870
+ if requested and task_id not in requested:
871
+ continue
872
+ if category_filter and manifest_category not in category_filter:
873
+ continue
874
+ task_path = tasks_root / f"{task_id}.md"
875
+ if not task_path.is_file():
876
+ raise BenchFormatError(f"pinchbench manifest references missing {task_path}")
877
+ task = parse_task_markdown(task_path.read_text(encoding="utf-8"), task_path.name)
878
+ # The manifest is upstream's declared source of truth. A few task
879
+ # frontmatters lag category moves, so use the manifest rather than
880
+ # making an otherwise runnable checkout fail export.
881
+ if variant == "automated-only" and task["grading_type"] != "automated":
882
+ continue
883
+ if variant == "core" and task_id not in core:
884
+ continue
885
+ rows.append(
886
+ self._row(
887
+ tasks_root.parent,
888
+ task,
889
+ manifest_category,
890
+ variant,
891
+ timeout_multiplier,
892
+ )
893
+ )
894
+ return rows
895
+
896
+ def _row(
897
+ self,
898
+ root: Path,
899
+ task: dict[str, Any],
900
+ category: str,
901
+ variant: str,
902
+ timeout_multiplier: float,
903
+ ) -> dict[str, Any]:
904
+ inputs: list[dict[str, Any]] = []
905
+ for index, item in enumerate(task["workspace_files"]):
906
+ if not isinstance(item, dict):
907
+ raise BenchFormatError(
908
+ f"{task['id']}: workspace_files[{index}] must be an object"
909
+ )
910
+ if "content" in item:
911
+ target = item.get("path") or item.get("dest")
912
+ if not isinstance(target, str) or not isinstance(item["content"], str):
913
+ raise BenchFormatError(
914
+ f"{task['id']}: inline workspace file needs string path/content"
915
+ )
916
+ inputs.append({"target": target, "content": item["content"]})
917
+ else:
918
+ source, target = item.get("source"), item.get("dest")
919
+ if not isinstance(source, str) or not isinstance(target, str):
920
+ raise BenchFormatError(
921
+ f"{task['id']}: workspace file needs string source/dest"
922
+ )
923
+ inputs.append({"source": source, "target": target})
924
+
925
+ stage: list[tuple[str, str]] = []
926
+ private_key = root / "assets" / "image_classification_answer_key.json"
927
+ if task["id"] == "task_image_identification" and private_key.is_file():
928
+ stage.append(
929
+ (str(private_key.resolve()), ".roller-private/image_classification_answer_key.json")
930
+ )
931
+ judge_required = task["grading_type"] in {"llm_judge", "hybrid"}
932
+ grade = GradeSpec(
933
+ code=_grader_source(task),
934
+ entrypoint="roller_grade",
935
+ env_names=JUDGE_ENV if judge_required else (),
936
+ optional_env_names=JUDGE_OPTIONAL_ENV if judge_required else (),
937
+ timeout_seconds=3600 if judge_required else 120,
938
+ stage=tuple(stage),
939
+ )
940
+ binding = BenchBinding(
941
+ self.name,
942
+ grade=grade,
943
+ category=category,
944
+ metadata={
945
+ "name": task["name"],
946
+ "grading_type": task["grading_type"],
947
+ "variant": variant,
948
+ "prerequisites": task["prerequisites"],
949
+ },
950
+ )
951
+ row: dict[str, Any] = {
952
+ "schema_version": "roller.task/v1",
953
+ "id": task["id"],
954
+ "harness": "openclaw",
955
+ "timeout_seconds": max(
956
+ 1, math.ceil(task["timeout_seconds"] * timeout_multiplier)
957
+ ),
958
+ "file_root": str((root / "assets").resolve()),
959
+ "files": {
960
+ "inputs": inputs,
961
+ "input_mode": "copy",
962
+ "output_dir": "outputs",
963
+ },
964
+ "bench": binding.as_dict(),
965
+ }
966
+ setup = _pinch_runtime_setup(task)
967
+ if setup:
968
+ row["runtime"] = {"setup": setup}
969
+ if task["sessions"]:
970
+ row["messages"] = task["sessions"]
971
+ else:
972
+ row["prompt"] = task["prompt"]
973
+ return row
974
+
975
+
976
+ __all__ = ["PinchBenchAdapter", "parse_task_markdown"]