roller-agent 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- roller/__init__.py +31 -0
- roller/__main__.py +6 -0
- roller/actf.py +431 -0
- roller/artifacts.py +124 -0
- roller/assets/deepseek/journal.mjs +44 -0
- roller/assets/deepseek/roller.patch.yml +15 -0
- roller/assets/deepseek/runner.py +149 -0
- roller/assets/hermes/runner.py +322 -0
- roller/assets/openclaw_capture.py +32 -0
- roller/assets/openclaw_drain.mjs +21 -0
- roller/assets/opencode/journal.mjs +42 -0
- roller/assets/opencode/runner.py +203 -0
- roller/assets/pi/runner.mjs +160 -0
- roller/assets/pi/runner.py +93 -0
- roller/benches/__init__.py +73 -0
- roller/benches/base.py +306 -0
- roller/benches/grading.py +553 -0
- roller/benches/pinchbench.py +976 -0
- roller/benches/qwenclaw/__init__.py +49 -0
- roller/benches/qwenclaw/adapter.py +167 -0
- roller/benches/qwenclaw/grader_runtime.py +395 -0
- roller/benches/qwenclaw/report.py +49 -0
- roller/benches/qwenclaw/task_parser.py +181 -0
- roller/benches/qwenclaw/workspace.py +202 -0
- roller/benches/report.py +166 -0
- roller/benches/skillsbench.py +708 -0
- roller/benches/transcripts.py +102 -0
- roller/benches/wildclaw.py +569 -0
- roller/bridge_capture.py +73 -0
- roller/bundles.py +545 -0
- roller/capture.py +385 -0
- roller/capture_protocol.py +301 -0
- roller/capture_runtime.py +48 -0
- roller/capture_states.py +37 -0
- roller/claude_capture.py +185 -0
- roller/claude_code_actf.py +594 -0
- roller/cli.py +1732 -0
- roller/codex_actf.py +584 -0
- roller/codex_capture.py +99 -0
- roller/codex_tools.py +137 -0
- roller/collector.py +818 -0
- roller/compaction_sft.py +672 -0
- roller/conversation.py +432 -0
- roller/deepseek_actf.py +135 -0
- roller/harness_registry.py +61 -0
- roller/harnesses/__init__.py +29 -0
- roller/harnesses/base.py +64 -0
- roller/harnesses/bridge.py +448 -0
- roller/harnesses/claude_code.py +839 -0
- roller/harnesses/codex.py +808 -0
- roller/harnesses/config_overrides.py +100 -0
- roller/harnesses/container.py +206 -0
- roller/harnesses/deepseek.py +104 -0
- roller/harnesses/docker.py +72 -0
- roller/harnesses/gateway_ports.py +155 -0
- roller/harnesses/hermes.py +119 -0
- roller/harnesses/openclaw.py +1605 -0
- roller/harnesses/opencode.py +193 -0
- roller/harnesses/pi.py +111 -0
- roller/hermes_actf.py +147 -0
- roller/image_build.py +206 -0
- roller/integrity.py +332 -0
- roller/model_protocols.py +201 -0
- roller/models.py +169 -0
- roller/openclaw_capture.py +140 -0
- roller/openclaw_delivery.py +211 -0
- roller/openclaw_usage.py +59 -0
- roller/opencode_actf.py +305 -0
- roller/opencode_capture.py +96 -0
- roller/pi_actf.py +172 -0
- roller/response_metadata.py +89 -0
- roller/run_lock.py +297 -0
- roller/sft.py +471 -0
- roller/storage.py +53 -0
- roller/tasks.py +689 -0
- roller/thinking.py +50 -0
- roller/thinking_capabilities.py +152 -0
- roller/trajectory_audit.py +386 -0
- roller/usage.py +168 -0
- roller/version.py +1 -0
- roller/viewer/__init__.py +3 -0
- roller/viewer/app.py +722 -0
- roller/viewer/static/app.js +752 -0
- roller/viewer/static/index.html +105 -0
- roller/viewer/static/style.css +213 -0
- roller/wire_trace.py +650 -0
- roller_agent-0.2.0.dist-info/METADATA +397 -0
- roller_agent-0.2.0.dist-info/RECORD +92 -0
- roller_agent-0.2.0.dist-info/WHEEL +5 -0
- roller_agent-0.2.0.dist-info/entry_points.txt +2 -0
- roller_agent-0.2.0.dist-info/licenses/LICENSE +201 -0
- roller_agent-0.2.0.dist-info/top_level.txt +1 -0
roller/tasks.py
ADDED
|
@@ -0,0 +1,689 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import re
|
|
6
|
+
from pathlib import Path, PurePosixPath
|
|
7
|
+
from typing import Any, Iterable
|
|
8
|
+
|
|
9
|
+
from roller.models import (
|
|
10
|
+
ContainerImageBuild,
|
|
11
|
+
ContextFile,
|
|
12
|
+
InputFile,
|
|
13
|
+
RunRequest,
|
|
14
|
+
Task,
|
|
15
|
+
UserSimulatorSpec,
|
|
16
|
+
)
|
|
17
|
+
from roller.harness_registry import HARNESS_NAMES
|
|
18
|
+
from roller.thinking import normalize_thinking
|
|
19
|
+
|
|
20
|
+
TASK_SCHEMA_V1 = "roller.task/v1"
|
|
21
|
+
CAPTURE_POLICIES = {"full", "hash", "none"}
|
|
22
|
+
DEFAULT_CONTEXT_TARGETS = {
|
|
23
|
+
"user": "USER.md",
|
|
24
|
+
"memory": "MEMORY.md",
|
|
25
|
+
"system": "SYSTEM.md",
|
|
26
|
+
"soul": "SOUL.md",
|
|
27
|
+
"agents": "AGENTS.md",
|
|
28
|
+
}
|
|
29
|
+
DEFAULT_CONTEXT_CAPTURE = {"user": "hash", "memory": "hash"}
|
|
30
|
+
GLOB_MARKERS = re.compile(r"[*?[]")
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class TaskFormatError(ValueError):
|
|
34
|
+
pass
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _parse_harness(value: Any, index: int, field_name: str) -> str:
|
|
38
|
+
if value is None:
|
|
39
|
+
return "openclaw"
|
|
40
|
+
if not isinstance(value, str) or value.strip() not in HARNESS_NAMES:
|
|
41
|
+
raise TaskFormatError(
|
|
42
|
+
f"task {index}: {field_name} must be one of {', '.join(HARNESS_NAMES)}"
|
|
43
|
+
)
|
|
44
|
+
return value.strip()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _string_list(value: Any, field_name: str, index: int) -> tuple[str, ...]:
|
|
48
|
+
if value is None or value == "":
|
|
49
|
+
return ()
|
|
50
|
+
if isinstance(value, str):
|
|
51
|
+
return tuple(line.strip() for line in value.splitlines() if line.strip())
|
|
52
|
+
if isinstance(value, list) and all(isinstance(item, str) for item in value):
|
|
53
|
+
return tuple(item.strip() for item in value if item.strip())
|
|
54
|
+
raise TaskFormatError(f"task {index}: {field_name!r} must be a string or string list")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def _parse_messages(value: Any, index: int) -> tuple[tuple[str, ...], tuple[bool, ...]]:
|
|
58
|
+
if not isinstance(value, list) or not value:
|
|
59
|
+
raise TaskFormatError(f"task {index}: 'messages' must be a non-empty list")
|
|
60
|
+
prompts: list[str] = []
|
|
61
|
+
new_session: list[bool] = []
|
|
62
|
+
for message_index, item in enumerate(value):
|
|
63
|
+
if isinstance(item, str):
|
|
64
|
+
content = item
|
|
65
|
+
reset = False
|
|
66
|
+
elif isinstance(item, dict):
|
|
67
|
+
role = item.get("role", "user")
|
|
68
|
+
content = item.get("content")
|
|
69
|
+
reset = item.get("new_session", False)
|
|
70
|
+
if role != "user":
|
|
71
|
+
raise TaskFormatError(
|
|
72
|
+
f"task {index}, message {message_index}: only user messages are accepted"
|
|
73
|
+
)
|
|
74
|
+
else:
|
|
75
|
+
raise TaskFormatError(f"task {index}, message {message_index}: invalid message")
|
|
76
|
+
if not isinstance(reset, bool):
|
|
77
|
+
raise TaskFormatError(
|
|
78
|
+
f"task {index}, message {message_index}: new_session must be boolean"
|
|
79
|
+
)
|
|
80
|
+
if message_index == 0 and reset:
|
|
81
|
+
raise TaskFormatError(
|
|
82
|
+
f"task {index}, message {message_index}: the first message already starts a new session"
|
|
83
|
+
)
|
|
84
|
+
if not isinstance(content, str) or not content.strip():
|
|
85
|
+
raise TaskFormatError(f"task {index}, message {message_index}: empty content")
|
|
86
|
+
prompts.append(content.strip())
|
|
87
|
+
new_session.append(reset)
|
|
88
|
+
return tuple(prompts), tuple(new_session)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _parse_user_simulator(value: Any, index: int) -> UserSimulatorSpec | None:
|
|
92
|
+
if value is None:
|
|
93
|
+
return None
|
|
94
|
+
if not isinstance(value, dict):
|
|
95
|
+
raise TaskFormatError(f"task {index}: user_simulator must be an object")
|
|
96
|
+
instruction = value.get("instruction")
|
|
97
|
+
if not isinstance(instruction, str) or not instruction.strip():
|
|
98
|
+
raise TaskFormatError(
|
|
99
|
+
f"task {index}: user_simulator.instruction must be a non-empty string"
|
|
100
|
+
)
|
|
101
|
+
harness = _parse_harness(value.get("harness"), index, "user_simulator.harness")
|
|
102
|
+
model = value.get("model")
|
|
103
|
+
if model is not None and (not isinstance(model, str) or not model.strip()):
|
|
104
|
+
raise TaskFormatError(
|
|
105
|
+
f"task {index}: user_simulator.model must be a non-empty string"
|
|
106
|
+
)
|
|
107
|
+
max_turns = value.get("max_turns", 6)
|
|
108
|
+
if (
|
|
109
|
+
isinstance(max_turns, bool)
|
|
110
|
+
or not isinstance(max_turns, int)
|
|
111
|
+
or not 1 <= max_turns <= 100
|
|
112
|
+
):
|
|
113
|
+
raise TaskFormatError(
|
|
114
|
+
f"task {index}: user_simulator.max_turns must be an integer from 1 to 100"
|
|
115
|
+
)
|
|
116
|
+
try:
|
|
117
|
+
thinking = normalize_thinking(
|
|
118
|
+
value.get("thinking"), field=f"task {index}: user_simulator.thinking"
|
|
119
|
+
)
|
|
120
|
+
except ValueError as exc:
|
|
121
|
+
raise TaskFormatError(str(exc)) from exc
|
|
122
|
+
return UserSimulatorSpec(
|
|
123
|
+
instruction=instruction.strip(),
|
|
124
|
+
harness=harness,
|
|
125
|
+
model=model.strip() if isinstance(model, str) else None,
|
|
126
|
+
max_turns=max_turns,
|
|
127
|
+
thinking=thinking,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def _read_rows(path: Path) -> list[dict[str, Any]]:
|
|
132
|
+
suffix = path.suffix.lower()
|
|
133
|
+
if suffix == ".jsonl":
|
|
134
|
+
rows: list[dict[str, Any]] = []
|
|
135
|
+
with path.open("r", encoding="utf-8-sig", errors="strict") as stream:
|
|
136
|
+
for line_number, raw_line in enumerate(stream, start=1):
|
|
137
|
+
line = raw_line.strip()
|
|
138
|
+
if not line:
|
|
139
|
+
continue
|
|
140
|
+
try:
|
|
141
|
+
row = json.loads(line)
|
|
142
|
+
except json.JSONDecodeError as exc:
|
|
143
|
+
raise TaskFormatError(f"{path}:{line_number}: invalid JSON: {exc.msg}") from exc
|
|
144
|
+
if not isinstance(row, dict):
|
|
145
|
+
raise TaskFormatError(f"{path}:{line_number}: each row must be an object")
|
|
146
|
+
rows.append(row)
|
|
147
|
+
return rows
|
|
148
|
+
|
|
149
|
+
if suffix == ".json":
|
|
150
|
+
try:
|
|
151
|
+
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
152
|
+
except json.JSONDecodeError as exc:
|
|
153
|
+
raise TaskFormatError(f"{path}: invalid JSON: {exc.msg}") from exc
|
|
154
|
+
if isinstance(payload, dict) and isinstance(payload.get("tasks"), list):
|
|
155
|
+
payload = payload["tasks"]
|
|
156
|
+
if not isinstance(payload, list) or not all(isinstance(row, dict) for row in payload):
|
|
157
|
+
raise TaskFormatError(f"{path}: expected a list of task objects or {{'tasks': [...]}}")
|
|
158
|
+
return payload
|
|
159
|
+
|
|
160
|
+
raise TaskFormatError(f"unsupported task-list extension {suffix!r}; use .jsonl or .json")
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _resolve_source(value: Any, base_dir: Path, label: str) -> Path:
|
|
164
|
+
if not isinstance(value, str) or not value.strip():
|
|
165
|
+
raise TaskFormatError(f"{label}: source must be a non-empty path string")
|
|
166
|
+
source = Path(value).expanduser()
|
|
167
|
+
if not source.is_absolute():
|
|
168
|
+
source = base_dir / source
|
|
169
|
+
return Path(os.path.abspath(source))
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _resolve_file_root(row: dict[str, Any], task_list: Path, index: int) -> Path:
|
|
173
|
+
value = row.get("file_root")
|
|
174
|
+
if value is None:
|
|
175
|
+
return task_list.parent
|
|
176
|
+
if not isinstance(value, str) or not value.strip():
|
|
177
|
+
raise TaskFormatError(f"task {index}: file_root must be a non-empty path string")
|
|
178
|
+
root = Path(value).expanduser()
|
|
179
|
+
if not root.is_absolute():
|
|
180
|
+
root = task_list.parent / root
|
|
181
|
+
root = Path(os.path.abspath(root))
|
|
182
|
+
if not root.is_dir():
|
|
183
|
+
raise TaskFormatError(f"task {index}: file_root is not a directory: {root}")
|
|
184
|
+
return root
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def normalize_relative_path(
|
|
188
|
+
value: Any,
|
|
189
|
+
label: str,
|
|
190
|
+
*,
|
|
191
|
+
allow_dot: bool = False,
|
|
192
|
+
allow_glob: bool = False,
|
|
193
|
+
) -> str:
|
|
194
|
+
if not isinstance(value, str) or not value.strip():
|
|
195
|
+
raise TaskFormatError(f"{label}: path must be a non-empty string")
|
|
196
|
+
raw = value.strip().replace("\\", "/")
|
|
197
|
+
if raw.startswith("/"):
|
|
198
|
+
raise TaskFormatError(f"{label}: absolute paths are not allowed: {raw!r}")
|
|
199
|
+
path = PurePosixPath(raw)
|
|
200
|
+
if any(part == ".." for part in path.parts):
|
|
201
|
+
raise TaskFormatError(f"{label}: parent traversal is not allowed: {raw!r}")
|
|
202
|
+
if not allow_glob and GLOB_MARKERS.search(raw):
|
|
203
|
+
raise TaskFormatError(f"{label}: glob characters are not allowed: {raw!r}")
|
|
204
|
+
normalized = path.as_posix().rstrip("/") or "."
|
|
205
|
+
if normalized == "." and not allow_dot:
|
|
206
|
+
raise TaskFormatError(f"{label}: '.' is not allowed here")
|
|
207
|
+
return normalized
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _parse_inputs(value: Any, base_dir: Path, index: int) -> tuple[InputFile, ...]:
|
|
211
|
+
if value is None:
|
|
212
|
+
return ()
|
|
213
|
+
if not isinstance(value, list):
|
|
214
|
+
raise TaskFormatError(f"task {index}: files.inputs must be a list")
|
|
215
|
+
inputs: list[InputFile] = []
|
|
216
|
+
for file_index, item in enumerate(value):
|
|
217
|
+
label = f"task {index}, input {file_index}"
|
|
218
|
+
if isinstance(item, str):
|
|
219
|
+
source_value = item
|
|
220
|
+
target_value = item
|
|
221
|
+
content = None
|
|
222
|
+
required = True
|
|
223
|
+
expected_sha256 = None
|
|
224
|
+
elif isinstance(item, dict):
|
|
225
|
+
source_value = item.get("source")
|
|
226
|
+
content = item.get("content")
|
|
227
|
+
if source_value is not None and content is not None:
|
|
228
|
+
raise TaskFormatError(
|
|
229
|
+
f"{label}: source and content are mutually exclusive"
|
|
230
|
+
)
|
|
231
|
+
if source_value is None and not isinstance(content, str):
|
|
232
|
+
raise TaskFormatError(
|
|
233
|
+
f"{label}: requires either source or string content"
|
|
234
|
+
)
|
|
235
|
+
target_value = item.get("target", item.get("path", source_value))
|
|
236
|
+
required = item.get("required", True)
|
|
237
|
+
expected_sha256 = item.get("sha256")
|
|
238
|
+
else:
|
|
239
|
+
raise TaskFormatError(f"{label}: expected a source string or object")
|
|
240
|
+
logical_path = normalize_relative_path(target_value, f"{label}.target")
|
|
241
|
+
source = None
|
|
242
|
+
if source_value is not None:
|
|
243
|
+
source_path = normalize_relative_path(source_value, f"{label}.source")
|
|
244
|
+
source = Path(os.path.abspath(base_dir / Path(source_path)))
|
|
245
|
+
if not isinstance(required, bool):
|
|
246
|
+
raise TaskFormatError(f"{label}: required must be boolean")
|
|
247
|
+
if expected_sha256 is not None:
|
|
248
|
+
if not isinstance(expected_sha256, str) or not re.fullmatch(
|
|
249
|
+
r"[0-9a-fA-F]{64}", expected_sha256
|
|
250
|
+
):
|
|
251
|
+
raise TaskFormatError(f"{label}: sha256 must contain exactly 64 hex characters")
|
|
252
|
+
expected_sha256 = expected_sha256.lower()
|
|
253
|
+
inputs.append(
|
|
254
|
+
InputFile(
|
|
255
|
+
source=source,
|
|
256
|
+
path=logical_path,
|
|
257
|
+
required=required,
|
|
258
|
+
expected_sha256=expected_sha256,
|
|
259
|
+
content=content,
|
|
260
|
+
)
|
|
261
|
+
)
|
|
262
|
+
return tuple(inputs)
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _parse_image_build(
|
|
266
|
+
value: Any, image: str | None, base_dir: Path, index: int
|
|
267
|
+
) -> ContainerImageBuild | None:
|
|
268
|
+
if value is None:
|
|
269
|
+
return None
|
|
270
|
+
label = f"task {index}: runtime.image_build"
|
|
271
|
+
if not isinstance(value, dict):
|
|
272
|
+
raise TaskFormatError(f"{label} must be an object")
|
|
273
|
+
if not image:
|
|
274
|
+
raise TaskFormatError(f"{label} requires runtime.image")
|
|
275
|
+
base_image = value.get("base_image")
|
|
276
|
+
if not isinstance(base_image, str) or not base_image.strip():
|
|
277
|
+
raise TaskFormatError(f"{label}.base_image must be a non-empty string")
|
|
278
|
+
context_value = value.get("context")
|
|
279
|
+
if not isinstance(context_value, str) or not context_value.strip():
|
|
280
|
+
raise TaskFormatError(f"{label}.context must be a non-empty path")
|
|
281
|
+
context = Path(context_value).expanduser()
|
|
282
|
+
if not context.is_absolute():
|
|
283
|
+
context = base_dir / context
|
|
284
|
+
context = context.resolve()
|
|
285
|
+
if not context.is_dir():
|
|
286
|
+
raise TaskFormatError(f"{label}.context is not a directory: {context}")
|
|
287
|
+
dockerfile = value.get("dockerfile")
|
|
288
|
+
if not isinstance(dockerfile, str) or not dockerfile.strip():
|
|
289
|
+
raise TaskFormatError(f"{label}.dockerfile must be non-empty Dockerfile source")
|
|
290
|
+
harness = value.get("harness")
|
|
291
|
+
if not isinstance(harness, str) or harness not in HARNESS_NAMES:
|
|
292
|
+
raise TaskFormatError(
|
|
293
|
+
f"{label}.harness must be one of {', '.join(HARNESS_NAMES)}"
|
|
294
|
+
)
|
|
295
|
+
build_args = value.get("args", {})
|
|
296
|
+
if not isinstance(build_args, dict) or not all(
|
|
297
|
+
isinstance(key, str) and isinstance(item, (str, int, float, bool))
|
|
298
|
+
for key, item in build_args.items()
|
|
299
|
+
):
|
|
300
|
+
raise TaskFormatError(f"{label}.args must be an object of scalar values")
|
|
301
|
+
timeout = value.get("timeout_seconds", 3600)
|
|
302
|
+
if isinstance(timeout, bool) or not isinstance(timeout, int) or timeout <= 0:
|
|
303
|
+
raise TaskFormatError(f"{label}.timeout_seconds must be a positive integer")
|
|
304
|
+
return ContainerImageBuild(
|
|
305
|
+
image=image,
|
|
306
|
+
base_image=base_image.strip(),
|
|
307
|
+
context=context,
|
|
308
|
+
dockerfile=dockerfile,
|
|
309
|
+
harness=harness,
|
|
310
|
+
build_args=tuple((key, str(item)) for key, item in sorted(build_args.items())),
|
|
311
|
+
timeout_seconds=timeout,
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _default_context_target(role: str, source: Path) -> str:
|
|
316
|
+
if role in DEFAULT_CONTEXT_TARGETS:
|
|
317
|
+
return DEFAULT_CONTEXT_TARGETS[role]
|
|
318
|
+
if role in {"skill", "skills"}:
|
|
319
|
+
return f"skills/{source.name}"
|
|
320
|
+
return source.name
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _parse_context(value: Any, base_dir: Path, index: int) -> tuple[ContextFile, ...]:
|
|
324
|
+
if value is None:
|
|
325
|
+
return ()
|
|
326
|
+
if not isinstance(value, list):
|
|
327
|
+
raise TaskFormatError(f"task {index}: files.context must be a list")
|
|
328
|
+
context_files: list[ContextFile] = []
|
|
329
|
+
for file_index, item in enumerate(value):
|
|
330
|
+
label = f"task {index}, context {file_index}"
|
|
331
|
+
if not isinstance(item, dict):
|
|
332
|
+
raise TaskFormatError(f"{label}: context entries must be objects")
|
|
333
|
+
role = item.get("role")
|
|
334
|
+
if not isinstance(role, str) or not role.strip():
|
|
335
|
+
raise TaskFormatError(f"{label}: role must be a non-empty string")
|
|
336
|
+
role = role.strip().lower()
|
|
337
|
+
source = _resolve_source(item.get("source"), base_dir, label)
|
|
338
|
+
target = normalize_relative_path(
|
|
339
|
+
item.get("target", _default_context_target(role, source)),
|
|
340
|
+
f"{label}.target",
|
|
341
|
+
allow_dot=source.is_dir(),
|
|
342
|
+
)
|
|
343
|
+
if PurePosixPath(target).parts and PurePosixPath(target).parts[0] in {
|
|
344
|
+
"inputs",
|
|
345
|
+
"outputs",
|
|
346
|
+
"context",
|
|
347
|
+
}:
|
|
348
|
+
raise TaskFormatError(
|
|
349
|
+
f"{label}.target: top-level inputs, outputs and context are reserved"
|
|
350
|
+
)
|
|
351
|
+
capture = item.get("capture", DEFAULT_CONTEXT_CAPTURE.get(role, "full"))
|
|
352
|
+
if capture not in CAPTURE_POLICIES:
|
|
353
|
+
raise TaskFormatError(
|
|
354
|
+
f"{label}: capture must be one of {sorted(CAPTURE_POLICIES)}"
|
|
355
|
+
)
|
|
356
|
+
required = item.get("required", True)
|
|
357
|
+
if not isinstance(required, bool):
|
|
358
|
+
raise TaskFormatError(f"{label}: required must be boolean")
|
|
359
|
+
if role in DEFAULT_CONTEXT_TARGETS and source.exists() and source.is_dir():
|
|
360
|
+
raise TaskFormatError(f"{label}: role {role!r} requires a file source")
|
|
361
|
+
context_files.append(ContextFile(role, source, target, capture, required))
|
|
362
|
+
return tuple(context_files)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def _parse_output_dir(value: Any, index: int) -> str:
|
|
366
|
+
output_dir = normalize_relative_path(
|
|
367
|
+
"outputs" if value is None else value,
|
|
368
|
+
f"task {index}, files.output_dir",
|
|
369
|
+
)
|
|
370
|
+
first_part = PurePosixPath(output_dir).parts[0]
|
|
371
|
+
if first_part in {"inputs", "context"}:
|
|
372
|
+
raise TaskFormatError(
|
|
373
|
+
f"task {index}, files.output_dir: top-level inputs and context are reserved"
|
|
374
|
+
)
|
|
375
|
+
return output_dir
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _validate_output_input_paths(
|
|
379
|
+
output_dir: str, inputs: tuple[InputFile, ...], index: int
|
|
380
|
+
) -> None:
|
|
381
|
+
output_prefix = f"{output_dir}/"
|
|
382
|
+
for input_file in inputs:
|
|
383
|
+
input_prefix = f"{input_file.path}/"
|
|
384
|
+
if (
|
|
385
|
+
input_file.path == output_dir
|
|
386
|
+
or input_file.path.startswith(output_prefix)
|
|
387
|
+
or output_dir.startswith(input_prefix)
|
|
388
|
+
):
|
|
389
|
+
raise TaskFormatError(
|
|
390
|
+
f"task {index}: files.output_dir {output_dir!r} overlaps input "
|
|
391
|
+
f"path {input_file.path!r}"
|
|
392
|
+
)
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _validate_output_context_paths(
|
|
396
|
+
output_dir: str, context_files: tuple[ContextFile, ...], index: int
|
|
397
|
+
) -> None:
|
|
398
|
+
output_prefix = f"{output_dir}/"
|
|
399
|
+
for context in context_files:
|
|
400
|
+
context_prefix = f"{context.target}/"
|
|
401
|
+
if (
|
|
402
|
+
context.target == output_dir
|
|
403
|
+
or context.target.startswith(output_prefix)
|
|
404
|
+
or output_dir.startswith(context_prefix)
|
|
405
|
+
):
|
|
406
|
+
raise TaskFormatError(
|
|
407
|
+
f"task {index}: files.output_dir {output_dir!r} overlaps context target "
|
|
408
|
+
f"{context.target!r}"
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _validate_input_context_paths(
|
|
413
|
+
inputs: tuple[InputFile, ...], context_files: tuple[ContextFile, ...], index: int
|
|
414
|
+
) -> None:
|
|
415
|
+
for input_file in inputs:
|
|
416
|
+
input_prefix = f"{input_file.path}/"
|
|
417
|
+
for context in context_files:
|
|
418
|
+
context_prefix = f"{context.target}/"
|
|
419
|
+
if (
|
|
420
|
+
input_file.path == context.target
|
|
421
|
+
or input_file.path.startswith(context_prefix)
|
|
422
|
+
or context.target.startswith(input_prefix)
|
|
423
|
+
):
|
|
424
|
+
raise TaskFormatError(
|
|
425
|
+
f"task {index}: input path {input_file.path!r} overlaps context "
|
|
426
|
+
f"target {context.target!r}"
|
|
427
|
+
)
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def _legacy_persona_context(
|
|
431
|
+
row: dict[str, Any], workspace: Path | None, index: int
|
|
432
|
+
) -> tuple[ContextFile, ...]:
|
|
433
|
+
persona = row.get("persona")
|
|
434
|
+
if not isinstance(persona, str) or not persona.strip() or workspace is None:
|
|
435
|
+
return ()
|
|
436
|
+
persona_root = (workspace / persona.strip()).resolve()
|
|
437
|
+
try:
|
|
438
|
+
persona_root.relative_to(workspace.resolve())
|
|
439
|
+
except ValueError as exc:
|
|
440
|
+
raise TaskFormatError(f"task {index}: persona escapes workspace") from exc
|
|
441
|
+
return (
|
|
442
|
+
ContextFile("user", persona_root / "USER.md", "USER.md", "hash", False),
|
|
443
|
+
ContextFile("memory", persona_root / "MEMORY.md", "MEMORY.md", "hash", False),
|
|
444
|
+
)
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def load_tasks(
|
|
448
|
+
path: Path,
|
|
449
|
+
*,
|
|
450
|
+
prompt_key: str = "prompt",
|
|
451
|
+
id_key: str = "id",
|
|
452
|
+
global_workspace: Path | None = None,
|
|
453
|
+
) -> list[Task]:
|
|
454
|
+
"""Load v1 tasks and normalize legacy Interns2_data rows into the same model."""
|
|
455
|
+
|
|
456
|
+
path = path.expanduser().resolve()
|
|
457
|
+
if not path.is_file():
|
|
458
|
+
raise FileNotFoundError(f"task list not found: {path}")
|
|
459
|
+
rows = _read_rows(path)
|
|
460
|
+
tasks: list[Task] = []
|
|
461
|
+
seen_ids: set[str] = set()
|
|
462
|
+
|
|
463
|
+
for index, row in enumerate(rows):
|
|
464
|
+
schema_version = row.get("schema_version", TASK_SCHEMA_V1)
|
|
465
|
+
if schema_version != TASK_SCHEMA_V1:
|
|
466
|
+
raise TaskFormatError(
|
|
467
|
+
f"task {index}: unsupported schema_version {schema_version!r}; expected {TASK_SCHEMA_V1!r}"
|
|
468
|
+
)
|
|
469
|
+
task_id_value = row.get(id_key, f"task_{index:08d}")
|
|
470
|
+
if not isinstance(task_id_value, (str, int)) or not str(task_id_value).strip():
|
|
471
|
+
raise TaskFormatError(f"task {index}: {id_key!r} must be a non-empty string or integer")
|
|
472
|
+
task_id = str(task_id_value).strip()
|
|
473
|
+
if task_id in seen_ids:
|
|
474
|
+
raise TaskFormatError(f"task {index}: duplicate task id {task_id!r}")
|
|
475
|
+
seen_ids.add(task_id)
|
|
476
|
+
|
|
477
|
+
user_simulator = _parse_user_simulator(row.get("user_simulator"), index)
|
|
478
|
+
if user_simulator is not None:
|
|
479
|
+
predefined_fields = tuple(
|
|
480
|
+
dict.fromkeys(
|
|
481
|
+
(
|
|
482
|
+
"prompt",
|
|
483
|
+
"question",
|
|
484
|
+
"messages",
|
|
485
|
+
*(() if prompt_key == "user_simulator" else (prompt_key,)),
|
|
486
|
+
)
|
|
487
|
+
)
|
|
488
|
+
)
|
|
489
|
+
conflicting = [
|
|
490
|
+
name
|
|
491
|
+
for name in predefined_fields
|
|
492
|
+
if row.get(name) is not None
|
|
493
|
+
]
|
|
494
|
+
if conflicting:
|
|
495
|
+
raise TaskFormatError(
|
|
496
|
+
f"task {index}: user_simulator cannot be combined with "
|
|
497
|
+
f"{', '.join(conflicting)}"
|
|
498
|
+
)
|
|
499
|
+
prompts = ()
|
|
500
|
+
new_session = ()
|
|
501
|
+
elif "messages" in row:
|
|
502
|
+
prompts, new_session = _parse_messages(row["messages"], index)
|
|
503
|
+
else:
|
|
504
|
+
prompt = row.get(prompt_key)
|
|
505
|
+
if prompt is None and prompt_key == "prompt":
|
|
506
|
+
prompt = row.get("question")
|
|
507
|
+
if not isinstance(prompt, str) or not prompt.strip():
|
|
508
|
+
raise TaskFormatError(
|
|
509
|
+
f"task {index}: missing non-empty {prompt_key!r} (or compatibility key 'question')"
|
|
510
|
+
)
|
|
511
|
+
prompts = (prompt.strip(),)
|
|
512
|
+
new_session = (False,)
|
|
513
|
+
|
|
514
|
+
workspace = global_workspace
|
|
515
|
+
row_workspace = row.get("workspace") or row.get("workspace_path")
|
|
516
|
+
if row_workspace is not None:
|
|
517
|
+
if not isinstance(row_workspace, str) or not row_workspace.strip():
|
|
518
|
+
raise TaskFormatError(f"task {index}: workspace must be a non-empty path string")
|
|
519
|
+
workspace = Path(row_workspace).expanduser()
|
|
520
|
+
if not workspace.is_absolute():
|
|
521
|
+
workspace = path.parent / workspace
|
|
522
|
+
workspace = workspace.resolve()
|
|
523
|
+
elif workspace is not None:
|
|
524
|
+
workspace = workspace.expanduser().resolve()
|
|
525
|
+
|
|
526
|
+
file_root = _resolve_file_root(row, path, index)
|
|
527
|
+
files = row.get("files", {})
|
|
528
|
+
if not isinstance(files, dict):
|
|
529
|
+
raise TaskFormatError(f"task {index}: files must be an object")
|
|
530
|
+
if "outputs" in files:
|
|
531
|
+
raise TaskFormatError(
|
|
532
|
+
f"task {index}: files.outputs is no longer supported; all files under "
|
|
533
|
+
"files.output_dir are collected automatically"
|
|
534
|
+
)
|
|
535
|
+
output_dir = _parse_output_dir(files.get("output_dir"), index)
|
|
536
|
+
inputs = _parse_inputs(files.get("inputs"), file_root, index)
|
|
537
|
+
input_mode = files.get("input_mode", "mount")
|
|
538
|
+
if input_mode not in {"mount", "copy"}:
|
|
539
|
+
raise TaskFormatError(
|
|
540
|
+
f"task {index}: files.input_mode must be 'mount' or 'copy'"
|
|
541
|
+
)
|
|
542
|
+
_validate_output_input_paths(output_dir, inputs, index)
|
|
543
|
+
legacy_workspace = workspace is not None and not inputs
|
|
544
|
+
if legacy_workspace:
|
|
545
|
+
inputs = (InputFile(workspace, ".", True, None),)
|
|
546
|
+
context_files = _parse_context(files.get("context"), file_root, index)
|
|
547
|
+
context_files += _legacy_persona_context(row, workspace, index)
|
|
548
|
+
_validate_input_context_paths(inputs, context_files, index)
|
|
549
|
+
_validate_output_context_paths(output_dir, context_files, index)
|
|
550
|
+
|
|
551
|
+
task_agent_value = row.get("task_agent")
|
|
552
|
+
agent_alias_value = row.get("agent")
|
|
553
|
+
if task_agent_value is not None and agent_alias_value is not None:
|
|
554
|
+
if task_agent_value != agent_alias_value:
|
|
555
|
+
raise TaskFormatError(
|
|
556
|
+
f"task {index}: 'task_agent' and 'agent' declarations must match"
|
|
557
|
+
)
|
|
558
|
+
agent_object = (
|
|
559
|
+
task_agent_value if task_agent_value is not None else agent_alias_value
|
|
560
|
+
)
|
|
561
|
+
if agent_object is not None and not isinstance(agent_object, dict):
|
|
562
|
+
raise TaskFormatError(f"task {index}: task_agent must be an object")
|
|
563
|
+
task_harness_value = row.get("harness")
|
|
564
|
+
agent_harness_value = row.get("agent_harness")
|
|
565
|
+
if isinstance(agent_object, dict):
|
|
566
|
+
object_harness = agent_object.get("harness")
|
|
567
|
+
if object_harness is not None:
|
|
568
|
+
for existing in (task_harness_value, agent_harness_value):
|
|
569
|
+
if existing is not None and existing != object_harness:
|
|
570
|
+
raise TaskFormatError(
|
|
571
|
+
f"task {index}: task-agent harness declarations must match"
|
|
572
|
+
)
|
|
573
|
+
task_harness_value = object_harness
|
|
574
|
+
object_model = agent_object.get("model")
|
|
575
|
+
else:
|
|
576
|
+
object_model = None
|
|
577
|
+
if task_harness_value is not None and agent_harness_value is not None:
|
|
578
|
+
if task_harness_value != agent_harness_value:
|
|
579
|
+
raise TaskFormatError(
|
|
580
|
+
f"task {index}: 'harness' and 'agent_harness' must match when both are set"
|
|
581
|
+
)
|
|
582
|
+
task_harness = _parse_harness(
|
|
583
|
+
task_harness_value if task_harness_value is not None else agent_harness_value,
|
|
584
|
+
index,
|
|
585
|
+
"harness",
|
|
586
|
+
)
|
|
587
|
+
|
|
588
|
+
model = row.get("model", object_model)
|
|
589
|
+
if model is None:
|
|
590
|
+
model = object_model
|
|
591
|
+
elif object_model is not None and model != object_model:
|
|
592
|
+
raise TaskFormatError(
|
|
593
|
+
f"task {index}: task-agent model declarations must match"
|
|
594
|
+
)
|
|
595
|
+
if model is not None and (not isinstance(model, str) or not model.strip()):
|
|
596
|
+
raise TaskFormatError(f"task {index}: model must be a non-empty string")
|
|
597
|
+
runtime = row.get("runtime", {})
|
|
598
|
+
if not isinstance(runtime, dict):
|
|
599
|
+
raise TaskFormatError(f"task {index}: runtime must be an object")
|
|
600
|
+
image = runtime.get("image")
|
|
601
|
+
if image is not None and (
|
|
602
|
+
not isinstance(image, str) or not image.strip()
|
|
603
|
+
):
|
|
604
|
+
raise TaskFormatError(f"task {index}: runtime.image must be a non-empty string")
|
|
605
|
+
docker_image = image.strip() if isinstance(image, str) else None
|
|
606
|
+
image_build = _parse_image_build(
|
|
607
|
+
runtime.get("image_build"), docker_image, file_root, index
|
|
608
|
+
)
|
|
609
|
+
workspace_container_path = runtime.get("workspace_path")
|
|
610
|
+
if workspace_container_path is not None and (
|
|
611
|
+
not isinstance(workspace_container_path, str)
|
|
612
|
+
or not workspace_container_path.startswith("/")
|
|
613
|
+
):
|
|
614
|
+
raise TaskFormatError(
|
|
615
|
+
f"task {index}: runtime.workspace_path must be an absolute container path"
|
|
616
|
+
)
|
|
617
|
+
try:
|
|
618
|
+
thinking = normalize_thinking(
|
|
619
|
+
runtime.get("thinking", row.get("thinking")),
|
|
620
|
+
field=f"task {index}: runtime.thinking",
|
|
621
|
+
)
|
|
622
|
+
except ValueError as exc:
|
|
623
|
+
raise TaskFormatError(str(exc)) from exc
|
|
624
|
+
|
|
625
|
+
tasks.append(
|
|
626
|
+
Task(
|
|
627
|
+
id=task_id,
|
|
628
|
+
prompts=prompts,
|
|
629
|
+
source_index=index,
|
|
630
|
+
raw=row,
|
|
631
|
+
schema_version=schema_version,
|
|
632
|
+
inputs=inputs,
|
|
633
|
+
context_files=context_files,
|
|
634
|
+
output_dir=output_dir,
|
|
635
|
+
legacy_workspace=legacy_workspace,
|
|
636
|
+
input_mode=input_mode,
|
|
637
|
+
workspace=workspace,
|
|
638
|
+
harness=task_harness,
|
|
639
|
+
model=model.strip() if isinstance(model, str) else None,
|
|
640
|
+
env_names=_string_list(runtime.get("env", row.get("env")), "runtime.env", index),
|
|
641
|
+
setup_commands=_string_list(
|
|
642
|
+
runtime.get("setup", row.get("setup", row.get("warmup"))),
|
|
643
|
+
"runtime.setup",
|
|
644
|
+
index,
|
|
645
|
+
),
|
|
646
|
+
new_session=new_session,
|
|
647
|
+
docker_image=docker_image,
|
|
648
|
+
image_build=image_build,
|
|
649
|
+
workspace_container_path=workspace_container_path,
|
|
650
|
+
thinking=thinking,
|
|
651
|
+
user_simulator=user_simulator,
|
|
652
|
+
)
|
|
653
|
+
)
|
|
654
|
+
return tasks
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def build_requests(
|
|
658
|
+
tasks: Iterable[Task],
|
|
659
|
+
models: list[str] | None,
|
|
660
|
+
timeout_seconds: int,
|
|
661
|
+
samples_per_task: int = 1,
|
|
662
|
+
) -> list[RunRequest]:
|
|
663
|
+
if timeout_seconds <= 0:
|
|
664
|
+
raise ValueError("timeout_seconds must be greater than zero")
|
|
665
|
+
if samples_per_task <= 0:
|
|
666
|
+
raise ValueError("samples_per_task must be greater than zero")
|
|
667
|
+
normalized_models = list(
|
|
668
|
+
dict.fromkeys(model.strip() for model in (models or []) if model.strip())
|
|
669
|
+
)
|
|
670
|
+
requests: list[RunRequest] = []
|
|
671
|
+
for task in tasks:
|
|
672
|
+
task_models = normalized_models or ([task.model] if task.model else [])
|
|
673
|
+
if not task_models:
|
|
674
|
+
raise TaskFormatError(
|
|
675
|
+
f"task {task.id!r} has no model; pass --model or set the task's 'model' field"
|
|
676
|
+
)
|
|
677
|
+
for model in task_models:
|
|
678
|
+
for sample_index in range(samples_per_task):
|
|
679
|
+
requests.append(
|
|
680
|
+
RunRequest(
|
|
681
|
+
len(requests),
|
|
682
|
+
task,
|
|
683
|
+
model,
|
|
684
|
+
timeout_seconds,
|
|
685
|
+
sample_index=sample_index,
|
|
686
|
+
sample_count=samples_per_task,
|
|
687
|
+
)
|
|
688
|
+
)
|
|
689
|
+
return requests
|