rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""Static features of recorded assistant calls, without executing arguments.
|
|
2
|
+
|
|
3
|
+
Command classes describe lexical forms, not successful operations. Feature names
|
|
4
|
+
contain tool identifiers and fixed categories, never argument values or paths.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
import shlex
|
|
12
|
+
from collections import Counter
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from itertools import pairwise
|
|
15
|
+
from pathlib import PurePosixPath, PureWindowsPath
|
|
16
|
+
from urllib.parse import urlsplit
|
|
17
|
+
|
|
18
|
+
_NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]{0,95}\Z")
|
|
19
|
+
_ARGUMENT_KEYS = frozenset(
|
|
20
|
+
{
|
|
21
|
+
"command",
|
|
22
|
+
"cmd",
|
|
23
|
+
"shell_command",
|
|
24
|
+
"path",
|
|
25
|
+
"file_path",
|
|
26
|
+
"filename",
|
|
27
|
+
"file_text",
|
|
28
|
+
"new_str",
|
|
29
|
+
"old_str",
|
|
30
|
+
"insert_line",
|
|
31
|
+
"view_range",
|
|
32
|
+
"timeout",
|
|
33
|
+
"is_input",
|
|
34
|
+
"message",
|
|
35
|
+
"thought",
|
|
36
|
+
"task_list",
|
|
37
|
+
"code",
|
|
38
|
+
"language",
|
|
39
|
+
}
|
|
40
|
+
)
|
|
41
|
+
_PATH_KEYS = {"path", "file_path", "filename"}
|
|
42
|
+
_COMMAND_KEYS = {"command", "cmd", "shell_command"}
|
|
43
|
+
_TYPES = {"null", "boolean", "number", "string", "array", "object"}
|
|
44
|
+
_PATH_KINDS = {"relative", "absolute", "home", "windows_absolute", "uri"}
|
|
45
|
+
_EXTENSIONS = frozenset(
|
|
46
|
+
[
|
|
47
|
+
"py",
|
|
48
|
+
"js",
|
|
49
|
+
"ts",
|
|
50
|
+
"jsx",
|
|
51
|
+
"tsx",
|
|
52
|
+
"json",
|
|
53
|
+
"toml",
|
|
54
|
+
"yaml",
|
|
55
|
+
"yml",
|
|
56
|
+
"ini",
|
|
57
|
+
"cfg",
|
|
58
|
+
"md",
|
|
59
|
+
"rst",
|
|
60
|
+
"txt",
|
|
61
|
+
"c",
|
|
62
|
+
"cpp",
|
|
63
|
+
"h",
|
|
64
|
+
"hpp",
|
|
65
|
+
"go",
|
|
66
|
+
"rs",
|
|
67
|
+
"sh",
|
|
68
|
+
"ipynb",
|
|
69
|
+
"csv",
|
|
70
|
+
"tsv",
|
|
71
|
+
"xml",
|
|
72
|
+
"html",
|
|
73
|
+
"css",
|
|
74
|
+
"scss",
|
|
75
|
+
"sql",
|
|
76
|
+
"java",
|
|
77
|
+
"kt",
|
|
78
|
+
"rb",
|
|
79
|
+
"php",
|
|
80
|
+
"none",
|
|
81
|
+
"other",
|
|
82
|
+
]
|
|
83
|
+
)
|
|
84
|
+
_COMMAND_CLASSES = {
|
|
85
|
+
"test",
|
|
86
|
+
"search",
|
|
87
|
+
"read",
|
|
88
|
+
"write",
|
|
89
|
+
"version_control",
|
|
90
|
+
"install",
|
|
91
|
+
"python",
|
|
92
|
+
"shell",
|
|
93
|
+
"other",
|
|
94
|
+
"compound",
|
|
95
|
+
"nontext",
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
@dataclass
|
|
100
|
+
class ToolCallFeatures:
|
|
101
|
+
dense: dict[str, float] = field(default_factory=dict)
|
|
102
|
+
sparse: set[str] = field(default_factory=set)
|
|
103
|
+
|
|
104
|
+
@property
|
|
105
|
+
def catalog(self) -> dict[str, dict[str, str]]:
|
|
106
|
+
return {
|
|
107
|
+
**{
|
|
108
|
+
name: {"kind": "continuous", "tier": "structural"}
|
|
109
|
+
for name in self.dense
|
|
110
|
+
},
|
|
111
|
+
**{name: {"kind": "binary", "tier": "structural"} for name in self.sparse},
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def valid_tool_feature(name: str, kind: str) -> bool:
|
|
116
|
+
"""Validate the complete exported tool feature grammar."""
|
|
117
|
+
if not isinstance(name, str):
|
|
118
|
+
return False
|
|
119
|
+
if kind == "continuous":
|
|
120
|
+
return name == "tool_calls_count" or (
|
|
121
|
+
name.startswith("tool_count:") and bool(_NAME.fullmatch(name[11:]))
|
|
122
|
+
)
|
|
123
|
+
if kind != "binary":
|
|
124
|
+
return False
|
|
125
|
+
parts = name.split(":")
|
|
126
|
+
if len(parts) < 2:
|
|
127
|
+
return False
|
|
128
|
+
if parts[0] == "tool_bigram":
|
|
129
|
+
names = parts[1].split(">")
|
|
130
|
+
return (
|
|
131
|
+
len(parts) == 2
|
|
132
|
+
and len(names) == 2
|
|
133
|
+
and all(_NAME.fullmatch(x) for x in names)
|
|
134
|
+
)
|
|
135
|
+
if not _NAME.fullmatch(parts[1]):
|
|
136
|
+
return False
|
|
137
|
+
if parts[0] == "tool":
|
|
138
|
+
return len(parts) == 2
|
|
139
|
+
if parts[0] == "tool_arg":
|
|
140
|
+
return (
|
|
141
|
+
len(parts) == 4
|
|
142
|
+
and parts[2] in _ARGUMENT_KEYS | {"other"}
|
|
143
|
+
and parts[3] in _TYPES
|
|
144
|
+
)
|
|
145
|
+
if parts[0] == "tool_path":
|
|
146
|
+
return len(parts) == 4 and parts[2] in _PATH_KINDS and parts[3] in _EXTENSIONS
|
|
147
|
+
if parts[0] == "tool_command":
|
|
148
|
+
return len(parts) == 3 and parts[2] in _COMMAND_CLASSES
|
|
149
|
+
return False
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _json_object_pairs(pairs):
|
|
153
|
+
result = {}
|
|
154
|
+
for key, value in pairs:
|
|
155
|
+
if key in result:
|
|
156
|
+
raise ValueError("duplicate tool argument key")
|
|
157
|
+
result[key] = value
|
|
158
|
+
return result
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def _finite_json(value):
|
|
162
|
+
if isinstance(value, dict):
|
|
163
|
+
if not all(isinstance(key, str) for key in value):
|
|
164
|
+
raise ValueError("tool arguments require string keys")
|
|
165
|
+
for item in value.values():
|
|
166
|
+
_finite_json(item)
|
|
167
|
+
elif isinstance(value, list):
|
|
168
|
+
for item in value:
|
|
169
|
+
_finite_json(item)
|
|
170
|
+
elif value is not None and type(value) not in {str, int, float, bool}:
|
|
171
|
+
raise ValueError("tool arguments must be finite JSON values")
|
|
172
|
+
json.dumps(value, allow_nan=False)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _parse_call(call):
|
|
176
|
+
if not isinstance(call, dict) or call.get("parse_error") is not None:
|
|
177
|
+
raise ValueError("malformed tool call")
|
|
178
|
+
if call.get("type", "function") != "function":
|
|
179
|
+
raise ValueError("only function tool calls are supported")
|
|
180
|
+
function = call.get("function")
|
|
181
|
+
if isinstance(function, dict):
|
|
182
|
+
name, arguments = function.get("name"), function.get("arguments")
|
|
183
|
+
else:
|
|
184
|
+
name, arguments = function, call.get("arguments")
|
|
185
|
+
if not isinstance(name, str) or not _NAME.fullmatch(name):
|
|
186
|
+
raise ValueError("tool name must be a safe identifier")
|
|
187
|
+
try:
|
|
188
|
+
if isinstance(arguments, str):
|
|
189
|
+
arguments = json.loads(arguments, object_pairs_hook=_json_object_pairs)
|
|
190
|
+
if not isinstance(arguments, dict):
|
|
191
|
+
raise ValueError("tool arguments must be an object")
|
|
192
|
+
_finite_json(arguments)
|
|
193
|
+
except (TypeError, ValueError, RecursionError):
|
|
194
|
+
raise ValueError("tool arguments must be a finite JSON object") from None
|
|
195
|
+
return name, arguments
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _located_calls(tool_calls, response_messages):
|
|
199
|
+
located = None
|
|
200
|
+
witnessed = False
|
|
201
|
+
if response_messages is not None:
|
|
202
|
+
if not isinstance(response_messages, list):
|
|
203
|
+
raise ValueError("response_messages must be a list")
|
|
204
|
+
located = []
|
|
205
|
+
for message_index, message in enumerate(response_messages):
|
|
206
|
+
if not isinstance(message, dict) or not isinstance(
|
|
207
|
+
message.get("role"), str
|
|
208
|
+
):
|
|
209
|
+
raise ValueError("malformed response message")
|
|
210
|
+
if message["role"] != "assistant":
|
|
211
|
+
continue
|
|
212
|
+
if "function_call" in message:
|
|
213
|
+
raise ValueError("legacy function_call messages are unsupported")
|
|
214
|
+
witnessed |= "tool_calls" in message
|
|
215
|
+
calls = message.get("tool_calls")
|
|
216
|
+
if calls is None:
|
|
217
|
+
continue
|
|
218
|
+
if not isinstance(calls, list):
|
|
219
|
+
raise ValueError("message tool_calls must be a list or null")
|
|
220
|
+
located.extend(
|
|
221
|
+
{"message_index": message_index, "call_index": index, "call": call}
|
|
222
|
+
for index, call in enumerate(calls)
|
|
223
|
+
)
|
|
224
|
+
if tool_calls is not None:
|
|
225
|
+
if not isinstance(tool_calls, list):
|
|
226
|
+
raise ValueError("tool_calls must be a list")
|
|
227
|
+
if located is not None and [item["call"] for item in located] != tool_calls:
|
|
228
|
+
raise ValueError("flattened and embedded tool calls disagree")
|
|
229
|
+
if located is None:
|
|
230
|
+
located = [
|
|
231
|
+
{"message_index": None, "call_index": index, "call": call}
|
|
232
|
+
for index, call in enumerate(tool_calls)
|
|
233
|
+
]
|
|
234
|
+
elif not witnessed:
|
|
235
|
+
raise ValueError("structured tool call evidence is unavailable")
|
|
236
|
+
return located
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _value_type(value):
|
|
240
|
+
if value is None:
|
|
241
|
+
return "null"
|
|
242
|
+
return {
|
|
243
|
+
bool: "boolean",
|
|
244
|
+
int: "number",
|
|
245
|
+
float: "number",
|
|
246
|
+
str: "string",
|
|
247
|
+
list: "array",
|
|
248
|
+
dict: "object",
|
|
249
|
+
}[type(value)]
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _path_category(value):
|
|
253
|
+
if re.match(r"[A-Za-z][A-Za-z0-9+.-]*://", value):
|
|
254
|
+
try:
|
|
255
|
+
path = PurePosixPath(urlsplit(value).path)
|
|
256
|
+
except ValueError:
|
|
257
|
+
raise ValueError("invalid tool path URI") from None
|
|
258
|
+
kind = "uri"
|
|
259
|
+
elif PureWindowsPath(value).is_absolute():
|
|
260
|
+
kind, path = "windows_absolute", PureWindowsPath(value)
|
|
261
|
+
else:
|
|
262
|
+
kind = (
|
|
263
|
+
"home"
|
|
264
|
+
if value.startswith("~")
|
|
265
|
+
else "absolute"
|
|
266
|
+
if value.startswith("/")
|
|
267
|
+
else "relative"
|
|
268
|
+
)
|
|
269
|
+
path = PurePosixPath(value)
|
|
270
|
+
extension = path.suffix.lower().lstrip(".") or "none"
|
|
271
|
+
return kind, extension if extension in _EXTENSIONS else "other"
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _command_class(words):
|
|
275
|
+
while words and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", words[0]):
|
|
276
|
+
words = words[1:]
|
|
277
|
+
if not words:
|
|
278
|
+
return "other"
|
|
279
|
+
executable = PurePosixPath(words[0]).name
|
|
280
|
+
args = words[1:]
|
|
281
|
+
if (
|
|
282
|
+
executable in {"pytest", "py.test", "nosetests", "unittest"}
|
|
283
|
+
or (
|
|
284
|
+
executable
|
|
285
|
+
in {"python", "python3", "python3.11", "python3.12", "python3.13"}
|
|
286
|
+
and len(args) >= 2
|
|
287
|
+
and args[:2] in (["-m", "pytest"], ["-m", "unittest"])
|
|
288
|
+
)
|
|
289
|
+
or (
|
|
290
|
+
executable in {"npm", "cargo", "go", "yarn", "make"}
|
|
291
|
+
and args
|
|
292
|
+
and args[0] == "test"
|
|
293
|
+
)
|
|
294
|
+
):
|
|
295
|
+
return "test"
|
|
296
|
+
if executable in {"rg", "grep", "egrep", "find", "fd"}:
|
|
297
|
+
return "search"
|
|
298
|
+
if executable in {
|
|
299
|
+
"cat",
|
|
300
|
+
"head",
|
|
301
|
+
"tail",
|
|
302
|
+
"less",
|
|
303
|
+
"more",
|
|
304
|
+
"sed",
|
|
305
|
+
"ls",
|
|
306
|
+
"pwd",
|
|
307
|
+
"view",
|
|
308
|
+
}:
|
|
309
|
+
return "read"
|
|
310
|
+
if executable in {
|
|
311
|
+
"touch",
|
|
312
|
+
"mkdir",
|
|
313
|
+
"rm",
|
|
314
|
+
"mv",
|
|
315
|
+
"cp",
|
|
316
|
+
"tee",
|
|
317
|
+
"create",
|
|
318
|
+
"insert",
|
|
319
|
+
"str_replace",
|
|
320
|
+
"undo_edit",
|
|
321
|
+
}:
|
|
322
|
+
return "write"
|
|
323
|
+
if executable == "git":
|
|
324
|
+
return "version_control"
|
|
325
|
+
if executable in {"pip", "pip3"} or (
|
|
326
|
+
executable in {"npm", "uv"} and args and args[0] in {"install", "pip"}
|
|
327
|
+
):
|
|
328
|
+
return "install"
|
|
329
|
+
if re.fullmatch(r"python(?:\d+(?:\.\d+)?)?", executable):
|
|
330
|
+
return "python"
|
|
331
|
+
if executable in {"sh", "bash", "zsh", "fish"}:
|
|
332
|
+
return "shell"
|
|
333
|
+
return "other"
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _command_classes(value):
|
|
337
|
+
if not isinstance(value, str):
|
|
338
|
+
return {"nontext"}
|
|
339
|
+
try:
|
|
340
|
+
lexer = shlex.shlex(value, posix=True, punctuation_chars=";&|()\n")
|
|
341
|
+
lexer.whitespace = " \t\r"
|
|
342
|
+
lexer.whitespace_split = True
|
|
343
|
+
words = list(lexer)
|
|
344
|
+
except ValueError:
|
|
345
|
+
raise ValueError("tool command cannot be parsed as shell text") from None
|
|
346
|
+
classes, segment = set(), []
|
|
347
|
+
for word in words:
|
|
348
|
+
if word and all(char in ";&|()\n" for char in word):
|
|
349
|
+
classes.add("compound")
|
|
350
|
+
if segment:
|
|
351
|
+
classes.add(_command_class(segment))
|
|
352
|
+
segment = []
|
|
353
|
+
else:
|
|
354
|
+
segment.append(word)
|
|
355
|
+
if segment:
|
|
356
|
+
classes.add(_command_class(segment))
|
|
357
|
+
return classes or {"other"}
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _call_features(name, arguments):
|
|
361
|
+
features = {f"tool:{name}"}
|
|
362
|
+
for key, value in arguments.items():
|
|
363
|
+
key_name = key if key in _ARGUMENT_KEYS else "other"
|
|
364
|
+
features.add(f"tool_arg:{name}:{key_name}:{_value_type(value)}")
|
|
365
|
+
if key in _PATH_KEYS and isinstance(value, str):
|
|
366
|
+
kind, extension = _path_category(value)
|
|
367
|
+
features.add(f"tool_path:{name}:{kind}:{extension}")
|
|
368
|
+
if key in _COMMAND_KEYS:
|
|
369
|
+
features.update(
|
|
370
|
+
f"tool_command:{name}:{kind}" for kind in _command_classes(value)
|
|
371
|
+
)
|
|
372
|
+
return features
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _extract(tool_calls, response_messages):
|
|
376
|
+
located = _located_calls(tool_calls, response_messages)
|
|
377
|
+
per_call, names = [], []
|
|
378
|
+
result = ToolCallFeatures(dense={"tool_calls_count": float(len(located))})
|
|
379
|
+
for item in located:
|
|
380
|
+
name, arguments = _parse_call(item["call"])
|
|
381
|
+
names.append(name)
|
|
382
|
+
features = _call_features(name, arguments)
|
|
383
|
+
per_call.append(features)
|
|
384
|
+
result.sparse.update(features)
|
|
385
|
+
result.dense.update(
|
|
386
|
+
{f"tool_count:{name}": float(count) for name, count in Counter(names).items()}
|
|
387
|
+
)
|
|
388
|
+
for index, (first, second) in enumerate(pairwise(names)):
|
|
389
|
+
feature = f"tool_bigram:{first}>{second}"
|
|
390
|
+
result.sparse.add(feature)
|
|
391
|
+
per_call[index].add(feature)
|
|
392
|
+
per_call[index + 1].add(feature)
|
|
393
|
+
return result, located, per_call, names
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def extract_tool_calls(tool_calls=None, response_messages=None) -> ToolCallFeatures:
|
|
397
|
+
"""Extract complete recorded calls; explicit [] denotes known absence."""
|
|
398
|
+
return _extract(tool_calls, response_messages)[0]
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def tool_call_evidence(
|
|
402
|
+
tool_calls=None, response_messages=None, *, feature=None
|
|
403
|
+
) -> list[dict]:
|
|
404
|
+
"""Return original calls and zero-based transcript/call positions for a feature."""
|
|
405
|
+
result, located, per_call, names = _extract(tool_calls, response_messages)
|
|
406
|
+
if feature is None or feature == "tool_calls_count":
|
|
407
|
+
return located
|
|
408
|
+
if valid_tool_feature(feature, "continuous"):
|
|
409
|
+
return [
|
|
410
|
+
item
|
|
411
|
+
for item, name in zip(located, names, strict=True)
|
|
412
|
+
if feature == f"tool_count:{name}"
|
|
413
|
+
]
|
|
414
|
+
if not valid_tool_feature(feature, "binary"):
|
|
415
|
+
raise ValueError("invalid tool feature")
|
|
416
|
+
if feature not in result.sparse:
|
|
417
|
+
return []
|
|
418
|
+
return [
|
|
419
|
+
item
|
|
420
|
+
for item, features in zip(located, per_call, strict=True)
|
|
421
|
+
if feature in features
|
|
422
|
+
]
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""Local, unstepped OpenHands and SWE-smith trajectory inputs for comparison."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from .adapters import _plain, _text
|
|
13
|
+
from .schema import CANONICAL_COLUMNS, validate_rollouts
|
|
14
|
+
from .tool_features import tool_call_evidence
|
|
15
|
+
|
|
16
|
+
_SUFFIXES = {".json", ".jsonl", ".parquet"}
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _unique_object(pairs):
|
|
20
|
+
result = {}
|
|
21
|
+
for name, value in pairs:
|
|
22
|
+
if name in result:
|
|
23
|
+
raise ValueError("duplicate trajectory JSON key")
|
|
24
|
+
result[name] = value
|
|
25
|
+
return result
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _decode(text):
|
|
29
|
+
return json.loads(text, object_pairs_hook=_unique_object)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _records(path):
|
|
33
|
+
path = Path(path)
|
|
34
|
+
if not path.exists():
|
|
35
|
+
raise FileNotFoundError(path)
|
|
36
|
+
files = (
|
|
37
|
+
sorted(
|
|
38
|
+
file
|
|
39
|
+
for file in path.iterdir()
|
|
40
|
+
if file.is_file() and file.suffix in _SUFFIXES
|
|
41
|
+
)
|
|
42
|
+
if path.is_dir()
|
|
43
|
+
else [path]
|
|
44
|
+
)
|
|
45
|
+
if not files:
|
|
46
|
+
raise ValueError("no trajectory files found")
|
|
47
|
+
for file in files:
|
|
48
|
+
if file.suffix not in _SUFFIXES:
|
|
49
|
+
raise ValueError("trajectory input must be JSON, JSONL, or Parquet")
|
|
50
|
+
if file.suffix == ".parquet":
|
|
51
|
+
rows = _plain(pd.read_parquet(file).to_dict("records"))
|
|
52
|
+
elif file.suffix == ".json":
|
|
53
|
+
try:
|
|
54
|
+
rows = _decode(file.read_text(encoding="utf-8"))
|
|
55
|
+
except (ValueError, UnicodeError):
|
|
56
|
+
raise ValueError(f"{file.name}: invalid trajectory JSON") from None
|
|
57
|
+
if not isinstance(rows, list):
|
|
58
|
+
raise ValueError(f"{file.name}: trajectory JSON must be a list")
|
|
59
|
+
else:
|
|
60
|
+
rows = []
|
|
61
|
+
try:
|
|
62
|
+
with file.open(encoding="utf-8") as stream:
|
|
63
|
+
for line_number, line in enumerate(stream, 1):
|
|
64
|
+
if not line.strip():
|
|
65
|
+
continue
|
|
66
|
+
try:
|
|
67
|
+
rows.append(_decode(line))
|
|
68
|
+
except ValueError:
|
|
69
|
+
raise ValueError(
|
|
70
|
+
f"{file.name}:{line_number}: invalid trajectory JSON"
|
|
71
|
+
) from None
|
|
72
|
+
except UnicodeError:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"{file.name}: invalid trajectory text encoding"
|
|
75
|
+
) from None
|
|
76
|
+
for position, row in enumerate(rows, 1):
|
|
77
|
+
if not isinstance(row, dict):
|
|
78
|
+
raise ValueError(
|
|
79
|
+
f"{file.name}:{position}: trajectory must be an object"
|
|
80
|
+
)
|
|
81
|
+
yield f"{file.name}:{position}", row
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def _identifier(value, name):
|
|
85
|
+
if not isinstance(value, str) or not value.strip():
|
|
86
|
+
raise ValueError(f"{name} must be a nonblank string")
|
|
87
|
+
return value
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _transcript(raw, format):
|
|
91
|
+
messages = raw.get("trajectory" if format == "openhands" else "messages")
|
|
92
|
+
if format == "swe-smith":
|
|
93
|
+
if not isinstance(messages, str):
|
|
94
|
+
raise ValueError("SWE-smith messages must be a JSON string")
|
|
95
|
+
try:
|
|
96
|
+
messages = _decode(messages)
|
|
97
|
+
except ValueError:
|
|
98
|
+
raise ValueError("SWE-smith messages contain invalid JSON") from None
|
|
99
|
+
if not isinstance(messages, list) or not messages:
|
|
100
|
+
raise ValueError("trajectory must contain messages")
|
|
101
|
+
# Validate every message's text, retaining the original roles and metadata.
|
|
102
|
+
_, messages = _text(messages)
|
|
103
|
+
try:
|
|
104
|
+
json.dumps(messages, allow_nan=False)
|
|
105
|
+
except (TypeError, ValueError, RecursionError):
|
|
106
|
+
raise ValueError(
|
|
107
|
+
"trajectory messages must contain finite JSON values"
|
|
108
|
+
) from None
|
|
109
|
+
if any(
|
|
110
|
+
message["role"]
|
|
111
|
+
not in {"system", "developer", "user", "assistant", "tool", "function"}
|
|
112
|
+
for message in messages
|
|
113
|
+
):
|
|
114
|
+
raise ValueError("unsupported trajectory message role")
|
|
115
|
+
first = next(
|
|
116
|
+
(
|
|
117
|
+
index
|
|
118
|
+
for index, message in enumerate(messages)
|
|
119
|
+
if message["role"] == "assistant"
|
|
120
|
+
),
|
|
121
|
+
None,
|
|
122
|
+
)
|
|
123
|
+
if first is None or not any(
|
|
124
|
+
message["role"] == "user" for message in messages[:first]
|
|
125
|
+
):
|
|
126
|
+
raise ValueError("trajectory requires a prompt followed by assistant messages")
|
|
127
|
+
prompt_messages = messages[:first]
|
|
128
|
+
assistant_messages = [
|
|
129
|
+
message for message in messages[first:] if message["role"] == "assistant"
|
|
130
|
+
]
|
|
131
|
+
prompt, _ = _text(prompt_messages)
|
|
132
|
+
response, _ = _text(assistant_messages)
|
|
133
|
+
calls = []
|
|
134
|
+
for message in assistant_messages:
|
|
135
|
+
value = message.get("tool_calls")
|
|
136
|
+
if value is not None:
|
|
137
|
+
if not isinstance(value, list):
|
|
138
|
+
raise ValueError("assistant tool_calls must be a list or null")
|
|
139
|
+
calls.extend(value)
|
|
140
|
+
tool_call_evidence(calls, messages)
|
|
141
|
+
return prompt, response, prompt_messages, messages, calls
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _reward(raw, column):
|
|
145
|
+
if column is None:
|
|
146
|
+
return None
|
|
147
|
+
if column not in raw:
|
|
148
|
+
raise ValueError("selected reward column is missing")
|
|
149
|
+
value = raw[column]
|
|
150
|
+
if value is None:
|
|
151
|
+
return None
|
|
152
|
+
if type(value) not in {bool, int, float}:
|
|
153
|
+
raise ValueError("selected reward must be numeric or null")
|
|
154
|
+
try:
|
|
155
|
+
value = float(value)
|
|
156
|
+
except OverflowError:
|
|
157
|
+
raise ValueError("selected reward must be finite") from None
|
|
158
|
+
if not math.isfinite(value):
|
|
159
|
+
raise ValueError("selected reward must be finite")
|
|
160
|
+
return value
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def load_trajectories(
|
|
164
|
+
path: str | Path,
|
|
165
|
+
*,
|
|
166
|
+
run: str,
|
|
167
|
+
format: str,
|
|
168
|
+
model: str | None = None,
|
|
169
|
+
reward_column: str | None = None,
|
|
170
|
+
) -> pd.DataFrame:
|
|
171
|
+
"""Read native JSONL/JSON-list/Parquet records with no training step.
|
|
172
|
+
|
|
173
|
+
Multiple SWE-smith models require explicit selection; OpenHands has no
|
|
174
|
+
per-record model metadata for filtering. `resolved` is only an
|
|
175
|
+
outcome: reward remains null unless reward_column explicitly selects it.
|
|
176
|
+
response_messages preserves the full transcript, including observations;
|
|
177
|
+
response contains assistant text only. Directories read all supported files.
|
|
178
|
+
"""
|
|
179
|
+
if format not in {"openhands", "swe-smith"}:
|
|
180
|
+
raise ValueError("unsupported trajectory format")
|
|
181
|
+
_identifier(run, "run")
|
|
182
|
+
if model is not None:
|
|
183
|
+
_identifier(model, "model")
|
|
184
|
+
if reward_column is not None:
|
|
185
|
+
_identifier(reward_column, "reward_column")
|
|
186
|
+
if format == "openhands" and model is not None:
|
|
187
|
+
raise ValueError(
|
|
188
|
+
"OpenHands model filtering is unavailable without per-record model metadata"
|
|
189
|
+
)
|
|
190
|
+
rows, selected_models = [], set()
|
|
191
|
+
for source, raw in _records(path):
|
|
192
|
+
try:
|
|
193
|
+
if format == "swe-smith":
|
|
194
|
+
source_model = _identifier(raw.get("model"), "model")
|
|
195
|
+
if model is not None and source_model != model:
|
|
196
|
+
continue
|
|
197
|
+
selected_models.add(source_model)
|
|
198
|
+
identifier = _identifier(
|
|
199
|
+
raw.get("trajectory_id" if format == "openhands" else "traj_id"),
|
|
200
|
+
"trajectory identity",
|
|
201
|
+
)
|
|
202
|
+
_identifier(raw.get("instance_id"), "instance_id")
|
|
203
|
+
prompt, response, prefix, messages, calls = _transcript(raw, format)
|
|
204
|
+
digest = hashlib.sha256(
|
|
205
|
+
json.dumps(
|
|
206
|
+
["chat", prefix],
|
|
207
|
+
sort_keys=True,
|
|
208
|
+
ensure_ascii=False,
|
|
209
|
+
separators=(",", ":"),
|
|
210
|
+
allow_nan=False,
|
|
211
|
+
).encode("utf-8")
|
|
212
|
+
).hexdigest()
|
|
213
|
+
rows.append(
|
|
214
|
+
{
|
|
215
|
+
"run": run,
|
|
216
|
+
"step": None,
|
|
217
|
+
"sample_id": identifier,
|
|
218
|
+
"prompt_id": digest,
|
|
219
|
+
"prompt": prompt,
|
|
220
|
+
"response": response,
|
|
221
|
+
"reward": _reward(raw, reward_column),
|
|
222
|
+
"prompt_messages": prefix,
|
|
223
|
+
"response_messages": messages,
|
|
224
|
+
"tool_calls": calls,
|
|
225
|
+
}
|
|
226
|
+
)
|
|
227
|
+
except (ValueError, TypeError) as exc:
|
|
228
|
+
raise ValueError(f"{source}: {exc}") from None
|
|
229
|
+
if not rows:
|
|
230
|
+
raise ValueError("no trajectories match the requested model or source")
|
|
231
|
+
if len(selected_models) > 1:
|
|
232
|
+
raise ValueError("select one SWE-smith model explicitly")
|
|
233
|
+
return validate_rollouts(
|
|
234
|
+
pd.DataFrame(rows, columns=CANONICAL_COLUMNS, dtype=object),
|
|
235
|
+
allow_unavailable=True,
|
|
236
|
+
)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"current_code_sha256": "5dd8fc2c9872cedaadaafd545eaac0fb17bbe271cbd5920d45b861fafc0c539b",
|
|
3
|
+
"legacy_code_sha256s": {
|
|
4
|
+
"e903f9e": "29405d4b3fb24805e7e129665177d84e679104139ec75477181cd53acd0f2b55",
|
|
5
|
+
"2cbfe5c": "7ef401a9b477749f21814f26de733721ece866ccee7b55801de01394ba87a395",
|
|
6
|
+
"f233cb2": "bd7b4438b8a6aba6ea2e32b4be8ff110c44b61c847f65713826376393e58f210",
|
|
7
|
+
"428b243": "80de635fce5273eec97ceca01f06c73cd5b92de1764ce35b9d8ae6369e484701",
|
|
8
|
+
"6827436": "ff795d565bf78030eb2a6509a74f71ac93dfc51767fa8db9a12ad4ff1ff653ab",
|
|
9
|
+
"7764671": "484f8ce03e30cdf3e0b97b2eba9c83acca30b697cf2da75a385dafe97cadf0e9",
|
|
10
|
+
"m3-final-wheel": "1293520e3d0ab30819fc5023afad662623573c6254de91b6b6250e5e91fda90d",
|
|
11
|
+
"b9ab589/8cdf116": "460d59c68d7a0dbc2e8633f74615e9da273e0ee6acf371c50f719242e2c7cc76"
|
|
12
|
+
}
|
|
13
|
+
}
|