rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,422 @@
1
+ """Static features of recorded assistant calls, without executing arguments.
2
+
3
+ Command classes describe lexical forms, not successful operations. Feature names
4
+ contain tool identifiers and fixed categories, never argument values or paths.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import json
10
+ import re
11
+ import shlex
12
+ from collections import Counter
13
+ from dataclasses import dataclass, field
14
+ from itertools import pairwise
15
+ from pathlib import PurePosixPath, PureWindowsPath
16
+ from urllib.parse import urlsplit
17
+
18
+ _NAME = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]{0,95}\Z")
19
+ _ARGUMENT_KEYS = frozenset(
20
+ {
21
+ "command",
22
+ "cmd",
23
+ "shell_command",
24
+ "path",
25
+ "file_path",
26
+ "filename",
27
+ "file_text",
28
+ "new_str",
29
+ "old_str",
30
+ "insert_line",
31
+ "view_range",
32
+ "timeout",
33
+ "is_input",
34
+ "message",
35
+ "thought",
36
+ "task_list",
37
+ "code",
38
+ "language",
39
+ }
40
+ )
41
+ _PATH_KEYS = {"path", "file_path", "filename"}
42
+ _COMMAND_KEYS = {"command", "cmd", "shell_command"}
43
+ _TYPES = {"null", "boolean", "number", "string", "array", "object"}
44
+ _PATH_KINDS = {"relative", "absolute", "home", "windows_absolute", "uri"}
45
+ _EXTENSIONS = frozenset(
46
+ [
47
+ "py",
48
+ "js",
49
+ "ts",
50
+ "jsx",
51
+ "tsx",
52
+ "json",
53
+ "toml",
54
+ "yaml",
55
+ "yml",
56
+ "ini",
57
+ "cfg",
58
+ "md",
59
+ "rst",
60
+ "txt",
61
+ "c",
62
+ "cpp",
63
+ "h",
64
+ "hpp",
65
+ "go",
66
+ "rs",
67
+ "sh",
68
+ "ipynb",
69
+ "csv",
70
+ "tsv",
71
+ "xml",
72
+ "html",
73
+ "css",
74
+ "scss",
75
+ "sql",
76
+ "java",
77
+ "kt",
78
+ "rb",
79
+ "php",
80
+ "none",
81
+ "other",
82
+ ]
83
+ )
84
+ _COMMAND_CLASSES = {
85
+ "test",
86
+ "search",
87
+ "read",
88
+ "write",
89
+ "version_control",
90
+ "install",
91
+ "python",
92
+ "shell",
93
+ "other",
94
+ "compound",
95
+ "nontext",
96
+ }
97
+
98
+
99
+ @dataclass
100
+ class ToolCallFeatures:
101
+ dense: dict[str, float] = field(default_factory=dict)
102
+ sparse: set[str] = field(default_factory=set)
103
+
104
+ @property
105
+ def catalog(self) -> dict[str, dict[str, str]]:
106
+ return {
107
+ **{
108
+ name: {"kind": "continuous", "tier": "structural"}
109
+ for name in self.dense
110
+ },
111
+ **{name: {"kind": "binary", "tier": "structural"} for name in self.sparse},
112
+ }
113
+
114
+
115
+ def valid_tool_feature(name: str, kind: str) -> bool:
116
+ """Validate the complete exported tool feature grammar."""
117
+ if not isinstance(name, str):
118
+ return False
119
+ if kind == "continuous":
120
+ return name == "tool_calls_count" or (
121
+ name.startswith("tool_count:") and bool(_NAME.fullmatch(name[11:]))
122
+ )
123
+ if kind != "binary":
124
+ return False
125
+ parts = name.split(":")
126
+ if len(parts) < 2:
127
+ return False
128
+ if parts[0] == "tool_bigram":
129
+ names = parts[1].split(">")
130
+ return (
131
+ len(parts) == 2
132
+ and len(names) == 2
133
+ and all(_NAME.fullmatch(x) for x in names)
134
+ )
135
+ if not _NAME.fullmatch(parts[1]):
136
+ return False
137
+ if parts[0] == "tool":
138
+ return len(parts) == 2
139
+ if parts[0] == "tool_arg":
140
+ return (
141
+ len(parts) == 4
142
+ and parts[2] in _ARGUMENT_KEYS | {"other"}
143
+ and parts[3] in _TYPES
144
+ )
145
+ if parts[0] == "tool_path":
146
+ return len(parts) == 4 and parts[2] in _PATH_KINDS and parts[3] in _EXTENSIONS
147
+ if parts[0] == "tool_command":
148
+ return len(parts) == 3 and parts[2] in _COMMAND_CLASSES
149
+ return False
150
+
151
+
152
+ def _json_object_pairs(pairs):
153
+ result = {}
154
+ for key, value in pairs:
155
+ if key in result:
156
+ raise ValueError("duplicate tool argument key")
157
+ result[key] = value
158
+ return result
159
+
160
+
161
+ def _finite_json(value):
162
+ if isinstance(value, dict):
163
+ if not all(isinstance(key, str) for key in value):
164
+ raise ValueError("tool arguments require string keys")
165
+ for item in value.values():
166
+ _finite_json(item)
167
+ elif isinstance(value, list):
168
+ for item in value:
169
+ _finite_json(item)
170
+ elif value is not None and type(value) not in {str, int, float, bool}:
171
+ raise ValueError("tool arguments must be finite JSON values")
172
+ json.dumps(value, allow_nan=False)
173
+
174
+
175
+ def _parse_call(call):
176
+ if not isinstance(call, dict) or call.get("parse_error") is not None:
177
+ raise ValueError("malformed tool call")
178
+ if call.get("type", "function") != "function":
179
+ raise ValueError("only function tool calls are supported")
180
+ function = call.get("function")
181
+ if isinstance(function, dict):
182
+ name, arguments = function.get("name"), function.get("arguments")
183
+ else:
184
+ name, arguments = function, call.get("arguments")
185
+ if not isinstance(name, str) or not _NAME.fullmatch(name):
186
+ raise ValueError("tool name must be a safe identifier")
187
+ try:
188
+ if isinstance(arguments, str):
189
+ arguments = json.loads(arguments, object_pairs_hook=_json_object_pairs)
190
+ if not isinstance(arguments, dict):
191
+ raise ValueError("tool arguments must be an object")
192
+ _finite_json(arguments)
193
+ except (TypeError, ValueError, RecursionError):
194
+ raise ValueError("tool arguments must be a finite JSON object") from None
195
+ return name, arguments
196
+
197
+
198
+ def _located_calls(tool_calls, response_messages):
199
+ located = None
200
+ witnessed = False
201
+ if response_messages is not None:
202
+ if not isinstance(response_messages, list):
203
+ raise ValueError("response_messages must be a list")
204
+ located = []
205
+ for message_index, message in enumerate(response_messages):
206
+ if not isinstance(message, dict) or not isinstance(
207
+ message.get("role"), str
208
+ ):
209
+ raise ValueError("malformed response message")
210
+ if message["role"] != "assistant":
211
+ continue
212
+ if "function_call" in message:
213
+ raise ValueError("legacy function_call messages are unsupported")
214
+ witnessed |= "tool_calls" in message
215
+ calls = message.get("tool_calls")
216
+ if calls is None:
217
+ continue
218
+ if not isinstance(calls, list):
219
+ raise ValueError("message tool_calls must be a list or null")
220
+ located.extend(
221
+ {"message_index": message_index, "call_index": index, "call": call}
222
+ for index, call in enumerate(calls)
223
+ )
224
+ if tool_calls is not None:
225
+ if not isinstance(tool_calls, list):
226
+ raise ValueError("tool_calls must be a list")
227
+ if located is not None and [item["call"] for item in located] != tool_calls:
228
+ raise ValueError("flattened and embedded tool calls disagree")
229
+ if located is None:
230
+ located = [
231
+ {"message_index": None, "call_index": index, "call": call}
232
+ for index, call in enumerate(tool_calls)
233
+ ]
234
+ elif not witnessed:
235
+ raise ValueError("structured tool call evidence is unavailable")
236
+ return located
237
+
238
+
239
+ def _value_type(value):
240
+ if value is None:
241
+ return "null"
242
+ return {
243
+ bool: "boolean",
244
+ int: "number",
245
+ float: "number",
246
+ str: "string",
247
+ list: "array",
248
+ dict: "object",
249
+ }[type(value)]
250
+
251
+
252
+ def _path_category(value):
253
+ if re.match(r"[A-Za-z][A-Za-z0-9+.-]*://", value):
254
+ try:
255
+ path = PurePosixPath(urlsplit(value).path)
256
+ except ValueError:
257
+ raise ValueError("invalid tool path URI") from None
258
+ kind = "uri"
259
+ elif PureWindowsPath(value).is_absolute():
260
+ kind, path = "windows_absolute", PureWindowsPath(value)
261
+ else:
262
+ kind = (
263
+ "home"
264
+ if value.startswith("~")
265
+ else "absolute"
266
+ if value.startswith("/")
267
+ else "relative"
268
+ )
269
+ path = PurePosixPath(value)
270
+ extension = path.suffix.lower().lstrip(".") or "none"
271
+ return kind, extension if extension in _EXTENSIONS else "other"
272
+
273
+
274
+ def _command_class(words):
275
+ while words and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", words[0]):
276
+ words = words[1:]
277
+ if not words:
278
+ return "other"
279
+ executable = PurePosixPath(words[0]).name
280
+ args = words[1:]
281
+ if (
282
+ executable in {"pytest", "py.test", "nosetests", "unittest"}
283
+ or (
284
+ executable
285
+ in {"python", "python3", "python3.11", "python3.12", "python3.13"}
286
+ and len(args) >= 2
287
+ and args[:2] in (["-m", "pytest"], ["-m", "unittest"])
288
+ )
289
+ or (
290
+ executable in {"npm", "cargo", "go", "yarn", "make"}
291
+ and args
292
+ and args[0] == "test"
293
+ )
294
+ ):
295
+ return "test"
296
+ if executable in {"rg", "grep", "egrep", "find", "fd"}:
297
+ return "search"
298
+ if executable in {
299
+ "cat",
300
+ "head",
301
+ "tail",
302
+ "less",
303
+ "more",
304
+ "sed",
305
+ "ls",
306
+ "pwd",
307
+ "view",
308
+ }:
309
+ return "read"
310
+ if executable in {
311
+ "touch",
312
+ "mkdir",
313
+ "rm",
314
+ "mv",
315
+ "cp",
316
+ "tee",
317
+ "create",
318
+ "insert",
319
+ "str_replace",
320
+ "undo_edit",
321
+ }:
322
+ return "write"
323
+ if executable == "git":
324
+ return "version_control"
325
+ if executable in {"pip", "pip3"} or (
326
+ executable in {"npm", "uv"} and args and args[0] in {"install", "pip"}
327
+ ):
328
+ return "install"
329
+ if re.fullmatch(r"python(?:\d+(?:\.\d+)?)?", executable):
330
+ return "python"
331
+ if executable in {"sh", "bash", "zsh", "fish"}:
332
+ return "shell"
333
+ return "other"
334
+
335
+
336
+ def _command_classes(value):
337
+ if not isinstance(value, str):
338
+ return {"nontext"}
339
+ try:
340
+ lexer = shlex.shlex(value, posix=True, punctuation_chars=";&|()\n")
341
+ lexer.whitespace = " \t\r"
342
+ lexer.whitespace_split = True
343
+ words = list(lexer)
344
+ except ValueError:
345
+ raise ValueError("tool command cannot be parsed as shell text") from None
346
+ classes, segment = set(), []
347
+ for word in words:
348
+ if word and all(char in ";&|()\n" for char in word):
349
+ classes.add("compound")
350
+ if segment:
351
+ classes.add(_command_class(segment))
352
+ segment = []
353
+ else:
354
+ segment.append(word)
355
+ if segment:
356
+ classes.add(_command_class(segment))
357
+ return classes or {"other"}
358
+
359
+
360
+ def _call_features(name, arguments):
361
+ features = {f"tool:{name}"}
362
+ for key, value in arguments.items():
363
+ key_name = key if key in _ARGUMENT_KEYS else "other"
364
+ features.add(f"tool_arg:{name}:{key_name}:{_value_type(value)}")
365
+ if key in _PATH_KEYS and isinstance(value, str):
366
+ kind, extension = _path_category(value)
367
+ features.add(f"tool_path:{name}:{kind}:{extension}")
368
+ if key in _COMMAND_KEYS:
369
+ features.update(
370
+ f"tool_command:{name}:{kind}" for kind in _command_classes(value)
371
+ )
372
+ return features
373
+
374
+
375
+ def _extract(tool_calls, response_messages):
376
+ located = _located_calls(tool_calls, response_messages)
377
+ per_call, names = [], []
378
+ result = ToolCallFeatures(dense={"tool_calls_count": float(len(located))})
379
+ for item in located:
380
+ name, arguments = _parse_call(item["call"])
381
+ names.append(name)
382
+ features = _call_features(name, arguments)
383
+ per_call.append(features)
384
+ result.sparse.update(features)
385
+ result.dense.update(
386
+ {f"tool_count:{name}": float(count) for name, count in Counter(names).items()}
387
+ )
388
+ for index, (first, second) in enumerate(pairwise(names)):
389
+ feature = f"tool_bigram:{first}>{second}"
390
+ result.sparse.add(feature)
391
+ per_call[index].add(feature)
392
+ per_call[index + 1].add(feature)
393
+ return result, located, per_call, names
394
+
395
+
396
+ def extract_tool_calls(tool_calls=None, response_messages=None) -> ToolCallFeatures:
397
+ """Extract complete recorded calls; explicit [] denotes known absence."""
398
+ return _extract(tool_calls, response_messages)[0]
399
+
400
+
401
+ def tool_call_evidence(
402
+ tool_calls=None, response_messages=None, *, feature=None
403
+ ) -> list[dict]:
404
+ """Return original calls and zero-based transcript/call positions for a feature."""
405
+ result, located, per_call, names = _extract(tool_calls, response_messages)
406
+ if feature is None or feature == "tool_calls_count":
407
+ return located
408
+ if valid_tool_feature(feature, "continuous"):
409
+ return [
410
+ item
411
+ for item, name in zip(located, names, strict=True)
412
+ if feature == f"tool_count:{name}"
413
+ ]
414
+ if not valid_tool_feature(feature, "binary"):
415
+ raise ValueError("invalid tool feature")
416
+ if feature not in result.sparse:
417
+ return []
418
+ return [
419
+ item
420
+ for item, features in zip(located, per_call, strict=True)
421
+ if feature in features
422
+ ]
@@ -0,0 +1,236 @@
1
+ """Local, unstepped OpenHands and SWE-smith trajectory inputs for comparison."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ from pathlib import Path
9
+
10
+ import pandas as pd
11
+
12
+ from .adapters import _plain, _text
13
+ from .schema import CANONICAL_COLUMNS, validate_rollouts
14
+ from .tool_features import tool_call_evidence
15
+
16
+ _SUFFIXES = {".json", ".jsonl", ".parquet"}
17
+
18
+
19
+ def _unique_object(pairs):
20
+ result = {}
21
+ for name, value in pairs:
22
+ if name in result:
23
+ raise ValueError("duplicate trajectory JSON key")
24
+ result[name] = value
25
+ return result
26
+
27
+
28
+ def _decode(text):
29
+ return json.loads(text, object_pairs_hook=_unique_object)
30
+
31
+
32
+ def _records(path):
33
+ path = Path(path)
34
+ if not path.exists():
35
+ raise FileNotFoundError(path)
36
+ files = (
37
+ sorted(
38
+ file
39
+ for file in path.iterdir()
40
+ if file.is_file() and file.suffix in _SUFFIXES
41
+ )
42
+ if path.is_dir()
43
+ else [path]
44
+ )
45
+ if not files:
46
+ raise ValueError("no trajectory files found")
47
+ for file in files:
48
+ if file.suffix not in _SUFFIXES:
49
+ raise ValueError("trajectory input must be JSON, JSONL, or Parquet")
50
+ if file.suffix == ".parquet":
51
+ rows = _plain(pd.read_parquet(file).to_dict("records"))
52
+ elif file.suffix == ".json":
53
+ try:
54
+ rows = _decode(file.read_text(encoding="utf-8"))
55
+ except (ValueError, UnicodeError):
56
+ raise ValueError(f"{file.name}: invalid trajectory JSON") from None
57
+ if not isinstance(rows, list):
58
+ raise ValueError(f"{file.name}: trajectory JSON must be a list")
59
+ else:
60
+ rows = []
61
+ try:
62
+ with file.open(encoding="utf-8") as stream:
63
+ for line_number, line in enumerate(stream, 1):
64
+ if not line.strip():
65
+ continue
66
+ try:
67
+ rows.append(_decode(line))
68
+ except ValueError:
69
+ raise ValueError(
70
+ f"{file.name}:{line_number}: invalid trajectory JSON"
71
+ ) from None
72
+ except UnicodeError:
73
+ raise ValueError(
74
+ f"{file.name}: invalid trajectory text encoding"
75
+ ) from None
76
+ for position, row in enumerate(rows, 1):
77
+ if not isinstance(row, dict):
78
+ raise ValueError(
79
+ f"{file.name}:{position}: trajectory must be an object"
80
+ )
81
+ yield f"{file.name}:{position}", row
82
+
83
+
84
+ def _identifier(value, name):
85
+ if not isinstance(value, str) or not value.strip():
86
+ raise ValueError(f"{name} must be a nonblank string")
87
+ return value
88
+
89
+
90
+ def _transcript(raw, format):
91
+ messages = raw.get("trajectory" if format == "openhands" else "messages")
92
+ if format == "swe-smith":
93
+ if not isinstance(messages, str):
94
+ raise ValueError("SWE-smith messages must be a JSON string")
95
+ try:
96
+ messages = _decode(messages)
97
+ except ValueError:
98
+ raise ValueError("SWE-smith messages contain invalid JSON") from None
99
+ if not isinstance(messages, list) or not messages:
100
+ raise ValueError("trajectory must contain messages")
101
+ # Validate every message's text, retaining the original roles and metadata.
102
+ _, messages = _text(messages)
103
+ try:
104
+ json.dumps(messages, allow_nan=False)
105
+ except (TypeError, ValueError, RecursionError):
106
+ raise ValueError(
107
+ "trajectory messages must contain finite JSON values"
108
+ ) from None
109
+ if any(
110
+ message["role"]
111
+ not in {"system", "developer", "user", "assistant", "tool", "function"}
112
+ for message in messages
113
+ ):
114
+ raise ValueError("unsupported trajectory message role")
115
+ first = next(
116
+ (
117
+ index
118
+ for index, message in enumerate(messages)
119
+ if message["role"] == "assistant"
120
+ ),
121
+ None,
122
+ )
123
+ if first is None or not any(
124
+ message["role"] == "user" for message in messages[:first]
125
+ ):
126
+ raise ValueError("trajectory requires a prompt followed by assistant messages")
127
+ prompt_messages = messages[:first]
128
+ assistant_messages = [
129
+ message for message in messages[first:] if message["role"] == "assistant"
130
+ ]
131
+ prompt, _ = _text(prompt_messages)
132
+ response, _ = _text(assistant_messages)
133
+ calls = []
134
+ for message in assistant_messages:
135
+ value = message.get("tool_calls")
136
+ if value is not None:
137
+ if not isinstance(value, list):
138
+ raise ValueError("assistant tool_calls must be a list or null")
139
+ calls.extend(value)
140
+ tool_call_evidence(calls, messages)
141
+ return prompt, response, prompt_messages, messages, calls
142
+
143
+
144
+ def _reward(raw, column):
145
+ if column is None:
146
+ return None
147
+ if column not in raw:
148
+ raise ValueError("selected reward column is missing")
149
+ value = raw[column]
150
+ if value is None:
151
+ return None
152
+ if type(value) not in {bool, int, float}:
153
+ raise ValueError("selected reward must be numeric or null")
154
+ try:
155
+ value = float(value)
156
+ except OverflowError:
157
+ raise ValueError("selected reward must be finite") from None
158
+ if not math.isfinite(value):
159
+ raise ValueError("selected reward must be finite")
160
+ return value
161
+
162
+
163
+ def load_trajectories(
164
+ path: str | Path,
165
+ *,
166
+ run: str,
167
+ format: str,
168
+ model: str | None = None,
169
+ reward_column: str | None = None,
170
+ ) -> pd.DataFrame:
171
+ """Read native JSONL/JSON-list/Parquet records with no training step.
172
+
173
+ Multiple SWE-smith models require explicit selection; OpenHands has no
174
+ per-record model metadata for filtering. `resolved` is only an
175
+ outcome: reward remains null unless reward_column explicitly selects it.
176
+ response_messages preserves the full transcript, including observations;
177
+ response contains assistant text only. Directories read all supported files.
178
+ """
179
+ if format not in {"openhands", "swe-smith"}:
180
+ raise ValueError("unsupported trajectory format")
181
+ _identifier(run, "run")
182
+ if model is not None:
183
+ _identifier(model, "model")
184
+ if reward_column is not None:
185
+ _identifier(reward_column, "reward_column")
186
+ if format == "openhands" and model is not None:
187
+ raise ValueError(
188
+ "OpenHands model filtering is unavailable without per-record model metadata"
189
+ )
190
+ rows, selected_models = [], set()
191
+ for source, raw in _records(path):
192
+ try:
193
+ if format == "swe-smith":
194
+ source_model = _identifier(raw.get("model"), "model")
195
+ if model is not None and source_model != model:
196
+ continue
197
+ selected_models.add(source_model)
198
+ identifier = _identifier(
199
+ raw.get("trajectory_id" if format == "openhands" else "traj_id"),
200
+ "trajectory identity",
201
+ )
202
+ _identifier(raw.get("instance_id"), "instance_id")
203
+ prompt, response, prefix, messages, calls = _transcript(raw, format)
204
+ digest = hashlib.sha256(
205
+ json.dumps(
206
+ ["chat", prefix],
207
+ sort_keys=True,
208
+ ensure_ascii=False,
209
+ separators=(",", ":"),
210
+ allow_nan=False,
211
+ ).encode("utf-8")
212
+ ).hexdigest()
213
+ rows.append(
214
+ {
215
+ "run": run,
216
+ "step": None,
217
+ "sample_id": identifier,
218
+ "prompt_id": digest,
219
+ "prompt": prompt,
220
+ "response": response,
221
+ "reward": _reward(raw, reward_column),
222
+ "prompt_messages": prefix,
223
+ "response_messages": messages,
224
+ "tool_calls": calls,
225
+ }
226
+ )
227
+ except (ValueError, TypeError) as exc:
228
+ raise ValueError(f"{source}: {exc}") from None
229
+ if not rows:
230
+ raise ValueError("no trajectories match the requested model or source")
231
+ if len(selected_models) > 1:
232
+ raise ValueError("select one SWE-smith model explicitly")
233
+ return validate_rollouts(
234
+ pd.DataFrame(rows, columns=CANONICAL_COLUMNS, dtype=object),
235
+ allow_unavailable=True,
236
+ )
@@ -0,0 +1,13 @@
1
+ {
2
+ "current_code_sha256": "5dd8fc2c9872cedaadaafd545eaac0fb17bbe271cbd5920d45b861fafc0c539b",
3
+ "legacy_code_sha256s": {
4
+ "e903f9e": "29405d4b3fb24805e7e129665177d84e679104139ec75477181cd53acd0f2b55",
5
+ "2cbfe5c": "7ef401a9b477749f21814f26de733721ece866ccee7b55801de01394ba87a395",
6
+ "f233cb2": "bd7b4438b8a6aba6ea2e32b4be8ff110c44b61c847f65713826376393e58f210",
7
+ "428b243": "80de635fce5273eec97ceca01f06c73cd5b92de1764ce35b9d8ae6369e484701",
8
+ "6827436": "ff795d565bf78030eb2a6509a74f71ac93dfc51767fa8db9a12ad4ff1ff653ab",
9
+ "7764671": "484f8ce03e30cdf3e0b97b2eba9c83acca30b697cf2da75a385dafe97cadf0e9",
10
+ "m3-final-wheel": "1293520e3d0ab30819fc5023afad662623573c6254de91b6b6250e5e91fda90d",
11
+ "b9ab589/8cdf116": "460d59c68d7a0dbc2e8633f74615e9da273e0ee6acf371c50f719242e2c7cc76"
12
+ }
13
+ }