rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,383 @@
1
+ """Read-only, text-only Inspect evaluation logs for cross-dump comparison."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import copy
6
+ import hashlib
7
+ import importlib
8
+ import json
9
+ import math
10
+ import os
11
+ import stat
12
+ import zipfile
13
+ from importlib.metadata import version
14
+ from io import BytesIO
15
+ from pathlib import Path
16
+
17
+ import pandas as pd
18
+
19
+ from .schema import CANONICAL_COLUMNS, validate_rollouts
20
+
21
+ _INSPECT_VERSION = "0.3.290"
22
+ _MAX_INPUT_BYTES = 64 * 1024 * 1024
23
+ _MAX_ARCHIVE_BYTES = 256 * 1024 * 1024
24
+ _MAX_ARCHIVE_MEMBERS = 100_000
25
+ _MESSAGE_FIELDS = {
26
+ "role",
27
+ "content",
28
+ "id",
29
+ "source",
30
+ "metadata",
31
+ "tool_calls",
32
+ "model",
33
+ "tool_call_id",
34
+ "function",
35
+ "error",
36
+ }
37
+
38
+
39
+ def _canonical(value):
40
+ try:
41
+ return json.dumps(
42
+ value,
43
+ ensure_ascii=False,
44
+ sort_keys=True,
45
+ separators=(",", ":"),
46
+ allow_nan=False,
47
+ )
48
+ except (ValueError, TypeError, OverflowError, RecursionError):
49
+ raise ValueError("Inspect messages must contain finite JSON values") from None
50
+
51
+
52
+ def _object(pairs):
53
+ value = {}
54
+ for key, item in pairs:
55
+ if key in value:
56
+ raise ValueError("duplicate Inspect JSON member")
57
+ value[key] = item
58
+ return value
59
+
60
+
61
+ def _local_bytes(path):
62
+ if "://" in str(path):
63
+ raise ValueError("Inspect input must be a local regular file")
64
+ path = Path(path)
65
+ if path.suffix not in {".json", ".eval"}:
66
+ raise ValueError("Inspect input must end in .json or .eval")
67
+ if not stat.S_ISREG(path.lstat().st_mode):
68
+ raise ValueError("Inspect input must be a regular file, without symlinks")
69
+ flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | os.O_NONBLOCK
70
+ with os.fdopen(os.open(path, flags), "rb") as stream:
71
+ before = os.fstat(stream.fileno())
72
+ if not stat.S_ISREG(before.st_mode):
73
+ raise ValueError("Inspect input must be a regular file")
74
+ if before.st_size > _MAX_INPUT_BYTES:
75
+ raise ValueError("Inspect input exceeds the supported file size")
76
+ data = stream.read(_MAX_INPUT_BYTES + 1)
77
+ after = os.fstat(stream.fileno())
78
+ if len(data) > _MAX_INPUT_BYTES:
79
+ raise ValueError("Inspect input exceeds the supported file size")
80
+ if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (
81
+ after.st_size,
82
+ after.st_mtime_ns,
83
+ after.st_ctime_ns,
84
+ ):
85
+ raise ValueError("Inspect source changed while loading")
86
+ return path, data
87
+
88
+
89
+ def _native(data):
90
+ try:
91
+ with zipfile.ZipFile(BytesIO(data)) as archive:
92
+ members = archive.infolist()
93
+ names = [member.filename for member in members]
94
+ if len(names) != len(set(names)):
95
+ raise ValueError("duplicate Inspect archive member")
96
+ if (
97
+ len(members) > _MAX_ARCHIVE_MEMBERS
98
+ or sum(member.file_size for member in members) > _MAX_ARCHIVE_BYTES
99
+ ):
100
+ raise ValueError("Inspect archive exceeds the supported decoded size")
101
+ if any(member.flag_bits & 1 for member in members):
102
+ raise ValueError("encrypted Inspect archives are unsupported")
103
+ except zipfile.BadZipFile:
104
+ raise ValueError("invalid Inspect archive") from None
105
+ try:
106
+ reader = importlib.import_module("inspect_ai.log")
107
+ except ImportError:
108
+ raise ValueError(
109
+ "native .eval input requires the optional rolloutdiff[inspect] installation"
110
+ ) from None
111
+ reader_version = version("inspect_ai")
112
+ if reader_version != _INSPECT_VERSION:
113
+ raise ValueError(f"native .eval input requires inspect-ai=={_INSPECT_VERSION}")
114
+ try:
115
+ # Resolve used content below: the native core walker treats string and
116
+ # block-form messages differently when resolving tool arguments.
117
+ value = reader.read_eval_log(
118
+ BytesIO(data), format="eval", resolve_attachments=False
119
+ ).model_dump(mode="json", exclude_unset=True, exclude_none=True)
120
+ except Exception: # noqa: BLE001 - reader diagnostics can include private source text.
121
+ # Framework errors can quote private samples or task configuration.
122
+ raise ValueError("Inspect reader could not decode the native log") from None
123
+ return value, reader_version
124
+
125
+
126
+ def _resolve_attachments(value, attachments):
127
+ def resolve(item, active):
128
+ if isinstance(item, str) and item.startswith(("attachment://", "tc://")):
129
+ key = item.split("://", 1)[1]
130
+ if key in active:
131
+ raise ValueError("cyclic Inspect content attachment")
132
+ replacement = attachments.get(key)
133
+ if not key or not isinstance(replacement, str):
134
+ raise ValueError("unresolved Inspect content attachment")
135
+ return resolve(replacement, active | {key})
136
+ if isinstance(item, list):
137
+ return [resolve(child, active) for child in item]
138
+ if isinstance(item, dict):
139
+ return {key: resolve(child, active) for key, child in item.items()}
140
+ return item
141
+
142
+ try:
143
+ return resolve(value, set())
144
+ except RecursionError:
145
+ raise ValueError("Inspect attachment nesting is too deep") from None
146
+
147
+
148
+ def _text(content, attachments):
149
+ if isinstance(content, str):
150
+ content = _resolve_attachments(content, attachments)
151
+ return content, content
152
+ if not isinstance(content, list):
153
+ raise ValueError("Inspect message content must be text")
154
+ blocks, texts = [], []
155
+ for block in content:
156
+ if not isinstance(block, dict) or block.get("type") != "text":
157
+ raise ValueError("only Inspect text content blocks are supported")
158
+ if not isinstance(block.get("text"), str):
159
+ raise ValueError("Inspect text blocks require text")
160
+ text, _ = _text(block["text"], attachments)
161
+ normalized = {key: value for key, value in block.items() if value is not None}
162
+ normalized["text"] = text
163
+ blocks.append(normalized)
164
+ texts.append(text)
165
+ return "\n".join(texts), blocks
166
+
167
+
168
+ def _call(call, attachments):
169
+ if not isinstance(call, dict):
170
+ raise ValueError("invalid Inspect tool call")
171
+ if any(
172
+ not isinstance(call.get(key), str) or not call[key].strip()
173
+ for key in ("id", "function")
174
+ ) or not isinstance(call.get("arguments"), dict):
175
+ raise ValueError("Inspect calls require id, function and argument object")
176
+ if call.get("parse_error"):
177
+ raise ValueError("Inspect log contains a tool call parsing error")
178
+ result = {key: value for key, value in call.items() if value is not None}
179
+ result["arguments"] = _resolve_attachments(call["arguments"], attachments)
180
+ result["type"] = call.get("type") or "function"
181
+ if result["type"] not in {"function", "custom"}:
182
+ raise ValueError("unsupported Inspect tool call type")
183
+ _canonical(result)
184
+ return result
185
+
186
+
187
+ def _messages(sample):
188
+ messages = sample.get("messages")
189
+ if not isinstance(messages, list) or not messages:
190
+ raise ValueError("Inspect sample requires a recorded conversation")
191
+ supplied_input = sample.get("input")
192
+ if isinstance(supplied_input, list) and any(
193
+ not isinstance(message, dict) or message.get("role") == "assistant"
194
+ for message in supplied_input
195
+ ):
196
+ raise ValueError("assistant input requires an explicit generation boundary")
197
+ attachments = sample.get("attachments") or {}
198
+ if not isinstance(attachments, dict):
199
+ raise ValueError("invalid Inspect attachments")
200
+ result, texts = [], []
201
+ for message in messages:
202
+ if not isinstance(message, dict) or message.get("role") not in {
203
+ "system",
204
+ "user",
205
+ "assistant",
206
+ "tool",
207
+ }:
208
+ raise ValueError("invalid Inspect chat message")
209
+ if message["role"] == "assistant" and message.get("source") == "input":
210
+ raise ValueError("assistant input requires an explicit generation boundary")
211
+ if set(message) - _MESSAGE_FIELDS:
212
+ raise ValueError("unsupported Inspect message field")
213
+ normalized = {
214
+ key: copy.deepcopy(value)
215
+ for key, value in message.items()
216
+ if value is not None and key != "tool_calls"
217
+ }
218
+ text, normalized["content"] = _text(message.get("content"), attachments)
219
+ calls = message.get("tool_calls")
220
+ if calls is not None:
221
+ if not isinstance(calls, list) or message["role"] != "assistant":
222
+ raise ValueError("Inspect calls require an assistant message list")
223
+ normalized["tool_calls"] = [_call(call, attachments) for call in calls]
224
+ _canonical(normalized)
225
+ result.append(normalized)
226
+ texts.append(text)
227
+ boundary = next(
228
+ (i for i, message in enumerate(result) if message["role"] == "assistant"), None
229
+ )
230
+ if (
231
+ boundary is None
232
+ or boundary == 0
233
+ or any(
234
+ message["role"] not in {"system", "user"} for message in result[:boundary]
235
+ )
236
+ ):
237
+ raise ValueError(
238
+ "Inspect conversation requires an initial prompt and assistant response"
239
+ )
240
+ return result, texts, boundary
241
+
242
+
243
+ def _reward(sample, score):
244
+ if score is None:
245
+ return None
246
+ scores = sample.get("scores")
247
+ if scores is None:
248
+ return None
249
+ if not isinstance(scores, dict):
250
+ raise ValueError("Inspect sample scores must be an object")
251
+ if score not in scores:
252
+ return None
253
+ selected = scores[score]
254
+ if not isinstance(selected, dict):
255
+ raise ValueError("selected Inspect score requires a finite numeric scalar")
256
+ value = selected.get("value")
257
+ if value is None:
258
+ return None
259
+ if type(value) not in {int, float}:
260
+ raise ValueError("selected Inspect score requires a finite numeric scalar")
261
+ try:
262
+ value = float(value)
263
+ except OverflowError:
264
+ raise ValueError(
265
+ "selected Inspect score requires a finite numeric scalar"
266
+ ) from None
267
+ if not math.isfinite(value):
268
+ raise ValueError("selected Inspect score requires a finite numeric scalar")
269
+ return value
270
+
271
+
272
+ def load_inspect(
273
+ path: str | Path, *, run: str, score: str | None = None
274
+ ) -> pd.DataFrame:
275
+ """Read completed text-only logs, preserving unknown training steps and rewards.
276
+
277
+ Scores are ignored unless selected explicitly as finite numeric scalars.
278
+ ``response`` projects assistant turns; ``response_messages`` retains the full
279
+ conversation, and ``attrs['inspect']`` records local source provenance.
280
+ Native .eval logs require the pinned optional Inspect reader.
281
+ """
282
+ if score is not None and (not isinstance(score, str) or not score.strip()):
283
+ raise ValueError("Inspect score must be a nonblank scorer name")
284
+ path, data = _local_bytes(path)
285
+ reader_version = None
286
+ if path.suffix == ".eval":
287
+ value, reader_version = _native(data)
288
+ else:
289
+ try:
290
+ value = json.loads(data, object_pairs_hook=_object)
291
+ except (UnicodeError, json.JSONDecodeError, RecursionError):
292
+ raise ValueError("invalid Inspect JSON log") from None
293
+ if (
294
+ not isinstance(value, dict)
295
+ or type(value.get("version")) is not int
296
+ or value["version"] != 2
297
+ ):
298
+ raise ValueError("only Inspect log schema version 2 is supported")
299
+ if value.get("status") != "success" or value.get("error") is not None:
300
+ raise ValueError("Inspect log must be completed successfully")
301
+ samples = value.get("samples")
302
+ if not isinstance(samples, list) or not samples:
303
+ raise ValueError("Inspect log must contain recorded samples")
304
+ rows, raw_messages, locators = [], {}, {}
305
+ for position, sample in enumerate(samples):
306
+ try:
307
+ if not isinstance(sample, dict):
308
+ raise ValueError("Inspect sample must be an object")
309
+ identity, epoch = sample.get("id"), sample.get("epoch")
310
+ if type(identity) not in {int, str} or (
311
+ isinstance(identity, str) and not identity.strip()
312
+ ):
313
+ raise ValueError(
314
+ "Inspect sample requires an integer or nonblank string id"
315
+ )
316
+ if type(epoch) is not int or epoch < 1:
317
+ raise ValueError("Inspect sample requires a positive epoch")
318
+ if (
319
+ sample.get("error") is not None
320
+ or sample.get("invalidation") is not None
321
+ ):
322
+ raise ValueError(
323
+ "errored or invalidated Inspect samples are unsupported"
324
+ )
325
+ sample_id = "inspect:" + _canonical([identity, epoch])
326
+ if sample_id in locators:
327
+ raise ValueError("duplicate Inspect sample id and epoch")
328
+ messages, texts, boundary = _messages(sample)
329
+ prompt_messages = messages[:boundary]
330
+ prompt = "\n".join(texts[:boundary])
331
+ response = "\n".join(
332
+ text
333
+ for text, message in zip(texts, messages, strict=True)
334
+ if message["role"] == "assistant"
335
+ )
336
+ calls = [
337
+ call
338
+ for message in messages
339
+ if message["role"] == "assistant"
340
+ for call in message.get("tool_calls", [])
341
+ ]
342
+ rows.append(
343
+ {
344
+ "run": run,
345
+ "step": None,
346
+ "sample_id": sample_id,
347
+ "prompt_id": hashlib.sha256(
348
+ _canonical(["chat", prompt_messages]).encode()
349
+ ).hexdigest(),
350
+ "prompt": prompt,
351
+ "response": response,
352
+ "reward": _reward(sample, score),
353
+ "prompt_messages": prompt_messages,
354
+ "response_messages": messages,
355
+ "tool_calls": calls,
356
+ }
357
+ )
358
+ raw_messages[sample_id] = copy.deepcopy(sample["messages"])
359
+ locators[sample_id] = {
360
+ "id": identity,
361
+ "epoch": epoch,
362
+ "uuid": sample.get("uuid"),
363
+ }
364
+ except ValueError as exc:
365
+ raise ValueError(f"Inspect sample {position + 1}: {exc}") from None
366
+ frame = pd.DataFrame(rows, columns=CANONICAL_COLUMNS, dtype=object)
367
+ out = validate_rollouts(frame, allow_unavailable=True)
368
+ out = out.sort_values("sample_id", kind="stable").reset_index(drop=True)
369
+ evaluation = value.get("eval")
370
+ evaluation = evaluation if isinstance(evaluation, dict) else {}
371
+ out.attrs["inspect"] = {
372
+ "schema_version": 2,
373
+ "reader_version": reader_version,
374
+ "file_sha256": hashlib.sha256(data).hexdigest(),
375
+ "size_bytes": len(data),
376
+ "score": score,
377
+ "training_steps": "unavailable",
378
+ "missing_rewards": sum(row["reward"] is None for row in rows),
379
+ "model": evaluation.get("model"),
380
+ "sample_locators": locators,
381
+ "raw_messages": raw_messages,
382
+ }
383
+ return out
@@ -0,0 +1,207 @@
1
+ """Resolve report exemplar references for explicitly requested local inspection."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import pandas as pd
6
+
7
+ from ._report_validation import validate_report
8
+ from .evidence import _reward_association_prepared
9
+ from .features import DENSE_BINARY, extract
10
+ from .schema import validate_rollouts
11
+
12
+
13
+ def _unavailable_comparison(reason: str) -> dict:
14
+ return {
15
+ "status": "unavailable",
16
+ "reason": reason,
17
+ "n_rollouts": None,
18
+ "n_present": None,
19
+ "n_absent": None,
20
+ "reward_present": None,
21
+ "reward_absent": None,
22
+ "difference": None,
23
+ "counterpart": None,
24
+ }
25
+
26
+
27
+ def _comparisons(event: dict, lookup: dict) -> None:
28
+ examples = [
29
+ example for member in event["members"] for example in member["exemplars"]
30
+ ]
31
+ groups = {
32
+ (example["run"], example["step"], example["prompt_id"]): []
33
+ for example in examples
34
+ if example["status"] == "selected" and example["kind"] == "binary"
35
+ }
36
+ for row in lookup.values():
37
+ key = row["run"], row["step"], row["prompt_id"]
38
+ if key in groups:
39
+ groups[key].append(row)
40
+ extracted = {}
41
+ for example in examples:
42
+ if example["status"] != "selected" or example["kind"] != "binary":
43
+ reason = (
44
+ example["status"]
45
+ if example["status"] != "selected"
46
+ else "unsupported_kind"
47
+ )
48
+ example["comparison"] = _unavailable_comparison(reason)
49
+ continue
50
+ group = sorted(
51
+ groups[(example["run"], example["step"], example["prompt_id"])],
52
+ key=lambda row: row["sample_id"],
53
+ )
54
+ expected_size = example["reward_group_size"]
55
+ if expected_size is not None and len(group) != expected_size:
56
+ raise ValueError("selected exemplar group size does not match rollouts")
57
+ feature, values = example["feature"], []
58
+ for row in group:
59
+ key = row["run"], row["sample_id"]
60
+ if key not in extracted:
61
+ extracted[key] = extract(row["response"])
62
+ features = extracted[key]
63
+ if feature == "defines_test_named":
64
+ # Historical reports use substring matching on parsed definitions.
65
+ value = float(
66
+ any(
67
+ name.startswith("def:") and "test" in name[4:].lower()
68
+ for name in features.sparse
69
+ )
70
+ )
71
+ elif feature in features.dense:
72
+ if feature not in DENSE_BINARY:
73
+ raise ValueError(
74
+ "reported binary feature is not binary in extractor"
75
+ )
76
+ value = features.dense[feature]
77
+ else:
78
+ value = float(feature in features.sparse)
79
+ if row["sample_id"] == example["sample_id"] and (
80
+ example["feature_value"] != 1 or value != 1
81
+ ):
82
+ raise ValueError(
83
+ "selected exemplar feature does not match extracted presence"
84
+ )
85
+ values.append(value)
86
+ frame = pd.DataFrame(
87
+ group, columns=["run", "step", "prompt_id", "sample_id", "reward"]
88
+ )
89
+ frame["_value"] = values
90
+ association = _reward_association_prepared(
91
+ frame,
92
+ "binary",
93
+ run=example["run"],
94
+ feature=feature,
95
+ window=(example["step"], example["step"]),
96
+ )
97
+ stats = association.groups.to_dict("records")[0]
98
+ comparison = _unavailable_comparison("no_absent_completion")
99
+ for name in (
100
+ "n_rollouts",
101
+ "n_present",
102
+ "n_absent",
103
+ "reward_present",
104
+ "reward_absent",
105
+ "difference",
106
+ ):
107
+ comparison[name] = None if pd.isna(stats[name]) else stats[name]
108
+ absent = next(
109
+ (row for row, value in zip(group, values, strict=True) if value == 0), None
110
+ )
111
+ if absent is not None:
112
+ comparison.update(
113
+ status="available",
114
+ reason=None,
115
+ counterpart={
116
+ **{
117
+ name: absent[name]
118
+ for name in (
119
+ "run",
120
+ "sample_id",
121
+ "step",
122
+ "prompt_id",
123
+ "reward",
124
+ "response",
125
+ )
126
+ },
127
+ "feature_value": 0.0,
128
+ },
129
+ )
130
+ example["comparison"] = comparison
131
+
132
+
133
+ def resolve_exemplars(
134
+ report: dict,
135
+ rollouts: pd.DataFrame,
136
+ *,
137
+ run: str,
138
+ event_rank: int,
139
+ with_absent: bool = False,
140
+ ) -> dict:
141
+ """Return an event copy with each exemplar's response, or None if unselected.
142
+
143
+ Resolve exact (run, sample_id) keys and require identical step, prompt_id,
144
+ and reward. Missing references, duplicates, or mismatches raise ValueError.
145
+ Members and exemplars follow report ordering rules; missing lists and
146
+ unsupported statuses remain intact. With with_absent, verify extracted
147
+ binary presence and add comparison counts/means over complete prompt-step
148
+ groups, choosing the absent counterpart by lexically smallest sample ID. Supply all
149
+ group completions; saved group sizes are checked when available. Comparisons
150
+ are descriptive. Inputs are unchanged; version 1 has no response fingerprint.
151
+ Historical defines_test_named uses substring matching for the entire group.
152
+ """
153
+ if type(with_absent) is not bool:
154
+ raise ValueError("with_absent must be a boolean")
155
+ if not isinstance(run, str) or not run.strip():
156
+ raise ValueError("run must be a nonempty string")
157
+ if type(event_rank) is not int or event_rank < 1:
158
+ raise ValueError("event_rank must be a positive integer")
159
+ report = validate_report(report)
160
+ selected_run = next((item for item in report["runs"] if item["run"] == run), None)
161
+ if selected_run is None:
162
+ raise ValueError("requested run is not present in report")
163
+ event = next(
164
+ (item for item in selected_run["events"] if item["rank"] == event_rank), None
165
+ )
166
+ if event is None:
167
+ raise ValueError("requested event rank is not present in report run")
168
+ if not isinstance(rollouts, pd.DataFrame):
169
+ raise ValueError("rollouts must be a DataFrame")
170
+ canonical = validate_rollouts(rollouts)
171
+ # Python scalar comparisons avoid NumPy rounding a large integer report reward.
172
+ lookup = {
173
+ (row["run"], row["sample_id"]): row for row in canonical.to_dict("records")
174
+ }
175
+ event["members"].sort(
176
+ key=lambda member: (
177
+ member["onset_step"],
178
+ member["alert_step"],
179
+ member["feature"],
180
+ )
181
+ )
182
+ for member in event["members"]:
183
+ member["exemplars"].sort(
184
+ key=lambda example: (
185
+ example["selection_order"] is None,
186
+ example["selection_order"] or 0,
187
+ example["sample_id"] or "",
188
+ )
189
+ )
190
+ for example in member["exemplars"]:
191
+ example["response"] = None
192
+ if example["status"] != "selected":
193
+ continue
194
+ if example["reward"] is None:
195
+ raise ValueError("selected exemplar reward is unavailable")
196
+ rollout = lookup.get((example["run"], example["sample_id"]))
197
+ if rollout is None:
198
+ raise ValueError("selected exemplar reference is missing from rollouts")
199
+ for field in ("step", "prompt_id", "reward"):
200
+ if example[field] != rollout[field]:
201
+ raise ValueError(
202
+ f"selected exemplar {field} does not match rollout"
203
+ )
204
+ example["response"] = rollout["response"]
205
+ if with_absent:
206
+ _comparisons(event, lookup)
207
+ return event