rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,383 @@
|
|
|
1
|
+
"""Read-only, text-only Inspect evaluation logs for cross-dump comparison."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import copy
|
|
6
|
+
import hashlib
|
|
7
|
+
import importlib
|
|
8
|
+
import json
|
|
9
|
+
import math
|
|
10
|
+
import os
|
|
11
|
+
import stat
|
|
12
|
+
import zipfile
|
|
13
|
+
from importlib.metadata import version
|
|
14
|
+
from io import BytesIO
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
import pandas as pd
|
|
18
|
+
|
|
19
|
+
from .schema import CANONICAL_COLUMNS, validate_rollouts
|
|
20
|
+
|
|
21
|
+
_INSPECT_VERSION = "0.3.290"
|
|
22
|
+
_MAX_INPUT_BYTES = 64 * 1024 * 1024
|
|
23
|
+
_MAX_ARCHIVE_BYTES = 256 * 1024 * 1024
|
|
24
|
+
_MAX_ARCHIVE_MEMBERS = 100_000
|
|
25
|
+
_MESSAGE_FIELDS = {
|
|
26
|
+
"role",
|
|
27
|
+
"content",
|
|
28
|
+
"id",
|
|
29
|
+
"source",
|
|
30
|
+
"metadata",
|
|
31
|
+
"tool_calls",
|
|
32
|
+
"model",
|
|
33
|
+
"tool_call_id",
|
|
34
|
+
"function",
|
|
35
|
+
"error",
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _canonical(value):
|
|
40
|
+
try:
|
|
41
|
+
return json.dumps(
|
|
42
|
+
value,
|
|
43
|
+
ensure_ascii=False,
|
|
44
|
+
sort_keys=True,
|
|
45
|
+
separators=(",", ":"),
|
|
46
|
+
allow_nan=False,
|
|
47
|
+
)
|
|
48
|
+
except (ValueError, TypeError, OverflowError, RecursionError):
|
|
49
|
+
raise ValueError("Inspect messages must contain finite JSON values") from None
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _object(pairs):
|
|
53
|
+
value = {}
|
|
54
|
+
for key, item in pairs:
|
|
55
|
+
if key in value:
|
|
56
|
+
raise ValueError("duplicate Inspect JSON member")
|
|
57
|
+
value[key] = item
|
|
58
|
+
return value
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _local_bytes(path):
|
|
62
|
+
if "://" in str(path):
|
|
63
|
+
raise ValueError("Inspect input must be a local regular file")
|
|
64
|
+
path = Path(path)
|
|
65
|
+
if path.suffix not in {".json", ".eval"}:
|
|
66
|
+
raise ValueError("Inspect input must end in .json or .eval")
|
|
67
|
+
if not stat.S_ISREG(path.lstat().st_mode):
|
|
68
|
+
raise ValueError("Inspect input must be a regular file, without symlinks")
|
|
69
|
+
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | os.O_NONBLOCK
|
|
70
|
+
with os.fdopen(os.open(path, flags), "rb") as stream:
|
|
71
|
+
before = os.fstat(stream.fileno())
|
|
72
|
+
if not stat.S_ISREG(before.st_mode):
|
|
73
|
+
raise ValueError("Inspect input must be a regular file")
|
|
74
|
+
if before.st_size > _MAX_INPUT_BYTES:
|
|
75
|
+
raise ValueError("Inspect input exceeds the supported file size")
|
|
76
|
+
data = stream.read(_MAX_INPUT_BYTES + 1)
|
|
77
|
+
after = os.fstat(stream.fileno())
|
|
78
|
+
if len(data) > _MAX_INPUT_BYTES:
|
|
79
|
+
raise ValueError("Inspect input exceeds the supported file size")
|
|
80
|
+
if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (
|
|
81
|
+
after.st_size,
|
|
82
|
+
after.st_mtime_ns,
|
|
83
|
+
after.st_ctime_ns,
|
|
84
|
+
):
|
|
85
|
+
raise ValueError("Inspect source changed while loading")
|
|
86
|
+
return path, data
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def _native(data):
|
|
90
|
+
try:
|
|
91
|
+
with zipfile.ZipFile(BytesIO(data)) as archive:
|
|
92
|
+
members = archive.infolist()
|
|
93
|
+
names = [member.filename for member in members]
|
|
94
|
+
if len(names) != len(set(names)):
|
|
95
|
+
raise ValueError("duplicate Inspect archive member")
|
|
96
|
+
if (
|
|
97
|
+
len(members) > _MAX_ARCHIVE_MEMBERS
|
|
98
|
+
or sum(member.file_size for member in members) > _MAX_ARCHIVE_BYTES
|
|
99
|
+
):
|
|
100
|
+
raise ValueError("Inspect archive exceeds the supported decoded size")
|
|
101
|
+
if any(member.flag_bits & 1 for member in members):
|
|
102
|
+
raise ValueError("encrypted Inspect archives are unsupported")
|
|
103
|
+
except zipfile.BadZipFile:
|
|
104
|
+
raise ValueError("invalid Inspect archive") from None
|
|
105
|
+
try:
|
|
106
|
+
reader = importlib.import_module("inspect_ai.log")
|
|
107
|
+
except ImportError:
|
|
108
|
+
raise ValueError(
|
|
109
|
+
"native .eval input requires the optional rolloutdiff[inspect] installation"
|
|
110
|
+
) from None
|
|
111
|
+
reader_version = version("inspect_ai")
|
|
112
|
+
if reader_version != _INSPECT_VERSION:
|
|
113
|
+
raise ValueError(f"native .eval input requires inspect-ai=={_INSPECT_VERSION}")
|
|
114
|
+
try:
|
|
115
|
+
# Resolve used content below: the native core walker treats string and
|
|
116
|
+
# block-form messages differently when resolving tool arguments.
|
|
117
|
+
value = reader.read_eval_log(
|
|
118
|
+
BytesIO(data), format="eval", resolve_attachments=False
|
|
119
|
+
).model_dump(mode="json", exclude_unset=True, exclude_none=True)
|
|
120
|
+
except Exception: # noqa: BLE001 - reader diagnostics can include private source text.
|
|
121
|
+
# Framework errors can quote private samples or task configuration.
|
|
122
|
+
raise ValueError("Inspect reader could not decode the native log") from None
|
|
123
|
+
return value, reader_version
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _resolve_attachments(value, attachments):
|
|
127
|
+
def resolve(item, active):
|
|
128
|
+
if isinstance(item, str) and item.startswith(("attachment://", "tc://")):
|
|
129
|
+
key = item.split("://", 1)[1]
|
|
130
|
+
if key in active:
|
|
131
|
+
raise ValueError("cyclic Inspect content attachment")
|
|
132
|
+
replacement = attachments.get(key)
|
|
133
|
+
if not key or not isinstance(replacement, str):
|
|
134
|
+
raise ValueError("unresolved Inspect content attachment")
|
|
135
|
+
return resolve(replacement, active | {key})
|
|
136
|
+
if isinstance(item, list):
|
|
137
|
+
return [resolve(child, active) for child in item]
|
|
138
|
+
if isinstance(item, dict):
|
|
139
|
+
return {key: resolve(child, active) for key, child in item.items()}
|
|
140
|
+
return item
|
|
141
|
+
|
|
142
|
+
try:
|
|
143
|
+
return resolve(value, set())
|
|
144
|
+
except RecursionError:
|
|
145
|
+
raise ValueError("Inspect attachment nesting is too deep") from None
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _text(content, attachments):
|
|
149
|
+
if isinstance(content, str):
|
|
150
|
+
content = _resolve_attachments(content, attachments)
|
|
151
|
+
return content, content
|
|
152
|
+
if not isinstance(content, list):
|
|
153
|
+
raise ValueError("Inspect message content must be text")
|
|
154
|
+
blocks, texts = [], []
|
|
155
|
+
for block in content:
|
|
156
|
+
if not isinstance(block, dict) or block.get("type") != "text":
|
|
157
|
+
raise ValueError("only Inspect text content blocks are supported")
|
|
158
|
+
if not isinstance(block.get("text"), str):
|
|
159
|
+
raise ValueError("Inspect text blocks require text")
|
|
160
|
+
text, _ = _text(block["text"], attachments)
|
|
161
|
+
normalized = {key: value for key, value in block.items() if value is not None}
|
|
162
|
+
normalized["text"] = text
|
|
163
|
+
blocks.append(normalized)
|
|
164
|
+
texts.append(text)
|
|
165
|
+
return "\n".join(texts), blocks
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def _call(call, attachments):
|
|
169
|
+
if not isinstance(call, dict):
|
|
170
|
+
raise ValueError("invalid Inspect tool call")
|
|
171
|
+
if any(
|
|
172
|
+
not isinstance(call.get(key), str) or not call[key].strip()
|
|
173
|
+
for key in ("id", "function")
|
|
174
|
+
) or not isinstance(call.get("arguments"), dict):
|
|
175
|
+
raise ValueError("Inspect calls require id, function and argument object")
|
|
176
|
+
if call.get("parse_error"):
|
|
177
|
+
raise ValueError("Inspect log contains a tool call parsing error")
|
|
178
|
+
result = {key: value for key, value in call.items() if value is not None}
|
|
179
|
+
result["arguments"] = _resolve_attachments(call["arguments"], attachments)
|
|
180
|
+
result["type"] = call.get("type") or "function"
|
|
181
|
+
if result["type"] not in {"function", "custom"}:
|
|
182
|
+
raise ValueError("unsupported Inspect tool call type")
|
|
183
|
+
_canonical(result)
|
|
184
|
+
return result
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _messages(sample):
|
|
188
|
+
messages = sample.get("messages")
|
|
189
|
+
if not isinstance(messages, list) or not messages:
|
|
190
|
+
raise ValueError("Inspect sample requires a recorded conversation")
|
|
191
|
+
supplied_input = sample.get("input")
|
|
192
|
+
if isinstance(supplied_input, list) and any(
|
|
193
|
+
not isinstance(message, dict) or message.get("role") == "assistant"
|
|
194
|
+
for message in supplied_input
|
|
195
|
+
):
|
|
196
|
+
raise ValueError("assistant input requires an explicit generation boundary")
|
|
197
|
+
attachments = sample.get("attachments") or {}
|
|
198
|
+
if not isinstance(attachments, dict):
|
|
199
|
+
raise ValueError("invalid Inspect attachments")
|
|
200
|
+
result, texts = [], []
|
|
201
|
+
for message in messages:
|
|
202
|
+
if not isinstance(message, dict) or message.get("role") not in {
|
|
203
|
+
"system",
|
|
204
|
+
"user",
|
|
205
|
+
"assistant",
|
|
206
|
+
"tool",
|
|
207
|
+
}:
|
|
208
|
+
raise ValueError("invalid Inspect chat message")
|
|
209
|
+
if message["role"] == "assistant" and message.get("source") == "input":
|
|
210
|
+
raise ValueError("assistant input requires an explicit generation boundary")
|
|
211
|
+
if set(message) - _MESSAGE_FIELDS:
|
|
212
|
+
raise ValueError("unsupported Inspect message field")
|
|
213
|
+
normalized = {
|
|
214
|
+
key: copy.deepcopy(value)
|
|
215
|
+
for key, value in message.items()
|
|
216
|
+
if value is not None and key != "tool_calls"
|
|
217
|
+
}
|
|
218
|
+
text, normalized["content"] = _text(message.get("content"), attachments)
|
|
219
|
+
calls = message.get("tool_calls")
|
|
220
|
+
if calls is not None:
|
|
221
|
+
if not isinstance(calls, list) or message["role"] != "assistant":
|
|
222
|
+
raise ValueError("Inspect calls require an assistant message list")
|
|
223
|
+
normalized["tool_calls"] = [_call(call, attachments) for call in calls]
|
|
224
|
+
_canonical(normalized)
|
|
225
|
+
result.append(normalized)
|
|
226
|
+
texts.append(text)
|
|
227
|
+
boundary = next(
|
|
228
|
+
(i for i, message in enumerate(result) if message["role"] == "assistant"), None
|
|
229
|
+
)
|
|
230
|
+
if (
|
|
231
|
+
boundary is None
|
|
232
|
+
or boundary == 0
|
|
233
|
+
or any(
|
|
234
|
+
message["role"] not in {"system", "user"} for message in result[:boundary]
|
|
235
|
+
)
|
|
236
|
+
):
|
|
237
|
+
raise ValueError(
|
|
238
|
+
"Inspect conversation requires an initial prompt and assistant response"
|
|
239
|
+
)
|
|
240
|
+
return result, texts, boundary
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def _reward(sample, score):
|
|
244
|
+
if score is None:
|
|
245
|
+
return None
|
|
246
|
+
scores = sample.get("scores")
|
|
247
|
+
if scores is None:
|
|
248
|
+
return None
|
|
249
|
+
if not isinstance(scores, dict):
|
|
250
|
+
raise ValueError("Inspect sample scores must be an object")
|
|
251
|
+
if score not in scores:
|
|
252
|
+
return None
|
|
253
|
+
selected = scores[score]
|
|
254
|
+
if not isinstance(selected, dict):
|
|
255
|
+
raise ValueError("selected Inspect score requires a finite numeric scalar")
|
|
256
|
+
value = selected.get("value")
|
|
257
|
+
if value is None:
|
|
258
|
+
return None
|
|
259
|
+
if type(value) not in {int, float}:
|
|
260
|
+
raise ValueError("selected Inspect score requires a finite numeric scalar")
|
|
261
|
+
try:
|
|
262
|
+
value = float(value)
|
|
263
|
+
except OverflowError:
|
|
264
|
+
raise ValueError(
|
|
265
|
+
"selected Inspect score requires a finite numeric scalar"
|
|
266
|
+
) from None
|
|
267
|
+
if not math.isfinite(value):
|
|
268
|
+
raise ValueError("selected Inspect score requires a finite numeric scalar")
|
|
269
|
+
return value
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def load_inspect(
|
|
273
|
+
path: str | Path, *, run: str, score: str | None = None
|
|
274
|
+
) -> pd.DataFrame:
|
|
275
|
+
"""Read completed text-only logs, preserving unknown training steps and rewards.
|
|
276
|
+
|
|
277
|
+
Scores are ignored unless selected explicitly as finite numeric scalars.
|
|
278
|
+
``response`` projects assistant turns; ``response_messages`` retains the full
|
|
279
|
+
conversation, and ``attrs['inspect']`` records local source provenance.
|
|
280
|
+
Native .eval logs require the pinned optional Inspect reader.
|
|
281
|
+
"""
|
|
282
|
+
if score is not None and (not isinstance(score, str) or not score.strip()):
|
|
283
|
+
raise ValueError("Inspect score must be a nonblank scorer name")
|
|
284
|
+
path, data = _local_bytes(path)
|
|
285
|
+
reader_version = None
|
|
286
|
+
if path.suffix == ".eval":
|
|
287
|
+
value, reader_version = _native(data)
|
|
288
|
+
else:
|
|
289
|
+
try:
|
|
290
|
+
value = json.loads(data, object_pairs_hook=_object)
|
|
291
|
+
except (UnicodeError, json.JSONDecodeError, RecursionError):
|
|
292
|
+
raise ValueError("invalid Inspect JSON log") from None
|
|
293
|
+
if (
|
|
294
|
+
not isinstance(value, dict)
|
|
295
|
+
or type(value.get("version")) is not int
|
|
296
|
+
or value["version"] != 2
|
|
297
|
+
):
|
|
298
|
+
raise ValueError("only Inspect log schema version 2 is supported")
|
|
299
|
+
if value.get("status") != "success" or value.get("error") is not None:
|
|
300
|
+
raise ValueError("Inspect log must be completed successfully")
|
|
301
|
+
samples = value.get("samples")
|
|
302
|
+
if not isinstance(samples, list) or not samples:
|
|
303
|
+
raise ValueError("Inspect log must contain recorded samples")
|
|
304
|
+
rows, raw_messages, locators = [], {}, {}
|
|
305
|
+
for position, sample in enumerate(samples):
|
|
306
|
+
try:
|
|
307
|
+
if not isinstance(sample, dict):
|
|
308
|
+
raise ValueError("Inspect sample must be an object")
|
|
309
|
+
identity, epoch = sample.get("id"), sample.get("epoch")
|
|
310
|
+
if type(identity) not in {int, str} or (
|
|
311
|
+
isinstance(identity, str) and not identity.strip()
|
|
312
|
+
):
|
|
313
|
+
raise ValueError(
|
|
314
|
+
"Inspect sample requires an integer or nonblank string id"
|
|
315
|
+
)
|
|
316
|
+
if type(epoch) is not int or epoch < 1:
|
|
317
|
+
raise ValueError("Inspect sample requires a positive epoch")
|
|
318
|
+
if (
|
|
319
|
+
sample.get("error") is not None
|
|
320
|
+
or sample.get("invalidation") is not None
|
|
321
|
+
):
|
|
322
|
+
raise ValueError(
|
|
323
|
+
"errored or invalidated Inspect samples are unsupported"
|
|
324
|
+
)
|
|
325
|
+
sample_id = "inspect:" + _canonical([identity, epoch])
|
|
326
|
+
if sample_id in locators:
|
|
327
|
+
raise ValueError("duplicate Inspect sample id and epoch")
|
|
328
|
+
messages, texts, boundary = _messages(sample)
|
|
329
|
+
prompt_messages = messages[:boundary]
|
|
330
|
+
prompt = "\n".join(texts[:boundary])
|
|
331
|
+
response = "\n".join(
|
|
332
|
+
text
|
|
333
|
+
for text, message in zip(texts, messages, strict=True)
|
|
334
|
+
if message["role"] == "assistant"
|
|
335
|
+
)
|
|
336
|
+
calls = [
|
|
337
|
+
call
|
|
338
|
+
for message in messages
|
|
339
|
+
if message["role"] == "assistant"
|
|
340
|
+
for call in message.get("tool_calls", [])
|
|
341
|
+
]
|
|
342
|
+
rows.append(
|
|
343
|
+
{
|
|
344
|
+
"run": run,
|
|
345
|
+
"step": None,
|
|
346
|
+
"sample_id": sample_id,
|
|
347
|
+
"prompt_id": hashlib.sha256(
|
|
348
|
+
_canonical(["chat", prompt_messages]).encode()
|
|
349
|
+
).hexdigest(),
|
|
350
|
+
"prompt": prompt,
|
|
351
|
+
"response": response,
|
|
352
|
+
"reward": _reward(sample, score),
|
|
353
|
+
"prompt_messages": prompt_messages,
|
|
354
|
+
"response_messages": messages,
|
|
355
|
+
"tool_calls": calls,
|
|
356
|
+
}
|
|
357
|
+
)
|
|
358
|
+
raw_messages[sample_id] = copy.deepcopy(sample["messages"])
|
|
359
|
+
locators[sample_id] = {
|
|
360
|
+
"id": identity,
|
|
361
|
+
"epoch": epoch,
|
|
362
|
+
"uuid": sample.get("uuid"),
|
|
363
|
+
}
|
|
364
|
+
except ValueError as exc:
|
|
365
|
+
raise ValueError(f"Inspect sample {position + 1}: {exc}") from None
|
|
366
|
+
frame = pd.DataFrame(rows, columns=CANONICAL_COLUMNS, dtype=object)
|
|
367
|
+
out = validate_rollouts(frame, allow_unavailable=True)
|
|
368
|
+
out = out.sort_values("sample_id", kind="stable").reset_index(drop=True)
|
|
369
|
+
evaluation = value.get("eval")
|
|
370
|
+
evaluation = evaluation if isinstance(evaluation, dict) else {}
|
|
371
|
+
out.attrs["inspect"] = {
|
|
372
|
+
"schema_version": 2,
|
|
373
|
+
"reader_version": reader_version,
|
|
374
|
+
"file_sha256": hashlib.sha256(data).hexdigest(),
|
|
375
|
+
"size_bytes": len(data),
|
|
376
|
+
"score": score,
|
|
377
|
+
"training_steps": "unavailable",
|
|
378
|
+
"missing_rewards": sum(row["reward"] is None for row in rows),
|
|
379
|
+
"model": evaluation.get("model"),
|
|
380
|
+
"sample_locators": locators,
|
|
381
|
+
"raw_messages": raw_messages,
|
|
382
|
+
}
|
|
383
|
+
return out
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
"""Resolve report exemplar references for explicitly requested local inspection."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pandas as pd
|
|
6
|
+
|
|
7
|
+
from ._report_validation import validate_report
|
|
8
|
+
from .evidence import _reward_association_prepared
|
|
9
|
+
from .features import DENSE_BINARY, extract
|
|
10
|
+
from .schema import validate_rollouts
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
def _unavailable_comparison(reason: str) -> dict:
|
|
14
|
+
return {
|
|
15
|
+
"status": "unavailable",
|
|
16
|
+
"reason": reason,
|
|
17
|
+
"n_rollouts": None,
|
|
18
|
+
"n_present": None,
|
|
19
|
+
"n_absent": None,
|
|
20
|
+
"reward_present": None,
|
|
21
|
+
"reward_absent": None,
|
|
22
|
+
"difference": None,
|
|
23
|
+
"counterpart": None,
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _comparisons(event: dict, lookup: dict) -> None:
|
|
28
|
+
examples = [
|
|
29
|
+
example for member in event["members"] for example in member["exemplars"]
|
|
30
|
+
]
|
|
31
|
+
groups = {
|
|
32
|
+
(example["run"], example["step"], example["prompt_id"]): []
|
|
33
|
+
for example in examples
|
|
34
|
+
if example["status"] == "selected" and example["kind"] == "binary"
|
|
35
|
+
}
|
|
36
|
+
for row in lookup.values():
|
|
37
|
+
key = row["run"], row["step"], row["prompt_id"]
|
|
38
|
+
if key in groups:
|
|
39
|
+
groups[key].append(row)
|
|
40
|
+
extracted = {}
|
|
41
|
+
for example in examples:
|
|
42
|
+
if example["status"] != "selected" or example["kind"] != "binary":
|
|
43
|
+
reason = (
|
|
44
|
+
example["status"]
|
|
45
|
+
if example["status"] != "selected"
|
|
46
|
+
else "unsupported_kind"
|
|
47
|
+
)
|
|
48
|
+
example["comparison"] = _unavailable_comparison(reason)
|
|
49
|
+
continue
|
|
50
|
+
group = sorted(
|
|
51
|
+
groups[(example["run"], example["step"], example["prompt_id"])],
|
|
52
|
+
key=lambda row: row["sample_id"],
|
|
53
|
+
)
|
|
54
|
+
expected_size = example["reward_group_size"]
|
|
55
|
+
if expected_size is not None and len(group) != expected_size:
|
|
56
|
+
raise ValueError("selected exemplar group size does not match rollouts")
|
|
57
|
+
feature, values = example["feature"], []
|
|
58
|
+
for row in group:
|
|
59
|
+
key = row["run"], row["sample_id"]
|
|
60
|
+
if key not in extracted:
|
|
61
|
+
extracted[key] = extract(row["response"])
|
|
62
|
+
features = extracted[key]
|
|
63
|
+
if feature == "defines_test_named":
|
|
64
|
+
# Historical reports use substring matching on parsed definitions.
|
|
65
|
+
value = float(
|
|
66
|
+
any(
|
|
67
|
+
name.startswith("def:") and "test" in name[4:].lower()
|
|
68
|
+
for name in features.sparse
|
|
69
|
+
)
|
|
70
|
+
)
|
|
71
|
+
elif feature in features.dense:
|
|
72
|
+
if feature not in DENSE_BINARY:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
"reported binary feature is not binary in extractor"
|
|
75
|
+
)
|
|
76
|
+
value = features.dense[feature]
|
|
77
|
+
else:
|
|
78
|
+
value = float(feature in features.sparse)
|
|
79
|
+
if row["sample_id"] == example["sample_id"] and (
|
|
80
|
+
example["feature_value"] != 1 or value != 1
|
|
81
|
+
):
|
|
82
|
+
raise ValueError(
|
|
83
|
+
"selected exemplar feature does not match extracted presence"
|
|
84
|
+
)
|
|
85
|
+
values.append(value)
|
|
86
|
+
frame = pd.DataFrame(
|
|
87
|
+
group, columns=["run", "step", "prompt_id", "sample_id", "reward"]
|
|
88
|
+
)
|
|
89
|
+
frame["_value"] = values
|
|
90
|
+
association = _reward_association_prepared(
|
|
91
|
+
frame,
|
|
92
|
+
"binary",
|
|
93
|
+
run=example["run"],
|
|
94
|
+
feature=feature,
|
|
95
|
+
window=(example["step"], example["step"]),
|
|
96
|
+
)
|
|
97
|
+
stats = association.groups.to_dict("records")[0]
|
|
98
|
+
comparison = _unavailable_comparison("no_absent_completion")
|
|
99
|
+
for name in (
|
|
100
|
+
"n_rollouts",
|
|
101
|
+
"n_present",
|
|
102
|
+
"n_absent",
|
|
103
|
+
"reward_present",
|
|
104
|
+
"reward_absent",
|
|
105
|
+
"difference",
|
|
106
|
+
):
|
|
107
|
+
comparison[name] = None if pd.isna(stats[name]) else stats[name]
|
|
108
|
+
absent = next(
|
|
109
|
+
(row for row, value in zip(group, values, strict=True) if value == 0), None
|
|
110
|
+
)
|
|
111
|
+
if absent is not None:
|
|
112
|
+
comparison.update(
|
|
113
|
+
status="available",
|
|
114
|
+
reason=None,
|
|
115
|
+
counterpart={
|
|
116
|
+
**{
|
|
117
|
+
name: absent[name]
|
|
118
|
+
for name in (
|
|
119
|
+
"run",
|
|
120
|
+
"sample_id",
|
|
121
|
+
"step",
|
|
122
|
+
"prompt_id",
|
|
123
|
+
"reward",
|
|
124
|
+
"response",
|
|
125
|
+
)
|
|
126
|
+
},
|
|
127
|
+
"feature_value": 0.0,
|
|
128
|
+
},
|
|
129
|
+
)
|
|
130
|
+
example["comparison"] = comparison
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def resolve_exemplars(
|
|
134
|
+
report: dict,
|
|
135
|
+
rollouts: pd.DataFrame,
|
|
136
|
+
*,
|
|
137
|
+
run: str,
|
|
138
|
+
event_rank: int,
|
|
139
|
+
with_absent: bool = False,
|
|
140
|
+
) -> dict:
|
|
141
|
+
"""Return an event copy with each exemplar's response, or None if unselected.
|
|
142
|
+
|
|
143
|
+
Resolve exact (run, sample_id) keys and require identical step, prompt_id,
|
|
144
|
+
and reward. Missing references, duplicates, or mismatches raise ValueError.
|
|
145
|
+
Members and exemplars follow report ordering rules; missing lists and
|
|
146
|
+
unsupported statuses remain intact. With with_absent, verify extracted
|
|
147
|
+
binary presence and add comparison counts/means over complete prompt-step
|
|
148
|
+
groups, choosing the absent counterpart by lexically smallest sample ID. Supply all
|
|
149
|
+
group completions; saved group sizes are checked when available. Comparisons
|
|
150
|
+
are descriptive. Inputs are unchanged; version 1 has no response fingerprint.
|
|
151
|
+
Historical defines_test_named uses substring matching for the entire group.
|
|
152
|
+
"""
|
|
153
|
+
if type(with_absent) is not bool:
|
|
154
|
+
raise ValueError("with_absent must be a boolean")
|
|
155
|
+
if not isinstance(run, str) or not run.strip():
|
|
156
|
+
raise ValueError("run must be a nonempty string")
|
|
157
|
+
if type(event_rank) is not int or event_rank < 1:
|
|
158
|
+
raise ValueError("event_rank must be a positive integer")
|
|
159
|
+
report = validate_report(report)
|
|
160
|
+
selected_run = next((item for item in report["runs"] if item["run"] == run), None)
|
|
161
|
+
if selected_run is None:
|
|
162
|
+
raise ValueError("requested run is not present in report")
|
|
163
|
+
event = next(
|
|
164
|
+
(item for item in selected_run["events"] if item["rank"] == event_rank), None
|
|
165
|
+
)
|
|
166
|
+
if event is None:
|
|
167
|
+
raise ValueError("requested event rank is not present in report run")
|
|
168
|
+
if not isinstance(rollouts, pd.DataFrame):
|
|
169
|
+
raise ValueError("rollouts must be a DataFrame")
|
|
170
|
+
canonical = validate_rollouts(rollouts)
|
|
171
|
+
# Python scalar comparisons avoid NumPy rounding a large integer report reward.
|
|
172
|
+
lookup = {
|
|
173
|
+
(row["run"], row["sample_id"]): row for row in canonical.to_dict("records")
|
|
174
|
+
}
|
|
175
|
+
event["members"].sort(
|
|
176
|
+
key=lambda member: (
|
|
177
|
+
member["onset_step"],
|
|
178
|
+
member["alert_step"],
|
|
179
|
+
member["feature"],
|
|
180
|
+
)
|
|
181
|
+
)
|
|
182
|
+
for member in event["members"]:
|
|
183
|
+
member["exemplars"].sort(
|
|
184
|
+
key=lambda example: (
|
|
185
|
+
example["selection_order"] is None,
|
|
186
|
+
example["selection_order"] or 0,
|
|
187
|
+
example["sample_id"] or "",
|
|
188
|
+
)
|
|
189
|
+
)
|
|
190
|
+
for example in member["exemplars"]:
|
|
191
|
+
example["response"] = None
|
|
192
|
+
if example["status"] != "selected":
|
|
193
|
+
continue
|
|
194
|
+
if example["reward"] is None:
|
|
195
|
+
raise ValueError("selected exemplar reward is unavailable")
|
|
196
|
+
rollout = lookup.get((example["run"], example["sample_id"]))
|
|
197
|
+
if rollout is None:
|
|
198
|
+
raise ValueError("selected exemplar reference is missing from rollouts")
|
|
199
|
+
for field in ("step", "prompt_id", "reward"):
|
|
200
|
+
if example[field] != rollout[field]:
|
|
201
|
+
raise ValueError(
|
|
202
|
+
f"selected exemplar {field} does not match rollout"
|
|
203
|
+
)
|
|
204
|
+
example["response"] = rollout["response"]
|
|
205
|
+
if with_absent:
|
|
206
|
+
_comparisons(event, lookup)
|
|
207
|
+
return event
|