rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,572 @@
|
|
|
1
|
+
"""Prompt-balanced comparisons of selected windows or two rollout dumps."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import re
|
|
9
|
+
from statistics import NormalDist
|
|
10
|
+
|
|
11
|
+
import numpy as np
|
|
12
|
+
import pandas as pd
|
|
13
|
+
|
|
14
|
+
from ._feature_values import _FeatureValueIndex
|
|
15
|
+
from ._sampling import cluster_standard_error, resolve_sampling_units
|
|
16
|
+
from .features import extract, featurize
|
|
17
|
+
from .prompt_mix import _keywords, _prompt_shift, _window
|
|
18
|
+
from .schema import validate_rollouts
|
|
19
|
+
from .series import _validate_features
|
|
20
|
+
|
|
21
|
+
_Z95 = NormalDist().inv_cdf(0.975)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _canonical_json(value) -> str:
|
|
25
|
+
try:
|
|
26
|
+
return json.dumps(
|
|
27
|
+
value,
|
|
28
|
+
sort_keys=True,
|
|
29
|
+
ensure_ascii=False,
|
|
30
|
+
separators=(",", ":"),
|
|
31
|
+
allow_nan=False,
|
|
32
|
+
)
|
|
33
|
+
except (TypeError, ValueError):
|
|
34
|
+
raise ValueError("canonical metadata must contain finite JSON values") from None
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _sha256(text: str) -> str:
|
|
38
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _select_run(rollouts: pd.DataFrame, run: str | None) -> pd.DataFrame:
|
|
42
|
+
if run is None:
|
|
43
|
+
runs = rollouts.run.unique()
|
|
44
|
+
if len(runs) != 1:
|
|
45
|
+
raise ValueError("select exactly one nonempty run")
|
|
46
|
+
run = runs[0]
|
|
47
|
+
if not isinstance(run, str) or not run.strip():
|
|
48
|
+
raise ValueError("run must be a nonblank string")
|
|
49
|
+
result = rollouts.loc[rollouts.run.eq(run)].sort_values(["step", "sample_id"])
|
|
50
|
+
if result.empty:
|
|
51
|
+
raise ValueError("selected run has no rollouts")
|
|
52
|
+
return result.reset_index(drop=True)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _fingerprint(frame: pd.DataFrame) -> str:
|
|
56
|
+
return _sha256(_canonical_json(frame.sort_values("sample_id").to_dict("records")))
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def canonical_fingerprint(
|
|
60
|
+
rollouts: pd.DataFrame, *, run: str | None = None, allow_unavailable: bool = False
|
|
61
|
+
) -> str:
|
|
62
|
+
"""Hash all canonical fields of one full run, independent of row order."""
|
|
63
|
+
return _fingerprint(
|
|
64
|
+
_select_run(
|
|
65
|
+
validate_rollouts(rollouts, allow_unavailable=allow_unavailable), run
|
|
66
|
+
)
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _content_ids(frame: pd.DataFrame) -> pd.Series:
|
|
71
|
+
return pd.Series(
|
|
72
|
+
[
|
|
73
|
+
_sha256(_canonical_json([prompt, messages]))
|
|
74
|
+
for prompt, messages in frame[["prompt", "prompt_messages"]].itertuples(
|
|
75
|
+
index=False, name=None
|
|
76
|
+
)
|
|
77
|
+
],
|
|
78
|
+
index=frame.index,
|
|
79
|
+
dtype=object,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _units_by_content(all_rows, full_run, content, mapping) -> pd.Series:
|
|
84
|
+
resolved = resolve_sampling_units(all_rows, mapping)
|
|
85
|
+
if mapping is None:
|
|
86
|
+
names = pd.Index(sorted(content.unique()), name="prompt_id")
|
|
87
|
+
return pd.Series(names, index=names, name="sampling_unit_id")
|
|
88
|
+
selected = resolved.loc[resolved.run.eq(full_run.run.iloc[0])]
|
|
89
|
+
aliases = pd.DataFrame(
|
|
90
|
+
{
|
|
91
|
+
"content": content,
|
|
92
|
+
"unit": full_run.prompt_id.map(
|
|
93
|
+
selected.set_index("prompt_id").sampling_unit_id
|
|
94
|
+
),
|
|
95
|
+
}
|
|
96
|
+
).drop_duplicates()
|
|
97
|
+
if aliases.groupby("content").unit.nunique().gt(1).any():
|
|
98
|
+
raise ValueError("sampling units conflict for identical prompt content")
|
|
99
|
+
return aliases.set_index("content").unit.sort_index()
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _compatible_units(a: pd.Series, b: pd.Series) -> bool:
|
|
103
|
+
shared = a.index.intersection(b.index)
|
|
104
|
+
pairs = pd.DataFrame({"a": a.loc[shared], "b": b.loc[shared]}).drop_duplicates()
|
|
105
|
+
return not pairs.a.duplicated().any() and not pairs.b.duplicated().any()
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _select_window(frame, content, units, window, side, fingerprint):
|
|
109
|
+
missing_steps = int(frame.step.isna().sum())
|
|
110
|
+
if missing_steps:
|
|
111
|
+
if window is not None or missing_steps != len(frame):
|
|
112
|
+
raise ValueError("unavailable training steps cannot select a step window")
|
|
113
|
+
mask = pd.Series(True, index=frame.index)
|
|
114
|
+
else:
|
|
115
|
+
window = (
|
|
116
|
+
(int(frame.step.min()), int(frame.step.max()))
|
|
117
|
+
if window is None
|
|
118
|
+
else _window(window)
|
|
119
|
+
)
|
|
120
|
+
mask = frame.step.between(*window)
|
|
121
|
+
if not mask.any():
|
|
122
|
+
raise ValueError("comparison window is empty")
|
|
123
|
+
original = frame.loc[mask].reset_index(drop=True)
|
|
124
|
+
identifiers = content.loc[mask].reset_index(drop=True)
|
|
125
|
+
internal = original.assign(run=side, prompt_id=identifiers)
|
|
126
|
+
if missing_steps:
|
|
127
|
+
# A single vocabulary pool is not an observed training step.
|
|
128
|
+
internal["step"] = 0
|
|
129
|
+
selected_units = units.loc[sorted(identifiers.unique())]
|
|
130
|
+
summary = {
|
|
131
|
+
"side": side,
|
|
132
|
+
"run": frame.run.iloc[0],
|
|
133
|
+
"window": list(window) if window is not None else None,
|
|
134
|
+
"observed_steps": sorted(int(step) for step in original.step.dropna().unique()),
|
|
135
|
+
"n_rollouts": len(original),
|
|
136
|
+
"n_prompts": len(selected_units),
|
|
137
|
+
"n_sampling_units": int(selected_units.nunique()),
|
|
138
|
+
"source": {
|
|
139
|
+
"canonical_sha256": fingerprint,
|
|
140
|
+
"format": "canonical",
|
|
141
|
+
"files": [],
|
|
142
|
+
"reward_column": None,
|
|
143
|
+
},
|
|
144
|
+
}
|
|
145
|
+
return original, internal, selected_units, summary
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def _mean(values) -> float | None:
|
|
149
|
+
values = np.asarray(values, dtype=float)
|
|
150
|
+
if not len(values) or not np.isfinite(values).all():
|
|
151
|
+
return None
|
|
152
|
+
scale = float(np.max(np.abs(values)))
|
|
153
|
+
if scale == 0:
|
|
154
|
+
return 0.0
|
|
155
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
156
|
+
ordinary = float(values.mean())
|
|
157
|
+
result = (
|
|
158
|
+
ordinary
|
|
159
|
+
if math.isfinite(ordinary)
|
|
160
|
+
else float((values / scale).mean() * scale)
|
|
161
|
+
)
|
|
162
|
+
return result if math.isfinite(result) else None
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _interval(estimate: float | None, se: float | None, status: str) -> dict:
|
|
166
|
+
result = {
|
|
167
|
+
"estimate": estimate,
|
|
168
|
+
"se": se,
|
|
169
|
+
"ci_low": None,
|
|
170
|
+
"ci_high": None,
|
|
171
|
+
"status": status,
|
|
172
|
+
}
|
|
173
|
+
if estimate is None:
|
|
174
|
+
result.update(se=None, status="numeric_unavailable")
|
|
175
|
+
elif se is not None:
|
|
176
|
+
low, high = estimate - _Z95 * se, estimate + _Z95 * se
|
|
177
|
+
if math.isfinite(low) and math.isfinite(high):
|
|
178
|
+
result.update(ci_low=low, ci_high=high)
|
|
179
|
+
else:
|
|
180
|
+
result["status"] = "numeric_unavailable"
|
|
181
|
+
return result
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _statistics(values, unit_ids) -> dict:
|
|
185
|
+
estimate = _mean(values)
|
|
186
|
+
if estimate is None:
|
|
187
|
+
return _interval(None, None, "numeric_unavailable")
|
|
188
|
+
try:
|
|
189
|
+
se = cluster_standard_error(values, unit_ids)
|
|
190
|
+
except ValueError:
|
|
191
|
+
return _interval(estimate, None, "numeric_unavailable")
|
|
192
|
+
return _interval(
|
|
193
|
+
estimate, se, "available" if se is not None else "insufficient_units"
|
|
194
|
+
)
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _side_estimate(
|
|
198
|
+
means: pd.Series, units: pd.Series, n_rollouts: int, kind: str
|
|
199
|
+
) -> dict:
|
|
200
|
+
result = _statistics(means.to_numpy(), units.to_numpy())
|
|
201
|
+
result["mean"] = result.pop("estimate")
|
|
202
|
+
result.update(
|
|
203
|
+
n_rollouts=n_rollouts,
|
|
204
|
+
n_prompts=len(means),
|
|
205
|
+
n_sampling_units=int(units.nunique()),
|
|
206
|
+
positive_prompts=int(means.gt(0).sum()) if kind == "binary" else None,
|
|
207
|
+
)
|
|
208
|
+
return result
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _paired(a: pd.Series, b: pd.Series, units_a: pd.Series, units_b: pd.Series) -> dict:
|
|
212
|
+
shared = a.index.intersection(b.index)
|
|
213
|
+
if not len(shared):
|
|
214
|
+
result = _interval(0.0, None, "no_shared_prompts")
|
|
215
|
+
result.update(estimate=None, n_prompts=0, n_sampling_units=0)
|
|
216
|
+
return result
|
|
217
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
218
|
+
changes = b.loc[shared].to_numpy() - a.loc[shared].to_numpy()
|
|
219
|
+
if not _compatible_units(units_a, units_b):
|
|
220
|
+
result = _interval(_mean(changes), None, "incompatible_units")
|
|
221
|
+
result.update(n_prompts=len(shared), n_sampling_units=None)
|
|
222
|
+
return result
|
|
223
|
+
shared_units = units_a.loc[shared]
|
|
224
|
+
result = _statistics(changes, shared_units.to_numpy())
|
|
225
|
+
result.update(n_prompts=len(shared), n_sampling_units=int(shared_units.nunique()))
|
|
226
|
+
return result
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _contrast(a, b, paired, units_a, units_b, *, same_run):
|
|
230
|
+
estimate = None if a["mean"] is None or b["mean"] is None else b["mean"] - a["mean"]
|
|
231
|
+
if estimate is None or not math.isfinite(estimate):
|
|
232
|
+
return _interval(None, None, "numeric_unavailable") | {"method": "unavailable"}
|
|
233
|
+
shared = paired["n_prompts"]
|
|
234
|
+
if shared == len(units_a) == len(units_b):
|
|
235
|
+
if paired["estimate"] is None or not math.isclose(
|
|
236
|
+
estimate, paired["estimate"], rel_tol=1e-10, abs_tol=1e-12
|
|
237
|
+
):
|
|
238
|
+
# Cancellation or per-prompt overflow prevents transferring paired
|
|
239
|
+
# uncertainty to the separately computed marginal difference.
|
|
240
|
+
return _interval(estimate, None, "numeric_unavailable") | {
|
|
241
|
+
"method": "unavailable"
|
|
242
|
+
}
|
|
243
|
+
result = {key: paired[key] for key in ("se", "ci_low", "ci_high", "status")}
|
|
244
|
+
# Center on B - A so the interval agrees with the displayed side means.
|
|
245
|
+
result = _interval(estimate, result["se"], result["status"])
|
|
246
|
+
method = (
|
|
247
|
+
"paired"
|
|
248
|
+
if result["status"] in {"available", "insufficient_units"}
|
|
249
|
+
else "unavailable"
|
|
250
|
+
)
|
|
251
|
+
return result | {"method": method}
|
|
252
|
+
if shared:
|
|
253
|
+
return _interval(estimate, None, "partial_overlap") | {"method": "unavailable"}
|
|
254
|
+
if same_run and set(units_a) & set(units_b):
|
|
255
|
+
return _interval(estimate, None, "shared_units") | {"method": "unavailable"}
|
|
256
|
+
if a["status"] == "numeric_unavailable" or b["status"] == "numeric_unavailable":
|
|
257
|
+
return _interval(estimate, None, "numeric_unavailable") | {
|
|
258
|
+
"method": "unavailable"
|
|
259
|
+
}
|
|
260
|
+
se = None if a["se"] is None or b["se"] is None else math.hypot(a["se"], b["se"])
|
|
261
|
+
if se is not None and not math.isfinite(se):
|
|
262
|
+
return _interval(estimate, None, "numeric_unavailable") | {
|
|
263
|
+
"method": "unavailable"
|
|
264
|
+
}
|
|
265
|
+
result = _interval(
|
|
266
|
+
estimate, se, "available" if se is not None else "insufficient_units"
|
|
267
|
+
)
|
|
268
|
+
return result | {
|
|
269
|
+
"method": "independent"
|
|
270
|
+
if result["status"] != "numeric_unavailable"
|
|
271
|
+
else "unavailable"
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
def _citations(original, aligned_values, kind, *, tool_count=False):
|
|
276
|
+
if kind != "binary" and not tool_count:
|
|
277
|
+
return {"status": "unsupported_kind", "items": []}
|
|
278
|
+
candidates = original.assign(
|
|
279
|
+
_value=aligned_values._value.to_numpy(),
|
|
280
|
+
_content=aligned_values.prompt_id.to_numpy(),
|
|
281
|
+
)
|
|
282
|
+
candidates = candidates.loc[candidates._value.gt(0)].sort_values(
|
|
283
|
+
["step", "sample_id"]
|
|
284
|
+
)
|
|
285
|
+
if candidates.empty:
|
|
286
|
+
return {"status": "no_positive_rollouts", "items": []}
|
|
287
|
+
selected = [
|
|
288
|
+
(
|
|
289
|
+
candidates.index[0],
|
|
290
|
+
"first_identity_positive"
|
|
291
|
+
if original.step.isna().all()
|
|
292
|
+
else "earliest_positive",
|
|
293
|
+
)
|
|
294
|
+
]
|
|
295
|
+
contents = {candidates.iloc[0]._content}
|
|
296
|
+
for index, row in candidates.iloc[1:].iterrows():
|
|
297
|
+
if row._content not in contents:
|
|
298
|
+
selected.append((index, "prompt_diversity"))
|
|
299
|
+
contents.add(row._content)
|
|
300
|
+
if len(selected) == 3:
|
|
301
|
+
break
|
|
302
|
+
if len(selected) < 3:
|
|
303
|
+
selected_indices = {index for index, _ in selected}
|
|
304
|
+
for index in candidates.index:
|
|
305
|
+
if index not in selected_indices:
|
|
306
|
+
selected.append((index, "additional_positive"))
|
|
307
|
+
if len(selected) == 3:
|
|
308
|
+
break
|
|
309
|
+
items = []
|
|
310
|
+
for index, reason in selected:
|
|
311
|
+
row = candidates.loc[index]
|
|
312
|
+
items.append(
|
|
313
|
+
{
|
|
314
|
+
"run": row.run,
|
|
315
|
+
"sample_id": row.sample_id,
|
|
316
|
+
"step": None if pd.isna(row.step) else int(row.step),
|
|
317
|
+
"prompt_id": row.prompt_id,
|
|
318
|
+
"feature_value": float(row._value),
|
|
319
|
+
"reward": None if pd.isna(row.reward) else float(row.reward),
|
|
320
|
+
"response_sha256": _sha256(row.response),
|
|
321
|
+
"selection_reason": reason,
|
|
322
|
+
}
|
|
323
|
+
)
|
|
324
|
+
return {"status": "selected", "items": items}
|
|
325
|
+
|
|
326
|
+
|
|
327
|
+
def _prompt_comparability(a, b, keywords):
|
|
328
|
+
grouped = []
|
|
329
|
+
for side, frame in (("before", a), ("after", b)):
|
|
330
|
+
grouped.append(frame.groupby("prompt_id").size().rename(f"{side}_n"))
|
|
331
|
+
detail = pd.concat(grouped, axis=1).fillna(0)
|
|
332
|
+
prompts = (
|
|
333
|
+
pd.concat([a, b]).drop_duplicates("prompt_id").set_index("prompt_id").prompt
|
|
334
|
+
)
|
|
335
|
+
detail["prompt_chars"] = prompts.str.len()
|
|
336
|
+
for word in keywords:
|
|
337
|
+
pattern = re.compile(r"(?<!\w)" + re.escape(word) + r"(?!\w)")
|
|
338
|
+
detail[f"keyword:{word}"] = prompts.str.casefold().str.contains(pattern)
|
|
339
|
+
shared = int((detail.before_n.gt(0) & detail.after_n.gt(0)).sum())
|
|
340
|
+
shift = _prompt_shift(detail, keywords, False)
|
|
341
|
+
return {
|
|
342
|
+
"status": "shared_prompts" if shared else "no_shared_prompts",
|
|
343
|
+
"shared_prompts": shared,
|
|
344
|
+
"only_a_prompts": int(detail.after_n.eq(0).sum()),
|
|
345
|
+
"only_b_prompts": int(detail.before_n.eq(0).sum()),
|
|
346
|
+
"prompt_shift": shift.to_dict("records"),
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
def compare_rollouts(
|
|
351
|
+
rollouts_a: pd.DataFrame,
|
|
352
|
+
rollouts_b: pd.DataFrame | None = None,
|
|
353
|
+
*,
|
|
354
|
+
a: tuple[int, int] | None = None,
|
|
355
|
+
b: tuple[int, int] | None = None,
|
|
356
|
+
run_a: str | None = None,
|
|
357
|
+
run_b: str | None = None,
|
|
358
|
+
sampling_units_a: pd.DataFrame | None = None,
|
|
359
|
+
sampling_units_b: pd.DataFrame | None = None,
|
|
360
|
+
keywords=(),
|
|
361
|
+
include_tool_calls: bool = False,
|
|
362
|
+
) -> dict:
|
|
363
|
+
"""Compare selected ranges with equal weighting of exact prompts.
|
|
364
|
+
|
|
365
|
+
Pool each prompt's completions within each range and select a shared vocabulary.
|
|
366
|
+
One input requires inclusive ranges, which may overlap or appear in either
|
|
367
|
+
order. Two inputs default to their full observed ranges. Differences are B - A.
|
|
368
|
+
Paired estimates cover shared prompts. Partial prompt overlap leaves the
|
|
369
|
+
overall interval unavailable. Complete-dump comparisons retain unknown steps
|
|
370
|
+
and rewards as null. Recorded tool features require include_tool_calls=True.
|
|
371
|
+
"""
|
|
372
|
+
words = _keywords(keywords)
|
|
373
|
+
same_run = rollouts_b is None
|
|
374
|
+
all_a = validate_rollouts(rollouts_a, allow_unavailable=True)
|
|
375
|
+
full_a = _select_run(all_a, run_a)
|
|
376
|
+
if same_run:
|
|
377
|
+
a, b = _window(a), _window(b)
|
|
378
|
+
if run_b is not None and run_b != full_a.run.iloc[0]:
|
|
379
|
+
raise ValueError("one-input windows must select the same run")
|
|
380
|
+
all_b, full_b = all_a, full_a
|
|
381
|
+
if sampling_units_a is None:
|
|
382
|
+
sampling_units_a = sampling_units_b
|
|
383
|
+
if sampling_units_b is None:
|
|
384
|
+
sampling_units_b = sampling_units_a
|
|
385
|
+
else:
|
|
386
|
+
all_b = validate_rollouts(rollouts_b, allow_unavailable=True)
|
|
387
|
+
full_b = _select_run(all_b, run_b)
|
|
388
|
+
fingerprint_a = _fingerprint(full_a)
|
|
389
|
+
fingerprint_b = fingerprint_a if same_run else _fingerprint(full_b)
|
|
390
|
+
same_source = (
|
|
391
|
+
full_a.run.iloc[0] == full_b.run.iloc[0] and fingerprint_a == fingerprint_b
|
|
392
|
+
)
|
|
393
|
+
content_a, content_b = _content_ids(full_a), _content_ids(full_b)
|
|
394
|
+
units_a = _units_by_content(all_a, full_a, content_a, sampling_units_a)
|
|
395
|
+
units_b = _units_by_content(all_b, full_b, content_b, sampling_units_b)
|
|
396
|
+
if same_source:
|
|
397
|
+
if not _compatible_units(units_a, units_b):
|
|
398
|
+
raise ValueError("sampling unit maps contradict within the same run")
|
|
399
|
+
units_b = units_a.copy()
|
|
400
|
+
sides = [
|
|
401
|
+
_select_window(full_a, content_a, units_a, a, "a", fingerprint_a),
|
|
402
|
+
_select_window(full_b, content_b, units_b, b, "b", fingerprint_b),
|
|
403
|
+
]
|
|
404
|
+
unavailable = any(side[0][["step", "reward"]].isna().any().any() for side in sides)
|
|
405
|
+
extended = unavailable or include_tool_calls
|
|
406
|
+
if extended:
|
|
407
|
+
for original, _, _, summary in sides:
|
|
408
|
+
summary["missing_steps"] = int(original.step.isna().sum())
|
|
409
|
+
summary["missing_rewards"] = int(original.reward.isna().sum())
|
|
410
|
+
internal = pd.concat([side[1] for side in sides], ignore_index=True)
|
|
411
|
+
features = featurize(internal, include_tool_calls=include_tool_calls)
|
|
412
|
+
_validate_features(internal, features)
|
|
413
|
+
index = _FeatureValueIndex(internal, features)
|
|
414
|
+
catalog = [
|
|
415
|
+
*features.catalog.to_dict("records"),
|
|
416
|
+
{"feature": "scalar:reward", "kind": "continuous", "tier": "structural"},
|
|
417
|
+
]
|
|
418
|
+
comparisons = []
|
|
419
|
+
for metadata in catalog:
|
|
420
|
+
name, kind = metadata["feature"], metadata["kind"]
|
|
421
|
+
values = [index.values(side, name)[0] for side in ("a", "b")]
|
|
422
|
+
means = [
|
|
423
|
+
frame.groupby("prompt_id", sort=True)._value.mean().reindex(side[2].index)
|
|
424
|
+
for frame, side in zip(values, sides, strict=True)
|
|
425
|
+
]
|
|
426
|
+
estimates = [
|
|
427
|
+
_side_estimate(mean, side[2], len(side[0]), kind)
|
|
428
|
+
for mean, side in zip(means, sides, strict=True)
|
|
429
|
+
]
|
|
430
|
+
paired = _paired(means[0], means[1], sides[0][2], sides[1][2])
|
|
431
|
+
missing_reward = name == "scalar:reward" and any(
|
|
432
|
+
side[0].reward.isna().any() for side in sides
|
|
433
|
+
)
|
|
434
|
+
if missing_reward:
|
|
435
|
+
for estimate, side in zip(estimates, sides, strict=True):
|
|
436
|
+
if side[0].reward.isna().any():
|
|
437
|
+
estimate.update(
|
|
438
|
+
mean=None,
|
|
439
|
+
se=None,
|
|
440
|
+
ci_low=None,
|
|
441
|
+
ci_high=None,
|
|
442
|
+
status="missing_observations",
|
|
443
|
+
)
|
|
444
|
+
if paired["n_prompts"]:
|
|
445
|
+
paired.update(
|
|
446
|
+
estimate=None,
|
|
447
|
+
se=None,
|
|
448
|
+
ci_low=None,
|
|
449
|
+
ci_high=None,
|
|
450
|
+
status="missing_observations",
|
|
451
|
+
)
|
|
452
|
+
difference = _contrast(
|
|
453
|
+
*estimates, paired, sides[0][2], sides[1][2], same_run=same_source
|
|
454
|
+
)
|
|
455
|
+
if missing_reward:
|
|
456
|
+
difference.update(status="missing_observations")
|
|
457
|
+
change = difference["estimate"]
|
|
458
|
+
direction = (
|
|
459
|
+
"unavailable"
|
|
460
|
+
if change is None
|
|
461
|
+
else "increase"
|
|
462
|
+
if change > 0
|
|
463
|
+
else "decrease"
|
|
464
|
+
if change < 0
|
|
465
|
+
else "unchanged"
|
|
466
|
+
)
|
|
467
|
+
comparisons.append(
|
|
468
|
+
{
|
|
469
|
+
"order": 0,
|
|
470
|
+
**metadata,
|
|
471
|
+
"direction": direction,
|
|
472
|
+
"a": estimates[0],
|
|
473
|
+
"b": estimates[1],
|
|
474
|
+
"difference": difference,
|
|
475
|
+
"paired": paired,
|
|
476
|
+
"citations": {
|
|
477
|
+
label: _citations(
|
|
478
|
+
side[0],
|
|
479
|
+
frame,
|
|
480
|
+
kind,
|
|
481
|
+
tool_count=include_tool_calls
|
|
482
|
+
and name.startswith(("tool_count:", "tool_calls_count")),
|
|
483
|
+
)
|
|
484
|
+
for label, side, frame in zip(
|
|
485
|
+
("a", "b"), sides, values, strict=True
|
|
486
|
+
)
|
|
487
|
+
},
|
|
488
|
+
}
|
|
489
|
+
)
|
|
490
|
+
comparisons.sort(
|
|
491
|
+
key=lambda row: (
|
|
492
|
+
row["kind"] != "binary",
|
|
493
|
+
-abs(row["difference"]["estimate"])
|
|
494
|
+
if row["kind"] == "binary" and row["difference"]["estimate"] is not None
|
|
495
|
+
else 0,
|
|
496
|
+
row["feature"],
|
|
497
|
+
)
|
|
498
|
+
)
|
|
499
|
+
for order, comparison in enumerate(comparisons, start=1):
|
|
500
|
+
comparison["order"] = order
|
|
501
|
+
report = {
|
|
502
|
+
"report_type": "comparison",
|
|
503
|
+
"schema_version": 2 if extended else 1,
|
|
504
|
+
"mode": "windows" if same_run else "dumps",
|
|
505
|
+
"sides": [side[3] for side in sides],
|
|
506
|
+
"prompt_mix": _prompt_comparability(sides[0][1], sides[1][1], words),
|
|
507
|
+
"features": comparisons,
|
|
508
|
+
}
|
|
509
|
+
if include_tool_calls:
|
|
510
|
+
report["tool_calls"] = "tool-calls-v1"
|
|
511
|
+
from .comparison_report import validate_comparison_report
|
|
512
|
+
|
|
513
|
+
return validate_comparison_report(report)
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def resolve_comparison_citations(report, rollouts, *, side: str, feature: str) -> dict:
|
|
517
|
+
"""Verify full-run provenance and every selected citation before returning text."""
|
|
518
|
+
from .comparison_report import validate_comparison_report
|
|
519
|
+
|
|
520
|
+
report = validate_comparison_report(report)
|
|
521
|
+
if not isinstance(side, str) or side not in {"a", "b"}:
|
|
522
|
+
raise ValueError("comparison side must be a or b")
|
|
523
|
+
matches = [item for item in report["features"] if item["feature"] == feature]
|
|
524
|
+
if not matches:
|
|
525
|
+
raise ValueError("comparison feature is unavailable")
|
|
526
|
+
source = next(item for item in report["sides"] if item["side"] == side)
|
|
527
|
+
full_run = _select_run(
|
|
528
|
+
validate_rollouts(rollouts, allow_unavailable=True), source["run"]
|
|
529
|
+
)
|
|
530
|
+
if _fingerprint(full_run) != source["source"]["canonical_sha256"]:
|
|
531
|
+
raise ValueError("comparison source fingerprint differs")
|
|
532
|
+
citations = matches[0]["citations"][side]
|
|
533
|
+
by_identity = full_run.set_index("sample_id")
|
|
534
|
+
items = []
|
|
535
|
+
for citation in citations["items"]:
|
|
536
|
+
if citation["sample_id"] not in by_identity.index:
|
|
537
|
+
raise ValueError("comparison citation identity is unavailable")
|
|
538
|
+
row = by_identity.loc[citation["sample_id"]]
|
|
539
|
+
if any(
|
|
540
|
+
citation[field] != row[field]
|
|
541
|
+
for field in ("run", "step", "prompt_id", "reward")
|
|
542
|
+
):
|
|
543
|
+
raise ValueError("comparison citation metadata differs")
|
|
544
|
+
if _sha256(row.response) != citation["response_sha256"]:
|
|
545
|
+
raise ValueError("comparison citation response differs")
|
|
546
|
+
is_tool = report.get("tool_calls") is not None and feature.startswith("tool")
|
|
547
|
+
if is_tool:
|
|
548
|
+
from .tool_features import extract_tool_calls
|
|
549
|
+
|
|
550
|
+
extracted = extract_tool_calls(row.tool_calls, row.response_messages)
|
|
551
|
+
else:
|
|
552
|
+
extracted = extract(row.response)
|
|
553
|
+
value = extracted.dense.get(feature, float(feature in extracted.sparse))
|
|
554
|
+
if value != citation["feature_value"] or (
|
|
555
|
+
value <= 0 if is_tool else value != 1.0
|
|
556
|
+
):
|
|
557
|
+
raise ValueError("comparison citation feature differs")
|
|
558
|
+
item = {**citation, "response": row.response}
|
|
559
|
+
if is_tool:
|
|
560
|
+
from .tool_features import tool_call_evidence
|
|
561
|
+
|
|
562
|
+
item["tool_evidence"] = tool_call_evidence(
|
|
563
|
+
row.tool_calls, row.response_messages, feature=feature
|
|
564
|
+
)
|
|
565
|
+
item["response_messages"] = row.response_messages
|
|
566
|
+
items.append(item)
|
|
567
|
+
return {
|
|
568
|
+
"side": side,
|
|
569
|
+
"feature": feature,
|
|
570
|
+
"status": citations["status"],
|
|
571
|
+
"items": items,
|
|
572
|
+
}
|