rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,572 @@
1
+ """Prompt-balanced comparisons of selected windows or two rollout dumps."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ import re
9
+ from statistics import NormalDist
10
+
11
+ import numpy as np
12
+ import pandas as pd
13
+
14
+ from ._feature_values import _FeatureValueIndex
15
+ from ._sampling import cluster_standard_error, resolve_sampling_units
16
+ from .features import extract, featurize
17
+ from .prompt_mix import _keywords, _prompt_shift, _window
18
+ from .schema import validate_rollouts
19
+ from .series import _validate_features
20
+
21
+ _Z95 = NormalDist().inv_cdf(0.975)
22
+
23
+
24
+ def _canonical_json(value) -> str:
25
+ try:
26
+ return json.dumps(
27
+ value,
28
+ sort_keys=True,
29
+ ensure_ascii=False,
30
+ separators=(",", ":"),
31
+ allow_nan=False,
32
+ )
33
+ except (TypeError, ValueError):
34
+ raise ValueError("canonical metadata must contain finite JSON values") from None
35
+
36
+
37
+ def _sha256(text: str) -> str:
38
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
39
+
40
+
41
+ def _select_run(rollouts: pd.DataFrame, run: str | None) -> pd.DataFrame:
42
+ if run is None:
43
+ runs = rollouts.run.unique()
44
+ if len(runs) != 1:
45
+ raise ValueError("select exactly one nonempty run")
46
+ run = runs[0]
47
+ if not isinstance(run, str) or not run.strip():
48
+ raise ValueError("run must be a nonblank string")
49
+ result = rollouts.loc[rollouts.run.eq(run)].sort_values(["step", "sample_id"])
50
+ if result.empty:
51
+ raise ValueError("selected run has no rollouts")
52
+ return result.reset_index(drop=True)
53
+
54
+
55
+ def _fingerprint(frame: pd.DataFrame) -> str:
56
+ return _sha256(_canonical_json(frame.sort_values("sample_id").to_dict("records")))
57
+
58
+
59
+ def canonical_fingerprint(
60
+ rollouts: pd.DataFrame, *, run: str | None = None, allow_unavailable: bool = False
61
+ ) -> str:
62
+ """Hash all canonical fields of one full run, independent of row order."""
63
+ return _fingerprint(
64
+ _select_run(
65
+ validate_rollouts(rollouts, allow_unavailable=allow_unavailable), run
66
+ )
67
+ )
68
+
69
+
70
+ def _content_ids(frame: pd.DataFrame) -> pd.Series:
71
+ return pd.Series(
72
+ [
73
+ _sha256(_canonical_json([prompt, messages]))
74
+ for prompt, messages in frame[["prompt", "prompt_messages"]].itertuples(
75
+ index=False, name=None
76
+ )
77
+ ],
78
+ index=frame.index,
79
+ dtype=object,
80
+ )
81
+
82
+
83
+ def _units_by_content(all_rows, full_run, content, mapping) -> pd.Series:
84
+ resolved = resolve_sampling_units(all_rows, mapping)
85
+ if mapping is None:
86
+ names = pd.Index(sorted(content.unique()), name="prompt_id")
87
+ return pd.Series(names, index=names, name="sampling_unit_id")
88
+ selected = resolved.loc[resolved.run.eq(full_run.run.iloc[0])]
89
+ aliases = pd.DataFrame(
90
+ {
91
+ "content": content,
92
+ "unit": full_run.prompt_id.map(
93
+ selected.set_index("prompt_id").sampling_unit_id
94
+ ),
95
+ }
96
+ ).drop_duplicates()
97
+ if aliases.groupby("content").unit.nunique().gt(1).any():
98
+ raise ValueError("sampling units conflict for identical prompt content")
99
+ return aliases.set_index("content").unit.sort_index()
100
+
101
+
102
+ def _compatible_units(a: pd.Series, b: pd.Series) -> bool:
103
+ shared = a.index.intersection(b.index)
104
+ pairs = pd.DataFrame({"a": a.loc[shared], "b": b.loc[shared]}).drop_duplicates()
105
+ return not pairs.a.duplicated().any() and not pairs.b.duplicated().any()
106
+
107
+
108
+ def _select_window(frame, content, units, window, side, fingerprint):
109
+ missing_steps = int(frame.step.isna().sum())
110
+ if missing_steps:
111
+ if window is not None or missing_steps != len(frame):
112
+ raise ValueError("unavailable training steps cannot select a step window")
113
+ mask = pd.Series(True, index=frame.index)
114
+ else:
115
+ window = (
116
+ (int(frame.step.min()), int(frame.step.max()))
117
+ if window is None
118
+ else _window(window)
119
+ )
120
+ mask = frame.step.between(*window)
121
+ if not mask.any():
122
+ raise ValueError("comparison window is empty")
123
+ original = frame.loc[mask].reset_index(drop=True)
124
+ identifiers = content.loc[mask].reset_index(drop=True)
125
+ internal = original.assign(run=side, prompt_id=identifiers)
126
+ if missing_steps:
127
+ # A single vocabulary pool is not an observed training step.
128
+ internal["step"] = 0
129
+ selected_units = units.loc[sorted(identifiers.unique())]
130
+ summary = {
131
+ "side": side,
132
+ "run": frame.run.iloc[0],
133
+ "window": list(window) if window is not None else None,
134
+ "observed_steps": sorted(int(step) for step in original.step.dropna().unique()),
135
+ "n_rollouts": len(original),
136
+ "n_prompts": len(selected_units),
137
+ "n_sampling_units": int(selected_units.nunique()),
138
+ "source": {
139
+ "canonical_sha256": fingerprint,
140
+ "format": "canonical",
141
+ "files": [],
142
+ "reward_column": None,
143
+ },
144
+ }
145
+ return original, internal, selected_units, summary
146
+
147
+
148
+ def _mean(values) -> float | None:
149
+ values = np.asarray(values, dtype=float)
150
+ if not len(values) or not np.isfinite(values).all():
151
+ return None
152
+ scale = float(np.max(np.abs(values)))
153
+ if scale == 0:
154
+ return 0.0
155
+ with np.errstate(over="ignore", invalid="ignore"):
156
+ ordinary = float(values.mean())
157
+ result = (
158
+ ordinary
159
+ if math.isfinite(ordinary)
160
+ else float((values / scale).mean() * scale)
161
+ )
162
+ return result if math.isfinite(result) else None
163
+
164
+
165
+ def _interval(estimate: float | None, se: float | None, status: str) -> dict:
166
+ result = {
167
+ "estimate": estimate,
168
+ "se": se,
169
+ "ci_low": None,
170
+ "ci_high": None,
171
+ "status": status,
172
+ }
173
+ if estimate is None:
174
+ result.update(se=None, status="numeric_unavailable")
175
+ elif se is not None:
176
+ low, high = estimate - _Z95 * se, estimate + _Z95 * se
177
+ if math.isfinite(low) and math.isfinite(high):
178
+ result.update(ci_low=low, ci_high=high)
179
+ else:
180
+ result["status"] = "numeric_unavailable"
181
+ return result
182
+
183
+
184
+ def _statistics(values, unit_ids) -> dict:
185
+ estimate = _mean(values)
186
+ if estimate is None:
187
+ return _interval(None, None, "numeric_unavailable")
188
+ try:
189
+ se = cluster_standard_error(values, unit_ids)
190
+ except ValueError:
191
+ return _interval(estimate, None, "numeric_unavailable")
192
+ return _interval(
193
+ estimate, se, "available" if se is not None else "insufficient_units"
194
+ )
195
+
196
+
197
+ def _side_estimate(
198
+ means: pd.Series, units: pd.Series, n_rollouts: int, kind: str
199
+ ) -> dict:
200
+ result = _statistics(means.to_numpy(), units.to_numpy())
201
+ result["mean"] = result.pop("estimate")
202
+ result.update(
203
+ n_rollouts=n_rollouts,
204
+ n_prompts=len(means),
205
+ n_sampling_units=int(units.nunique()),
206
+ positive_prompts=int(means.gt(0).sum()) if kind == "binary" else None,
207
+ )
208
+ return result
209
+
210
+
211
+ def _paired(a: pd.Series, b: pd.Series, units_a: pd.Series, units_b: pd.Series) -> dict:
212
+ shared = a.index.intersection(b.index)
213
+ if not len(shared):
214
+ result = _interval(0.0, None, "no_shared_prompts")
215
+ result.update(estimate=None, n_prompts=0, n_sampling_units=0)
216
+ return result
217
+ with np.errstate(over="ignore", invalid="ignore"):
218
+ changes = b.loc[shared].to_numpy() - a.loc[shared].to_numpy()
219
+ if not _compatible_units(units_a, units_b):
220
+ result = _interval(_mean(changes), None, "incompatible_units")
221
+ result.update(n_prompts=len(shared), n_sampling_units=None)
222
+ return result
223
+ shared_units = units_a.loc[shared]
224
+ result = _statistics(changes, shared_units.to_numpy())
225
+ result.update(n_prompts=len(shared), n_sampling_units=int(shared_units.nunique()))
226
+ return result
227
+
228
+
229
+ def _contrast(a, b, paired, units_a, units_b, *, same_run):
230
+ estimate = None if a["mean"] is None or b["mean"] is None else b["mean"] - a["mean"]
231
+ if estimate is None or not math.isfinite(estimate):
232
+ return _interval(None, None, "numeric_unavailable") | {"method": "unavailable"}
233
+ shared = paired["n_prompts"]
234
+ if shared == len(units_a) == len(units_b):
235
+ if paired["estimate"] is None or not math.isclose(
236
+ estimate, paired["estimate"], rel_tol=1e-10, abs_tol=1e-12
237
+ ):
238
+ # Cancellation or per-prompt overflow prevents transferring paired
239
+ # uncertainty to the separately computed marginal difference.
240
+ return _interval(estimate, None, "numeric_unavailable") | {
241
+ "method": "unavailable"
242
+ }
243
+ result = {key: paired[key] for key in ("se", "ci_low", "ci_high", "status")}
244
+ # Center on B - A so the interval agrees with the displayed side means.
245
+ result = _interval(estimate, result["se"], result["status"])
246
+ method = (
247
+ "paired"
248
+ if result["status"] in {"available", "insufficient_units"}
249
+ else "unavailable"
250
+ )
251
+ return result | {"method": method}
252
+ if shared:
253
+ return _interval(estimate, None, "partial_overlap") | {"method": "unavailable"}
254
+ if same_run and set(units_a) & set(units_b):
255
+ return _interval(estimate, None, "shared_units") | {"method": "unavailable"}
256
+ if a["status"] == "numeric_unavailable" or b["status"] == "numeric_unavailable":
257
+ return _interval(estimate, None, "numeric_unavailable") | {
258
+ "method": "unavailable"
259
+ }
260
+ se = None if a["se"] is None or b["se"] is None else math.hypot(a["se"], b["se"])
261
+ if se is not None and not math.isfinite(se):
262
+ return _interval(estimate, None, "numeric_unavailable") | {
263
+ "method": "unavailable"
264
+ }
265
+ result = _interval(
266
+ estimate, se, "available" if se is not None else "insufficient_units"
267
+ )
268
+ return result | {
269
+ "method": "independent"
270
+ if result["status"] != "numeric_unavailable"
271
+ else "unavailable"
272
+ }
273
+
274
+
275
+ def _citations(original, aligned_values, kind, *, tool_count=False):
276
+ if kind != "binary" and not tool_count:
277
+ return {"status": "unsupported_kind", "items": []}
278
+ candidates = original.assign(
279
+ _value=aligned_values._value.to_numpy(),
280
+ _content=aligned_values.prompt_id.to_numpy(),
281
+ )
282
+ candidates = candidates.loc[candidates._value.gt(0)].sort_values(
283
+ ["step", "sample_id"]
284
+ )
285
+ if candidates.empty:
286
+ return {"status": "no_positive_rollouts", "items": []}
287
+ selected = [
288
+ (
289
+ candidates.index[0],
290
+ "first_identity_positive"
291
+ if original.step.isna().all()
292
+ else "earliest_positive",
293
+ )
294
+ ]
295
+ contents = {candidates.iloc[0]._content}
296
+ for index, row in candidates.iloc[1:].iterrows():
297
+ if row._content not in contents:
298
+ selected.append((index, "prompt_diversity"))
299
+ contents.add(row._content)
300
+ if len(selected) == 3:
301
+ break
302
+ if len(selected) < 3:
303
+ selected_indices = {index for index, _ in selected}
304
+ for index in candidates.index:
305
+ if index not in selected_indices:
306
+ selected.append((index, "additional_positive"))
307
+ if len(selected) == 3:
308
+ break
309
+ items = []
310
+ for index, reason in selected:
311
+ row = candidates.loc[index]
312
+ items.append(
313
+ {
314
+ "run": row.run,
315
+ "sample_id": row.sample_id,
316
+ "step": None if pd.isna(row.step) else int(row.step),
317
+ "prompt_id": row.prompt_id,
318
+ "feature_value": float(row._value),
319
+ "reward": None if pd.isna(row.reward) else float(row.reward),
320
+ "response_sha256": _sha256(row.response),
321
+ "selection_reason": reason,
322
+ }
323
+ )
324
+ return {"status": "selected", "items": items}
325
+
326
+
327
+ def _prompt_comparability(a, b, keywords):
328
+ grouped = []
329
+ for side, frame in (("before", a), ("after", b)):
330
+ grouped.append(frame.groupby("prompt_id").size().rename(f"{side}_n"))
331
+ detail = pd.concat(grouped, axis=1).fillna(0)
332
+ prompts = (
333
+ pd.concat([a, b]).drop_duplicates("prompt_id").set_index("prompt_id").prompt
334
+ )
335
+ detail["prompt_chars"] = prompts.str.len()
336
+ for word in keywords:
337
+ pattern = re.compile(r"(?<!\w)" + re.escape(word) + r"(?!\w)")
338
+ detail[f"keyword:{word}"] = prompts.str.casefold().str.contains(pattern)
339
+ shared = int((detail.before_n.gt(0) & detail.after_n.gt(0)).sum())
340
+ shift = _prompt_shift(detail, keywords, False)
341
+ return {
342
+ "status": "shared_prompts" if shared else "no_shared_prompts",
343
+ "shared_prompts": shared,
344
+ "only_a_prompts": int(detail.after_n.eq(0).sum()),
345
+ "only_b_prompts": int(detail.before_n.eq(0).sum()),
346
+ "prompt_shift": shift.to_dict("records"),
347
+ }
348
+
349
+
350
+ def compare_rollouts(
351
+ rollouts_a: pd.DataFrame,
352
+ rollouts_b: pd.DataFrame | None = None,
353
+ *,
354
+ a: tuple[int, int] | None = None,
355
+ b: tuple[int, int] | None = None,
356
+ run_a: str | None = None,
357
+ run_b: str | None = None,
358
+ sampling_units_a: pd.DataFrame | None = None,
359
+ sampling_units_b: pd.DataFrame | None = None,
360
+ keywords=(),
361
+ include_tool_calls: bool = False,
362
+ ) -> dict:
363
+ """Compare selected ranges with equal weighting of exact prompts.
364
+
365
+ Pool each prompt's completions within each range and select a shared vocabulary.
366
+ One input requires inclusive ranges, which may overlap or appear in either
367
+ order. Two inputs default to their full observed ranges. Differences are B - A.
368
+ Paired estimates cover shared prompts. Partial prompt overlap leaves the
369
+ overall interval unavailable. Complete-dump comparisons retain unknown steps
370
+ and rewards as null. Recorded tool features require include_tool_calls=True.
371
+ """
372
+ words = _keywords(keywords)
373
+ same_run = rollouts_b is None
374
+ all_a = validate_rollouts(rollouts_a, allow_unavailable=True)
375
+ full_a = _select_run(all_a, run_a)
376
+ if same_run:
377
+ a, b = _window(a), _window(b)
378
+ if run_b is not None and run_b != full_a.run.iloc[0]:
379
+ raise ValueError("one-input windows must select the same run")
380
+ all_b, full_b = all_a, full_a
381
+ if sampling_units_a is None:
382
+ sampling_units_a = sampling_units_b
383
+ if sampling_units_b is None:
384
+ sampling_units_b = sampling_units_a
385
+ else:
386
+ all_b = validate_rollouts(rollouts_b, allow_unavailable=True)
387
+ full_b = _select_run(all_b, run_b)
388
+ fingerprint_a = _fingerprint(full_a)
389
+ fingerprint_b = fingerprint_a if same_run else _fingerprint(full_b)
390
+ same_source = (
391
+ full_a.run.iloc[0] == full_b.run.iloc[0] and fingerprint_a == fingerprint_b
392
+ )
393
+ content_a, content_b = _content_ids(full_a), _content_ids(full_b)
394
+ units_a = _units_by_content(all_a, full_a, content_a, sampling_units_a)
395
+ units_b = _units_by_content(all_b, full_b, content_b, sampling_units_b)
396
+ if same_source:
397
+ if not _compatible_units(units_a, units_b):
398
+ raise ValueError("sampling unit maps contradict within the same run")
399
+ units_b = units_a.copy()
400
+ sides = [
401
+ _select_window(full_a, content_a, units_a, a, "a", fingerprint_a),
402
+ _select_window(full_b, content_b, units_b, b, "b", fingerprint_b),
403
+ ]
404
+ unavailable = any(side[0][["step", "reward"]].isna().any().any() for side in sides)
405
+ extended = unavailable or include_tool_calls
406
+ if extended:
407
+ for original, _, _, summary in sides:
408
+ summary["missing_steps"] = int(original.step.isna().sum())
409
+ summary["missing_rewards"] = int(original.reward.isna().sum())
410
+ internal = pd.concat([side[1] for side in sides], ignore_index=True)
411
+ features = featurize(internal, include_tool_calls=include_tool_calls)
412
+ _validate_features(internal, features)
413
+ index = _FeatureValueIndex(internal, features)
414
+ catalog = [
415
+ *features.catalog.to_dict("records"),
416
+ {"feature": "scalar:reward", "kind": "continuous", "tier": "structural"},
417
+ ]
418
+ comparisons = []
419
+ for metadata in catalog:
420
+ name, kind = metadata["feature"], metadata["kind"]
421
+ values = [index.values(side, name)[0] for side in ("a", "b")]
422
+ means = [
423
+ frame.groupby("prompt_id", sort=True)._value.mean().reindex(side[2].index)
424
+ for frame, side in zip(values, sides, strict=True)
425
+ ]
426
+ estimates = [
427
+ _side_estimate(mean, side[2], len(side[0]), kind)
428
+ for mean, side in zip(means, sides, strict=True)
429
+ ]
430
+ paired = _paired(means[0], means[1], sides[0][2], sides[1][2])
431
+ missing_reward = name == "scalar:reward" and any(
432
+ side[0].reward.isna().any() for side in sides
433
+ )
434
+ if missing_reward:
435
+ for estimate, side in zip(estimates, sides, strict=True):
436
+ if side[0].reward.isna().any():
437
+ estimate.update(
438
+ mean=None,
439
+ se=None,
440
+ ci_low=None,
441
+ ci_high=None,
442
+ status="missing_observations",
443
+ )
444
+ if paired["n_prompts"]:
445
+ paired.update(
446
+ estimate=None,
447
+ se=None,
448
+ ci_low=None,
449
+ ci_high=None,
450
+ status="missing_observations",
451
+ )
452
+ difference = _contrast(
453
+ *estimates, paired, sides[0][2], sides[1][2], same_run=same_source
454
+ )
455
+ if missing_reward:
456
+ difference.update(status="missing_observations")
457
+ change = difference["estimate"]
458
+ direction = (
459
+ "unavailable"
460
+ if change is None
461
+ else "increase"
462
+ if change > 0
463
+ else "decrease"
464
+ if change < 0
465
+ else "unchanged"
466
+ )
467
+ comparisons.append(
468
+ {
469
+ "order": 0,
470
+ **metadata,
471
+ "direction": direction,
472
+ "a": estimates[0],
473
+ "b": estimates[1],
474
+ "difference": difference,
475
+ "paired": paired,
476
+ "citations": {
477
+ label: _citations(
478
+ side[0],
479
+ frame,
480
+ kind,
481
+ tool_count=include_tool_calls
482
+ and name.startswith(("tool_count:", "tool_calls_count")),
483
+ )
484
+ for label, side, frame in zip(
485
+ ("a", "b"), sides, values, strict=True
486
+ )
487
+ },
488
+ }
489
+ )
490
+ comparisons.sort(
491
+ key=lambda row: (
492
+ row["kind"] != "binary",
493
+ -abs(row["difference"]["estimate"])
494
+ if row["kind"] == "binary" and row["difference"]["estimate"] is not None
495
+ else 0,
496
+ row["feature"],
497
+ )
498
+ )
499
+ for order, comparison in enumerate(comparisons, start=1):
500
+ comparison["order"] = order
501
+ report = {
502
+ "report_type": "comparison",
503
+ "schema_version": 2 if extended else 1,
504
+ "mode": "windows" if same_run else "dumps",
505
+ "sides": [side[3] for side in sides],
506
+ "prompt_mix": _prompt_comparability(sides[0][1], sides[1][1], words),
507
+ "features": comparisons,
508
+ }
509
+ if include_tool_calls:
510
+ report["tool_calls"] = "tool-calls-v1"
511
+ from .comparison_report import validate_comparison_report
512
+
513
+ return validate_comparison_report(report)
514
+
515
+
516
+ def resolve_comparison_citations(report, rollouts, *, side: str, feature: str) -> dict:
517
+ """Verify full-run provenance and every selected citation before returning text."""
518
+ from .comparison_report import validate_comparison_report
519
+
520
+ report = validate_comparison_report(report)
521
+ if not isinstance(side, str) or side not in {"a", "b"}:
522
+ raise ValueError("comparison side must be a or b")
523
+ matches = [item for item in report["features"] if item["feature"] == feature]
524
+ if not matches:
525
+ raise ValueError("comparison feature is unavailable")
526
+ source = next(item for item in report["sides"] if item["side"] == side)
527
+ full_run = _select_run(
528
+ validate_rollouts(rollouts, allow_unavailable=True), source["run"]
529
+ )
530
+ if _fingerprint(full_run) != source["source"]["canonical_sha256"]:
531
+ raise ValueError("comparison source fingerprint differs")
532
+ citations = matches[0]["citations"][side]
533
+ by_identity = full_run.set_index("sample_id")
534
+ items = []
535
+ for citation in citations["items"]:
536
+ if citation["sample_id"] not in by_identity.index:
537
+ raise ValueError("comparison citation identity is unavailable")
538
+ row = by_identity.loc[citation["sample_id"]]
539
+ if any(
540
+ citation[field] != row[field]
541
+ for field in ("run", "step", "prompt_id", "reward")
542
+ ):
543
+ raise ValueError("comparison citation metadata differs")
544
+ if _sha256(row.response) != citation["response_sha256"]:
545
+ raise ValueError("comparison citation response differs")
546
+ is_tool = report.get("tool_calls") is not None and feature.startswith("tool")
547
+ if is_tool:
548
+ from .tool_features import extract_tool_calls
549
+
550
+ extracted = extract_tool_calls(row.tool_calls, row.response_messages)
551
+ else:
552
+ extracted = extract(row.response)
553
+ value = extracted.dense.get(feature, float(feature in extracted.sparse))
554
+ if value != citation["feature_value"] or (
555
+ value <= 0 if is_tool else value != 1.0
556
+ ):
557
+ raise ValueError("comparison citation feature differs")
558
+ item = {**citation, "response": row.response}
559
+ if is_tool:
560
+ from .tool_features import tool_call_evidence
561
+
562
+ item["tool_evidence"] = tool_call_evidence(
563
+ row.tool_calls, row.response_messages, feature=feature
564
+ )
565
+ item["response_messages"] = row.response_messages
566
+ items.append(item)
567
+ return {
568
+ "side": side,
569
+ "feature": feature,
570
+ "status": citations["status"],
571
+ "items": items,
572
+ }