rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,982 @@
1
+ """Validated comparison JSON and self-contained HTML reports."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ import math
8
+ from base64 import b64encode
9
+ from html import escape
10
+ from pathlib import Path
11
+ from statistics import NormalDist
12
+ from typing import Annotated, Literal
13
+
14
+ from pydantic import AfterValidator, BaseModel, ConfigDict, Field, ValidationError
15
+
16
+ from ._report_validation import _constant, _object
17
+ from .features import DENSE_BINARY, _tier, extract
18
+ from .html_report import _STYLE
19
+ from .report import _identifier
20
+ from .tool_features import valid_tool_feature
21
+
22
+ Identifier = Annotated[str, AfterValidator(_identifier)]
23
+ Count = Annotated[int, Field(ge=0)]
24
+ PositiveCount = Annotated[int, Field(ge=1)]
25
+ Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
26
+ Nonnegative = Annotated[float, Field(ge=0)]
27
+ _Z = NormalDist().inv_cdf(0.975)
28
+ _CONTINUOUS = (extract("").dense.keys() - DENSE_BINARY) | {"scalar:reward"}
29
+ _SPARSE_PREFIXES = (
30
+ "call:",
31
+ "def:",
32
+ "class:",
33
+ "import:",
34
+ "setattr:",
35
+ "strlit:",
36
+ "file:",
37
+ "w:",
38
+ "b:",
39
+ )
40
+
41
+
42
+ class _Model(BaseModel):
43
+ model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
44
+
45
+
46
+ class _File(_Model):
47
+ sha256: Digest
48
+ size_bytes: Count
49
+
50
+
51
+ class _Source(_Model):
52
+ canonical_sha256: Digest
53
+ format: Literal["canonical", "trl", "verl", "openhands", "swe-smith", "inspect"]
54
+ files: list[_File]
55
+ reward_column: Identifier | None
56
+ model: Identifier | None = None
57
+
58
+
59
+ class _Side(_Model):
60
+ side: Literal["a", "b"]
61
+ run: Identifier
62
+ window: Annotated[list[Count], Field(min_length=2, max_length=2)] | None
63
+ observed_steps: list[Count]
64
+ n_rollouts: PositiveCount
65
+ n_prompts: PositiveCount
66
+ n_sampling_units: PositiveCount
67
+ source: _Source
68
+ missing_steps: Count | None = None
69
+ missing_rewards: Count | None = None
70
+
71
+
72
+ class _Shift(_Model):
73
+ descriptor: Identifier
74
+ statistic: Identifier
75
+ before: float | None
76
+ after: float | None
77
+ delta: float | None
78
+ distance: Annotated[float, Field(ge=0, le=1)] | None
79
+ n_before: Count
80
+ n_after: Count
81
+ missing_before: Count
82
+ missing_after: Count
83
+
84
+
85
+ class _PromptMix(_Model):
86
+ status: Literal["shared_prompts", "no_shared_prompts"]
87
+ shared_prompts: Count
88
+ only_a_prompts: Count
89
+ only_b_prompts: Count
90
+ prompt_shift: list[_Shift]
91
+
92
+
93
+ class _Interval(_Model):
94
+ se: Nonnegative | None
95
+ ci_low: float | None
96
+ ci_high: float | None
97
+
98
+
99
+ class _Estimate(_Interval):
100
+ mean: float | None
101
+ n_rollouts: PositiveCount
102
+ n_prompts: PositiveCount
103
+ n_sampling_units: PositiveCount
104
+ positive_prompts: Count | None
105
+ status: Literal[
106
+ "available", "insufficient_units", "numeric_unavailable", "missing_observations"
107
+ ]
108
+
109
+
110
+ class _Contrast(_Interval):
111
+ estimate: float | None
112
+ status: Literal[
113
+ "available",
114
+ "insufficient_units",
115
+ "partial_overlap",
116
+ "shared_units",
117
+ "incompatible_units",
118
+ "numeric_unavailable",
119
+ "missing_observations",
120
+ ]
121
+ method: Literal["independent", "paired", "unavailable"]
122
+
123
+
124
+ class _Paired(_Interval):
125
+ estimate: float | None
126
+ n_prompts: Count
127
+ n_sampling_units: Count | None
128
+ status: Literal[
129
+ "available",
130
+ "insufficient_units",
131
+ "no_shared_prompts",
132
+ "incompatible_units",
133
+ "numeric_unavailable",
134
+ "missing_observations",
135
+ ]
136
+
137
+
138
+ class _Citation(_Model):
139
+ run: Identifier
140
+ sample_id: Identifier
141
+ step: Count | None
142
+ prompt_id: Identifier
143
+ feature_value: float
144
+ reward: float | None
145
+ response_sha256: Digest
146
+ selection_reason: Literal[
147
+ "earliest_positive",
148
+ "first_identity_positive",
149
+ "prompt_diversity",
150
+ "additional_positive",
151
+ ]
152
+
153
+
154
+ class _Citations(_Model):
155
+ status: Literal["selected", "no_positive_rollouts", "unsupported_kind"]
156
+ items: Annotated[list[_Citation], Field(max_length=3)]
157
+
158
+
159
+ class _CitationSides(_Model):
160
+ a: _Citations
161
+ b: _Citations
162
+
163
+
164
+ class _Feature(_Model):
165
+ order: PositiveCount
166
+ feature: Identifier
167
+ tier: Literal["structural", "prose"]
168
+ kind: Literal["binary", "continuous"]
169
+ direction: Literal["increase", "decrease", "unchanged", "unavailable"]
170
+ a: _Estimate
171
+ b: _Estimate
172
+ difference: _Contrast
173
+ paired: _Paired
174
+ citations: _CitationSides
175
+
176
+
177
+ class _Report(_Model):
178
+ report_type: Literal["comparison"]
179
+ schema_version: Literal[1, 2]
180
+ mode: Literal["windows", "dumps"]
181
+ sides: Annotated[list[_Side], Field(min_length=2, max_length=2)]
182
+ prompt_mix: _PromptMix
183
+ features: list[_Feature]
184
+ tool_calls: Literal["tool-calls-v1"] | None = None
185
+
186
+
187
+ def _require(condition: bool, message: str) -> None:
188
+ if not condition:
189
+ raise ValueError(f"invalid comparison report: {message}")
190
+
191
+
192
+ def _same(left: float | None, right: float | None) -> bool:
193
+ if left is None or right is None:
194
+ return left is right
195
+ return math.isclose(left, right, rel_tol=1e-10, abs_tol=1e-12)
196
+
197
+
198
+ def _check_interval(value: _Interval, center: float | None, status: str) -> None:
199
+ bounds = value.ci_low, value.ci_high
200
+ if status == "available":
201
+ _require(
202
+ center is not None
203
+ and value.se is not None
204
+ and all(v is not None for v in bounds),
205
+ "available interval requires finite measurements",
206
+ )
207
+ _require(
208
+ _same(value.ci_low, center - _Z * value.se)
209
+ and _same(value.ci_high, center + _Z * value.se),
210
+ "interval does not match its estimate and standard error",
211
+ )
212
+ else:
213
+ _require(bounds == (None, None), "unavailable interval requires null bounds")
214
+ if status != "numeric_unavailable" or center is None:
215
+ _require(
216
+ value.se is None, "unavailable uncertainty requires null standard error"
217
+ )
218
+ if status == "missing_observations":
219
+ _require(center is None, "missing observations cannot have an estimate")
220
+ elif status not in {"numeric_unavailable", "no_shared_prompts"}:
221
+ _require(center is not None, "observed estimate is missing")
222
+
223
+
224
+ def _check_side(side: _Side) -> None:
225
+ if side.window is None:
226
+ _require(
227
+ not side.observed_steps and side.missing_steps == side.n_rollouts,
228
+ "unavailable steps require an unavailable window",
229
+ )
230
+ else:
231
+ _require(side.window[0] <= side.window[1], "window bounds are reversed")
232
+ _require(
233
+ side.observed_steps
234
+ and side.observed_steps == sorted(set(side.observed_steps))
235
+ and all(
236
+ side.window[0] <= step <= side.window[1] for step in side.observed_steps
237
+ )
238
+ and side.missing_steps in (None, 0),
239
+ "observed steps must be ordered, unique and within the window",
240
+ )
241
+ _require(
242
+ side.missing_rewards is None or side.missing_rewards <= side.n_rollouts,
243
+ "missing reward count exceeds selected rollouts",
244
+ )
245
+ _require(
246
+ side.n_sampling_units <= side.n_prompts <= side.n_rollouts
247
+ and len(side.observed_steps) <= side.n_rollouts,
248
+ "side counts are inconsistent",
249
+ )
250
+
251
+
252
+ def _check_estimate(value: _Estimate, side: _Side, kind: str) -> None:
253
+ _require(
254
+ all(
255
+ getattr(value, key) == getattr(side, key)
256
+ for key in ("n_rollouts", "n_prompts", "n_sampling_units")
257
+ ),
258
+ "feature counts differ from side counts",
259
+ )
260
+ _check_interval(value, value.mean, value.status)
261
+ if value.status == "available":
262
+ _require(
263
+ value.n_sampling_units >= 2,
264
+ "available uncertainty requires at least two units",
265
+ )
266
+ if value.status == "insufficient_units":
267
+ _require(
268
+ value.n_sampling_units < 2, "insufficient uncertainty has enough units"
269
+ )
270
+ if kind == "continuous":
271
+ _require(
272
+ value.positive_prompts is None,
273
+ "continuous features have no positive-prompt count",
274
+ )
275
+ else:
276
+ _require(
277
+ value.positive_prompts is not None
278
+ and value.positive_prompts <= value.n_prompts,
279
+ "binary positive-prompt count is invalid",
280
+ )
281
+ _require(
282
+ value.mean is not None and 0 <= value.mean <= 1,
283
+ "binary mean must be a prevalence",
284
+ )
285
+ _require(
286
+ (value.mean == 0) == (value.positive_prompts == 0)
287
+ and value.mean <= value.positive_prompts / value.n_prompts + 1e-12,
288
+ "binary mean and positive-prompt count disagree",
289
+ )
290
+
291
+
292
+ def _check_citations(value: _Citations, feature: _Feature, side: _Side) -> None:
293
+ estimate = getattr(feature, side.side)
294
+ tool_count = valid_tool_feature(feature.feature, "continuous")
295
+ if feature.kind == "continuous" and not tool_count:
296
+ _require(
297
+ value.status == "unsupported_kind" and not value.items,
298
+ "continuous citations must be unsupported",
299
+ )
300
+ return
301
+ positive = (
302
+ estimate.mean is not None and estimate.mean > 0
303
+ if tool_count
304
+ else bool(estimate.positive_prompts)
305
+ )
306
+ if not positive:
307
+ _require(
308
+ value.status == "no_positive_rollouts" and not value.items,
309
+ "absent binary feature must have no citations",
310
+ )
311
+ return
312
+ _require(
313
+ value.status == "selected" and bool(value.items),
314
+ "positive binary feature requires citations",
315
+ )
316
+ _require(len(value.items) <= side.n_rollouts, "too many citations for side")
317
+ seen_samples, seen_prompts = set(), set()
318
+ remaining = False
319
+ first = value.items[0]
320
+ for index, item in enumerate(value.items):
321
+ _require(
322
+ item.run == side.run
323
+ and (
324
+ item.step in side.observed_steps
325
+ if side.window is not None
326
+ else item.step is None
327
+ )
328
+ and (
329
+ item.feature_value >= 1 and item.feature_value.is_integer()
330
+ if tool_count
331
+ else item.feature_value == 1
332
+ )
333
+ and item.sample_id not in seen_samples,
334
+ "citation identity or feature value is inconsistent",
335
+ )
336
+ _require(
337
+ item.reward is not None or bool(side.missing_rewards),
338
+ "citation reward is missing without missing source rewards",
339
+ )
340
+ _require(
341
+ (item.step or 0, item.sample_id) >= (first.step or 0, first.sample_id),
342
+ "first citation must be earliest",
343
+ )
344
+ if index == 0:
345
+ _require(
346
+ item.selection_reason
347
+ == (
348
+ "earliest_positive"
349
+ if side.window is not None
350
+ else "first_identity_positive"
351
+ ),
352
+ "first citation requires earliest-positive reason",
353
+ )
354
+ elif item.selection_reason == "prompt_diversity":
355
+ _require(
356
+ not remaining and item.prompt_id not in seen_prompts,
357
+ "diversity citation repeats a prompt or follows additional citations",
358
+ )
359
+ else:
360
+ _require(
361
+ item.selection_reason == "additional_positive",
362
+ "later citation has invalid selection reason",
363
+ )
364
+ remaining = True
365
+ seen_samples.add(item.sample_id)
366
+ seen_prompts.add(item.prompt_id)
367
+ diversity = sum(item.selection_reason == "prompt_diversity" for item in value.items)
368
+ _require(
369
+ diversity < (estimate.n_prompts if tool_count else estimate.positive_prompts),
370
+ "diversity citations exceed positive prompt count",
371
+ )
372
+ _require(
373
+ tool_count or len(value.items) >= min(3, estimate.positive_prompts),
374
+ "citations omit known positive prompts",
375
+ )
376
+
377
+
378
+ def _check_feature(feature: _Feature, report: _Report) -> None:
379
+ name = feature.feature
380
+ known = (
381
+ name in DENSE_BINARY
382
+ or name in _CONTINUOUS
383
+ or any(
384
+ name.startswith(prefix) and len(name) > len(prefix)
385
+ for prefix in _SPARSE_PREFIXES
386
+ )
387
+ )
388
+ tool = report.tool_calls is not None and valid_tool_feature(name, feature.kind)
389
+ _require(
390
+ (known or tool) and feature.tier == _tier(name),
391
+ "feature identity or tier is invalid",
392
+ )
393
+ _require(
394
+ tool or feature.kind == ("continuous" if name in _CONTINUOUS else "binary"),
395
+ "feature kind is inconsistent",
396
+ )
397
+ for side in report.sides:
398
+ _check_estimate(getattr(feature, side.side), side, feature.kind)
399
+ _check_citations(getattr(feature.citations, side.side), feature, side)
400
+ if tool and feature.kind == "continuous":
401
+ mean = getattr(feature, side.side).mean
402
+ _require(mean is None or mean >= 0, "tool counts cannot be negative")
403
+ _require(
404
+ (getattr(feature, side.side).status == "missing_observations")
405
+ == (name == "scalar:reward" and bool(side.missing_rewards)),
406
+ "missing reward observations differ from side counts",
407
+ )
408
+ difference, paired = feature.difference, feature.paired
409
+ expected = (
410
+ None
411
+ if feature.a.mean is None or feature.b.mean is None
412
+ else feature.b.mean - feature.a.mean
413
+ )
414
+ if expected is not None and not math.isfinite(expected):
415
+ expected = None
416
+ _require(_same(difference.estimate, expected), "difference must equal B minus A")
417
+ direction = (
418
+ "unavailable"
419
+ if expected is None
420
+ else "increase"
421
+ if expected > 0
422
+ else "decrease"
423
+ if expected < 0
424
+ else "unchanged"
425
+ )
426
+ _require(feature.direction == direction, "direction differs from observed effect")
427
+ _check_interval(difference, difference.estimate, difference.status)
428
+ _check_interval(paired, paired.estimate, paired.status)
429
+ if feature.kind == "binary":
430
+ _require(
431
+ paired.estimate is None or -1 <= paired.estimate <= 1,
432
+ "paired binary effect must be a prevalence difference",
433
+ )
434
+ if name == "scalar:reward" and any(side.missing_rewards for side in report.sides):
435
+ _require(
436
+ difference.estimate is None
437
+ and difference.status == "missing_observations"
438
+ and difference.method == "unavailable"
439
+ and paired.estimate is None
440
+ and paired.status
441
+ == (
442
+ "missing_observations"
443
+ if report.prompt_mix.shared_prompts
444
+ else "no_shared_prompts"
445
+ )
446
+ and paired.n_prompts == report.prompt_mix.shared_prompts,
447
+ "incomplete reward observations require unavailable comparisons",
448
+ )
449
+ _require(
450
+ paired.n_sampling_units == 0
451
+ if not paired.n_prompts
452
+ else (
453
+ paired.n_sampling_units is None
454
+ or 1
455
+ <= paired.n_sampling_units
456
+ <= min(
457
+ paired.n_prompts,
458
+ feature.a.n_sampling_units,
459
+ feature.b.n_sampling_units,
460
+ )
461
+ ),
462
+ "paired unit counts are invalid",
463
+ )
464
+ return
465
+ mix = report.prompt_mix
466
+ _require(
467
+ paired.n_prompts == mix.shared_prompts,
468
+ "paired count differs from shared-prompt count",
469
+ )
470
+ if not mix.shared_prompts:
471
+ _require(
472
+ paired.status == "no_shared_prompts"
473
+ and paired.n_sampling_units == 0
474
+ and paired.estimate is None,
475
+ "unmatched prompts require unavailable paired effect",
476
+ )
477
+ elif paired.status == "incompatible_units" or (
478
+ paired.status == "numeric_unavailable" and paired.n_sampling_units is None
479
+ ):
480
+ _require(
481
+ paired.n_sampling_units is None,
482
+ "incompatible paired units must have unavailable count",
483
+ )
484
+ else:
485
+ _require(
486
+ paired.status != "no_shared_prompts"
487
+ and paired.n_sampling_units is not None
488
+ and 1
489
+ <= paired.n_sampling_units
490
+ <= min(
491
+ paired.n_prompts, feature.a.n_sampling_units, feature.b.n_sampling_units
492
+ ),
493
+ "paired unit counts are invalid",
494
+ )
495
+ if paired.status == "available":
496
+ _require(
497
+ paired.n_sampling_units >= 2,
498
+ "available paired uncertainty requires two units",
499
+ )
500
+ if paired.status == "insufficient_units":
501
+ _require(
502
+ paired.n_sampling_units < 2,
503
+ "insufficient paired uncertainty has enough units",
504
+ )
505
+ full_overlap = (
506
+ mix.shared_prompts > 0 and mix.only_a_prompts == mix.only_b_prompts == 0
507
+ )
508
+ if full_overlap:
509
+ if not _same(paired.estimate, difference.estimate):
510
+ _require(
511
+ difference.status == "numeric_unavailable"
512
+ and difference.method == "unavailable"
513
+ and all(
514
+ getattr(difference, key) is None
515
+ for key in ("se", "ci_low", "ci_high")
516
+ ),
517
+ "inconsistent numeric effects require unavailable marginal uncertainty",
518
+ )
519
+ elif paired.status != "incompatible_units":
520
+ method = (
521
+ "unavailable" if paired.status == "numeric_unavailable" else "paired"
522
+ )
523
+ _require(
524
+ difference.method == method
525
+ and difference.status == paired.status
526
+ and all(
527
+ _same(getattr(difference, key), getattr(paired, key))
528
+ for key in ("se", "ci_low", "ci_high")
529
+ ),
530
+ "fully paired uncertainty must match paired measurements",
531
+ )
532
+ else:
533
+ _require(
534
+ difference.method == "unavailable"
535
+ and difference.status == "incompatible_units",
536
+ "incompatible paired uncertainty must remain unavailable",
537
+ )
538
+ elif mix.shared_prompts:
539
+ _require(
540
+ difference.method == "unavailable"
541
+ and difference.status in {"partial_overlap", "numeric_unavailable"},
542
+ "partial overlap requires unavailable marginal uncertainty",
543
+ )
544
+ elif difference.method == "independent":
545
+ _require(
546
+ difference.status
547
+ in {"available", "insufficient_units", "numeric_unavailable"},
548
+ "independent contrast has invalid status",
549
+ )
550
+ if difference.status == "available":
551
+ _require(
552
+ feature.a.status == feature.b.status == "available"
553
+ and _same(difference.se, math.hypot(feature.a.se, feature.b.se)),
554
+ "independent uncertainty differs from side uncertainty",
555
+ )
556
+ if difference.status == "insufficient_units":
557
+ _require(
558
+ min(feature.a.n_sampling_units, feature.b.n_sampling_units) < 2,
559
+ "independent contrast has sufficient units",
560
+ )
561
+ else:
562
+ _require(
563
+ difference.method == "unavailable"
564
+ and difference.status in {"shared_units", "numeric_unavailable"},
565
+ "unmatched contrast requires independent or shared-unit uncertainty",
566
+ )
567
+
568
+
569
+ def validate_comparison_report(value: object) -> dict:
570
+ """Validate comparison fields and semantic links without changing the input."""
571
+ _require(isinstance(value, dict), "expected a JSON object")
572
+ _require(
573
+ type(value.get("schema_version")) is int and value["schema_version"] in (1, 2),
574
+ "unsupported schema version",
575
+ )
576
+ if value["schema_version"] == 1:
577
+ _require(
578
+ "tool_calls" not in value
579
+ and isinstance(value.get("sides"), list)
580
+ and all(
581
+ isinstance(side, dict)
582
+ and not {"missing_steps", "missing_rewards"}.intersection(side)
583
+ and isinstance(side.get("source"), dict)
584
+ and "model" not in side["source"]
585
+ for side in value.get("sides", [])
586
+ ),
587
+ "comparison version 1 cannot contain version 2 fields",
588
+ )
589
+ try:
590
+ report = _Report.model_validate(value)
591
+ except (ValidationError, RecursionError, OverflowError):
592
+ raise ValueError("invalid comparison report fields or types") from None
593
+ _require(
594
+ [side.side for side in report.sides] == ["a", "b"],
595
+ "sides must be ordered A then B",
596
+ )
597
+ a, b = report.sides
598
+ same_source = a.run == b.run and (
599
+ a.source.canonical_sha256 == b.source.canonical_sha256
600
+ )
601
+ for side in report.sides:
602
+ _check_side(side)
603
+ if report.schema_version == 1:
604
+ _require(
605
+ side.window is not None
606
+ and side.missing_steps is None
607
+ and side.missing_rewards is None
608
+ and report.tool_calls is None
609
+ and side.source.model is None
610
+ and side.source.format in {"canonical", "trl", "verl"},
611
+ "unavailable source metadata requires comparison version 2",
612
+ )
613
+ else:
614
+ _require(
615
+ side.missing_steps is not None and side.missing_rewards is not None,
616
+ "comparison version 2 requires source coverage counts",
617
+ )
618
+ if report.mode == "windows":
619
+ _require(
620
+ a.run == b.run
621
+ and a.source == b.source
622
+ and a.window is not None
623
+ and b.window is not None,
624
+ "windows must select the same source run",
625
+ )
626
+ if same_source and a.window is not None and b.window is not None:
627
+ _require(
628
+ {step for step in a.observed_steps if b.window[0] <= step <= b.window[1]}
629
+ == {
630
+ step for step in b.observed_steps if a.window[0] <= step <= a.window[1]
631
+ },
632
+ "overlapping windows disagree about observed steps",
633
+ )
634
+ mix = report.prompt_mix
635
+ if same_source and set(a.observed_steps) & set(b.observed_steps):
636
+ _require(mix.shared_prompts > 0, "shared observations require shared prompts")
637
+ _require(
638
+ mix.shared_prompts + mix.only_a_prompts == a.n_prompts
639
+ and mix.shared_prompts + mix.only_b_prompts == b.n_prompts,
640
+ "prompt overlap counts differ from side counts",
641
+ )
642
+ _require(
643
+ mix.status == ("shared_prompts" if mix.shared_prompts else "no_shared_prompts"),
644
+ "prompt overlap status differs from counts",
645
+ )
646
+ descriptors = set()
647
+ for shift in mix.prompt_shift:
648
+ _require(
649
+ shift.descriptor not in descriptors, "prompt descriptors must be unique"
650
+ )
651
+ descriptors.add(shift.descriptor)
652
+ _require(
653
+ shift.n_before + shift.missing_before == a.n_prompts
654
+ and shift.n_after + shift.missing_after == b.n_prompts,
655
+ "prompt descriptor counts differ from side counts",
656
+ )
657
+ if shift.before is not None and shift.after is not None:
658
+ _require(
659
+ _same(shift.delta, shift.after - shift.before),
660
+ "prompt shift differs from its means",
661
+ )
662
+ else:
663
+ _require(shift.delta is None, "prompt delta requires both means")
664
+ _require(
665
+ len({feature.feature for feature in report.features}) == len(report.features),
666
+ "features must be unique",
667
+ )
668
+ _require(
669
+ [feature.order for feature in report.features]
670
+ == list(range(1, len(report.features) + 1)),
671
+ "feature order must be contiguous",
672
+ )
673
+ ordered = sorted(
674
+ report.features,
675
+ key=lambda f: (
676
+ f.kind != "binary",
677
+ f.difference.estimate is None if f.kind == "binary" else False,
678
+ -abs(f.difference.estimate)
679
+ if f.kind == "binary" and f.difference.estimate is not None
680
+ else 0,
681
+ f.feature,
682
+ ),
683
+ )
684
+ _require(report.features == ordered, "feature order differs from effect ordering")
685
+ citation_metadata = {}
686
+ for feature in report.features:
687
+ _check_feature(feature, report)
688
+ if same_source and a.observed_steps == b.observed_steps:
689
+ _require(
690
+ feature.a == feature.b
691
+ and feature.citations.a == feature.citations.b
692
+ and feature.paired.estimate in (None, 0.0)
693
+ and feature.paired.se in (None, 0.0),
694
+ "identical observations require identical side evidence and zero measured change",
695
+ )
696
+ for side in ("a", "b"):
697
+ for item in getattr(feature.citations, side).items:
698
+ key = (
699
+ None if same_source else side,
700
+ item.run,
701
+ item.sample_id,
702
+ )
703
+ metadata = item.step, item.prompt_id, item.reward, item.response_sha256
704
+ _require(
705
+ key not in citation_metadata or citation_metadata[key] == metadata,
706
+ "canonical citation metadata differs across features",
707
+ )
708
+ citation_metadata[key] = metadata
709
+ result = report.model_dump()
710
+ if report.schema_version == 1:
711
+ for side in result["sides"]:
712
+ side.pop("missing_steps")
713
+ side.pop("missing_rewards")
714
+ side["source"].pop("model")
715
+ result.pop("tool_calls")
716
+ return result
717
+
718
+
719
+ def comparison_json(report: dict) -> str:
720
+ """Encode a validated comparison as deterministic, strict UTF-8 JSON."""
721
+ return (
722
+ json.dumps(
723
+ validate_comparison_report(report),
724
+ ensure_ascii=False,
725
+ allow_nan=False,
726
+ sort_keys=True,
727
+ indent=2,
728
+ )
729
+ + "\n"
730
+ )
731
+
732
+
733
+ def load_comparison_report(path: str | Path) -> dict:
734
+ """Read strict JSON with duplicate-key checks and errors without payloads."""
735
+ try:
736
+ with Path(path).open(encoding="utf-8") as stream:
737
+ value = json.load(
738
+ stream, object_pairs_hook=_object, parse_constant=_constant
739
+ )
740
+ except (ValueError, RecursionError):
741
+ raise ValueError("invalid comparison report JSON") from None
742
+ return validate_comparison_report(value)
743
+
744
+
745
+ _SCRIPT = """
746
+ (() => {
747
+ const cards = Array.from(document.querySelectorAll('.comparison-feature'));
748
+ const text = cards.map(card => card.textContent.toLowerCase());
749
+ const search = document.getElementById('feature-search');
750
+ const kind = document.getElementById('feature-kind');
751
+ const count = document.getElementById('result-count');
752
+ function update() {
753
+ const query = search.value.trim().toLowerCase();
754
+ let visible = 0;
755
+ cards.forEach((card, index) => {
756
+ card.hidden = !text[index].includes(query)
757
+ || (kind.value !== 'all' && card.dataset.kind !== kind.value);
758
+ if (!card.hidden) visible++;
759
+ });
760
+ count.textContent = `${visible} of ${cards.length} features shown`;
761
+ document.getElementById('no-matches').hidden = visible !== 0 || cards.length === 0;
762
+ }
763
+ search.addEventListener('input', update);
764
+ kind.addEventListener('change', update);
765
+ document.getElementById('reset-filters').addEventListener('click', () => {
766
+ search.value = ''; kind.value = 'all'; update(); search.focus();
767
+ });
768
+ document.getElementById('expand-features').addEventListener('click', () => {
769
+ const visible = cards.filter(card => !card.hidden);
770
+ const expand = visible.some(card => !card.open);
771
+ visible.forEach(card => { card.open = expand; });
772
+ });
773
+ document.getElementById('controls').hidden = false;
774
+ update();
775
+ })();
776
+ """
777
+ _COMPARISON_STYLE = """
778
+ .table-wrap { overflow-x: auto; margin: 14px 0; }
779
+ table { width: 100%; border-collapse: collapse; font-size: 13px; }
780
+ th, td { padding: 9px 10px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; }
781
+ th { font-weight: 650; } td { font-variant-numeric: tabular-nums; }
782
+ .delta { font-weight: 700; font-variant-numeric: tabular-nums; }
783
+ .comparison-feature .event-body > p { margin-bottom: 10px; }
784
+ .sources { margin: 14px 0 24px; }
785
+ .sources > summary { padding: 8px 0; }
786
+ """
787
+
788
+
789
+ def _text(value: object) -> str:
790
+ return escape(str(value), quote=True)
791
+
792
+
793
+ def _number(value: float | None, *, scale: float = 1, signed: bool = False) -> str:
794
+ if value is None:
795
+ return '<span class="unavailable">Unavailable</span>'
796
+ number = value * scale
797
+ spec = ".4g" if abs(number) >= 1e6 or 0 < abs(number) < 0.0001 else ".2f"
798
+ return format(number, ("+" if signed else "") + spec)
799
+
800
+
801
+ def _measurement(row: dict, kind: str, *, difference: bool = False) -> str:
802
+ scale = 100 if kind == "binary" else 1
803
+ unit = (" pp" if difference else "%") if kind == "binary" else ""
804
+ center = row["estimate" if difference else "mean"]
805
+ mean = _number(center, scale=scale, signed=difference)
806
+ if center is not None:
807
+ mean += unit
808
+ interval = "Unavailable"
809
+ if row["ci_low"] is not None:
810
+ interval = f"[{_number(row['ci_low'], scale=scale)}, {_number(row['ci_high'], scale=scale)}]{unit}"
811
+ status = {
812
+ "available": "Available",
813
+ "insufficient_units": "Fewer than two sampling units",
814
+ "partial_overlap": "Only some prompts are shared",
815
+ "shared_units": "Shared sampling units without paired prompts",
816
+ "incompatible_units": "Sampling-unit groupings differ",
817
+ "numeric_unavailable": "Numerical limits",
818
+ "no_shared_prompts": "No shared prompts",
819
+ "missing_observations": "Reward observations missing",
820
+ }[row["status"]]
821
+ return f"<td>{mean}</td><td>{_number(row['se'], scale=scale)}{unit if row['se'] is not None else ''}</td><td>{interval}</td><td>{status}</td>"
822
+
823
+
824
+ def _citation_html(citations: dict, side: str) -> str:
825
+ parts = [f'<section class="metric"><h4>Side {side.upper()} citations</h4>']
826
+ if not citations["items"]:
827
+ parts.append(
828
+ '<p class="note">Continuous features have no selected citations.</p>'
829
+ if citations["status"] == "unsupported_kind"
830
+ else '<p class="note">No positive rollouts in this range.</p>'
831
+ )
832
+ for item in citations["items"]:
833
+ parts.append(
834
+ f'<details class="exemplar"><summary><code>{_text(item["sample_id"])}</code> · step {item["step"] if item["step"] is not None else "unavailable"}</summary><dl>'
835
+ )
836
+ for key, label in (
837
+ ("run", "Run"),
838
+ ("prompt_id", "Prompt ID"),
839
+ ("feature_value", "Feature value"),
840
+ ("reward", "Reward"),
841
+ ("selection_reason", "Selection reason"),
842
+ ("response_sha256", "Response SHA-256"),
843
+ ):
844
+ value = (
845
+ item[key].replace("_", " ") if key == "selection_reason" else item[key]
846
+ )
847
+ if value is None:
848
+ value = "Unavailable"
849
+ parts.append(f"<dt>{label}</dt><dd><code>{_text(value)}</code></dd>")
850
+ parts.append("</dl></details>")
851
+ parts.append("</section>")
852
+ return "".join(parts)
853
+
854
+
855
+ def _feature_html(feature: dict) -> str:
856
+ kind, difference = feature["kind"], feature["difference"]
857
+ effect = _number(
858
+ difference["estimate"], scale=100 if kind == "binary" else 1, signed=True
859
+ )
860
+ if difference["estimate"] is not None:
861
+ effect += " pp" if kind == "binary" else " native units"
862
+ parts = [
863
+ f'<details class="event-card comparison-feature" data-kind="{kind}" id="feature-{feature["order"]}">',
864
+ f'<summary><span class="rank">{feature["order"]}</span><span><code class="representative">{_text(feature["feature"])}</code>',
865
+ f'<span class="anchors">{feature["tier"]} · {kind} · {feature["direction"]}</span></span>',
866
+ f'<span class="delta">{effect}</span></summary><div class="event-body">',
867
+ f'<p class="note">B - A interval method: {_text(difference["method"])}. {"Binary prevalence and percentage-point differences." if kind == "binary" else "Continuous measurements retain native units."}</p>',
868
+ '<div class="table-wrap"><table><thead><tr><th scope="col">Measurement</th><th scope="col">Estimate</th><th scope="col">Standard error</th><th scope="col">95% normal interval</th><th scope="col">Interval status</th></tr></thead><tbody>',
869
+ ]
870
+ for side in ("a", "b"):
871
+ parts.append(
872
+ f'<tr><th scope="row">Side {side.upper()}</th>{_measurement(feature[side], kind)}</tr>'
873
+ )
874
+ parts.append(
875
+ f'<tr><th scope="row">B - A</th>{_measurement(difference, kind, difference=True)}</tr>'
876
+ )
877
+ parts.append(
878
+ f'<tr><th scope="row">Shared-prompt change</th>{_measurement(feature["paired"], kind, difference=True)}</tr></tbody></table></div>'
879
+ )
880
+ paired = feature["paired"]
881
+ parts.append(
882
+ f'<p class="note">Shared prompts: {paired["n_prompts"]}. Sampling units: {"unavailable" if paired["n_sampling_units"] is None else paired["n_sampling_units"]}.</p>'
883
+ )
884
+ if kind == "binary":
885
+ parts.append(
886
+ f'<p class="note">Positive prompts: A {feature["a"]["positive_prompts"]} of {feature["a"]["n_prompts"]}; B {feature["b"]["positive_prompts"]} of {feature["b"]["n_prompts"]}.</p>'
887
+ )
888
+ parts.append('<div class="metrics">')
889
+ parts.extend(
890
+ _citation_html(feature["citations"][side], side) for side in ("a", "b")
891
+ )
892
+ parts.append("</div></div></details>")
893
+ return "".join(parts)
894
+
895
+
896
+ def render_comparison_report(report: dict) -> str:
897
+ """Render validated comparison evidence as escaped HTML."""
898
+ report = validate_comparison_report(report)
899
+ script_hash = b64encode(hashlib.sha256(_SCRIPT.encode("utf-8")).digest()).decode(
900
+ "ascii"
901
+ )
902
+ policy = f"default-src 'none'; style-src 'unsafe-inline'; script-src 'sha256-{script_hash}'; base-uri 'none'; form-action 'none'"
903
+ parts = [
904
+ '<!doctype html>\n<html lang="en"><head><meta charset="utf-8">',
905
+ '<meta name="viewport" content="width=device-width, initial-scale=1">',
906
+ f'<meta http-equiv="Content-Security-Policy" content="{escape(policy, quote=True)}">',
907
+ "<title>Rollout Diff comparison report</title>",
908
+ f"<style>{_STYLE}{_COMPARISON_STYLE}</style></head><body><main>",
909
+ f'<header class="intro"><p class="eyebrow">Rollout Diff · Comparison version {report["schema_version"]}</p><h1>Comparison report</h1>',
910
+ "<p>Differences are B - A. Each exact prompt receives equal weight after pooling its completions within each selected range. Both ranges share one feature vocabulary. Binary features are ordered by absolute prevalence change, followed by continuous features in name order.</p>",
911
+ '<p class="note">Pointwise 95% intervals use a normal approximation and measured standard errors. Supplied sampling units account for dependence among prompts while preserving equal prompt weights. Intervals are unadjusted for comparisons across features. No standard-error floor is applied. Causal and task-success claims require separate evidence.</p>',
912
+ '<p class="note">Comparing rewards across dumps requires the same reward definition and scale.</p></header>',
913
+ '<div class="table-wrap"><table><thead><tr><th>Side</th><th>Run</th><th>Inclusive steps</th><th>Rollouts</th><th>Exact prompts</th><th>Sampling units</th></tr></thead><tbody>',
914
+ ]
915
+ for side in report["sides"]:
916
+ window = (
917
+ f"{side['window'][0]} to {side['window'][1]} ({len(side['observed_steps'])} observed)"
918
+ if side["window"] is not None
919
+ else "Training steps unavailable"
920
+ )
921
+ parts.append(
922
+ f'<tr><th scope="row">{side["side"].upper()}</th><td><code>{_text(side["run"])}</code></td><td>{window}</td><td>{side["n_rollouts"]}</td><td>{side["n_prompts"]}</td><td>{side["n_sampling_units"]}</td></tr>'
923
+ )
924
+ parts.append("</tbody></table></div>")
925
+ if report["schema_version"] == 2:
926
+ for side in report["sides"]:
927
+ parts.append(
928
+ f'<p class="note">Side {side["side"].upper()}: {side["missing_steps"]} records without training steps; {side["missing_rewards"]} without reward. Missing reward observations leave the full-side reward comparison unavailable.</p>'
929
+ )
930
+ mix = report["prompt_mix"]
931
+ parts.append(
932
+ f"<section><h2>Prompt overlap</h2><p>{mix['shared_prompts']} shared prompts, {mix['only_a_prompts']} only in A, {mix['only_b_prompts']} only in B.</p>"
933
+ )
934
+ if not mix["shared_prompts"]:
935
+ parts.append(
936
+ '<p class="warning-note">No exact prompts are shared, so a paired change is unavailable. Independent-side intervals require disjoint sampling units. Differences may reflect prompt composition.</p>'
937
+ )
938
+ elif mix["only_a_prompts"] or mix["only_b_prompts"]:
939
+ parts.append(
940
+ '<p class="warning-note">Partial overlap leaves the overall interval unavailable. The paired comparison covers only shared prompts. Differences may reflect prompt composition.</p>'
941
+ )
942
+ else:
943
+ parts.append(
944
+ '<p class="note">All exact prompts are shared. Paired uncertainty requires compatible sampling-unit groupings.</p>'
945
+ )
946
+ parts.append(
947
+ '<p class="note">Intervals need at least two sampling units. Partial prompt overlap, shared units without paired prompts, incompatible groupings or numerical limits can leave an interval unavailable. Available estimates are still shown. A zero measured standard error does not establish zero uncertainty.</p>'
948
+ )
949
+ if mix["prompt_shift"]:
950
+ parts.append(
951
+ '<div class="table-wrap"><table><thead><tr><th>Prompt descriptor</th><th>Statistic</th><th>A mean</th><th>B mean</th><th>B - A</th><th>Distance</th><th>Observed A / B</th><th>Missing A / B</th></tr></thead><tbody>'
952
+ )
953
+ for shift in mix["prompt_shift"]:
954
+ parts.append(
955
+ f"<tr><th><code>{_text(shift['descriptor'])}</code></th><td>{_text(shift['statistic'])}</td><td>{_number(shift['before'])}</td><td>{_number(shift['after'])}</td><td>{_number(shift['delta'], signed=True)}</td><td>{_number(shift['distance'])}</td><td>{shift['n_before']} / {shift['n_after']}</td><td>{shift['missing_before']} / {shift['missing_after']}</td></tr>"
956
+ )
957
+ parts.append("</tbody></table></div>")
958
+ parts.append(
959
+ '</section><details class="sources"><summary>Source fingerprints</summary><div class="metrics">'
960
+ )
961
+ for side in report["sides"]:
962
+ source = side["source"]
963
+ parts.append(
964
+ f'<section class="metric"><h3>Side {side["side"].upper()}</h3><dl><dt>Canonical SHA-256</dt><dd><code>{source["canonical_sha256"]}</code></dd><dt>Format</dt><dd>{source["format"]}</dd><dt>Reward column</dt><dd>{_text(source["reward_column"]) if source["reward_column"] is not None else "Unspecified"}</dd></dl>'
965
+ )
966
+ for file in source["files"]:
967
+ parts.append(
968
+ f'<p class="reference-note"><code>{file["sha256"]}</code> · {file["size_bytes"]} bytes</p>'
969
+ )
970
+ parts.append("</section>")
971
+ parts.extend(
972
+ [
973
+ "</div></details><section><h2>Feature differences</h2>",
974
+ '<p class="note">Citations are deterministic feature-positive examples selected without reward ranking. Their selection does not establish representativeness. Inspect responses in the original dump by side and feature.</p>',
975
+ '<div class="controls" id="controls" hidden><label class="search-label">Search features<input id="feature-search" type="search" placeholder="Feature or sample ID" autocomplete="off"></label><label>Feature kind<select id="feature-kind"><option value="all">All features</option><option value="binary">Binary</option><option value="continuous">Continuous</option></select></label><button id="reset-filters" type="button">Reset filters</button><button id="expand-features" type="button">Expand / collapse</button><p class="result-count" id="result-count" role="status" aria-live="polite"></p></div><p class="empty" id="no-matches" hidden>No matching features.</p>',
976
+ ]
977
+ )
978
+ if not report["features"]:
979
+ parts.append('<p class="empty">No features in this comparison.</p>')
980
+ parts.extend(_feature_html(feature) for feature in report["features"])
981
+ parts.append(f"</section></main><script>{_SCRIPT}</script></body></html>\n")
982
+ return "".join(parts)