rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,982 @@
|
|
|
1
|
+
"""Validated comparison JSON and self-contained HTML reports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
from base64 import b64encode
|
|
9
|
+
from html import escape
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from statistics import NormalDist
|
|
12
|
+
from typing import Annotated, Literal
|
|
13
|
+
|
|
14
|
+
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, ValidationError
|
|
15
|
+
|
|
16
|
+
from ._report_validation import _constant, _object
|
|
17
|
+
from .features import DENSE_BINARY, _tier, extract
|
|
18
|
+
from .html_report import _STYLE
|
|
19
|
+
from .report import _identifier
|
|
20
|
+
from .tool_features import valid_tool_feature
|
|
21
|
+
|
|
22
|
+
Identifier = Annotated[str, AfterValidator(_identifier)]
|
|
23
|
+
Count = Annotated[int, Field(ge=0)]
|
|
24
|
+
PositiveCount = Annotated[int, Field(ge=1)]
|
|
25
|
+
Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
|
26
|
+
Nonnegative = Annotated[float, Field(ge=0)]
|
|
27
|
+
_Z = NormalDist().inv_cdf(0.975)
|
|
28
|
+
_CONTINUOUS = (extract("").dense.keys() - DENSE_BINARY) | {"scalar:reward"}
|
|
29
|
+
_SPARSE_PREFIXES = (
|
|
30
|
+
"call:",
|
|
31
|
+
"def:",
|
|
32
|
+
"class:",
|
|
33
|
+
"import:",
|
|
34
|
+
"setattr:",
|
|
35
|
+
"strlit:",
|
|
36
|
+
"file:",
|
|
37
|
+
"w:",
|
|
38
|
+
"b:",
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class _Model(BaseModel):
|
|
43
|
+
model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class _File(_Model):
|
|
47
|
+
sha256: Digest
|
|
48
|
+
size_bytes: Count
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class _Source(_Model):
|
|
52
|
+
canonical_sha256: Digest
|
|
53
|
+
format: Literal["canonical", "trl", "verl", "openhands", "swe-smith", "inspect"]
|
|
54
|
+
files: list[_File]
|
|
55
|
+
reward_column: Identifier | None
|
|
56
|
+
model: Identifier | None = None
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _Side(_Model):
|
|
60
|
+
side: Literal["a", "b"]
|
|
61
|
+
run: Identifier
|
|
62
|
+
window: Annotated[list[Count], Field(min_length=2, max_length=2)] | None
|
|
63
|
+
observed_steps: list[Count]
|
|
64
|
+
n_rollouts: PositiveCount
|
|
65
|
+
n_prompts: PositiveCount
|
|
66
|
+
n_sampling_units: PositiveCount
|
|
67
|
+
source: _Source
|
|
68
|
+
missing_steps: Count | None = None
|
|
69
|
+
missing_rewards: Count | None = None
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class _Shift(_Model):
|
|
73
|
+
descriptor: Identifier
|
|
74
|
+
statistic: Identifier
|
|
75
|
+
before: float | None
|
|
76
|
+
after: float | None
|
|
77
|
+
delta: float | None
|
|
78
|
+
distance: Annotated[float, Field(ge=0, le=1)] | None
|
|
79
|
+
n_before: Count
|
|
80
|
+
n_after: Count
|
|
81
|
+
missing_before: Count
|
|
82
|
+
missing_after: Count
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
class _PromptMix(_Model):
|
|
86
|
+
status: Literal["shared_prompts", "no_shared_prompts"]
|
|
87
|
+
shared_prompts: Count
|
|
88
|
+
only_a_prompts: Count
|
|
89
|
+
only_b_prompts: Count
|
|
90
|
+
prompt_shift: list[_Shift]
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class _Interval(_Model):
|
|
94
|
+
se: Nonnegative | None
|
|
95
|
+
ci_low: float | None
|
|
96
|
+
ci_high: float | None
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class _Estimate(_Interval):
|
|
100
|
+
mean: float | None
|
|
101
|
+
n_rollouts: PositiveCount
|
|
102
|
+
n_prompts: PositiveCount
|
|
103
|
+
n_sampling_units: PositiveCount
|
|
104
|
+
positive_prompts: Count | None
|
|
105
|
+
status: Literal[
|
|
106
|
+
"available", "insufficient_units", "numeric_unavailable", "missing_observations"
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class _Contrast(_Interval):
|
|
111
|
+
estimate: float | None
|
|
112
|
+
status: Literal[
|
|
113
|
+
"available",
|
|
114
|
+
"insufficient_units",
|
|
115
|
+
"partial_overlap",
|
|
116
|
+
"shared_units",
|
|
117
|
+
"incompatible_units",
|
|
118
|
+
"numeric_unavailable",
|
|
119
|
+
"missing_observations",
|
|
120
|
+
]
|
|
121
|
+
method: Literal["independent", "paired", "unavailable"]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _Paired(_Interval):
|
|
125
|
+
estimate: float | None
|
|
126
|
+
n_prompts: Count
|
|
127
|
+
n_sampling_units: Count | None
|
|
128
|
+
status: Literal[
|
|
129
|
+
"available",
|
|
130
|
+
"insufficient_units",
|
|
131
|
+
"no_shared_prompts",
|
|
132
|
+
"incompatible_units",
|
|
133
|
+
"numeric_unavailable",
|
|
134
|
+
"missing_observations",
|
|
135
|
+
]
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class _Citation(_Model):
|
|
139
|
+
run: Identifier
|
|
140
|
+
sample_id: Identifier
|
|
141
|
+
step: Count | None
|
|
142
|
+
prompt_id: Identifier
|
|
143
|
+
feature_value: float
|
|
144
|
+
reward: float | None
|
|
145
|
+
response_sha256: Digest
|
|
146
|
+
selection_reason: Literal[
|
|
147
|
+
"earliest_positive",
|
|
148
|
+
"first_identity_positive",
|
|
149
|
+
"prompt_diversity",
|
|
150
|
+
"additional_positive",
|
|
151
|
+
]
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
class _Citations(_Model):
|
|
155
|
+
status: Literal["selected", "no_positive_rollouts", "unsupported_kind"]
|
|
156
|
+
items: Annotated[list[_Citation], Field(max_length=3)]
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class _CitationSides(_Model):
|
|
160
|
+
a: _Citations
|
|
161
|
+
b: _Citations
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
class _Feature(_Model):
|
|
165
|
+
order: PositiveCount
|
|
166
|
+
feature: Identifier
|
|
167
|
+
tier: Literal["structural", "prose"]
|
|
168
|
+
kind: Literal["binary", "continuous"]
|
|
169
|
+
direction: Literal["increase", "decrease", "unchanged", "unavailable"]
|
|
170
|
+
a: _Estimate
|
|
171
|
+
b: _Estimate
|
|
172
|
+
difference: _Contrast
|
|
173
|
+
paired: _Paired
|
|
174
|
+
citations: _CitationSides
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
class _Report(_Model):
|
|
178
|
+
report_type: Literal["comparison"]
|
|
179
|
+
schema_version: Literal[1, 2]
|
|
180
|
+
mode: Literal["windows", "dumps"]
|
|
181
|
+
sides: Annotated[list[_Side], Field(min_length=2, max_length=2)]
|
|
182
|
+
prompt_mix: _PromptMix
|
|
183
|
+
features: list[_Feature]
|
|
184
|
+
tool_calls: Literal["tool-calls-v1"] | None = None
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def _require(condition: bool, message: str) -> None:
|
|
188
|
+
if not condition:
|
|
189
|
+
raise ValueError(f"invalid comparison report: {message}")
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _same(left: float | None, right: float | None) -> bool:
|
|
193
|
+
if left is None or right is None:
|
|
194
|
+
return left is right
|
|
195
|
+
return math.isclose(left, right, rel_tol=1e-10, abs_tol=1e-12)
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _check_interval(value: _Interval, center: float | None, status: str) -> None:
|
|
199
|
+
bounds = value.ci_low, value.ci_high
|
|
200
|
+
if status == "available":
|
|
201
|
+
_require(
|
|
202
|
+
center is not None
|
|
203
|
+
and value.se is not None
|
|
204
|
+
and all(v is not None for v in bounds),
|
|
205
|
+
"available interval requires finite measurements",
|
|
206
|
+
)
|
|
207
|
+
_require(
|
|
208
|
+
_same(value.ci_low, center - _Z * value.se)
|
|
209
|
+
and _same(value.ci_high, center + _Z * value.se),
|
|
210
|
+
"interval does not match its estimate and standard error",
|
|
211
|
+
)
|
|
212
|
+
else:
|
|
213
|
+
_require(bounds == (None, None), "unavailable interval requires null bounds")
|
|
214
|
+
if status != "numeric_unavailable" or center is None:
|
|
215
|
+
_require(
|
|
216
|
+
value.se is None, "unavailable uncertainty requires null standard error"
|
|
217
|
+
)
|
|
218
|
+
if status == "missing_observations":
|
|
219
|
+
_require(center is None, "missing observations cannot have an estimate")
|
|
220
|
+
elif status not in {"numeric_unavailable", "no_shared_prompts"}:
|
|
221
|
+
_require(center is not None, "observed estimate is missing")
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _check_side(side: _Side) -> None:
|
|
225
|
+
if side.window is None:
|
|
226
|
+
_require(
|
|
227
|
+
not side.observed_steps and side.missing_steps == side.n_rollouts,
|
|
228
|
+
"unavailable steps require an unavailable window",
|
|
229
|
+
)
|
|
230
|
+
else:
|
|
231
|
+
_require(side.window[0] <= side.window[1], "window bounds are reversed")
|
|
232
|
+
_require(
|
|
233
|
+
side.observed_steps
|
|
234
|
+
and side.observed_steps == sorted(set(side.observed_steps))
|
|
235
|
+
and all(
|
|
236
|
+
side.window[0] <= step <= side.window[1] for step in side.observed_steps
|
|
237
|
+
)
|
|
238
|
+
and side.missing_steps in (None, 0),
|
|
239
|
+
"observed steps must be ordered, unique and within the window",
|
|
240
|
+
)
|
|
241
|
+
_require(
|
|
242
|
+
side.missing_rewards is None or side.missing_rewards <= side.n_rollouts,
|
|
243
|
+
"missing reward count exceeds selected rollouts",
|
|
244
|
+
)
|
|
245
|
+
_require(
|
|
246
|
+
side.n_sampling_units <= side.n_prompts <= side.n_rollouts
|
|
247
|
+
and len(side.observed_steps) <= side.n_rollouts,
|
|
248
|
+
"side counts are inconsistent",
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _check_estimate(value: _Estimate, side: _Side, kind: str) -> None:
|
|
253
|
+
_require(
|
|
254
|
+
all(
|
|
255
|
+
getattr(value, key) == getattr(side, key)
|
|
256
|
+
for key in ("n_rollouts", "n_prompts", "n_sampling_units")
|
|
257
|
+
),
|
|
258
|
+
"feature counts differ from side counts",
|
|
259
|
+
)
|
|
260
|
+
_check_interval(value, value.mean, value.status)
|
|
261
|
+
if value.status == "available":
|
|
262
|
+
_require(
|
|
263
|
+
value.n_sampling_units >= 2,
|
|
264
|
+
"available uncertainty requires at least two units",
|
|
265
|
+
)
|
|
266
|
+
if value.status == "insufficient_units":
|
|
267
|
+
_require(
|
|
268
|
+
value.n_sampling_units < 2, "insufficient uncertainty has enough units"
|
|
269
|
+
)
|
|
270
|
+
if kind == "continuous":
|
|
271
|
+
_require(
|
|
272
|
+
value.positive_prompts is None,
|
|
273
|
+
"continuous features have no positive-prompt count",
|
|
274
|
+
)
|
|
275
|
+
else:
|
|
276
|
+
_require(
|
|
277
|
+
value.positive_prompts is not None
|
|
278
|
+
and value.positive_prompts <= value.n_prompts,
|
|
279
|
+
"binary positive-prompt count is invalid",
|
|
280
|
+
)
|
|
281
|
+
_require(
|
|
282
|
+
value.mean is not None and 0 <= value.mean <= 1,
|
|
283
|
+
"binary mean must be a prevalence",
|
|
284
|
+
)
|
|
285
|
+
_require(
|
|
286
|
+
(value.mean == 0) == (value.positive_prompts == 0)
|
|
287
|
+
and value.mean <= value.positive_prompts / value.n_prompts + 1e-12,
|
|
288
|
+
"binary mean and positive-prompt count disagree",
|
|
289
|
+
)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _check_citations(value: _Citations, feature: _Feature, side: _Side) -> None:
|
|
293
|
+
estimate = getattr(feature, side.side)
|
|
294
|
+
tool_count = valid_tool_feature(feature.feature, "continuous")
|
|
295
|
+
if feature.kind == "continuous" and not tool_count:
|
|
296
|
+
_require(
|
|
297
|
+
value.status == "unsupported_kind" and not value.items,
|
|
298
|
+
"continuous citations must be unsupported",
|
|
299
|
+
)
|
|
300
|
+
return
|
|
301
|
+
positive = (
|
|
302
|
+
estimate.mean is not None and estimate.mean > 0
|
|
303
|
+
if tool_count
|
|
304
|
+
else bool(estimate.positive_prompts)
|
|
305
|
+
)
|
|
306
|
+
if not positive:
|
|
307
|
+
_require(
|
|
308
|
+
value.status == "no_positive_rollouts" and not value.items,
|
|
309
|
+
"absent binary feature must have no citations",
|
|
310
|
+
)
|
|
311
|
+
return
|
|
312
|
+
_require(
|
|
313
|
+
value.status == "selected" and bool(value.items),
|
|
314
|
+
"positive binary feature requires citations",
|
|
315
|
+
)
|
|
316
|
+
_require(len(value.items) <= side.n_rollouts, "too many citations for side")
|
|
317
|
+
seen_samples, seen_prompts = set(), set()
|
|
318
|
+
remaining = False
|
|
319
|
+
first = value.items[0]
|
|
320
|
+
for index, item in enumerate(value.items):
|
|
321
|
+
_require(
|
|
322
|
+
item.run == side.run
|
|
323
|
+
and (
|
|
324
|
+
item.step in side.observed_steps
|
|
325
|
+
if side.window is not None
|
|
326
|
+
else item.step is None
|
|
327
|
+
)
|
|
328
|
+
and (
|
|
329
|
+
item.feature_value >= 1 and item.feature_value.is_integer()
|
|
330
|
+
if tool_count
|
|
331
|
+
else item.feature_value == 1
|
|
332
|
+
)
|
|
333
|
+
and item.sample_id not in seen_samples,
|
|
334
|
+
"citation identity or feature value is inconsistent",
|
|
335
|
+
)
|
|
336
|
+
_require(
|
|
337
|
+
item.reward is not None or bool(side.missing_rewards),
|
|
338
|
+
"citation reward is missing without missing source rewards",
|
|
339
|
+
)
|
|
340
|
+
_require(
|
|
341
|
+
(item.step or 0, item.sample_id) >= (first.step or 0, first.sample_id),
|
|
342
|
+
"first citation must be earliest",
|
|
343
|
+
)
|
|
344
|
+
if index == 0:
|
|
345
|
+
_require(
|
|
346
|
+
item.selection_reason
|
|
347
|
+
== (
|
|
348
|
+
"earliest_positive"
|
|
349
|
+
if side.window is not None
|
|
350
|
+
else "first_identity_positive"
|
|
351
|
+
),
|
|
352
|
+
"first citation requires earliest-positive reason",
|
|
353
|
+
)
|
|
354
|
+
elif item.selection_reason == "prompt_diversity":
|
|
355
|
+
_require(
|
|
356
|
+
not remaining and item.prompt_id not in seen_prompts,
|
|
357
|
+
"diversity citation repeats a prompt or follows additional citations",
|
|
358
|
+
)
|
|
359
|
+
else:
|
|
360
|
+
_require(
|
|
361
|
+
item.selection_reason == "additional_positive",
|
|
362
|
+
"later citation has invalid selection reason",
|
|
363
|
+
)
|
|
364
|
+
remaining = True
|
|
365
|
+
seen_samples.add(item.sample_id)
|
|
366
|
+
seen_prompts.add(item.prompt_id)
|
|
367
|
+
diversity = sum(item.selection_reason == "prompt_diversity" for item in value.items)
|
|
368
|
+
_require(
|
|
369
|
+
diversity < (estimate.n_prompts if tool_count else estimate.positive_prompts),
|
|
370
|
+
"diversity citations exceed positive prompt count",
|
|
371
|
+
)
|
|
372
|
+
_require(
|
|
373
|
+
tool_count or len(value.items) >= min(3, estimate.positive_prompts),
|
|
374
|
+
"citations omit known positive prompts",
|
|
375
|
+
)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _check_feature(feature: _Feature, report: _Report) -> None:
|
|
379
|
+
name = feature.feature
|
|
380
|
+
known = (
|
|
381
|
+
name in DENSE_BINARY
|
|
382
|
+
or name in _CONTINUOUS
|
|
383
|
+
or any(
|
|
384
|
+
name.startswith(prefix) and len(name) > len(prefix)
|
|
385
|
+
for prefix in _SPARSE_PREFIXES
|
|
386
|
+
)
|
|
387
|
+
)
|
|
388
|
+
tool = report.tool_calls is not None and valid_tool_feature(name, feature.kind)
|
|
389
|
+
_require(
|
|
390
|
+
(known or tool) and feature.tier == _tier(name),
|
|
391
|
+
"feature identity or tier is invalid",
|
|
392
|
+
)
|
|
393
|
+
_require(
|
|
394
|
+
tool or feature.kind == ("continuous" if name in _CONTINUOUS else "binary"),
|
|
395
|
+
"feature kind is inconsistent",
|
|
396
|
+
)
|
|
397
|
+
for side in report.sides:
|
|
398
|
+
_check_estimate(getattr(feature, side.side), side, feature.kind)
|
|
399
|
+
_check_citations(getattr(feature.citations, side.side), feature, side)
|
|
400
|
+
if tool and feature.kind == "continuous":
|
|
401
|
+
mean = getattr(feature, side.side).mean
|
|
402
|
+
_require(mean is None or mean >= 0, "tool counts cannot be negative")
|
|
403
|
+
_require(
|
|
404
|
+
(getattr(feature, side.side).status == "missing_observations")
|
|
405
|
+
== (name == "scalar:reward" and bool(side.missing_rewards)),
|
|
406
|
+
"missing reward observations differ from side counts",
|
|
407
|
+
)
|
|
408
|
+
difference, paired = feature.difference, feature.paired
|
|
409
|
+
expected = (
|
|
410
|
+
None
|
|
411
|
+
if feature.a.mean is None or feature.b.mean is None
|
|
412
|
+
else feature.b.mean - feature.a.mean
|
|
413
|
+
)
|
|
414
|
+
if expected is not None and not math.isfinite(expected):
|
|
415
|
+
expected = None
|
|
416
|
+
_require(_same(difference.estimate, expected), "difference must equal B minus A")
|
|
417
|
+
direction = (
|
|
418
|
+
"unavailable"
|
|
419
|
+
if expected is None
|
|
420
|
+
else "increase"
|
|
421
|
+
if expected > 0
|
|
422
|
+
else "decrease"
|
|
423
|
+
if expected < 0
|
|
424
|
+
else "unchanged"
|
|
425
|
+
)
|
|
426
|
+
_require(feature.direction == direction, "direction differs from observed effect")
|
|
427
|
+
_check_interval(difference, difference.estimate, difference.status)
|
|
428
|
+
_check_interval(paired, paired.estimate, paired.status)
|
|
429
|
+
if feature.kind == "binary":
|
|
430
|
+
_require(
|
|
431
|
+
paired.estimate is None or -1 <= paired.estimate <= 1,
|
|
432
|
+
"paired binary effect must be a prevalence difference",
|
|
433
|
+
)
|
|
434
|
+
if name == "scalar:reward" and any(side.missing_rewards for side in report.sides):
|
|
435
|
+
_require(
|
|
436
|
+
difference.estimate is None
|
|
437
|
+
and difference.status == "missing_observations"
|
|
438
|
+
and difference.method == "unavailable"
|
|
439
|
+
and paired.estimate is None
|
|
440
|
+
and paired.status
|
|
441
|
+
== (
|
|
442
|
+
"missing_observations"
|
|
443
|
+
if report.prompt_mix.shared_prompts
|
|
444
|
+
else "no_shared_prompts"
|
|
445
|
+
)
|
|
446
|
+
and paired.n_prompts == report.prompt_mix.shared_prompts,
|
|
447
|
+
"incomplete reward observations require unavailable comparisons",
|
|
448
|
+
)
|
|
449
|
+
_require(
|
|
450
|
+
paired.n_sampling_units == 0
|
|
451
|
+
if not paired.n_prompts
|
|
452
|
+
else (
|
|
453
|
+
paired.n_sampling_units is None
|
|
454
|
+
or 1
|
|
455
|
+
<= paired.n_sampling_units
|
|
456
|
+
<= min(
|
|
457
|
+
paired.n_prompts,
|
|
458
|
+
feature.a.n_sampling_units,
|
|
459
|
+
feature.b.n_sampling_units,
|
|
460
|
+
)
|
|
461
|
+
),
|
|
462
|
+
"paired unit counts are invalid",
|
|
463
|
+
)
|
|
464
|
+
return
|
|
465
|
+
mix = report.prompt_mix
|
|
466
|
+
_require(
|
|
467
|
+
paired.n_prompts == mix.shared_prompts,
|
|
468
|
+
"paired count differs from shared-prompt count",
|
|
469
|
+
)
|
|
470
|
+
if not mix.shared_prompts:
|
|
471
|
+
_require(
|
|
472
|
+
paired.status == "no_shared_prompts"
|
|
473
|
+
and paired.n_sampling_units == 0
|
|
474
|
+
and paired.estimate is None,
|
|
475
|
+
"unmatched prompts require unavailable paired effect",
|
|
476
|
+
)
|
|
477
|
+
elif paired.status == "incompatible_units" or (
|
|
478
|
+
paired.status == "numeric_unavailable" and paired.n_sampling_units is None
|
|
479
|
+
):
|
|
480
|
+
_require(
|
|
481
|
+
paired.n_sampling_units is None,
|
|
482
|
+
"incompatible paired units must have unavailable count",
|
|
483
|
+
)
|
|
484
|
+
else:
|
|
485
|
+
_require(
|
|
486
|
+
paired.status != "no_shared_prompts"
|
|
487
|
+
and paired.n_sampling_units is not None
|
|
488
|
+
and 1
|
|
489
|
+
<= paired.n_sampling_units
|
|
490
|
+
<= min(
|
|
491
|
+
paired.n_prompts, feature.a.n_sampling_units, feature.b.n_sampling_units
|
|
492
|
+
),
|
|
493
|
+
"paired unit counts are invalid",
|
|
494
|
+
)
|
|
495
|
+
if paired.status == "available":
|
|
496
|
+
_require(
|
|
497
|
+
paired.n_sampling_units >= 2,
|
|
498
|
+
"available paired uncertainty requires two units",
|
|
499
|
+
)
|
|
500
|
+
if paired.status == "insufficient_units":
|
|
501
|
+
_require(
|
|
502
|
+
paired.n_sampling_units < 2,
|
|
503
|
+
"insufficient paired uncertainty has enough units",
|
|
504
|
+
)
|
|
505
|
+
full_overlap = (
|
|
506
|
+
mix.shared_prompts > 0 and mix.only_a_prompts == mix.only_b_prompts == 0
|
|
507
|
+
)
|
|
508
|
+
if full_overlap:
|
|
509
|
+
if not _same(paired.estimate, difference.estimate):
|
|
510
|
+
_require(
|
|
511
|
+
difference.status == "numeric_unavailable"
|
|
512
|
+
and difference.method == "unavailable"
|
|
513
|
+
and all(
|
|
514
|
+
getattr(difference, key) is None
|
|
515
|
+
for key in ("se", "ci_low", "ci_high")
|
|
516
|
+
),
|
|
517
|
+
"inconsistent numeric effects require unavailable marginal uncertainty",
|
|
518
|
+
)
|
|
519
|
+
elif paired.status != "incompatible_units":
|
|
520
|
+
method = (
|
|
521
|
+
"unavailable" if paired.status == "numeric_unavailable" else "paired"
|
|
522
|
+
)
|
|
523
|
+
_require(
|
|
524
|
+
difference.method == method
|
|
525
|
+
and difference.status == paired.status
|
|
526
|
+
and all(
|
|
527
|
+
_same(getattr(difference, key), getattr(paired, key))
|
|
528
|
+
for key in ("se", "ci_low", "ci_high")
|
|
529
|
+
),
|
|
530
|
+
"fully paired uncertainty must match paired measurements",
|
|
531
|
+
)
|
|
532
|
+
else:
|
|
533
|
+
_require(
|
|
534
|
+
difference.method == "unavailable"
|
|
535
|
+
and difference.status == "incompatible_units",
|
|
536
|
+
"incompatible paired uncertainty must remain unavailable",
|
|
537
|
+
)
|
|
538
|
+
elif mix.shared_prompts:
|
|
539
|
+
_require(
|
|
540
|
+
difference.method == "unavailable"
|
|
541
|
+
and difference.status in {"partial_overlap", "numeric_unavailable"},
|
|
542
|
+
"partial overlap requires unavailable marginal uncertainty",
|
|
543
|
+
)
|
|
544
|
+
elif difference.method == "independent":
|
|
545
|
+
_require(
|
|
546
|
+
difference.status
|
|
547
|
+
in {"available", "insufficient_units", "numeric_unavailable"},
|
|
548
|
+
"independent contrast has invalid status",
|
|
549
|
+
)
|
|
550
|
+
if difference.status == "available":
|
|
551
|
+
_require(
|
|
552
|
+
feature.a.status == feature.b.status == "available"
|
|
553
|
+
and _same(difference.se, math.hypot(feature.a.se, feature.b.se)),
|
|
554
|
+
"independent uncertainty differs from side uncertainty",
|
|
555
|
+
)
|
|
556
|
+
if difference.status == "insufficient_units":
|
|
557
|
+
_require(
|
|
558
|
+
min(feature.a.n_sampling_units, feature.b.n_sampling_units) < 2,
|
|
559
|
+
"independent contrast has sufficient units",
|
|
560
|
+
)
|
|
561
|
+
else:
|
|
562
|
+
_require(
|
|
563
|
+
difference.method == "unavailable"
|
|
564
|
+
and difference.status in {"shared_units", "numeric_unavailable"},
|
|
565
|
+
"unmatched contrast requires independent or shared-unit uncertainty",
|
|
566
|
+
)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def validate_comparison_report(value: object) -> dict:
|
|
570
|
+
"""Validate comparison fields and semantic links without changing the input."""
|
|
571
|
+
_require(isinstance(value, dict), "expected a JSON object")
|
|
572
|
+
_require(
|
|
573
|
+
type(value.get("schema_version")) is int and value["schema_version"] in (1, 2),
|
|
574
|
+
"unsupported schema version",
|
|
575
|
+
)
|
|
576
|
+
if value["schema_version"] == 1:
|
|
577
|
+
_require(
|
|
578
|
+
"tool_calls" not in value
|
|
579
|
+
and isinstance(value.get("sides"), list)
|
|
580
|
+
and all(
|
|
581
|
+
isinstance(side, dict)
|
|
582
|
+
and not {"missing_steps", "missing_rewards"}.intersection(side)
|
|
583
|
+
and isinstance(side.get("source"), dict)
|
|
584
|
+
and "model" not in side["source"]
|
|
585
|
+
for side in value.get("sides", [])
|
|
586
|
+
),
|
|
587
|
+
"comparison version 1 cannot contain version 2 fields",
|
|
588
|
+
)
|
|
589
|
+
try:
|
|
590
|
+
report = _Report.model_validate(value)
|
|
591
|
+
except (ValidationError, RecursionError, OverflowError):
|
|
592
|
+
raise ValueError("invalid comparison report fields or types") from None
|
|
593
|
+
_require(
|
|
594
|
+
[side.side for side in report.sides] == ["a", "b"],
|
|
595
|
+
"sides must be ordered A then B",
|
|
596
|
+
)
|
|
597
|
+
a, b = report.sides
|
|
598
|
+
same_source = a.run == b.run and (
|
|
599
|
+
a.source.canonical_sha256 == b.source.canonical_sha256
|
|
600
|
+
)
|
|
601
|
+
for side in report.sides:
|
|
602
|
+
_check_side(side)
|
|
603
|
+
if report.schema_version == 1:
|
|
604
|
+
_require(
|
|
605
|
+
side.window is not None
|
|
606
|
+
and side.missing_steps is None
|
|
607
|
+
and side.missing_rewards is None
|
|
608
|
+
and report.tool_calls is None
|
|
609
|
+
and side.source.model is None
|
|
610
|
+
and side.source.format in {"canonical", "trl", "verl"},
|
|
611
|
+
"unavailable source metadata requires comparison version 2",
|
|
612
|
+
)
|
|
613
|
+
else:
|
|
614
|
+
_require(
|
|
615
|
+
side.missing_steps is not None and side.missing_rewards is not None,
|
|
616
|
+
"comparison version 2 requires source coverage counts",
|
|
617
|
+
)
|
|
618
|
+
if report.mode == "windows":
|
|
619
|
+
_require(
|
|
620
|
+
a.run == b.run
|
|
621
|
+
and a.source == b.source
|
|
622
|
+
and a.window is not None
|
|
623
|
+
and b.window is not None,
|
|
624
|
+
"windows must select the same source run",
|
|
625
|
+
)
|
|
626
|
+
if same_source and a.window is not None and b.window is not None:
|
|
627
|
+
_require(
|
|
628
|
+
{step for step in a.observed_steps if b.window[0] <= step <= b.window[1]}
|
|
629
|
+
== {
|
|
630
|
+
step for step in b.observed_steps if a.window[0] <= step <= a.window[1]
|
|
631
|
+
},
|
|
632
|
+
"overlapping windows disagree about observed steps",
|
|
633
|
+
)
|
|
634
|
+
mix = report.prompt_mix
|
|
635
|
+
if same_source and set(a.observed_steps) & set(b.observed_steps):
|
|
636
|
+
_require(mix.shared_prompts > 0, "shared observations require shared prompts")
|
|
637
|
+
_require(
|
|
638
|
+
mix.shared_prompts + mix.only_a_prompts == a.n_prompts
|
|
639
|
+
and mix.shared_prompts + mix.only_b_prompts == b.n_prompts,
|
|
640
|
+
"prompt overlap counts differ from side counts",
|
|
641
|
+
)
|
|
642
|
+
_require(
|
|
643
|
+
mix.status == ("shared_prompts" if mix.shared_prompts else "no_shared_prompts"),
|
|
644
|
+
"prompt overlap status differs from counts",
|
|
645
|
+
)
|
|
646
|
+
descriptors = set()
|
|
647
|
+
for shift in mix.prompt_shift:
|
|
648
|
+
_require(
|
|
649
|
+
shift.descriptor not in descriptors, "prompt descriptors must be unique"
|
|
650
|
+
)
|
|
651
|
+
descriptors.add(shift.descriptor)
|
|
652
|
+
_require(
|
|
653
|
+
shift.n_before + shift.missing_before == a.n_prompts
|
|
654
|
+
and shift.n_after + shift.missing_after == b.n_prompts,
|
|
655
|
+
"prompt descriptor counts differ from side counts",
|
|
656
|
+
)
|
|
657
|
+
if shift.before is not None and shift.after is not None:
|
|
658
|
+
_require(
|
|
659
|
+
_same(shift.delta, shift.after - shift.before),
|
|
660
|
+
"prompt shift differs from its means",
|
|
661
|
+
)
|
|
662
|
+
else:
|
|
663
|
+
_require(shift.delta is None, "prompt delta requires both means")
|
|
664
|
+
_require(
|
|
665
|
+
len({feature.feature for feature in report.features}) == len(report.features),
|
|
666
|
+
"features must be unique",
|
|
667
|
+
)
|
|
668
|
+
_require(
|
|
669
|
+
[feature.order for feature in report.features]
|
|
670
|
+
== list(range(1, len(report.features) + 1)),
|
|
671
|
+
"feature order must be contiguous",
|
|
672
|
+
)
|
|
673
|
+
ordered = sorted(
|
|
674
|
+
report.features,
|
|
675
|
+
key=lambda f: (
|
|
676
|
+
f.kind != "binary",
|
|
677
|
+
f.difference.estimate is None if f.kind == "binary" else False,
|
|
678
|
+
-abs(f.difference.estimate)
|
|
679
|
+
if f.kind == "binary" and f.difference.estimate is not None
|
|
680
|
+
else 0,
|
|
681
|
+
f.feature,
|
|
682
|
+
),
|
|
683
|
+
)
|
|
684
|
+
_require(report.features == ordered, "feature order differs from effect ordering")
|
|
685
|
+
citation_metadata = {}
|
|
686
|
+
for feature in report.features:
|
|
687
|
+
_check_feature(feature, report)
|
|
688
|
+
if same_source and a.observed_steps == b.observed_steps:
|
|
689
|
+
_require(
|
|
690
|
+
feature.a == feature.b
|
|
691
|
+
and feature.citations.a == feature.citations.b
|
|
692
|
+
and feature.paired.estimate in (None, 0.0)
|
|
693
|
+
and feature.paired.se in (None, 0.0),
|
|
694
|
+
"identical observations require identical side evidence and zero measured change",
|
|
695
|
+
)
|
|
696
|
+
for side in ("a", "b"):
|
|
697
|
+
for item in getattr(feature.citations, side).items:
|
|
698
|
+
key = (
|
|
699
|
+
None if same_source else side,
|
|
700
|
+
item.run,
|
|
701
|
+
item.sample_id,
|
|
702
|
+
)
|
|
703
|
+
metadata = item.step, item.prompt_id, item.reward, item.response_sha256
|
|
704
|
+
_require(
|
|
705
|
+
key not in citation_metadata or citation_metadata[key] == metadata,
|
|
706
|
+
"canonical citation metadata differs across features",
|
|
707
|
+
)
|
|
708
|
+
citation_metadata[key] = metadata
|
|
709
|
+
result = report.model_dump()
|
|
710
|
+
if report.schema_version == 1:
|
|
711
|
+
for side in result["sides"]:
|
|
712
|
+
side.pop("missing_steps")
|
|
713
|
+
side.pop("missing_rewards")
|
|
714
|
+
side["source"].pop("model")
|
|
715
|
+
result.pop("tool_calls")
|
|
716
|
+
return result
|
|
717
|
+
|
|
718
|
+
|
|
719
|
+
def comparison_json(report: dict) -> str:
|
|
720
|
+
"""Encode a validated comparison as deterministic, strict UTF-8 JSON."""
|
|
721
|
+
return (
|
|
722
|
+
json.dumps(
|
|
723
|
+
validate_comparison_report(report),
|
|
724
|
+
ensure_ascii=False,
|
|
725
|
+
allow_nan=False,
|
|
726
|
+
sort_keys=True,
|
|
727
|
+
indent=2,
|
|
728
|
+
)
|
|
729
|
+
+ "\n"
|
|
730
|
+
)
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def load_comparison_report(path: str | Path) -> dict:
|
|
734
|
+
"""Read strict JSON with duplicate-key checks and errors without payloads."""
|
|
735
|
+
try:
|
|
736
|
+
with Path(path).open(encoding="utf-8") as stream:
|
|
737
|
+
value = json.load(
|
|
738
|
+
stream, object_pairs_hook=_object, parse_constant=_constant
|
|
739
|
+
)
|
|
740
|
+
except (ValueError, RecursionError):
|
|
741
|
+
raise ValueError("invalid comparison report JSON") from None
|
|
742
|
+
return validate_comparison_report(value)
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
_SCRIPT = """
|
|
746
|
+
(() => {
|
|
747
|
+
const cards = Array.from(document.querySelectorAll('.comparison-feature'));
|
|
748
|
+
const text = cards.map(card => card.textContent.toLowerCase());
|
|
749
|
+
const search = document.getElementById('feature-search');
|
|
750
|
+
const kind = document.getElementById('feature-kind');
|
|
751
|
+
const count = document.getElementById('result-count');
|
|
752
|
+
function update() {
|
|
753
|
+
const query = search.value.trim().toLowerCase();
|
|
754
|
+
let visible = 0;
|
|
755
|
+
cards.forEach((card, index) => {
|
|
756
|
+
card.hidden = !text[index].includes(query)
|
|
757
|
+
|| (kind.value !== 'all' && card.dataset.kind !== kind.value);
|
|
758
|
+
if (!card.hidden) visible++;
|
|
759
|
+
});
|
|
760
|
+
count.textContent = `${visible} of ${cards.length} features shown`;
|
|
761
|
+
document.getElementById('no-matches').hidden = visible !== 0 || cards.length === 0;
|
|
762
|
+
}
|
|
763
|
+
search.addEventListener('input', update);
|
|
764
|
+
kind.addEventListener('change', update);
|
|
765
|
+
document.getElementById('reset-filters').addEventListener('click', () => {
|
|
766
|
+
search.value = ''; kind.value = 'all'; update(); search.focus();
|
|
767
|
+
});
|
|
768
|
+
document.getElementById('expand-features').addEventListener('click', () => {
|
|
769
|
+
const visible = cards.filter(card => !card.hidden);
|
|
770
|
+
const expand = visible.some(card => !card.open);
|
|
771
|
+
visible.forEach(card => { card.open = expand; });
|
|
772
|
+
});
|
|
773
|
+
document.getElementById('controls').hidden = false;
|
|
774
|
+
update();
|
|
775
|
+
})();
|
|
776
|
+
"""
|
|
777
|
+
_COMPARISON_STYLE = """
|
|
778
|
+
.table-wrap { overflow-x: auto; margin: 14px 0; }
|
|
779
|
+
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
|
780
|
+
th, td { padding: 9px 10px; border-bottom: 1px solid var(--line); text-align: left; vertical-align: top; }
|
|
781
|
+
th { font-weight: 650; } td { font-variant-numeric: tabular-nums; }
|
|
782
|
+
.delta { font-weight: 700; font-variant-numeric: tabular-nums; }
|
|
783
|
+
.comparison-feature .event-body > p { margin-bottom: 10px; }
|
|
784
|
+
.sources { margin: 14px 0 24px; }
|
|
785
|
+
.sources > summary { padding: 8px 0; }
|
|
786
|
+
"""
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
def _text(value: object) -> str:
|
|
790
|
+
return escape(str(value), quote=True)
|
|
791
|
+
|
|
792
|
+
|
|
793
|
+
def _number(value: float | None, *, scale: float = 1, signed: bool = False) -> str:
|
|
794
|
+
if value is None:
|
|
795
|
+
return '<span class="unavailable">Unavailable</span>'
|
|
796
|
+
number = value * scale
|
|
797
|
+
spec = ".4g" if abs(number) >= 1e6 or 0 < abs(number) < 0.0001 else ".2f"
|
|
798
|
+
return format(number, ("+" if signed else "") + spec)
|
|
799
|
+
|
|
800
|
+
|
|
801
|
+
def _measurement(row: dict, kind: str, *, difference: bool = False) -> str:
|
|
802
|
+
scale = 100 if kind == "binary" else 1
|
|
803
|
+
unit = (" pp" if difference else "%") if kind == "binary" else ""
|
|
804
|
+
center = row["estimate" if difference else "mean"]
|
|
805
|
+
mean = _number(center, scale=scale, signed=difference)
|
|
806
|
+
if center is not None:
|
|
807
|
+
mean += unit
|
|
808
|
+
interval = "Unavailable"
|
|
809
|
+
if row["ci_low"] is not None:
|
|
810
|
+
interval = f"[{_number(row['ci_low'], scale=scale)}, {_number(row['ci_high'], scale=scale)}]{unit}"
|
|
811
|
+
status = {
|
|
812
|
+
"available": "Available",
|
|
813
|
+
"insufficient_units": "Fewer than two sampling units",
|
|
814
|
+
"partial_overlap": "Only some prompts are shared",
|
|
815
|
+
"shared_units": "Shared sampling units without paired prompts",
|
|
816
|
+
"incompatible_units": "Sampling-unit groupings differ",
|
|
817
|
+
"numeric_unavailable": "Numerical limits",
|
|
818
|
+
"no_shared_prompts": "No shared prompts",
|
|
819
|
+
"missing_observations": "Reward observations missing",
|
|
820
|
+
}[row["status"]]
|
|
821
|
+
return f"<td>{mean}</td><td>{_number(row['se'], scale=scale)}{unit if row['se'] is not None else ''}</td><td>{interval}</td><td>{status}</td>"
|
|
822
|
+
|
|
823
|
+
|
|
824
|
+
def _citation_html(citations: dict, side: str) -> str:
|
|
825
|
+
parts = [f'<section class="metric"><h4>Side {side.upper()} citations</h4>']
|
|
826
|
+
if not citations["items"]:
|
|
827
|
+
parts.append(
|
|
828
|
+
'<p class="note">Continuous features have no selected citations.</p>'
|
|
829
|
+
if citations["status"] == "unsupported_kind"
|
|
830
|
+
else '<p class="note">No positive rollouts in this range.</p>'
|
|
831
|
+
)
|
|
832
|
+
for item in citations["items"]:
|
|
833
|
+
parts.append(
|
|
834
|
+
f'<details class="exemplar"><summary><code>{_text(item["sample_id"])}</code> · step {item["step"] if item["step"] is not None else "unavailable"}</summary><dl>'
|
|
835
|
+
)
|
|
836
|
+
for key, label in (
|
|
837
|
+
("run", "Run"),
|
|
838
|
+
("prompt_id", "Prompt ID"),
|
|
839
|
+
("feature_value", "Feature value"),
|
|
840
|
+
("reward", "Reward"),
|
|
841
|
+
("selection_reason", "Selection reason"),
|
|
842
|
+
("response_sha256", "Response SHA-256"),
|
|
843
|
+
):
|
|
844
|
+
value = (
|
|
845
|
+
item[key].replace("_", " ") if key == "selection_reason" else item[key]
|
|
846
|
+
)
|
|
847
|
+
if value is None:
|
|
848
|
+
value = "Unavailable"
|
|
849
|
+
parts.append(f"<dt>{label}</dt><dd><code>{_text(value)}</code></dd>")
|
|
850
|
+
parts.append("</dl></details>")
|
|
851
|
+
parts.append("</section>")
|
|
852
|
+
return "".join(parts)
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def _feature_html(feature: dict) -> str:
|
|
856
|
+
kind, difference = feature["kind"], feature["difference"]
|
|
857
|
+
effect = _number(
|
|
858
|
+
difference["estimate"], scale=100 if kind == "binary" else 1, signed=True
|
|
859
|
+
)
|
|
860
|
+
if difference["estimate"] is not None:
|
|
861
|
+
effect += " pp" if kind == "binary" else " native units"
|
|
862
|
+
parts = [
|
|
863
|
+
f'<details class="event-card comparison-feature" data-kind="{kind}" id="feature-{feature["order"]}">',
|
|
864
|
+
f'<summary><span class="rank">{feature["order"]}</span><span><code class="representative">{_text(feature["feature"])}</code>',
|
|
865
|
+
f'<span class="anchors">{feature["tier"]} · {kind} · {feature["direction"]}</span></span>',
|
|
866
|
+
f'<span class="delta">{effect}</span></summary><div class="event-body">',
|
|
867
|
+
f'<p class="note">B - A interval method: {_text(difference["method"])}. {"Binary prevalence and percentage-point differences." if kind == "binary" else "Continuous measurements retain native units."}</p>',
|
|
868
|
+
'<div class="table-wrap"><table><thead><tr><th scope="col">Measurement</th><th scope="col">Estimate</th><th scope="col">Standard error</th><th scope="col">95% normal interval</th><th scope="col">Interval status</th></tr></thead><tbody>',
|
|
869
|
+
]
|
|
870
|
+
for side in ("a", "b"):
|
|
871
|
+
parts.append(
|
|
872
|
+
f'<tr><th scope="row">Side {side.upper()}</th>{_measurement(feature[side], kind)}</tr>'
|
|
873
|
+
)
|
|
874
|
+
parts.append(
|
|
875
|
+
f'<tr><th scope="row">B - A</th>{_measurement(difference, kind, difference=True)}</tr>'
|
|
876
|
+
)
|
|
877
|
+
parts.append(
|
|
878
|
+
f'<tr><th scope="row">Shared-prompt change</th>{_measurement(feature["paired"], kind, difference=True)}</tr></tbody></table></div>'
|
|
879
|
+
)
|
|
880
|
+
paired = feature["paired"]
|
|
881
|
+
parts.append(
|
|
882
|
+
f'<p class="note">Shared prompts: {paired["n_prompts"]}. Sampling units: {"unavailable" if paired["n_sampling_units"] is None else paired["n_sampling_units"]}.</p>'
|
|
883
|
+
)
|
|
884
|
+
if kind == "binary":
|
|
885
|
+
parts.append(
|
|
886
|
+
f'<p class="note">Positive prompts: A {feature["a"]["positive_prompts"]} of {feature["a"]["n_prompts"]}; B {feature["b"]["positive_prompts"]} of {feature["b"]["n_prompts"]}.</p>'
|
|
887
|
+
)
|
|
888
|
+
parts.append('<div class="metrics">')
|
|
889
|
+
parts.extend(
|
|
890
|
+
_citation_html(feature["citations"][side], side) for side in ("a", "b")
|
|
891
|
+
)
|
|
892
|
+
parts.append("</div></div></details>")
|
|
893
|
+
return "".join(parts)
|
|
894
|
+
|
|
895
|
+
|
|
896
|
+
def render_comparison_report(report: dict) -> str:
|
|
897
|
+
"""Render validated comparison evidence as escaped HTML."""
|
|
898
|
+
report = validate_comparison_report(report)
|
|
899
|
+
script_hash = b64encode(hashlib.sha256(_SCRIPT.encode("utf-8")).digest()).decode(
|
|
900
|
+
"ascii"
|
|
901
|
+
)
|
|
902
|
+
policy = f"default-src 'none'; style-src 'unsafe-inline'; script-src 'sha256-{script_hash}'; base-uri 'none'; form-action 'none'"
|
|
903
|
+
parts = [
|
|
904
|
+
'<!doctype html>\n<html lang="en"><head><meta charset="utf-8">',
|
|
905
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
|
906
|
+
f'<meta http-equiv="Content-Security-Policy" content="{escape(policy, quote=True)}">',
|
|
907
|
+
"<title>Rollout Diff comparison report</title>",
|
|
908
|
+
f"<style>{_STYLE}{_COMPARISON_STYLE}</style></head><body><main>",
|
|
909
|
+
f'<header class="intro"><p class="eyebrow">Rollout Diff · Comparison version {report["schema_version"]}</p><h1>Comparison report</h1>',
|
|
910
|
+
"<p>Differences are B - A. Each exact prompt receives equal weight after pooling its completions within each selected range. Both ranges share one feature vocabulary. Binary features are ordered by absolute prevalence change, followed by continuous features in name order.</p>",
|
|
911
|
+
'<p class="note">Pointwise 95% intervals use a normal approximation and measured standard errors. Supplied sampling units account for dependence among prompts while preserving equal prompt weights. Intervals are unadjusted for comparisons across features. No standard-error floor is applied. Causal and task-success claims require separate evidence.</p>',
|
|
912
|
+
'<p class="note">Comparing rewards across dumps requires the same reward definition and scale.</p></header>',
|
|
913
|
+
'<div class="table-wrap"><table><thead><tr><th>Side</th><th>Run</th><th>Inclusive steps</th><th>Rollouts</th><th>Exact prompts</th><th>Sampling units</th></tr></thead><tbody>',
|
|
914
|
+
]
|
|
915
|
+
for side in report["sides"]:
|
|
916
|
+
window = (
|
|
917
|
+
f"{side['window'][0]} to {side['window'][1]} ({len(side['observed_steps'])} observed)"
|
|
918
|
+
if side["window"] is not None
|
|
919
|
+
else "Training steps unavailable"
|
|
920
|
+
)
|
|
921
|
+
parts.append(
|
|
922
|
+
f'<tr><th scope="row">{side["side"].upper()}</th><td><code>{_text(side["run"])}</code></td><td>{window}</td><td>{side["n_rollouts"]}</td><td>{side["n_prompts"]}</td><td>{side["n_sampling_units"]}</td></tr>'
|
|
923
|
+
)
|
|
924
|
+
parts.append("</tbody></table></div>")
|
|
925
|
+
if report["schema_version"] == 2:
|
|
926
|
+
for side in report["sides"]:
|
|
927
|
+
parts.append(
|
|
928
|
+
f'<p class="note">Side {side["side"].upper()}: {side["missing_steps"]} records without training steps; {side["missing_rewards"]} without reward. Missing reward observations leave the full-side reward comparison unavailable.</p>'
|
|
929
|
+
)
|
|
930
|
+
mix = report["prompt_mix"]
|
|
931
|
+
parts.append(
|
|
932
|
+
f"<section><h2>Prompt overlap</h2><p>{mix['shared_prompts']} shared prompts, {mix['only_a_prompts']} only in A, {mix['only_b_prompts']} only in B.</p>"
|
|
933
|
+
)
|
|
934
|
+
if not mix["shared_prompts"]:
|
|
935
|
+
parts.append(
|
|
936
|
+
'<p class="warning-note">No exact prompts are shared, so a paired change is unavailable. Independent-side intervals require disjoint sampling units. Differences may reflect prompt composition.</p>'
|
|
937
|
+
)
|
|
938
|
+
elif mix["only_a_prompts"] or mix["only_b_prompts"]:
|
|
939
|
+
parts.append(
|
|
940
|
+
'<p class="warning-note">Partial overlap leaves the overall interval unavailable. The paired comparison covers only shared prompts. Differences may reflect prompt composition.</p>'
|
|
941
|
+
)
|
|
942
|
+
else:
|
|
943
|
+
parts.append(
|
|
944
|
+
'<p class="note">All exact prompts are shared. Paired uncertainty requires compatible sampling-unit groupings.</p>'
|
|
945
|
+
)
|
|
946
|
+
parts.append(
|
|
947
|
+
'<p class="note">Intervals need at least two sampling units. Partial prompt overlap, shared units without paired prompts, incompatible groupings or numerical limits can leave an interval unavailable. Available estimates are still shown. A zero measured standard error does not establish zero uncertainty.</p>'
|
|
948
|
+
)
|
|
949
|
+
if mix["prompt_shift"]:
|
|
950
|
+
parts.append(
|
|
951
|
+
'<div class="table-wrap"><table><thead><tr><th>Prompt descriptor</th><th>Statistic</th><th>A mean</th><th>B mean</th><th>B - A</th><th>Distance</th><th>Observed A / B</th><th>Missing A / B</th></tr></thead><tbody>'
|
|
952
|
+
)
|
|
953
|
+
for shift in mix["prompt_shift"]:
|
|
954
|
+
parts.append(
|
|
955
|
+
f"<tr><th><code>{_text(shift['descriptor'])}</code></th><td>{_text(shift['statistic'])}</td><td>{_number(shift['before'])}</td><td>{_number(shift['after'])}</td><td>{_number(shift['delta'], signed=True)}</td><td>{_number(shift['distance'])}</td><td>{shift['n_before']} / {shift['n_after']}</td><td>{shift['missing_before']} / {shift['missing_after']}</td></tr>"
|
|
956
|
+
)
|
|
957
|
+
parts.append("</tbody></table></div>")
|
|
958
|
+
parts.append(
|
|
959
|
+
'</section><details class="sources"><summary>Source fingerprints</summary><div class="metrics">'
|
|
960
|
+
)
|
|
961
|
+
for side in report["sides"]:
|
|
962
|
+
source = side["source"]
|
|
963
|
+
parts.append(
|
|
964
|
+
f'<section class="metric"><h3>Side {side["side"].upper()}</h3><dl><dt>Canonical SHA-256</dt><dd><code>{source["canonical_sha256"]}</code></dd><dt>Format</dt><dd>{source["format"]}</dd><dt>Reward column</dt><dd>{_text(source["reward_column"]) if source["reward_column"] is not None else "Unspecified"}</dd></dl>'
|
|
965
|
+
)
|
|
966
|
+
for file in source["files"]:
|
|
967
|
+
parts.append(
|
|
968
|
+
f'<p class="reference-note"><code>{file["sha256"]}</code> · {file["size_bytes"]} bytes</p>'
|
|
969
|
+
)
|
|
970
|
+
parts.append("</section>")
|
|
971
|
+
parts.extend(
|
|
972
|
+
[
|
|
973
|
+
"</div></details><section><h2>Feature differences</h2>",
|
|
974
|
+
'<p class="note">Citations are deterministic feature-positive examples selected without reward ranking. Their selection does not establish representativeness. Inspect responses in the original dump by side and feature.</p>',
|
|
975
|
+
'<div class="controls" id="controls" hidden><label class="search-label">Search features<input id="feature-search" type="search" placeholder="Feature or sample ID" autocomplete="off"></label><label>Feature kind<select id="feature-kind"><option value="all">All features</option><option value="binary">Binary</option><option value="continuous">Continuous</option></select></label><button id="reset-filters" type="button">Reset filters</button><button id="expand-features" type="button">Expand / collapse</button><p class="result-count" id="result-count" role="status" aria-live="polite"></p></div><p class="empty" id="no-matches" hidden>No matching features.</p>',
|
|
976
|
+
]
|
|
977
|
+
)
|
|
978
|
+
if not report["features"]:
|
|
979
|
+
parts.append('<p class="empty">No features in this comparison.</p>')
|
|
980
|
+
parts.extend(_feature_html(feature) for feature in report["features"])
|
|
981
|
+
parts.append(f"</section></main><script>{_SCRIPT}</script></body></html>\n")
|
|
982
|
+
return "".join(parts)
|