rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,726 @@
|
|
|
1
|
+
"""Static, self-contained presentation of version-1 investigation reports."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from base64 import b64encode
|
|
8
|
+
from html import escape
|
|
9
|
+
|
|
10
|
+
from ._report_validation import load_report, validate_report
|
|
11
|
+
|
|
12
|
+
__all__ = ["load_report", "render_report"]
|
|
13
|
+
|
|
14
|
+
_STYLE = """
|
|
15
|
+
:root { color-scheme: light; --ink: #182d35; --muted: #52646c;
|
|
16
|
+
--line: #d8e0e2; --accent: #17695f; --paper: #f5f7f7; }
|
|
17
|
+
* { box-sizing: border-box; }
|
|
18
|
+
body { margin: 0; background: var(--paper); color: var(--ink);
|
|
19
|
+
font: 15px/1.5 system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; }
|
|
20
|
+
main { max-width: 1200px; margin: auto; padding: 32px 28px 60px; }
|
|
21
|
+
h1, h2, h3, h4, p { margin: 0; }
|
|
22
|
+
h1 { font-size: 30px; letter-spacing: -.7px; line-height: 1.2; margin: 4px 0 12px; }
|
|
23
|
+
h2 { font-size: 22px; margin: 0 0 12px; }
|
|
24
|
+
h3 { font-size: 16px; margin-bottom: 10px; }
|
|
25
|
+
h4 { font-size: 14px; margin-bottom: 10px; }
|
|
26
|
+
code { font: .92em/1.5 ui-monospace, SFMono-Regular, Consolas, monospace;
|
|
27
|
+
overflow-wrap: anywhere; }
|
|
28
|
+
.eyebrow { color: var(--accent); font-weight: 700; font-size: 12px;
|
|
29
|
+
letter-spacing: 1.2px; text-transform: uppercase; }
|
|
30
|
+
.muted, .note, .unavailable { color: var(--muted); }
|
|
31
|
+
.note { font-size: 13px; }
|
|
32
|
+
.intro { max-width: 850px; margin-bottom: 22px; }
|
|
33
|
+
.controls { position: sticky; top: 0; z-index: 2; background: #f5f7f7;
|
|
34
|
+
border-block: 1px solid var(--line); padding: 12px 0;
|
|
35
|
+
display: grid; grid-template-columns: minmax(180px, 1fr) 180px auto auto;
|
|
36
|
+
gap: 12px; align-items: end; }
|
|
37
|
+
label { display: block; font-size: 12px; font-weight: 650; }
|
|
38
|
+
input[type=search], select, button { font: inherit; color: var(--ink);
|
|
39
|
+
background: white; border: 1px solid #9caaaf; border-radius: 5px; padding: 8px 10px; }
|
|
40
|
+
input[type=search], select { width: 100%; margin-top: 4px; }
|
|
41
|
+
input[type=checkbox] { accent-color: var(--accent); }
|
|
42
|
+
.checkbox { padding-bottom: 10px; white-space: nowrap; }
|
|
43
|
+
button, summary { cursor: pointer; }
|
|
44
|
+
button:hover { background: #e8f1ef; }
|
|
45
|
+
:focus-visible { outline: 3px solid #157b9e; outline-offset: 3px; }
|
|
46
|
+
.result-count { grid-column: 1 / -1; font-size: 12px; color: var(--muted); }
|
|
47
|
+
.run { margin-top: 30px; }
|
|
48
|
+
.stats { display: grid; grid-template-columns: repeat(4, 1fr); gap: 12px;
|
|
49
|
+
margin: 14px 0 10px; }
|
|
50
|
+
.stat { background: white; border: 1px solid var(--line); border-radius: 6px;
|
|
51
|
+
padding: 12px 16px; }
|
|
52
|
+
.stat strong { display: block; font-size: 24px; font-variant-numeric: tabular-nums; }
|
|
53
|
+
.stat span { font-size: 12px; color: var(--muted); }
|
|
54
|
+
.run-counts { margin-bottom: 18px; }
|
|
55
|
+
.event-card { background: white; border: 1px solid var(--line);
|
|
56
|
+
border-radius: 7px; margin: 10px 0; overflow: clip; }
|
|
57
|
+
.event-card > summary { display: grid; grid-template-columns: 36px minmax(0, 1fr) auto 14px;
|
|
58
|
+
gap: 12px; padding: 16px; align-items: start; list-style: none; }
|
|
59
|
+
summary::-webkit-details-marker { display: none; }
|
|
60
|
+
.event-card > summary::after { content: "+"; color: var(--muted); font-size: 18px; }
|
|
61
|
+
.event-card[open] > summary::after { content: "-"; }
|
|
62
|
+
.event-card[open] > summary { border-bottom: 1px solid var(--line); }
|
|
63
|
+
.rank { font-size: 13px; color: var(--muted); padding-top: 3px; }
|
|
64
|
+
.representative { display: block; font-weight: 650; font-size: 16px; }
|
|
65
|
+
.anchors { display: block; font-size: 12px; color: var(--muted); margin-top: 5px; }
|
|
66
|
+
.badges { display: flex; gap: 6px; flex-wrap: wrap; justify-content: end; max-width: 270px; }
|
|
67
|
+
.badge { font-size: 11px; border-radius: 4px; background: #e9f2ef;
|
|
68
|
+
color: #1c584e; padding: 3px 7px; white-space: nowrap; }
|
|
69
|
+
.badge.contextual, .badge.other { background: #eef0f3; color: #4d5d70; }
|
|
70
|
+
.badge.warning { background: #fff1ce; color: #785015; }
|
|
71
|
+
.event-body { padding: 18px; }
|
|
72
|
+
.metrics { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr));
|
|
73
|
+
gap: 14px; margin: 14px 0; }
|
|
74
|
+
.metric { border: 1px solid var(--line); border-radius: 5px; padding: 14px; }
|
|
75
|
+
dl { display: grid; grid-template-columns: minmax(130px, 1fr) minmax(0, 1fr);
|
|
76
|
+
gap: 6px 16px; font-size: 13px; margin: 0; }
|
|
77
|
+
dt { color: var(--muted); }
|
|
78
|
+
dd { margin: 0; overflow-wrap: anywhere; font-variant-numeric: tabular-nums; }
|
|
79
|
+
.member { margin-top: 20px; padding-top: 20px; border-top: 1px solid var(--line); }
|
|
80
|
+
.member-title { display: flex; gap: 10px; flex-wrap: wrap; align-items: baseline; }
|
|
81
|
+
.member-title h3 { margin-bottom: 4px; }
|
|
82
|
+
.member-meta { margin: 10px 0; }
|
|
83
|
+
.warning-note { background: #fff7e2; border-left: 3px solid #c69936;
|
|
84
|
+
padding: 8px 12px; font-size: 13px; margin: 10px 0; overflow-wrap: anywhere; }
|
|
85
|
+
.reference-note { font-size: 12px; color: var(--muted); margin: 8px 0; }
|
|
86
|
+
.exemplar { border: 1px solid var(--line); border-radius: 5px; margin-top: 7px; }
|
|
87
|
+
.exemplar > summary { padding: 10px 12px; font-size: 13px; }
|
|
88
|
+
.exemplar > dl { padding: 4px 14px 14px; }
|
|
89
|
+
.empty { padding: 24px 0; color: var(--muted); }
|
|
90
|
+
.table-scroll { overflow-x: auto; margin: 12px 0; }
|
|
91
|
+
table { width: 100%; border-collapse: collapse; font-size: 13px; }
|
|
92
|
+
th, td { text-align: left; padding: 6px 10px; border-bottom: 1px solid var(--line); }
|
|
93
|
+
[hidden] { display: none !important; }
|
|
94
|
+
@media (max-width: 700px) {
|
|
95
|
+
main { padding: 22px 16px 40px; } h1 { font-size: 26px; }
|
|
96
|
+
.controls { position: static; grid-template-columns: 1fr 1fr; }
|
|
97
|
+
.search-label { grid-column: 1 / -1; } .checkbox { white-space: normal; }
|
|
98
|
+
.stats { grid-template-columns: 1fr 1fr; gap: 8px; }
|
|
99
|
+
.event-card > summary { grid-template-columns: 26px minmax(0, 1fr) 12px; gap: 8px; padding: 12px; }
|
|
100
|
+
.badges { grid-column: 2; justify-content: start; max-width: none; }
|
|
101
|
+
.event-card > summary::after { grid-column: 3; grid-row: 1; }
|
|
102
|
+
.event-body { padding: 12px; } .metrics { grid-template-columns: 1fr; }
|
|
103
|
+
dl { grid-template-columns: minmax(110px, 1fr) minmax(0, 1fr); gap: 6px 10px; }
|
|
104
|
+
}
|
|
105
|
+
@media print {
|
|
106
|
+
.controls { display: none; } main { max-width: none; padding: 0; }
|
|
107
|
+
body { background: white; } .event-card { break-inside: avoid; }
|
|
108
|
+
}
|
|
109
|
+
"""
|
|
110
|
+
|
|
111
|
+
_SCRIPT = """
|
|
112
|
+
(() => {
|
|
113
|
+
const cards = Array.from(document.querySelectorAll('.event-card'));
|
|
114
|
+
const text = cards.map(card => card.textContent.toLowerCase());
|
|
115
|
+
const search = document.getElementById('event-search');
|
|
116
|
+
const status = document.getElementById('event-status');
|
|
117
|
+
const floor = document.getElementById('floor-only');
|
|
118
|
+
const count = document.getElementById('result-count');
|
|
119
|
+
function update() {
|
|
120
|
+
const query = search.value.trim().toLowerCase();
|
|
121
|
+
let visible = 0;
|
|
122
|
+
cards.forEach((card, index) => {
|
|
123
|
+
card.hidden = !text[index].includes(query)
|
|
124
|
+
|| (status.value !== 'all' && card.dataset.category !== status.value)
|
|
125
|
+
|| (floor.checked && card.dataset.floor !== 'true');
|
|
126
|
+
if (!card.hidden) visible++;
|
|
127
|
+
});
|
|
128
|
+
count.textContent = `${visible} of ${cards.length} events shown`;
|
|
129
|
+
document.getElementById('no-matches').hidden = visible !== 0 || cards.length === 0;
|
|
130
|
+
}
|
|
131
|
+
search.addEventListener('input', update);
|
|
132
|
+
status.addEventListener('change', update);
|
|
133
|
+
floor.addEventListener('change', update);
|
|
134
|
+
document.getElementById('reset-filters').addEventListener('click', () => {
|
|
135
|
+
search.value = '';
|
|
136
|
+
status.value = 'all';
|
|
137
|
+
floor.checked = false;
|
|
138
|
+
update();
|
|
139
|
+
search.focus();
|
|
140
|
+
});
|
|
141
|
+
document.getElementById('controls').hidden = false;
|
|
142
|
+
update();
|
|
143
|
+
})();
|
|
144
|
+
"""
|
|
145
|
+
|
|
146
|
+
_CONTROLS = """
|
|
147
|
+
<div class="controls" id="controls" hidden>
|
|
148
|
+
<label class="search-label">Search events
|
|
149
|
+
<input id="event-search" type="search" placeholder="Feature, event or sample ID" autocomplete="off">
|
|
150
|
+
</label>
|
|
151
|
+
<label>Ranking status<select id="event-status">
|
|
152
|
+
<option value="all">All events</option><option value="behavioral">Behavioral</option>
|
|
153
|
+
<option value="contextual">Contextual</option><option value="other">Other / unavailable</option>
|
|
154
|
+
</select></label>
|
|
155
|
+
<label class="checkbox"><input type="checkbox" id="floor-only"> Numerical floor only</label>
|
|
156
|
+
<button id="reset-filters" type="button">Reset filters</button>
|
|
157
|
+
<p class="result-count" id="result-count" role="status" aria-live="polite"></p>
|
|
158
|
+
</div>
|
|
159
|
+
<p class="empty" id="no-matches" hidden>No matching events. Adjust or reset the filters.</p>
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
_REPLICATION = (
|
|
163
|
+
("status", "Status"),
|
|
164
|
+
("replicates", "Replicates"),
|
|
165
|
+
("horizon_complete", "Horizon complete"),
|
|
166
|
+
("coverage_policy", "Coverage policy"),
|
|
167
|
+
("cutoff_step", "Cutoff step"),
|
|
168
|
+
)
|
|
169
|
+
_PERSISTENCE = (
|
|
170
|
+
("status", "Status"),
|
|
171
|
+
("fraction", "Retained fraction"),
|
|
172
|
+
("coverage", "Evaluable coverage"),
|
|
173
|
+
("n_observations", "Observed observations"),
|
|
174
|
+
("n_evaluable", "Evaluable observations"),
|
|
175
|
+
("n_retained", "Retained observations"),
|
|
176
|
+
("n_unavailable", "Unavailable observations"),
|
|
177
|
+
)
|
|
178
|
+
_REWARD = (
|
|
179
|
+
("status", "Status"),
|
|
180
|
+
("mean_difference", "Mean reward difference"),
|
|
181
|
+
("n_groups", "Groups"),
|
|
182
|
+
("n_mixed_groups", "Mixed groups"),
|
|
183
|
+
("n_rollouts", "Window rollouts"),
|
|
184
|
+
("n_included_rollouts", "Included rollouts"),
|
|
185
|
+
("window_start", "Window start"),
|
|
186
|
+
("window_end", "Window end"),
|
|
187
|
+
)
|
|
188
|
+
_BREADTH = (
|
|
189
|
+
("positive_prompts", "Positive prompts"),
|
|
190
|
+
("total_prompts", "Total prompts"),
|
|
191
|
+
("promptmix_status", "Shared-prompt status"),
|
|
192
|
+
)
|
|
193
|
+
_EXEMPLAR = (
|
|
194
|
+
("run", "Run"),
|
|
195
|
+
("sample_id", "Sample ID"),
|
|
196
|
+
("prompt_id", "Prompt ID"),
|
|
197
|
+
("step", "Original step"),
|
|
198
|
+
("status", "Status"),
|
|
199
|
+
("selection_order", "Selection order"),
|
|
200
|
+
("selection_reason", "Selection reason"),
|
|
201
|
+
("feature_value", "Feature value"),
|
|
202
|
+
("reward", "Reward"),
|
|
203
|
+
("reward_selection_status", "Reward selection status"),
|
|
204
|
+
("reward_group_size", "Reward group size"),
|
|
205
|
+
("reward_group_mean", "Reward group mean"),
|
|
206
|
+
("reward_deviation", "Reward deviation"),
|
|
207
|
+
("reward_deviation_status", "Reward deviation status"),
|
|
208
|
+
("n_window_rollouts", "Window rollouts"),
|
|
209
|
+
("n_candidates", "Candidates"),
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _value(value: object, *, identifier: bool = False) -> str:
|
|
214
|
+
if value is None:
|
|
215
|
+
return '<span class="unavailable">Unavailable</span>'
|
|
216
|
+
if isinstance(value, bool):
|
|
217
|
+
return "Yes" if value else "No"
|
|
218
|
+
text = escape(str(value))
|
|
219
|
+
return f"<code>{text}</code>" if identifier else text
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _fields(record: dict, fields: tuple) -> str:
|
|
223
|
+
return (
|
|
224
|
+
"<dl>"
|
|
225
|
+
+ "".join(
|
|
226
|
+
f"<dt>{label}</dt><dd>{_value(record[key], identifier=key in {'run', 'sample_id', 'prompt_id'})}</dd>"
|
|
227
|
+
for key, label in fields
|
|
228
|
+
)
|
|
229
|
+
+ "</dl>"
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _status(value: str) -> str:
|
|
234
|
+
return escape(value.replace("_", " "))
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _references(references: list[dict]) -> str:
|
|
238
|
+
return ", ".join(
|
|
239
|
+
_value(item["feature"], identifier=True)
|
|
240
|
+
for item in sorted(references, key=lambda item: (item["run"], item["feature"]))
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _prompt_mix(mix: dict | None) -> str:
|
|
245
|
+
result = ['<section class="metric"><h4>Shared-prompt change</h4>']
|
|
246
|
+
if mix is None:
|
|
247
|
+
return (
|
|
248
|
+
"".join(result)
|
|
249
|
+
+ '<p class="unavailable">Prompt-mix detail was not recorded.</p></section>'
|
|
250
|
+
)
|
|
251
|
+
result.extend(
|
|
252
|
+
[
|
|
253
|
+
'<p class="note">Baseline window: ',
|
|
254
|
+
f"{_value(mix['before_start'])} to {_value(mix['before_end'])}. ",
|
|
255
|
+
f"Alert window: {_value(mix['after_start'])} to {_value(mix['after_end'])}.</p>",
|
|
256
|
+
_fields(
|
|
257
|
+
mix,
|
|
258
|
+
(
|
|
259
|
+
("status", "Status"),
|
|
260
|
+
("shared_prompts", "Exact shared prompts"),
|
|
261
|
+
("shared_sampling_units", "Shared sampling units"),
|
|
262
|
+
("shared_change", "Mean within-prompt change"),
|
|
263
|
+
("shared_se", "Shared-change SE"),
|
|
264
|
+
),
|
|
265
|
+
),
|
|
266
|
+
]
|
|
267
|
+
)
|
|
268
|
+
if mix["status"] == "no_shared_prompts":
|
|
269
|
+
result.append('<p class="note">No exact prompts appear in both windows.</p>')
|
|
270
|
+
if mix["shared_se"] == 0:
|
|
271
|
+
result.append(
|
|
272
|
+
'<p class="note">A zero measured SE does not establish zero uncertainty.</p>'
|
|
273
|
+
)
|
|
274
|
+
result.append("<h4>Prompt-side shift</h4>")
|
|
275
|
+
if mix["prompt_shift_status"] == "missing":
|
|
276
|
+
result.append(
|
|
277
|
+
'<p class="unavailable">Prompt-side measurements were not recorded.</p>'
|
|
278
|
+
)
|
|
279
|
+
else:
|
|
280
|
+
columns = (
|
|
281
|
+
("descriptor", "Descriptor"),
|
|
282
|
+
("statistic", "Statistic"),
|
|
283
|
+
("before", "Before mean"),
|
|
284
|
+
("after", "After mean"),
|
|
285
|
+
("delta", "Difference"),
|
|
286
|
+
("distance", "Distance"),
|
|
287
|
+
("n_before", "Observed before"),
|
|
288
|
+
("n_after", "Observed after"),
|
|
289
|
+
("missing_before", "Missing before"),
|
|
290
|
+
("missing_after", "Missing after"),
|
|
291
|
+
)
|
|
292
|
+
result.append('<div class="table-scroll" tabindex="0"><table><thead><tr>')
|
|
293
|
+
result.extend(f'<th scope="col">{title}</th>' for _, title in columns)
|
|
294
|
+
result.append("</tr></thead><tbody>")
|
|
295
|
+
for row in sorted(mix["prompt_shift"], key=lambda row: row["descriptor"]):
|
|
296
|
+
result.append("<tr>")
|
|
297
|
+
result.extend(f"<td>{_value(row[key])}</td>" for key, _ in columns)
|
|
298
|
+
result.append("</tr>")
|
|
299
|
+
result.append("</tbody></table></div>")
|
|
300
|
+
result.append(
|
|
301
|
+
'<p class="reference-note">Prompt composition can contribute to the overall change.</p></section>'
|
|
302
|
+
)
|
|
303
|
+
return "".join(result)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _member(member: dict) -> str:
|
|
307
|
+
evidence = member["evidence"]
|
|
308
|
+
result = [
|
|
309
|
+
'<article class="member"><div class="member-title">',
|
|
310
|
+
f"<h3>{_value(member['feature'], identifier=True)}</h3>",
|
|
311
|
+
f'<span class="note">{_status(member["kind"])} · {_status(member["direction"])}</span></div>',
|
|
312
|
+
'<div class="member-meta">',
|
|
313
|
+
_fields(
|
|
314
|
+
member,
|
|
315
|
+
(
|
|
316
|
+
("run", "Run"),
|
|
317
|
+
("onset_step", "Onset step"),
|
|
318
|
+
("alert_step", "Alert step"),
|
|
319
|
+
("status", "Membership status"),
|
|
320
|
+
),
|
|
321
|
+
),
|
|
322
|
+
"</div>",
|
|
323
|
+
_fields(
|
|
324
|
+
evidence,
|
|
325
|
+
(
|
|
326
|
+
("status", "Evidence record"),
|
|
327
|
+
("tier", "Feature tier"),
|
|
328
|
+
("scale_source", "Scale source"),
|
|
329
|
+
("numerical_floor_dominant", "Numerical floor dominant"),
|
|
330
|
+
),
|
|
331
|
+
),
|
|
332
|
+
]
|
|
333
|
+
if evidence["numerical_floor_dominant"] is True:
|
|
334
|
+
result.append(
|
|
335
|
+
'<p class="warning-note">The numerical floor dominates this feature\'s baseline scale.</p>'
|
|
336
|
+
)
|
|
337
|
+
if evidence.get("retrospective") is not None:
|
|
338
|
+
result.append(
|
|
339
|
+
_fields(
|
|
340
|
+
evidence["retrospective"],
|
|
341
|
+
(
|
|
342
|
+
("status", "Retrospective status"),
|
|
343
|
+
("changepoint_step", "Retrospective boundary step"),
|
|
344
|
+
),
|
|
345
|
+
)
|
|
346
|
+
)
|
|
347
|
+
result.append('<div class="metrics">')
|
|
348
|
+
for key, heading, fields in (
|
|
349
|
+
("replication", "Split-half replication", _REPLICATION),
|
|
350
|
+
("persistence", "Persistence", _PERSISTENCE),
|
|
351
|
+
("reward_association", "Signed reward association", _REWARD),
|
|
352
|
+
("prompt_breadth", "Prompt breadth", _BREADTH),
|
|
353
|
+
):
|
|
354
|
+
result.append(f'<section class="metric"><h4>{heading}</h4>')
|
|
355
|
+
result.append(_fields(evidence[key], fields))
|
|
356
|
+
if key == "reward_association":
|
|
357
|
+
result.append(
|
|
358
|
+
'<p class="reference-note">Present-minus-absent reward in mixed groups.</p>'
|
|
359
|
+
)
|
|
360
|
+
result.append("</section>")
|
|
361
|
+
result.append("</div>")
|
|
362
|
+
result.append(_prompt_mix(evidence.get("prompt_mix")))
|
|
363
|
+
result.append("<h4>Exemplar references</h4>")
|
|
364
|
+
result.append(
|
|
365
|
+
'<p class="reference-note">References only; raw rollout text is not included. Selected examples are descriptive, not representative estimates.</p>'
|
|
366
|
+
)
|
|
367
|
+
result.append(
|
|
368
|
+
f'<p class="note">Record status: {_status(member["exemplar_status"])}</p>'
|
|
369
|
+
)
|
|
370
|
+
if not member["exemplars"]:
|
|
371
|
+
result.append('<p class="note">No exemplar records in this report.</p>')
|
|
372
|
+
for exemplar in sorted(
|
|
373
|
+
member["exemplars"],
|
|
374
|
+
key=lambda row: (
|
|
375
|
+
row["selection_order"] is None,
|
|
376
|
+
row["selection_order"] or 0,
|
|
377
|
+
row["sample_id"] or "",
|
|
378
|
+
),
|
|
379
|
+
):
|
|
380
|
+
result.append('<details class="exemplar"><summary>')
|
|
381
|
+
if exemplar["sample_id"] is None:
|
|
382
|
+
result.append(f"No selected reference · {_status(exemplar['status'])}")
|
|
383
|
+
else:
|
|
384
|
+
result.append(
|
|
385
|
+
f"{_value(exemplar['sample_id'], identifier=True)} · step {_value(exemplar['step'])}"
|
|
386
|
+
)
|
|
387
|
+
result.extend(["</summary>", _fields(exemplar, _EXEMPLAR), "</details>"])
|
|
388
|
+
result.append("</article>")
|
|
389
|
+
return "".join(result)
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
def _event(event: dict, run_index: int) -> str:
|
|
393
|
+
ranking = event["ranking"]
|
|
394
|
+
category = event["ranking_status"]
|
|
395
|
+
if category not in {"behavioral", "contextual"}:
|
|
396
|
+
category = "other"
|
|
397
|
+
floors = ranking["numerical_floor_members"]
|
|
398
|
+
result = [
|
|
399
|
+
f'<details class="event-card" id="event-{run_index}-{event["rank"]}" data-category="{category}" data-floor="{str(bool(floors)).lower()}">',
|
|
400
|
+
f'<summary><span class="rank">#{event["rank"]}</span><span>',
|
|
401
|
+
f'<code class="representative">{escape(event["representative_feature"])}</code>',
|
|
402
|
+
f'<span class="anchors">{_value(event["event_id"], identifier=True)} · {event["n_members"]} members · onset {event["onset_step"]} · alert {event["alert_step"]}</span></span>',
|
|
403
|
+
f'<span class="badges"><span class="badge {category}">{_status(event["ranking_status"])}</span>',
|
|
404
|
+
]
|
|
405
|
+
if floors:
|
|
406
|
+
result.append(
|
|
407
|
+
f'<span class="badge warning">Numerical floor · {len(floors)}</span>'
|
|
408
|
+
)
|
|
409
|
+
if ranking["numerical_floor_unavailable_members"]:
|
|
410
|
+
result.append('<span class="badge other">Floor evidence unavailable</span>')
|
|
411
|
+
result.append('</span></summary><div class="event-body">')
|
|
412
|
+
result.append(
|
|
413
|
+
f'<p class="note">Event onset range: {event["onset_start"]} to {event["onset_end"]}. Representative evidence availability:</p>'
|
|
414
|
+
)
|
|
415
|
+
result.append(
|
|
416
|
+
_fields(
|
|
417
|
+
ranking,
|
|
418
|
+
(
|
|
419
|
+
("n_prioritizable_members", "Prioritizable members"),
|
|
420
|
+
("replication_availability", "Replication"),
|
|
421
|
+
("persistence_availability", "Persistence"),
|
|
422
|
+
("reward_availability", "Reward association"),
|
|
423
|
+
("breadth_availability", "Prompt breadth"),
|
|
424
|
+
),
|
|
425
|
+
)
|
|
426
|
+
)
|
|
427
|
+
for key, label in (
|
|
428
|
+
("numerical_floor_members", "Numerical floor"),
|
|
429
|
+
("numerical_floor_unavailable_members", "Floor evidence unavailable"),
|
|
430
|
+
("missing_evidence_members", "Missing evidence"),
|
|
431
|
+
):
|
|
432
|
+
if ranking[key]:
|
|
433
|
+
result.append(
|
|
434
|
+
f'<p class="warning-note">{label}: {_references(ranking[key])}</p>'
|
|
435
|
+
)
|
|
436
|
+
for member in sorted(
|
|
437
|
+
event["members"],
|
|
438
|
+
key=lambda row: (row["onset_step"], row["alert_step"], row["feature"]),
|
|
439
|
+
):
|
|
440
|
+
result.append(_member(member))
|
|
441
|
+
result.append("</div></details>")
|
|
442
|
+
return "".join(result)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def _run(run: dict, index: int) -> str:
|
|
446
|
+
counts = run["counts"]
|
|
447
|
+
result = [
|
|
448
|
+
f'<section class="run" id="run-{index}"><h2>Run {_value(run["run"], identifier=True)}</h2>',
|
|
449
|
+
f'<p class="note">Status: {_status(run["status"])}</p><div class="stats">',
|
|
450
|
+
]
|
|
451
|
+
for key, label in (
|
|
452
|
+
("rollouts", "Rollouts"),
|
|
453
|
+
("screened_series", "Screened series"),
|
|
454
|
+
("sequential_alerts", "Sequential alerts"),
|
|
455
|
+
("events", "Ranked events"),
|
|
456
|
+
):
|
|
457
|
+
result.append(
|
|
458
|
+
f'<div class="stat"><strong>{counts[key]:,}</strong><span>{label}</span></div>'
|
|
459
|
+
)
|
|
460
|
+
result.append('</div><p class="note run-counts">')
|
|
461
|
+
result.append(
|
|
462
|
+
" · ".join(
|
|
463
|
+
f"{counts[key]:,} {label}"
|
|
464
|
+
for key, label in (
|
|
465
|
+
("series", "series"),
|
|
466
|
+
("unscorable_series", "unscorable"),
|
|
467
|
+
("observed_steps", "observed steps"),
|
|
468
|
+
("prompts", "prompts"),
|
|
469
|
+
("behavioral_events", "behavioral events"),
|
|
470
|
+
("contextual_events", "contextual events"),
|
|
471
|
+
("other_events", "other events"),
|
|
472
|
+
)
|
|
473
|
+
)
|
|
474
|
+
)
|
|
475
|
+
result.append("</p>")
|
|
476
|
+
retrospective = run.get("retrospective")
|
|
477
|
+
if retrospective is not None and retrospective["status"] == "ran":
|
|
478
|
+
result.append(
|
|
479
|
+
'<section class="metric"><h3>Experimental retrospective boundaries</h3>'
|
|
480
|
+
)
|
|
481
|
+
result.append(
|
|
482
|
+
_fields(
|
|
483
|
+
retrospective["screening"],
|
|
484
|
+
(
|
|
485
|
+
("n_series", "Series"),
|
|
486
|
+
("n_screened", "Screened"),
|
|
487
|
+
("n_changes", "Selected boundaries"),
|
|
488
|
+
("n_unscorable", "Unscorable"),
|
|
489
|
+
),
|
|
490
|
+
)
|
|
491
|
+
)
|
|
492
|
+
if retrospective["changes"]:
|
|
493
|
+
result.append(
|
|
494
|
+
"<details><summary>Selected features and boundary steps</summary><ul>"
|
|
495
|
+
)
|
|
496
|
+
for change in sorted(
|
|
497
|
+
retrospective["changes"],
|
|
498
|
+
key=lambda row: (row["changepoint_step"], row["feature"]),
|
|
499
|
+
):
|
|
500
|
+
result.append(
|
|
501
|
+
f"<li>{_value(change['feature'], identifier=True)} at step {_value(change['changepoint_step'])}</li>"
|
|
502
|
+
)
|
|
503
|
+
result.append("</ul></details>")
|
|
504
|
+
result.append("</section>")
|
|
505
|
+
if not run["events"]:
|
|
506
|
+
result.append('<p class="empty">No sequential events in this run.</p>')
|
|
507
|
+
result.extend(
|
|
508
|
+
_event(event, index)
|
|
509
|
+
for event in sorted(run["events"], key=lambda row: row["rank"])
|
|
510
|
+
)
|
|
511
|
+
result.append("</section>")
|
|
512
|
+
return "".join(result)
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _benchmark(reference: dict | None) -> str:
|
|
516
|
+
if reference is None:
|
|
517
|
+
return '<p class="note">Benchmark figures not supplied.</p>'
|
|
518
|
+
details = []
|
|
519
|
+
heading = "Supplied benchmark evidence"
|
|
520
|
+
qualification = "These benchmark figures are not a guarantee for this run. Provenance fields are supplied by the caller."
|
|
521
|
+
if reference["schema_version"] == 2:
|
|
522
|
+
applicability = reference.get("applicability") or {}
|
|
523
|
+
if applicability.get("source") == "bundled":
|
|
524
|
+
heading = "Bundled benchmark reference"
|
|
525
|
+
qualification = (
|
|
526
|
+
"Historical benchmark rates do not calibrate this run. "
|
|
527
|
+
"Source and configuration checks are recorded claims from report creation."
|
|
528
|
+
)
|
|
529
|
+
details.extend(
|
|
530
|
+
[
|
|
531
|
+
f"<p>Reference: {_value(reference['reference_id'])}</p>",
|
|
532
|
+
f"<p>Configuration check recorded at analysis: {_value(applicability.get('status', 'unverified'))}</p>",
|
|
533
|
+
]
|
|
534
|
+
)
|
|
535
|
+
if applicability.get("reasons"):
|
|
536
|
+
details.append(f"<p>{_value(', '.join(applicability['reasons']))}</p>")
|
|
537
|
+
acceptance = reference["provenance"]["original_acceptance"]
|
|
538
|
+
ceiling = acceptance["max_null_false_alert_rate"]
|
|
539
|
+
for row in reference["results"]["synthetic_null"]:
|
|
540
|
+
if row["n_scorable"] and row["n_alerts"] / row["n_scorable"] > ceiling:
|
|
541
|
+
details.append(
|
|
542
|
+
f"<p><strong>{row['detector']} exceeds the original {100 * ceiling:g}% null ceiling.</strong></p>"
|
|
543
|
+
)
|
|
544
|
+
for row in acceptance["planted_recovery"]:
|
|
545
|
+
timing = "timely" if row["detector"] == "sequential" else "localized"
|
|
546
|
+
details.append(
|
|
547
|
+
f"<p>{row['detector']}: {row['n_within_tolerance']}/{row['n_planted']} {timing} planted changes.</p>"
|
|
548
|
+
)
|
|
549
|
+
details.append(
|
|
550
|
+
"<details><summary>Recorded protocol and provenance</summary><pre>"
|
|
551
|
+
+ escape(json.dumps(reference["provenance"], indent=2, allow_nan=False))
|
|
552
|
+
+ "</pre></details>"
|
|
553
|
+
)
|
|
554
|
+
if reference.get("file_sha256"):
|
|
555
|
+
details.append(
|
|
556
|
+
_fields(reference, (("file_sha256", "Reference file SHA-256"),))
|
|
557
|
+
)
|
|
558
|
+
reference = reference["results"]
|
|
559
|
+
result = [
|
|
560
|
+
f'<details class="metric"><summary>{heading}</summary>',
|
|
561
|
+
f'<p class="note">{qualification}</p>',
|
|
562
|
+
*details,
|
|
563
|
+
"<h3>Synthetic-null alert rates</h3>",
|
|
564
|
+
'<p class="note">Alerts per scorable generated no-change series. Unscorable trials remain listed.</p>',
|
|
565
|
+
'<div class="table-scroll" tabindex="0"><table><thead><tr><th>Detector</th><th>Trials</th><th>Scorable</th><th>Alerts</th><th>Rate</th></tr></thead><tbody>',
|
|
566
|
+
]
|
|
567
|
+
for row in reference["synthetic_null"]:
|
|
568
|
+
rate = (
|
|
569
|
+
f"{100 * row['n_alerts'] / row['n_scorable']:.4g}%"
|
|
570
|
+
if row["n_scorable"]
|
|
571
|
+
else None
|
|
572
|
+
)
|
|
573
|
+
result.append(
|
|
574
|
+
f"<tr><th>{_value(row['detector'])}</th><td>{row['n_trials']}</td><td>{row['n_scorable']}</td><td>{row['n_alerts']}</td><td>{_value(rate)}</td></tr>"
|
|
575
|
+
)
|
|
576
|
+
result.append(
|
|
577
|
+
'</tbody></table></div><h3>Public pre-onset alerts</h3><p class="note">These counts precede each reference label onset and describe observed behavioral alerts.</p>'
|
|
578
|
+
)
|
|
579
|
+
result.append(
|
|
580
|
+
'<div class="table-scroll" tabindex="0"><table><thead><tr><th>Run</th><th>Label</th><th>Reference onset</th><th>Status</th><th>Alerts</th></tr></thead><tbody>'
|
|
581
|
+
)
|
|
582
|
+
for row in reference["pre_onset"]:
|
|
583
|
+
result.append(
|
|
584
|
+
f"<tr><th>{_value(row['run'], identifier=True)}</th><td>{_status(row['label'])}</td><td>{_value(row['reference_onset'])}</td><td>{_status(row['status'])}</td><td>{_value(row['n_alerts'])}</td></tr>"
|
|
585
|
+
)
|
|
586
|
+
result.append("</tbody></table></div><h3>Provenance</h3>")
|
|
587
|
+
result.append(
|
|
588
|
+
_fields(
|
|
589
|
+
reference,
|
|
590
|
+
(
|
|
591
|
+
("source_commit", "Source commit"),
|
|
592
|
+
("protocol_sha256", "Protocol SHA-256"),
|
|
593
|
+
),
|
|
594
|
+
)
|
|
595
|
+
)
|
|
596
|
+
if "file_sha256" in reference:
|
|
597
|
+
result.append(_fields(reference, (("file_sha256", "Reference file SHA-256"),)))
|
|
598
|
+
result.append('<p class="note">Source artifact SHA-256:</p><ul>')
|
|
599
|
+
result.extend(
|
|
600
|
+
f"<li><code>{digest}</code></li>" for digest in reference["artifact_sha256s"]
|
|
601
|
+
)
|
|
602
|
+
result.append('</ul><p class="note">Public input SHA-256:</p><ul>')
|
|
603
|
+
for row in reference["pre_onset"]:
|
|
604
|
+
result.extend(
|
|
605
|
+
f"<li>{_value(row['run'], identifier=True)}: <code>{digest}</code></li>"
|
|
606
|
+
for digest in row["source_sha256s"]
|
|
607
|
+
)
|
|
608
|
+
result.append("</ul></details>")
|
|
609
|
+
return "".join(result)
|
|
610
|
+
|
|
611
|
+
|
|
612
|
+
def render_report(report: dict) -> str:
|
|
613
|
+
"""Validate an analysis report and render escaped, self-contained HTML.
|
|
614
|
+
|
|
615
|
+
Existing within-run ranks are preserved. Unknown JSON fields are omitted.
|
|
616
|
+
"""
|
|
617
|
+
report = validate_report(report)
|
|
618
|
+
script_hash = b64encode(hashlib.sha256(_SCRIPT.encode("utf-8")).digest()).decode(
|
|
619
|
+
"ascii"
|
|
620
|
+
)
|
|
621
|
+
policy = f"default-src 'none'; style-src 'unsafe-inline'; script-src 'sha256-{script_hash}'; base-uri 'none'; form-action 'none'"
|
|
622
|
+
result = [
|
|
623
|
+
'<!doctype html>\n<html lang="en"><head><meta charset="utf-8">',
|
|
624
|
+
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
|
625
|
+
f'<meta http-equiv="Content-Security-Policy" content="{escape(policy, quote=True)}">',
|
|
626
|
+
"<title>Rollout Diff investigation report</title>",
|
|
627
|
+
f"<style>{_STYLE}</style></head><body><main>",
|
|
628
|
+
f'<header class="intro"><p class="eyebrow">Rollout Diff · Report version {report["schema_version"]}</p>',
|
|
629
|
+
"<h1>Investigation report</h1>",
|
|
630
|
+
'<p class="note">Events are ranked using a heuristic assessment of supporting evidence. Grouped features share similar onset times and overlapping rollouts, providing leads for further investigation. Expand an event to examine its measurements, evidence availability, and exemplar references.</p></header>',
|
|
631
|
+
_benchmark(report.get("benchmark")),
|
|
632
|
+
_CONTROLS,
|
|
633
|
+
]
|
|
634
|
+
retrospective = report.get("retrospective")
|
|
635
|
+
if retrospective is not None:
|
|
636
|
+
if retrospective["status"] == "disabled":
|
|
637
|
+
result.insert(
|
|
638
|
+
-2,
|
|
639
|
+
'<p class="note">Retrospective localization was disabled. No boundaries were assessed.</p>',
|
|
640
|
+
)
|
|
641
|
+
else:
|
|
642
|
+
result.insert(
|
|
643
|
+
-2,
|
|
644
|
+
'<section class="metric"><h2>Experimental retrospective localization</h2>'
|
|
645
|
+
+ f"<p>{_value(retrospective['warning'])}</p>"
|
|
646
|
+
+ f"<p>Algorithm <code>{_value(retrospective['algorithm'])}</code></p>"
|
|
647
|
+
+ "<details><summary>Effective retrospective settings</summary><pre>"
|
|
648
|
+
+ escape(
|
|
649
|
+
json.dumps(
|
|
650
|
+
retrospective["effective_settings"], indent=2, allow_nan=False
|
|
651
|
+
)
|
|
652
|
+
)
|
|
653
|
+
+ "</pre></details></section>",
|
|
654
|
+
)
|
|
655
|
+
if report.get("sensitivity") is not None:
|
|
656
|
+
result.insert(-2, _sensitivity(report["sensitivity"], report["runs"]))
|
|
657
|
+
if not report["runs"]:
|
|
658
|
+
result.append('<p class="empty">No runs in this report.</p>')
|
|
659
|
+
result.extend(
|
|
660
|
+
_run(run, index)
|
|
661
|
+
for index, run in enumerate(sorted(report["runs"], key=lambda row: row["run"]))
|
|
662
|
+
)
|
|
663
|
+
result.append(f"</main><script>{_SCRIPT}</script></body></html>\n")
|
|
664
|
+
return "".join(result)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def _sensitivity(profile: dict, runs: list[dict]) -> str:
|
|
668
|
+
result = [
|
|
669
|
+
'<section class="metric"><h2>Experimental sensitivity</h2>',
|
|
670
|
+
f"<p><code>{_value(profile['id'])}</code></p>",
|
|
671
|
+
'<p class="note">Reconstructed on current semantics using full-run vocabulary. Alert dates are offline observations. Historical benchmark figures describe the default procedure and do not calibrate this profile.</p>',
|
|
672
|
+
'<p class="note">Structural binary features use a zero minimum prevalence increase. Continuous/reward context retains default behavior. Other binary features are excluded from sequential screening.</p>',
|
|
673
|
+
'<div class="table-scroll" tabindex="0"><table><thead><tr><th>Run</th><th>Comparison</th><th>Series</th><th>Screened</th><th>Alerts</th><th>Unscorable</th><th>Excluded series</th></tr></thead><tbody>',
|
|
674
|
+
]
|
|
675
|
+
for group, label in (
|
|
676
|
+
("default_full", "Default, full"),
|
|
677
|
+
("default_eligible", "Default, same cohort"),
|
|
678
|
+
("experimental_eligible", "Experimental, same cohort"),
|
|
679
|
+
):
|
|
680
|
+
for row in profile["counts"][group]:
|
|
681
|
+
result.append(
|
|
682
|
+
f"<tr><th>{_value(row['run'], identifier=True)}</th><td>{label}</td>"
|
|
683
|
+
+ "".join(
|
|
684
|
+
f"<td>{row[key]}</td>"
|
|
685
|
+
for key in ("n_series", "n_screened", "n_alerted", "n_unscorable")
|
|
686
|
+
)
|
|
687
|
+
+ "<td>0</td></tr>"
|
|
688
|
+
)
|
|
689
|
+
for run in runs:
|
|
690
|
+
counts = run["counts"]
|
|
691
|
+
result.append(
|
|
692
|
+
f"<tr><th>{_value(run['run'], identifier=True)}</th><td>Selected output with context</td>"
|
|
693
|
+
+ "".join(
|
|
694
|
+
f"<td>{counts[key]}</td>"
|
|
695
|
+
for key in (
|
|
696
|
+
"series",
|
|
697
|
+
"screened_series",
|
|
698
|
+
"sequential_alerts",
|
|
699
|
+
"unscorable_series",
|
|
700
|
+
"excluded_series",
|
|
701
|
+
)
|
|
702
|
+
)
|
|
703
|
+
+ "</tr>"
|
|
704
|
+
)
|
|
705
|
+
result.extend(
|
|
706
|
+
[
|
|
707
|
+
"</tbody></table></div><details><summary>Effective settings and profile provenance</summary><pre>",
|
|
708
|
+
escape(
|
|
709
|
+
json.dumps(
|
|
710
|
+
{
|
|
711
|
+
key: profile[key]
|
|
712
|
+
for key in (
|
|
713
|
+
"effective_settings",
|
|
714
|
+
"eligibility",
|
|
715
|
+
"source_checkpoint",
|
|
716
|
+
"provenance",
|
|
717
|
+
)
|
|
718
|
+
},
|
|
719
|
+
indent=2,
|
|
720
|
+
allow_nan=False,
|
|
721
|
+
)
|
|
722
|
+
),
|
|
723
|
+
"</pre></details></section>",
|
|
724
|
+
]
|
|
725
|
+
)
|
|
726
|
+
return "".join(result)
|