rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/report.py
ADDED
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
"""Versioned investigation reports from completed analysis tables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import math
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import tempfile
|
|
10
|
+
from collections import Counter, defaultdict
|
|
11
|
+
from copy import deepcopy
|
|
12
|
+
from numbers import Integral, Real
|
|
13
|
+
from pathlib import Path, PureWindowsPath
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
import pandas as pd
|
|
17
|
+
|
|
18
|
+
from .pipeline import AnalysisResult
|
|
19
|
+
from .series import _require_columns
|
|
20
|
+
|
|
21
|
+
_BUNDLED_BENCHMARK = object()
|
|
22
|
+
|
|
23
|
+
_EVENT = [
|
|
24
|
+
"event_id",
|
|
25
|
+
"run",
|
|
26
|
+
"rank",
|
|
27
|
+
"representative_feature",
|
|
28
|
+
"ranking_status",
|
|
29
|
+
"onset_start",
|
|
30
|
+
"onset_end",
|
|
31
|
+
"onset_step",
|
|
32
|
+
"alert_step",
|
|
33
|
+
"n_members",
|
|
34
|
+
]
|
|
35
|
+
_RANKING = [
|
|
36
|
+
"n_prioritizable_members",
|
|
37
|
+
"replication_availability",
|
|
38
|
+
"persistence_availability",
|
|
39
|
+
"reward_availability",
|
|
40
|
+
"breadth_availability",
|
|
41
|
+
]
|
|
42
|
+
_REFERENCES = [
|
|
43
|
+
"missing_evidence_members",
|
|
44
|
+
"numerical_floor_members",
|
|
45
|
+
"numerical_floor_unavailable_members",
|
|
46
|
+
]
|
|
47
|
+
_MEMBER = ["run", "feature", "kind", "direction", "onset_step", "alert_step", "status"]
|
|
48
|
+
_EXEMPLAR = [
|
|
49
|
+
"run",
|
|
50
|
+
"feature",
|
|
51
|
+
"kind",
|
|
52
|
+
"direction",
|
|
53
|
+
"onset_step",
|
|
54
|
+
"alert_step",
|
|
55
|
+
"status",
|
|
56
|
+
"n_window_rollouts",
|
|
57
|
+
"n_candidates",
|
|
58
|
+
"reward_selection_status",
|
|
59
|
+
"selection_order",
|
|
60
|
+
"sample_id",
|
|
61
|
+
"step",
|
|
62
|
+
"prompt_id",
|
|
63
|
+
"feature_value",
|
|
64
|
+
"reward",
|
|
65
|
+
"reward_group_size",
|
|
66
|
+
"reward_group_mean",
|
|
67
|
+
"reward_deviation",
|
|
68
|
+
"reward_deviation_status",
|
|
69
|
+
"selection_reason",
|
|
70
|
+
]
|
|
71
|
+
_EVIDENCE = {
|
|
72
|
+
"replication": {
|
|
73
|
+
**{
|
|
74
|
+
name: f"replication_{name}"
|
|
75
|
+
for name in ["status", "coverage_policy", "horizon_complete", "cutoff_step"]
|
|
76
|
+
},
|
|
77
|
+
"replicates": "replicates",
|
|
78
|
+
},
|
|
79
|
+
"persistence": {
|
|
80
|
+
name: f"persistence_{name}"
|
|
81
|
+
for name in [
|
|
82
|
+
"status",
|
|
83
|
+
"fraction",
|
|
84
|
+
"coverage",
|
|
85
|
+
"n_observations",
|
|
86
|
+
"n_evaluable",
|
|
87
|
+
"n_retained",
|
|
88
|
+
"n_unavailable",
|
|
89
|
+
]
|
|
90
|
+
},
|
|
91
|
+
"reward_association": {
|
|
92
|
+
name: f"reward_{name}"
|
|
93
|
+
for name in [
|
|
94
|
+
"status",
|
|
95
|
+
"mean_difference",
|
|
96
|
+
"n_groups",
|
|
97
|
+
"n_mixed_groups",
|
|
98
|
+
"n_rollouts",
|
|
99
|
+
"n_included_rollouts",
|
|
100
|
+
"window_start",
|
|
101
|
+
"window_end",
|
|
102
|
+
]
|
|
103
|
+
},
|
|
104
|
+
"prompt_breadth": {
|
|
105
|
+
"positive_prompts": "prompt_positive_prompts",
|
|
106
|
+
"total_prompts": "prompt_total_prompts",
|
|
107
|
+
"promptmix_status": "promptmix_status",
|
|
108
|
+
},
|
|
109
|
+
}
|
|
110
|
+
_FEATURE_EVIDENCE = ["tier", "scale_source", "numerical_floor_dominant"]
|
|
111
|
+
_PROMPT_MIX = {
|
|
112
|
+
"status": "promptmix_status",
|
|
113
|
+
**{
|
|
114
|
+
name: f"prompt_{name}"
|
|
115
|
+
for name in [
|
|
116
|
+
"before_start",
|
|
117
|
+
"before_end",
|
|
118
|
+
"after_start",
|
|
119
|
+
"after_end",
|
|
120
|
+
"shared_prompts",
|
|
121
|
+
"shared_sampling_units",
|
|
122
|
+
"shared_change",
|
|
123
|
+
"shared_se",
|
|
124
|
+
]
|
|
125
|
+
},
|
|
126
|
+
}
|
|
127
|
+
_PROMPT_SHIFT = [
|
|
128
|
+
"descriptor",
|
|
129
|
+
"statistic",
|
|
130
|
+
"before",
|
|
131
|
+
"after",
|
|
132
|
+
"delta",
|
|
133
|
+
"distance",
|
|
134
|
+
"n_before",
|
|
135
|
+
"n_after",
|
|
136
|
+
"missing_before",
|
|
137
|
+
"missing_after",
|
|
138
|
+
]
|
|
139
|
+
_SCREENING = {
|
|
140
|
+
"series": "n_series",
|
|
141
|
+
"screened_series": "n_screened",
|
|
142
|
+
"sequential_alerts": "n_alerted",
|
|
143
|
+
"unscorable_series": "n_unscorable",
|
|
144
|
+
}
|
|
145
|
+
_STEPS = {
|
|
146
|
+
"changepoint_step",
|
|
147
|
+
"step",
|
|
148
|
+
"onset_start",
|
|
149
|
+
"onset_end",
|
|
150
|
+
"onset_step",
|
|
151
|
+
"alert_step",
|
|
152
|
+
"replication_cutoff_step",
|
|
153
|
+
"reward_window_start",
|
|
154
|
+
"reward_window_end",
|
|
155
|
+
"prompt_before_start",
|
|
156
|
+
"prompt_before_end",
|
|
157
|
+
"prompt_after_start",
|
|
158
|
+
"prompt_after_end",
|
|
159
|
+
}
|
|
160
|
+
_INTEGERS = _STEPS | {
|
|
161
|
+
"rank",
|
|
162
|
+
"selection_order",
|
|
163
|
+
"reward_group_size",
|
|
164
|
+
"prompt_positive_prompts",
|
|
165
|
+
"prompt_total_prompts",
|
|
166
|
+
"prompt_shared_prompts",
|
|
167
|
+
"prompt_shared_sampling_units",
|
|
168
|
+
"missing_before",
|
|
169
|
+
"missing_after",
|
|
170
|
+
}
|
|
171
|
+
_IDENTIFIERS = {
|
|
172
|
+
"run",
|
|
173
|
+
"event_id",
|
|
174
|
+
"feature",
|
|
175
|
+
"representative_feature",
|
|
176
|
+
"sample_id",
|
|
177
|
+
"prompt_id",
|
|
178
|
+
}
|
|
179
|
+
# POSIX roots are recognized at identifier component boundaries.
|
|
180
|
+
_EMBEDDED_PATH = re.compile(
|
|
181
|
+
r"(?i:(?<!\w)file://)"
|
|
182
|
+
r"|[A-Za-z]:(?:\\|/(?!/))"
|
|
183
|
+
r"|\\\\[^\\/\s]+[\\/][^\\/\s]+"
|
|
184
|
+
r"""|(?:^|(?<=[\s=,;|"'(\[{<]))//[^/\\\s]+[\\/][^/\\\s]+"""
|
|
185
|
+
r"""|(?:^|(?<=[\s:=,;|"'(\[{<]))(?:/|~[\\/])(?=[^/\\\s])"""
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def _identifier(value: str) -> str:
|
|
190
|
+
if not isinstance(value, str) or not value.strip():
|
|
191
|
+
raise ValueError("report identifiers must be nonempty strings")
|
|
192
|
+
candidate = value.removeprefix("file:").removeprefix("strlit:")
|
|
193
|
+
if (
|
|
194
|
+
candidate.startswith(("/", "\\", "~/", "~\\"))
|
|
195
|
+
or PureWindowsPath(candidate).is_absolute()
|
|
196
|
+
or _EMBEDDED_PATH.search(value)
|
|
197
|
+
):
|
|
198
|
+
raise ValueError(
|
|
199
|
+
"report identifiers must not contain absolute filesystem paths"
|
|
200
|
+
)
|
|
201
|
+
return value
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _integer(value, *, identity=False):
|
|
205
|
+
if value is None or value is pd.NA or pd.isna(value):
|
|
206
|
+
return None
|
|
207
|
+
if isinstance(value, (bool, np.bool_)):
|
|
208
|
+
raise ValueError("report integer fields require exact integers")
|
|
209
|
+
if (isinstance(value, Integral) and not isinstance(value, (bool, np.bool_))) or (
|
|
210
|
+
not identity
|
|
211
|
+
and isinstance(value, Real)
|
|
212
|
+
and math.isfinite(value)
|
|
213
|
+
and abs(value) < 2**53
|
|
214
|
+
and int(value) == value
|
|
215
|
+
):
|
|
216
|
+
number = int(value)
|
|
217
|
+
else:
|
|
218
|
+
raise ValueError("report integer fields require exact integers")
|
|
219
|
+
if number < 0:
|
|
220
|
+
raise ValueError("report integer fields must be nonnegative")
|
|
221
|
+
return number
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _value(name, value):
|
|
225
|
+
if value is None or value is pd.NA or value is pd.NaT:
|
|
226
|
+
return None
|
|
227
|
+
if (
|
|
228
|
+
isinstance(value, Real)
|
|
229
|
+
and not isinstance(value, (bool, np.bool_))
|
|
230
|
+
and not math.isfinite(value)
|
|
231
|
+
):
|
|
232
|
+
return None
|
|
233
|
+
if name in _IDENTIFIERS:
|
|
234
|
+
return _identifier(value)
|
|
235
|
+
if name in _INTEGERS or name.startswith("n_") or "_n_" in name:
|
|
236
|
+
return _integer(value, identity=name in _STEPS)
|
|
237
|
+
if isinstance(value, (bool, np.bool_)):
|
|
238
|
+
return bool(value)
|
|
239
|
+
if isinstance(value, Integral):
|
|
240
|
+
return int(value)
|
|
241
|
+
if isinstance(value, Real):
|
|
242
|
+
return float(value)
|
|
243
|
+
if isinstance(value, str):
|
|
244
|
+
return value
|
|
245
|
+
raise ValueError(f"unsupported report value for {name}")
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _fields(row, names):
|
|
249
|
+
mapping = dict(zip(names, names, strict=True)) if isinstance(names, list) else names
|
|
250
|
+
return {
|
|
251
|
+
output: _value(source, row.get(source)) for output, source in mapping.items()
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _index(frame, keys, columns, name):
|
|
256
|
+
_require_columns(frame, list(dict.fromkeys([*keys, *columns])), name)
|
|
257
|
+
if frame.duplicated(keys).any():
|
|
258
|
+
raise ValueError(f"duplicate {name} keys")
|
|
259
|
+
result = {}
|
|
260
|
+
for row in frame[list(dict.fromkeys([*keys, *columns]))].to_dict("records"):
|
|
261
|
+
key = tuple(row[k] for k in keys)
|
|
262
|
+
if not all(isinstance(v, str) and v.strip() for v in key):
|
|
263
|
+
raise ValueError(f"invalid {name} keys")
|
|
264
|
+
result[key] = row
|
|
265
|
+
return result
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _references(values, allowed):
|
|
269
|
+
references = []
|
|
270
|
+
for run, feature in values:
|
|
271
|
+
if (run, feature) not in allowed:
|
|
272
|
+
raise ValueError("ranking reference is not an event member")
|
|
273
|
+
references.append({"run": _identifier(run), "feature": _identifier(feature)})
|
|
274
|
+
return sorted(references, key=lambda row: (row["run"], row["feature"]))
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _member_report(member, source, exemplars, shifts, policy):
|
|
278
|
+
row = _fields(member, _MEMBER)
|
|
279
|
+
evidence = source if source is not None else {}
|
|
280
|
+
row["evidence"] = {
|
|
281
|
+
"status": "recorded" if source is not None else "missing",
|
|
282
|
+
**_fields(evidence, _FEATURE_EVIDENCE),
|
|
283
|
+
**{name: _fields(evidence, columns) for name, columns in _EVIDENCE.items()},
|
|
284
|
+
"prompt_mix": {
|
|
285
|
+
**_fields(evidence, _PROMPT_MIX),
|
|
286
|
+
"prompt_shift_status": "recorded" if shifts else "missing",
|
|
287
|
+
"prompt_shift": shifts,
|
|
288
|
+
},
|
|
289
|
+
}
|
|
290
|
+
if policy is not None:
|
|
291
|
+
detail = _fields(
|
|
292
|
+
evidence,
|
|
293
|
+
{"status": "retrospective_status", "changepoint_step": "changepoint_step"},
|
|
294
|
+
)
|
|
295
|
+
if source is None:
|
|
296
|
+
detail["status"] = (
|
|
297
|
+
"disabled" if policy["status"] == "disabled" else "missing"
|
|
298
|
+
)
|
|
299
|
+
row["evidence"]["retrospective"] = detail
|
|
300
|
+
row["exemplar_status"] = "recorded" if exemplars else "missing"
|
|
301
|
+
row["exemplars"] = sorted(
|
|
302
|
+
[_fields(example, _EXEMPLAR) for example in exemplars],
|
|
303
|
+
key=lambda example: (
|
|
304
|
+
example["selection_order"] is None,
|
|
305
|
+
example["selection_order"] or 0,
|
|
306
|
+
example["sample_id"] or "",
|
|
307
|
+
),
|
|
308
|
+
)
|
|
309
|
+
return row
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _prompt_shifts(frame, evidence):
|
|
313
|
+
grouped = defaultdict(list)
|
|
314
|
+
if frame.empty:
|
|
315
|
+
return grouped
|
|
316
|
+
windows = ["before_start", "before_end", "after_start", "after_end"]
|
|
317
|
+
_require_columns(
|
|
318
|
+
frame, ["run", "feature", *_PROMPT_SHIFT, *windows], "prompt shifts"
|
|
319
|
+
)
|
|
320
|
+
if frame.duplicated(["run", "feature", "descriptor"]).any():
|
|
321
|
+
raise ValueError("duplicate prompt-shift descriptors")
|
|
322
|
+
for shift in frame.to_dict("records"):
|
|
323
|
+
key = shift["run"], shift["feature"]
|
|
324
|
+
source = evidence.get(key)
|
|
325
|
+
if source is None:
|
|
326
|
+
continue
|
|
327
|
+
if any(shift[name] != source[f"prompt_{name}"] for name in windows):
|
|
328
|
+
raise ValueError("prompt-shift windows do not match feature evidence")
|
|
329
|
+
row = _fields(shift, _PROMPT_SHIFT)
|
|
330
|
+
row["descriptor"] = _identifier(row["descriptor"])
|
|
331
|
+
grouped[key].append(row)
|
|
332
|
+
for rows in grouped.values():
|
|
333
|
+
rows.sort(key=lambda row: row["descriptor"])
|
|
334
|
+
return grouped
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _check_retrospective_tables(analysis, settings, changes, screening, evidence):
|
|
338
|
+
"""Check source identities and detector outcomes before exporting selected boundaries."""
|
|
339
|
+
keys = ["run", "feature"]
|
|
340
|
+
series = analysis.aggregation.series
|
|
341
|
+
groups = (
|
|
342
|
+
series.sort_values("step", kind="stable")
|
|
343
|
+
.groupby(keys, sort=False)
|
|
344
|
+
.agg(
|
|
345
|
+
tier=("tier", "first"),
|
|
346
|
+
kind=("kind", "first"),
|
|
347
|
+
steps=("step", list),
|
|
348
|
+
)
|
|
349
|
+
.to_dict("index")
|
|
350
|
+
)
|
|
351
|
+
diagnostics = _index(
|
|
352
|
+
analysis.retrospective.diagnostics,
|
|
353
|
+
keys,
|
|
354
|
+
[
|
|
355
|
+
"tier",
|
|
356
|
+
"kind",
|
|
357
|
+
"status",
|
|
358
|
+
"n_observations",
|
|
359
|
+
"candidate_step",
|
|
360
|
+
"previous_step",
|
|
361
|
+
"n_before",
|
|
362
|
+
"n_after",
|
|
363
|
+
"min_segment_steps",
|
|
364
|
+
"penalty_multiplier",
|
|
365
|
+
"min_effect",
|
|
366
|
+
"dependence",
|
|
367
|
+
],
|
|
368
|
+
"retrospective diagnostics",
|
|
369
|
+
)
|
|
370
|
+
if (
|
|
371
|
+
diagnostics.keys() != groups.keys()
|
|
372
|
+
or changes.keys() - groups.keys()
|
|
373
|
+
or evidence.keys() - groups.keys()
|
|
374
|
+
):
|
|
375
|
+
raise ValueError("retrospective rows conflict with the aggregated series")
|
|
376
|
+
counts = defaultdict(Counter)
|
|
377
|
+
for key, row in diagnostics.items():
|
|
378
|
+
group = groups[key]
|
|
379
|
+
status = row["status"]
|
|
380
|
+
selected = status == "change"
|
|
381
|
+
scorable = status in {"change", "no_change"}
|
|
382
|
+
steps = group["steps"]
|
|
383
|
+
if (
|
|
384
|
+
status
|
|
385
|
+
not in {
|
|
386
|
+
"change",
|
|
387
|
+
"no_change",
|
|
388
|
+
"unavailable_uncertainty",
|
|
389
|
+
"insufficient_data",
|
|
390
|
+
}
|
|
391
|
+
or any(row[name] != group[name] for name in ("kind", "tier"))
|
|
392
|
+
or row["n_observations"] != len(steps)
|
|
393
|
+
or selected != (key in changes)
|
|
394
|
+
):
|
|
395
|
+
raise ValueError(
|
|
396
|
+
"retrospective diagnostics conflict with the series or selected changes"
|
|
397
|
+
)
|
|
398
|
+
candidate = _integer(row["candidate_step"], identity=True)
|
|
399
|
+
if (candidate is not None) != scorable:
|
|
400
|
+
raise ValueError("retrospective candidate conflicts with scorable status")
|
|
401
|
+
if scorable:
|
|
402
|
+
before = _integer(row["n_before"])
|
|
403
|
+
if (
|
|
404
|
+
before is None
|
|
405
|
+
or not 0 < before < len(steps)
|
|
406
|
+
or steps[before] != candidate
|
|
407
|
+
or steps[before - 1] != row["previous_step"]
|
|
408
|
+
or row["n_after"] != len(steps) - before
|
|
409
|
+
):
|
|
410
|
+
raise ValueError("retrospective segments conflict with observed steps")
|
|
411
|
+
effect_name = (
|
|
412
|
+
"min_prevalence_change"
|
|
413
|
+
if row["kind"] == "binary"
|
|
414
|
+
else "min_continuous_change"
|
|
415
|
+
)
|
|
416
|
+
if (
|
|
417
|
+
any(
|
|
418
|
+
row[name] != settings[name]
|
|
419
|
+
for name in ("min_segment_steps", "penalty_multiplier", "dependence")
|
|
420
|
+
)
|
|
421
|
+
or row["min_effect"] != settings[effect_name]
|
|
422
|
+
):
|
|
423
|
+
raise ValueError(
|
|
424
|
+
"retrospective diagnostics conflict with effective settings"
|
|
425
|
+
)
|
|
426
|
+
boundary = None
|
|
427
|
+
if selected:
|
|
428
|
+
change = changes[key]
|
|
429
|
+
boundary = _integer(change["changepoint_step"], identity=True)
|
|
430
|
+
if (
|
|
431
|
+
candidate is None
|
|
432
|
+
or boundary != candidate
|
|
433
|
+
or any(
|
|
434
|
+
change[k] != row[k]
|
|
435
|
+
for k in ("kind", "tier", "previous_step", "n_before", "n_after")
|
|
436
|
+
)
|
|
437
|
+
):
|
|
438
|
+
raise ValueError("retrospective boundary conflicts with its diagnostic")
|
|
439
|
+
if key in evidence and (
|
|
440
|
+
evidence[key]["retrospective_status"] != status
|
|
441
|
+
or _integer(evidence[key]["changepoint_step"], identity=True) != boundary
|
|
442
|
+
):
|
|
443
|
+
raise ValueError(
|
|
444
|
+
"retrospective evidence conflicts with the detector outcome"
|
|
445
|
+
)
|
|
446
|
+
counts[key[0]].update(
|
|
447
|
+
n_series=1,
|
|
448
|
+
n_screened=int(scorable),
|
|
449
|
+
n_changes=int(selected),
|
|
450
|
+
n_unscorable=int(not scorable),
|
|
451
|
+
)
|
|
452
|
+
for (run,), row in screening.items():
|
|
453
|
+
if any(
|
|
454
|
+
counts[run][name] != row[name]
|
|
455
|
+
for name in ("n_series", "n_screened", "n_changes", "n_unscorable")
|
|
456
|
+
):
|
|
457
|
+
raise ValueError(
|
|
458
|
+
"retrospective screening counts conflict with diagnostic outcomes"
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _retrospective_runs(analysis, policy, runs, actual_runs, evidence):
|
|
463
|
+
result = analysis.retrospective
|
|
464
|
+
if policy["status"] == "disabled":
|
|
465
|
+
if any(
|
|
466
|
+
not table.empty
|
|
467
|
+
for table in (result.diagnostics, result.screening, result.changepoints)
|
|
468
|
+
):
|
|
469
|
+
raise ValueError(
|
|
470
|
+
"disabled retrospective analysis contains detector results"
|
|
471
|
+
)
|
|
472
|
+
if (
|
|
473
|
+
not analysis.evidence.retrospective_status.eq("disabled")
|
|
474
|
+
.fillna(False)
|
|
475
|
+
.all()
|
|
476
|
+
or analysis.evidence.changepoint_step.notna().any()
|
|
477
|
+
):
|
|
478
|
+
raise ValueError(
|
|
479
|
+
"disabled retrospective analysis contains contradictory evidence"
|
|
480
|
+
)
|
|
481
|
+
return {
|
|
482
|
+
run: {"status": "disabled", "screening": None, "changes": None}
|
|
483
|
+
for run in runs
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
counts = ["n_series", "n_screened", "n_changes", "n_unscorable"]
|
|
487
|
+
screening = _index(result.screening, ["run"], counts, "retrospective screening")
|
|
488
|
+
if {key[0] for key in screening} != actual_runs:
|
|
489
|
+
raise ValueError("retrospective screening must cover all analyzed runs")
|
|
490
|
+
selected = _index(
|
|
491
|
+
result.changepoints,
|
|
492
|
+
["run", "feature"],
|
|
493
|
+
["tier", "kind", "changepoint_step", "previous_step", "n_before", "n_after"],
|
|
494
|
+
"retrospective changes",
|
|
495
|
+
)
|
|
496
|
+
_check_retrospective_tables(
|
|
497
|
+
analysis, policy["effective_settings"], selected, screening, evidence
|
|
498
|
+
)
|
|
499
|
+
changes = defaultdict(list)
|
|
500
|
+
for (run, _), row in selected.items():
|
|
501
|
+
changes[run].append(_fields(row, ["feature", "changepoint_step"]))
|
|
502
|
+
return {
|
|
503
|
+
run: {
|
|
504
|
+
"status": "ran",
|
|
505
|
+
"screening": _fields(screening[(run,)], counts)
|
|
506
|
+
if run in actual_runs
|
|
507
|
+
else dict.fromkeys(counts, 0),
|
|
508
|
+
"changes": sorted(
|
|
509
|
+
changes[run], key=lambda row: (row["changepoint_step"], row["feature"])
|
|
510
|
+
),
|
|
511
|
+
}
|
|
512
|
+
for run in runs
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def build_report(
|
|
517
|
+
analysis: AnalysisResult,
|
|
518
|
+
*,
|
|
519
|
+
run: str | None = None,
|
|
520
|
+
benchmark: dict | None = _BUNDLED_BENCHMARK,
|
|
521
|
+
) -> dict:
|
|
522
|
+
"""Build a v3 report, or preserve v1/v2 for analysis predating detector policy.
|
|
523
|
+
|
|
524
|
+
Run/event/member order is canonical; unavailable numbers become null.
|
|
525
|
+
`run` supplies identity for empty input and must match any analyzed run.
|
|
526
|
+
Absolute path-shaped identifiers fail export rather than changing references.
|
|
527
|
+
Omitted benchmark evidence uses the bundled historical reference. Pass None
|
|
528
|
+
to omit it or a reference dictionary to supply separate benchmark evidence.
|
|
529
|
+
"""
|
|
530
|
+
policy = analysis.retrospective_metadata
|
|
531
|
+
metadata_settings = (analysis.benchmark_metadata or {}).get("settings", {})
|
|
532
|
+
if policy is not None:
|
|
533
|
+
from ._report_validation import _RetrospectivePolicy
|
|
534
|
+
|
|
535
|
+
try:
|
|
536
|
+
policy = _RetrospectivePolicy.model_validate(policy).model_dump()
|
|
537
|
+
except ValueError:
|
|
538
|
+
raise ValueError(
|
|
539
|
+
"invalid retrospective execution policy metadata"
|
|
540
|
+
) from None
|
|
541
|
+
if (
|
|
542
|
+
isinstance(metadata_settings, dict)
|
|
543
|
+
and "retrospective" in metadata_settings
|
|
544
|
+
and metadata_settings["retrospective"] != policy["effective_settings"]
|
|
545
|
+
):
|
|
546
|
+
raise ValueError(
|
|
547
|
+
"benchmark settings conflict with retrospective execution policy"
|
|
548
|
+
)
|
|
549
|
+
if policy is None and (
|
|
550
|
+
(
|
|
551
|
+
"retrospective_status" in analysis.evidence
|
|
552
|
+
and analysis.evidence.retrospective_status.eq("disabled").any()
|
|
553
|
+
)
|
|
554
|
+
or (
|
|
555
|
+
isinstance(metadata_settings, dict)
|
|
556
|
+
and metadata_settings.get("retrospective", {}) is None
|
|
557
|
+
)
|
|
558
|
+
):
|
|
559
|
+
raise ValueError(
|
|
560
|
+
"disabled retrospective analysis requires execution policy metadata"
|
|
561
|
+
)
|
|
562
|
+
if analysis.sensitivity_metadata is None and (
|
|
563
|
+
"n_excluded" in analysis.sequential.screening
|
|
564
|
+
or (analysis.benchmark_metadata or {}).get("sensitivity") is not None
|
|
565
|
+
or analysis.sequential.diagnostics.status.eq("profile_excluded").any()
|
|
566
|
+
):
|
|
567
|
+
raise ValueError("experimental analysis requires sensitivity metadata")
|
|
568
|
+
canonical = _index(analysis.features.dense, ["run", "sample_id"], [], "rollouts")
|
|
569
|
+
if analysis.sensitivity_metadata is not None:
|
|
570
|
+
_require_columns(
|
|
571
|
+
analysis.sequential.screening, ["n_excluded"], "sensitivity screening"
|
|
572
|
+
)
|
|
573
|
+
screening = _index(
|
|
574
|
+
analysis.sequential.screening,
|
|
575
|
+
["run"],
|
|
576
|
+
[
|
|
577
|
+
*list(_SCREENING.values()),
|
|
578
|
+
*(["n_excluded"] if analysis.sensitivity_metadata is not None else []),
|
|
579
|
+
],
|
|
580
|
+
"screening",
|
|
581
|
+
)
|
|
582
|
+
events = _index(
|
|
583
|
+
analysis.events,
|
|
584
|
+
["event_id"],
|
|
585
|
+
["run", "onset_start", "onset_end", "n_members", "members"],
|
|
586
|
+
"events",
|
|
587
|
+
)
|
|
588
|
+
ranking = _index(
|
|
589
|
+
analysis.event_ranking,
|
|
590
|
+
["event_id"],
|
|
591
|
+
[*_EVENT, *_RANKING, *_REFERENCES],
|
|
592
|
+
"event ranking",
|
|
593
|
+
)
|
|
594
|
+
members = _index(
|
|
595
|
+
analysis.event_members,
|
|
596
|
+
["run", "feature"],
|
|
597
|
+
["event_id", *_MEMBER],
|
|
598
|
+
"event members",
|
|
599
|
+
)
|
|
600
|
+
evidence = _index(
|
|
601
|
+
analysis.evidence,
|
|
602
|
+
["run", "feature"],
|
|
603
|
+
[
|
|
604
|
+
*_FEATURE_EVIDENCE,
|
|
605
|
+
*(
|
|
606
|
+
["retrospective_status", "changepoint_step"]
|
|
607
|
+
if policy is not None
|
|
608
|
+
else []
|
|
609
|
+
),
|
|
610
|
+
*[column for group in _EVIDENCE.values() for column in group.values()],
|
|
611
|
+
*_PROMPT_MIX.values(),
|
|
612
|
+
],
|
|
613
|
+
"evidence",
|
|
614
|
+
)
|
|
615
|
+
alerts = _index(
|
|
616
|
+
analysis.sequential.alerts, ["run", "feature"], [], "sequential alerts"
|
|
617
|
+
)
|
|
618
|
+
shifts = _prompt_shifts(analysis.prompt_shifts, evidence)
|
|
619
|
+
if set(ranking) != set(events) or set(members) != set(alerts):
|
|
620
|
+
raise ValueError("event, ranking and sequential membership coverage must agree")
|
|
621
|
+
actual_runs = {key[0] for key in canonical}
|
|
622
|
+
if {key[0] for key in screening} != actual_runs:
|
|
623
|
+
raise ValueError("screening must cover all analyzed runs")
|
|
624
|
+
runs = set(actual_runs)
|
|
625
|
+
if run is not None:
|
|
626
|
+
_identifier(run)
|
|
627
|
+
if actual_runs and actual_runs != {run}:
|
|
628
|
+
raise ValueError("requested run does not match analyzed runs")
|
|
629
|
+
runs.add(run)
|
|
630
|
+
|
|
631
|
+
grouped_members, grouped_examples = defaultdict(list), defaultdict(list)
|
|
632
|
+
for member in members.values():
|
|
633
|
+
event = events.get((member["event_id"],))
|
|
634
|
+
if event is None or event["run"] != member["run"]:
|
|
635
|
+
raise ValueError("event member has no matching event")
|
|
636
|
+
grouped_members[member["event_id"]].append(member)
|
|
637
|
+
_require_columns(analysis.exemplars, _EXEMPLAR, "exemplars")
|
|
638
|
+
if analysis.exemplars.duplicated(["run", "feature", "selection_order"]).any():
|
|
639
|
+
raise ValueError("duplicate exemplar selection order")
|
|
640
|
+
for example in analysis.exemplars[_EXEMPLAR].to_dict("records"):
|
|
641
|
+
key = example["run"], example["feature"]
|
|
642
|
+
if key not in members:
|
|
643
|
+
raise ValueError("exemplar has no matching event member")
|
|
644
|
+
if (
|
|
645
|
+
example["status"] == "selected"
|
|
646
|
+
and (example["run"], example["sample_id"]) not in canonical
|
|
647
|
+
):
|
|
648
|
+
raise ValueError(
|
|
649
|
+
"exemplar reference does not resolve to a canonical rollout"
|
|
650
|
+
)
|
|
651
|
+
grouped_examples[key].append(example)
|
|
652
|
+
|
|
653
|
+
grouped_events = defaultdict(list)
|
|
654
|
+
for key, ordered in ranking.items():
|
|
655
|
+
event = events[key]
|
|
656
|
+
group = sorted(
|
|
657
|
+
grouped_members[event["event_id"]],
|
|
658
|
+
key=lambda m: (m["onset_step"], m["alert_step"], m["feature"]),
|
|
659
|
+
)
|
|
660
|
+
references = {(member["run"], member["feature"]) for member in group}
|
|
661
|
+
if (
|
|
662
|
+
event["run"] not in actual_runs
|
|
663
|
+
or ordered["run"] != event["run"]
|
|
664
|
+
or event["n_members"] != len(group)
|
|
665
|
+
or ordered["n_members"] != len(group)
|
|
666
|
+
or set(event["members"]) != references
|
|
667
|
+
or (ordered["run"], ordered["representative_feature"]) not in references
|
|
668
|
+
):
|
|
669
|
+
raise ValueError("event summary conflicts with its members or ranking")
|
|
670
|
+
row = _fields(ordered, _EVENT)
|
|
671
|
+
row["ranking"] = {
|
|
672
|
+
**_fields(ordered, _RANKING),
|
|
673
|
+
**{name: _references(ordered[name], references) for name in _REFERENCES},
|
|
674
|
+
}
|
|
675
|
+
row["members"] = [
|
|
676
|
+
_member_report(
|
|
677
|
+
member,
|
|
678
|
+
evidence.get((member["run"], member["feature"])),
|
|
679
|
+
grouped_examples[(member["run"], member["feature"])],
|
|
680
|
+
shifts[(member["run"], member["feature"])],
|
|
681
|
+
policy,
|
|
682
|
+
)
|
|
683
|
+
for member in group
|
|
684
|
+
]
|
|
685
|
+
grouped_events[event["run"]].append(row)
|
|
686
|
+
|
|
687
|
+
_require_columns(
|
|
688
|
+
analysis.aggregation.prompt_schedule,
|
|
689
|
+
["run", "step", "prompt_id"],
|
|
690
|
+
"prompt schedule",
|
|
691
|
+
)
|
|
692
|
+
steps, prompts = defaultdict(set), defaultdict(set)
|
|
693
|
+
for row in analysis.aggregation.prompt_schedule[
|
|
694
|
+
["run", "step", "prompt_id"]
|
|
695
|
+
].to_dict("records"):
|
|
696
|
+
steps[row["run"]].add(row["step"])
|
|
697
|
+
prompts[row["run"]].add(row["prompt_id"])
|
|
698
|
+
rollout_counts = Counter(key[0] for key in canonical)
|
|
699
|
+
retrospective_runs = (
|
|
700
|
+
_retrospective_runs(analysis, policy, runs, actual_runs, evidence)
|
|
701
|
+
if policy is not None
|
|
702
|
+
else {}
|
|
703
|
+
)
|
|
704
|
+
output = []
|
|
705
|
+
for name in sorted(runs):
|
|
706
|
+
ordered = sorted(
|
|
707
|
+
grouped_events[name], key=lambda event: (event["rank"], event["event_id"])
|
|
708
|
+
)
|
|
709
|
+
if [event["rank"] for event in ordered] != list(range(1, len(ordered) + 1)):
|
|
710
|
+
raise ValueError("event ranks must be contiguous within each run")
|
|
711
|
+
counts = (
|
|
712
|
+
_fields(screening[(name,)], _SCREENING)
|
|
713
|
+
if (name,) in screening
|
|
714
|
+
else dict.fromkeys(_SCREENING, 0)
|
|
715
|
+
)
|
|
716
|
+
if counts["sequential_alerts"] != sum(key[0] == name for key in alerts):
|
|
717
|
+
raise ValueError("screening alert count does not match sequential alerts")
|
|
718
|
+
behavioral = sum(e["ranking_status"] == "behavioral" for e in ordered)
|
|
719
|
+
contextual = sum(e["ranking_status"] == "contextual" for e in ordered)
|
|
720
|
+
counts.update(
|
|
721
|
+
rollouts=rollout_counts[name],
|
|
722
|
+
observed_steps=len(steps[name]),
|
|
723
|
+
prompts=len(prompts[name]),
|
|
724
|
+
events=len(ordered),
|
|
725
|
+
behavioral_events=behavioral,
|
|
726
|
+
contextual_events=contextual,
|
|
727
|
+
other_events=len(ordered) - behavioral - contextual,
|
|
728
|
+
)
|
|
729
|
+
if analysis.sensitivity_metadata is not None:
|
|
730
|
+
counts["excluded_series"] = (
|
|
731
|
+
_integer(screening[(name,)]["n_excluded"])
|
|
732
|
+
if (name,) in screening
|
|
733
|
+
else 0
|
|
734
|
+
)
|
|
735
|
+
output.append(
|
|
736
|
+
{
|
|
737
|
+
"run": _identifier(name),
|
|
738
|
+
"status": "analyzed" if rollout_counts[name] else "empty",
|
|
739
|
+
"counts": counts,
|
|
740
|
+
"events": ordered,
|
|
741
|
+
**(
|
|
742
|
+
{"retrospective": retrospective_runs[name]}
|
|
743
|
+
if policy is not None
|
|
744
|
+
else {}
|
|
745
|
+
),
|
|
746
|
+
}
|
|
747
|
+
)
|
|
748
|
+
report = {"schema_version": 1, "runs": output}
|
|
749
|
+
metadata = analysis.benchmark_metadata
|
|
750
|
+
if analysis.sensitivity_metadata is not None:
|
|
751
|
+
report["schema_version"] = 2
|
|
752
|
+
profile = deepcopy(analysis.sensitivity_metadata)
|
|
753
|
+
for rows in profile["counts"].values():
|
|
754
|
+
for entry in output:
|
|
755
|
+
if entry["status"] == "empty" and not any(
|
|
756
|
+
r["run"] == entry["run"] for r in rows
|
|
757
|
+
):
|
|
758
|
+
rows.append(
|
|
759
|
+
{
|
|
760
|
+
"run": entry["run"],
|
|
761
|
+
"n_series": 0,
|
|
762
|
+
"n_screened": 0,
|
|
763
|
+
"n_alerted": 0,
|
|
764
|
+
"n_unscorable": 0,
|
|
765
|
+
}
|
|
766
|
+
)
|
|
767
|
+
rows.sort(key=lambda row: row["run"])
|
|
768
|
+
report["sensitivity"] = profile
|
|
769
|
+
metadata = {**(metadata or {}), "sensitivity": profile["id"]}
|
|
770
|
+
if policy is not None:
|
|
771
|
+
report.update(schema_version=3, retrospective=deepcopy(policy))
|
|
772
|
+
if benchmark is _BUNDLED_BENCHMARK:
|
|
773
|
+
from .benchmark_reference import load_bundled_benchmark_reference
|
|
774
|
+
|
|
775
|
+
benchmark = load_bundled_benchmark_reference(
|
|
776
|
+
metadata,
|
|
777
|
+
retrospective_disabled=policy is not None
|
|
778
|
+
and policy["status"] == "disabled",
|
|
779
|
+
)
|
|
780
|
+
report["benchmark"] = benchmark
|
|
781
|
+
elif benchmark is not None:
|
|
782
|
+
from .benchmark_reference import validate_supplied_benchmark_reference
|
|
783
|
+
|
|
784
|
+
report["benchmark"] = validate_supplied_benchmark_reference(benchmark)
|
|
785
|
+
if report["schema_version"] >= 2:
|
|
786
|
+
from ._report_validation import validate_report
|
|
787
|
+
|
|
788
|
+
report = validate_report(report)
|
|
789
|
+
return report
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def report_json(report: dict) -> str:
|
|
793
|
+
"""Encode a report as deterministic UTF-8-compatible strict JSON."""
|
|
794
|
+
return (
|
|
795
|
+
json.dumps(
|
|
796
|
+
report, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2
|
|
797
|
+
)
|
|
798
|
+
+ "\n"
|
|
799
|
+
)
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def _check_output_path(path: Path) -> None:
|
|
803
|
+
if path.exists() or path.is_symlink():
|
|
804
|
+
raise FileExistsError(f"output already exists: {path}")
|
|
805
|
+
if not path.parent.is_dir():
|
|
806
|
+
raise FileNotFoundError(f"output directory does not exist: {path.parent}")
|
|
807
|
+
|
|
808
|
+
|
|
809
|
+
def write_report(report: dict, path: str | Path) -> None:
|
|
810
|
+
"""Publish a complete report atomically; never replace an existing path."""
|
|
811
|
+
path = Path(path)
|
|
812
|
+
_check_output_path(path)
|
|
813
|
+
_write_text(report_json(report), path)
|
|
814
|
+
|
|
815
|
+
|
|
816
|
+
def _write_text(text: str, path: str | Path) -> None:
|
|
817
|
+
path = Path(path)
|
|
818
|
+
_check_output_path(path)
|
|
819
|
+
temporary = None
|
|
820
|
+
try:
|
|
821
|
+
with tempfile.NamedTemporaryFile(
|
|
822
|
+
mode="w",
|
|
823
|
+
encoding="utf-8",
|
|
824
|
+
newline="\n",
|
|
825
|
+
dir=path.parent,
|
|
826
|
+
prefix=".rolloutdiff-",
|
|
827
|
+
suffix=".tmp",
|
|
828
|
+
delete=False,
|
|
829
|
+
) as stream:
|
|
830
|
+
temporary = Path(stream.name)
|
|
831
|
+
stream.write(text)
|
|
832
|
+
stream.flush()
|
|
833
|
+
os.fsync(stream.fileno())
|
|
834
|
+
# A same-filesystem hard link publishes without an overwrite race.
|
|
835
|
+
os.link(temporary, path)
|
|
836
|
+
finally:
|
|
837
|
+
if temporary is not None:
|
|
838
|
+
temporary.unlink(missing_ok=True)
|