rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/report.py ADDED
@@ -0,0 +1,838 @@
1
+ """Versioned investigation reports from completed analysis tables."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import math
7
+ import os
8
+ import re
9
+ import tempfile
10
+ from collections import Counter, defaultdict
11
+ from copy import deepcopy
12
+ from numbers import Integral, Real
13
+ from pathlib import Path, PureWindowsPath
14
+
15
+ import numpy as np
16
+ import pandas as pd
17
+
18
+ from .pipeline import AnalysisResult
19
+ from .series import _require_columns
20
+
21
+ _BUNDLED_BENCHMARK = object()
22
+
23
+ _EVENT = [
24
+ "event_id",
25
+ "run",
26
+ "rank",
27
+ "representative_feature",
28
+ "ranking_status",
29
+ "onset_start",
30
+ "onset_end",
31
+ "onset_step",
32
+ "alert_step",
33
+ "n_members",
34
+ ]
35
+ _RANKING = [
36
+ "n_prioritizable_members",
37
+ "replication_availability",
38
+ "persistence_availability",
39
+ "reward_availability",
40
+ "breadth_availability",
41
+ ]
42
+ _REFERENCES = [
43
+ "missing_evidence_members",
44
+ "numerical_floor_members",
45
+ "numerical_floor_unavailable_members",
46
+ ]
47
+ _MEMBER = ["run", "feature", "kind", "direction", "onset_step", "alert_step", "status"]
48
+ _EXEMPLAR = [
49
+ "run",
50
+ "feature",
51
+ "kind",
52
+ "direction",
53
+ "onset_step",
54
+ "alert_step",
55
+ "status",
56
+ "n_window_rollouts",
57
+ "n_candidates",
58
+ "reward_selection_status",
59
+ "selection_order",
60
+ "sample_id",
61
+ "step",
62
+ "prompt_id",
63
+ "feature_value",
64
+ "reward",
65
+ "reward_group_size",
66
+ "reward_group_mean",
67
+ "reward_deviation",
68
+ "reward_deviation_status",
69
+ "selection_reason",
70
+ ]
71
+ _EVIDENCE = {
72
+ "replication": {
73
+ **{
74
+ name: f"replication_{name}"
75
+ for name in ["status", "coverage_policy", "horizon_complete", "cutoff_step"]
76
+ },
77
+ "replicates": "replicates",
78
+ },
79
+ "persistence": {
80
+ name: f"persistence_{name}"
81
+ for name in [
82
+ "status",
83
+ "fraction",
84
+ "coverage",
85
+ "n_observations",
86
+ "n_evaluable",
87
+ "n_retained",
88
+ "n_unavailable",
89
+ ]
90
+ },
91
+ "reward_association": {
92
+ name: f"reward_{name}"
93
+ for name in [
94
+ "status",
95
+ "mean_difference",
96
+ "n_groups",
97
+ "n_mixed_groups",
98
+ "n_rollouts",
99
+ "n_included_rollouts",
100
+ "window_start",
101
+ "window_end",
102
+ ]
103
+ },
104
+ "prompt_breadth": {
105
+ "positive_prompts": "prompt_positive_prompts",
106
+ "total_prompts": "prompt_total_prompts",
107
+ "promptmix_status": "promptmix_status",
108
+ },
109
+ }
110
+ _FEATURE_EVIDENCE = ["tier", "scale_source", "numerical_floor_dominant"]
111
+ _PROMPT_MIX = {
112
+ "status": "promptmix_status",
113
+ **{
114
+ name: f"prompt_{name}"
115
+ for name in [
116
+ "before_start",
117
+ "before_end",
118
+ "after_start",
119
+ "after_end",
120
+ "shared_prompts",
121
+ "shared_sampling_units",
122
+ "shared_change",
123
+ "shared_se",
124
+ ]
125
+ },
126
+ }
127
+ _PROMPT_SHIFT = [
128
+ "descriptor",
129
+ "statistic",
130
+ "before",
131
+ "after",
132
+ "delta",
133
+ "distance",
134
+ "n_before",
135
+ "n_after",
136
+ "missing_before",
137
+ "missing_after",
138
+ ]
139
+ _SCREENING = {
140
+ "series": "n_series",
141
+ "screened_series": "n_screened",
142
+ "sequential_alerts": "n_alerted",
143
+ "unscorable_series": "n_unscorable",
144
+ }
145
+ _STEPS = {
146
+ "changepoint_step",
147
+ "step",
148
+ "onset_start",
149
+ "onset_end",
150
+ "onset_step",
151
+ "alert_step",
152
+ "replication_cutoff_step",
153
+ "reward_window_start",
154
+ "reward_window_end",
155
+ "prompt_before_start",
156
+ "prompt_before_end",
157
+ "prompt_after_start",
158
+ "prompt_after_end",
159
+ }
160
+ _INTEGERS = _STEPS | {
161
+ "rank",
162
+ "selection_order",
163
+ "reward_group_size",
164
+ "prompt_positive_prompts",
165
+ "prompt_total_prompts",
166
+ "prompt_shared_prompts",
167
+ "prompt_shared_sampling_units",
168
+ "missing_before",
169
+ "missing_after",
170
+ }
171
+ _IDENTIFIERS = {
172
+ "run",
173
+ "event_id",
174
+ "feature",
175
+ "representative_feature",
176
+ "sample_id",
177
+ "prompt_id",
178
+ }
179
+ # POSIX roots are recognized at identifier component boundaries.
180
+ _EMBEDDED_PATH = re.compile(
181
+ r"(?i:(?<!\w)file://)"
182
+ r"|[A-Za-z]:(?:\\|/(?!/))"
183
+ r"|\\\\[^\\/\s]+[\\/][^\\/\s]+"
184
+ r"""|(?:^|(?<=[\s=,;|"'(\[{<]))//[^/\\\s]+[\\/][^/\\\s]+"""
185
+ r"""|(?:^|(?<=[\s:=,;|"'(\[{<]))(?:/|~[\\/])(?=[^/\\\s])"""
186
+ )
187
+
188
+
189
+ def _identifier(value: str) -> str:
190
+ if not isinstance(value, str) or not value.strip():
191
+ raise ValueError("report identifiers must be nonempty strings")
192
+ candidate = value.removeprefix("file:").removeprefix("strlit:")
193
+ if (
194
+ candidate.startswith(("/", "\\", "~/", "~\\"))
195
+ or PureWindowsPath(candidate).is_absolute()
196
+ or _EMBEDDED_PATH.search(value)
197
+ ):
198
+ raise ValueError(
199
+ "report identifiers must not contain absolute filesystem paths"
200
+ )
201
+ return value
202
+
203
+
204
+ def _integer(value, *, identity=False):
205
+ if value is None or value is pd.NA or pd.isna(value):
206
+ return None
207
+ if isinstance(value, (bool, np.bool_)):
208
+ raise ValueError("report integer fields require exact integers")
209
+ if (isinstance(value, Integral) and not isinstance(value, (bool, np.bool_))) or (
210
+ not identity
211
+ and isinstance(value, Real)
212
+ and math.isfinite(value)
213
+ and abs(value) < 2**53
214
+ and int(value) == value
215
+ ):
216
+ number = int(value)
217
+ else:
218
+ raise ValueError("report integer fields require exact integers")
219
+ if number < 0:
220
+ raise ValueError("report integer fields must be nonnegative")
221
+ return number
222
+
223
+
224
+ def _value(name, value):
225
+ if value is None or value is pd.NA or value is pd.NaT:
226
+ return None
227
+ if (
228
+ isinstance(value, Real)
229
+ and not isinstance(value, (bool, np.bool_))
230
+ and not math.isfinite(value)
231
+ ):
232
+ return None
233
+ if name in _IDENTIFIERS:
234
+ return _identifier(value)
235
+ if name in _INTEGERS or name.startswith("n_") or "_n_" in name:
236
+ return _integer(value, identity=name in _STEPS)
237
+ if isinstance(value, (bool, np.bool_)):
238
+ return bool(value)
239
+ if isinstance(value, Integral):
240
+ return int(value)
241
+ if isinstance(value, Real):
242
+ return float(value)
243
+ if isinstance(value, str):
244
+ return value
245
+ raise ValueError(f"unsupported report value for {name}")
246
+
247
+
248
+ def _fields(row, names):
249
+ mapping = dict(zip(names, names, strict=True)) if isinstance(names, list) else names
250
+ return {
251
+ output: _value(source, row.get(source)) for output, source in mapping.items()
252
+ }
253
+
254
+
255
+ def _index(frame, keys, columns, name):
256
+ _require_columns(frame, list(dict.fromkeys([*keys, *columns])), name)
257
+ if frame.duplicated(keys).any():
258
+ raise ValueError(f"duplicate {name} keys")
259
+ result = {}
260
+ for row in frame[list(dict.fromkeys([*keys, *columns]))].to_dict("records"):
261
+ key = tuple(row[k] for k in keys)
262
+ if not all(isinstance(v, str) and v.strip() for v in key):
263
+ raise ValueError(f"invalid {name} keys")
264
+ result[key] = row
265
+ return result
266
+
267
+
268
+ def _references(values, allowed):
269
+ references = []
270
+ for run, feature in values:
271
+ if (run, feature) not in allowed:
272
+ raise ValueError("ranking reference is not an event member")
273
+ references.append({"run": _identifier(run), "feature": _identifier(feature)})
274
+ return sorted(references, key=lambda row: (row["run"], row["feature"]))
275
+
276
+
277
+ def _member_report(member, source, exemplars, shifts, policy):
278
+ row = _fields(member, _MEMBER)
279
+ evidence = source if source is not None else {}
280
+ row["evidence"] = {
281
+ "status": "recorded" if source is not None else "missing",
282
+ **_fields(evidence, _FEATURE_EVIDENCE),
283
+ **{name: _fields(evidence, columns) for name, columns in _EVIDENCE.items()},
284
+ "prompt_mix": {
285
+ **_fields(evidence, _PROMPT_MIX),
286
+ "prompt_shift_status": "recorded" if shifts else "missing",
287
+ "prompt_shift": shifts,
288
+ },
289
+ }
290
+ if policy is not None:
291
+ detail = _fields(
292
+ evidence,
293
+ {"status": "retrospective_status", "changepoint_step": "changepoint_step"},
294
+ )
295
+ if source is None:
296
+ detail["status"] = (
297
+ "disabled" if policy["status"] == "disabled" else "missing"
298
+ )
299
+ row["evidence"]["retrospective"] = detail
300
+ row["exemplar_status"] = "recorded" if exemplars else "missing"
301
+ row["exemplars"] = sorted(
302
+ [_fields(example, _EXEMPLAR) for example in exemplars],
303
+ key=lambda example: (
304
+ example["selection_order"] is None,
305
+ example["selection_order"] or 0,
306
+ example["sample_id"] or "",
307
+ ),
308
+ )
309
+ return row
310
+
311
+
312
+ def _prompt_shifts(frame, evidence):
313
+ grouped = defaultdict(list)
314
+ if frame.empty:
315
+ return grouped
316
+ windows = ["before_start", "before_end", "after_start", "after_end"]
317
+ _require_columns(
318
+ frame, ["run", "feature", *_PROMPT_SHIFT, *windows], "prompt shifts"
319
+ )
320
+ if frame.duplicated(["run", "feature", "descriptor"]).any():
321
+ raise ValueError("duplicate prompt-shift descriptors")
322
+ for shift in frame.to_dict("records"):
323
+ key = shift["run"], shift["feature"]
324
+ source = evidence.get(key)
325
+ if source is None:
326
+ continue
327
+ if any(shift[name] != source[f"prompt_{name}"] for name in windows):
328
+ raise ValueError("prompt-shift windows do not match feature evidence")
329
+ row = _fields(shift, _PROMPT_SHIFT)
330
+ row["descriptor"] = _identifier(row["descriptor"])
331
+ grouped[key].append(row)
332
+ for rows in grouped.values():
333
+ rows.sort(key=lambda row: row["descriptor"])
334
+ return grouped
335
+
336
+
337
+ def _check_retrospective_tables(analysis, settings, changes, screening, evidence):
338
+ """Check source identities and detector outcomes before exporting selected boundaries."""
339
+ keys = ["run", "feature"]
340
+ series = analysis.aggregation.series
341
+ groups = (
342
+ series.sort_values("step", kind="stable")
343
+ .groupby(keys, sort=False)
344
+ .agg(
345
+ tier=("tier", "first"),
346
+ kind=("kind", "first"),
347
+ steps=("step", list),
348
+ )
349
+ .to_dict("index")
350
+ )
351
+ diagnostics = _index(
352
+ analysis.retrospective.diagnostics,
353
+ keys,
354
+ [
355
+ "tier",
356
+ "kind",
357
+ "status",
358
+ "n_observations",
359
+ "candidate_step",
360
+ "previous_step",
361
+ "n_before",
362
+ "n_after",
363
+ "min_segment_steps",
364
+ "penalty_multiplier",
365
+ "min_effect",
366
+ "dependence",
367
+ ],
368
+ "retrospective diagnostics",
369
+ )
370
+ if (
371
+ diagnostics.keys() != groups.keys()
372
+ or changes.keys() - groups.keys()
373
+ or evidence.keys() - groups.keys()
374
+ ):
375
+ raise ValueError("retrospective rows conflict with the aggregated series")
376
+ counts = defaultdict(Counter)
377
+ for key, row in diagnostics.items():
378
+ group = groups[key]
379
+ status = row["status"]
380
+ selected = status == "change"
381
+ scorable = status in {"change", "no_change"}
382
+ steps = group["steps"]
383
+ if (
384
+ status
385
+ not in {
386
+ "change",
387
+ "no_change",
388
+ "unavailable_uncertainty",
389
+ "insufficient_data",
390
+ }
391
+ or any(row[name] != group[name] for name in ("kind", "tier"))
392
+ or row["n_observations"] != len(steps)
393
+ or selected != (key in changes)
394
+ ):
395
+ raise ValueError(
396
+ "retrospective diagnostics conflict with the series or selected changes"
397
+ )
398
+ candidate = _integer(row["candidate_step"], identity=True)
399
+ if (candidate is not None) != scorable:
400
+ raise ValueError("retrospective candidate conflicts with scorable status")
401
+ if scorable:
402
+ before = _integer(row["n_before"])
403
+ if (
404
+ before is None
405
+ or not 0 < before < len(steps)
406
+ or steps[before] != candidate
407
+ or steps[before - 1] != row["previous_step"]
408
+ or row["n_after"] != len(steps) - before
409
+ ):
410
+ raise ValueError("retrospective segments conflict with observed steps")
411
+ effect_name = (
412
+ "min_prevalence_change"
413
+ if row["kind"] == "binary"
414
+ else "min_continuous_change"
415
+ )
416
+ if (
417
+ any(
418
+ row[name] != settings[name]
419
+ for name in ("min_segment_steps", "penalty_multiplier", "dependence")
420
+ )
421
+ or row["min_effect"] != settings[effect_name]
422
+ ):
423
+ raise ValueError(
424
+ "retrospective diagnostics conflict with effective settings"
425
+ )
426
+ boundary = None
427
+ if selected:
428
+ change = changes[key]
429
+ boundary = _integer(change["changepoint_step"], identity=True)
430
+ if (
431
+ candidate is None
432
+ or boundary != candidate
433
+ or any(
434
+ change[k] != row[k]
435
+ for k in ("kind", "tier", "previous_step", "n_before", "n_after")
436
+ )
437
+ ):
438
+ raise ValueError("retrospective boundary conflicts with its diagnostic")
439
+ if key in evidence and (
440
+ evidence[key]["retrospective_status"] != status
441
+ or _integer(evidence[key]["changepoint_step"], identity=True) != boundary
442
+ ):
443
+ raise ValueError(
444
+ "retrospective evidence conflicts with the detector outcome"
445
+ )
446
+ counts[key[0]].update(
447
+ n_series=1,
448
+ n_screened=int(scorable),
449
+ n_changes=int(selected),
450
+ n_unscorable=int(not scorable),
451
+ )
452
+ for (run,), row in screening.items():
453
+ if any(
454
+ counts[run][name] != row[name]
455
+ for name in ("n_series", "n_screened", "n_changes", "n_unscorable")
456
+ ):
457
+ raise ValueError(
458
+ "retrospective screening counts conflict with diagnostic outcomes"
459
+ )
460
+
461
+
462
+ def _retrospective_runs(analysis, policy, runs, actual_runs, evidence):
463
+ result = analysis.retrospective
464
+ if policy["status"] == "disabled":
465
+ if any(
466
+ not table.empty
467
+ for table in (result.diagnostics, result.screening, result.changepoints)
468
+ ):
469
+ raise ValueError(
470
+ "disabled retrospective analysis contains detector results"
471
+ )
472
+ if (
473
+ not analysis.evidence.retrospective_status.eq("disabled")
474
+ .fillna(False)
475
+ .all()
476
+ or analysis.evidence.changepoint_step.notna().any()
477
+ ):
478
+ raise ValueError(
479
+ "disabled retrospective analysis contains contradictory evidence"
480
+ )
481
+ return {
482
+ run: {"status": "disabled", "screening": None, "changes": None}
483
+ for run in runs
484
+ }
485
+
486
+ counts = ["n_series", "n_screened", "n_changes", "n_unscorable"]
487
+ screening = _index(result.screening, ["run"], counts, "retrospective screening")
488
+ if {key[0] for key in screening} != actual_runs:
489
+ raise ValueError("retrospective screening must cover all analyzed runs")
490
+ selected = _index(
491
+ result.changepoints,
492
+ ["run", "feature"],
493
+ ["tier", "kind", "changepoint_step", "previous_step", "n_before", "n_after"],
494
+ "retrospective changes",
495
+ )
496
+ _check_retrospective_tables(
497
+ analysis, policy["effective_settings"], selected, screening, evidence
498
+ )
499
+ changes = defaultdict(list)
500
+ for (run, _), row in selected.items():
501
+ changes[run].append(_fields(row, ["feature", "changepoint_step"]))
502
+ return {
503
+ run: {
504
+ "status": "ran",
505
+ "screening": _fields(screening[(run,)], counts)
506
+ if run in actual_runs
507
+ else dict.fromkeys(counts, 0),
508
+ "changes": sorted(
509
+ changes[run], key=lambda row: (row["changepoint_step"], row["feature"])
510
+ ),
511
+ }
512
+ for run in runs
513
+ }
514
+
515
+
516
+ def build_report(
517
+ analysis: AnalysisResult,
518
+ *,
519
+ run: str | None = None,
520
+ benchmark: dict | None = _BUNDLED_BENCHMARK,
521
+ ) -> dict:
522
+ """Build a v3 report, or preserve v1/v2 for analysis predating detector policy.
523
+
524
+ Run/event/member order is canonical; unavailable numbers become null.
525
+ `run` supplies identity for empty input and must match any analyzed run.
526
+ Absolute path-shaped identifiers fail export rather than changing references.
527
+ Omitted benchmark evidence uses the bundled historical reference. Pass None
528
+ to omit it or a reference dictionary to supply separate benchmark evidence.
529
+ """
530
+ policy = analysis.retrospective_metadata
531
+ metadata_settings = (analysis.benchmark_metadata or {}).get("settings", {})
532
+ if policy is not None:
533
+ from ._report_validation import _RetrospectivePolicy
534
+
535
+ try:
536
+ policy = _RetrospectivePolicy.model_validate(policy).model_dump()
537
+ except ValueError:
538
+ raise ValueError(
539
+ "invalid retrospective execution policy metadata"
540
+ ) from None
541
+ if (
542
+ isinstance(metadata_settings, dict)
543
+ and "retrospective" in metadata_settings
544
+ and metadata_settings["retrospective"] != policy["effective_settings"]
545
+ ):
546
+ raise ValueError(
547
+ "benchmark settings conflict with retrospective execution policy"
548
+ )
549
+ if policy is None and (
550
+ (
551
+ "retrospective_status" in analysis.evidence
552
+ and analysis.evidence.retrospective_status.eq("disabled").any()
553
+ )
554
+ or (
555
+ isinstance(metadata_settings, dict)
556
+ and metadata_settings.get("retrospective", {}) is None
557
+ )
558
+ ):
559
+ raise ValueError(
560
+ "disabled retrospective analysis requires execution policy metadata"
561
+ )
562
+ if analysis.sensitivity_metadata is None and (
563
+ "n_excluded" in analysis.sequential.screening
564
+ or (analysis.benchmark_metadata or {}).get("sensitivity") is not None
565
+ or analysis.sequential.diagnostics.status.eq("profile_excluded").any()
566
+ ):
567
+ raise ValueError("experimental analysis requires sensitivity metadata")
568
+ canonical = _index(analysis.features.dense, ["run", "sample_id"], [], "rollouts")
569
+ if analysis.sensitivity_metadata is not None:
570
+ _require_columns(
571
+ analysis.sequential.screening, ["n_excluded"], "sensitivity screening"
572
+ )
573
+ screening = _index(
574
+ analysis.sequential.screening,
575
+ ["run"],
576
+ [
577
+ *list(_SCREENING.values()),
578
+ *(["n_excluded"] if analysis.sensitivity_metadata is not None else []),
579
+ ],
580
+ "screening",
581
+ )
582
+ events = _index(
583
+ analysis.events,
584
+ ["event_id"],
585
+ ["run", "onset_start", "onset_end", "n_members", "members"],
586
+ "events",
587
+ )
588
+ ranking = _index(
589
+ analysis.event_ranking,
590
+ ["event_id"],
591
+ [*_EVENT, *_RANKING, *_REFERENCES],
592
+ "event ranking",
593
+ )
594
+ members = _index(
595
+ analysis.event_members,
596
+ ["run", "feature"],
597
+ ["event_id", *_MEMBER],
598
+ "event members",
599
+ )
600
+ evidence = _index(
601
+ analysis.evidence,
602
+ ["run", "feature"],
603
+ [
604
+ *_FEATURE_EVIDENCE,
605
+ *(
606
+ ["retrospective_status", "changepoint_step"]
607
+ if policy is not None
608
+ else []
609
+ ),
610
+ *[column for group in _EVIDENCE.values() for column in group.values()],
611
+ *_PROMPT_MIX.values(),
612
+ ],
613
+ "evidence",
614
+ )
615
+ alerts = _index(
616
+ analysis.sequential.alerts, ["run", "feature"], [], "sequential alerts"
617
+ )
618
+ shifts = _prompt_shifts(analysis.prompt_shifts, evidence)
619
+ if set(ranking) != set(events) or set(members) != set(alerts):
620
+ raise ValueError("event, ranking and sequential membership coverage must agree")
621
+ actual_runs = {key[0] for key in canonical}
622
+ if {key[0] for key in screening} != actual_runs:
623
+ raise ValueError("screening must cover all analyzed runs")
624
+ runs = set(actual_runs)
625
+ if run is not None:
626
+ _identifier(run)
627
+ if actual_runs and actual_runs != {run}:
628
+ raise ValueError("requested run does not match analyzed runs")
629
+ runs.add(run)
630
+
631
+ grouped_members, grouped_examples = defaultdict(list), defaultdict(list)
632
+ for member in members.values():
633
+ event = events.get((member["event_id"],))
634
+ if event is None or event["run"] != member["run"]:
635
+ raise ValueError("event member has no matching event")
636
+ grouped_members[member["event_id"]].append(member)
637
+ _require_columns(analysis.exemplars, _EXEMPLAR, "exemplars")
638
+ if analysis.exemplars.duplicated(["run", "feature", "selection_order"]).any():
639
+ raise ValueError("duplicate exemplar selection order")
640
+ for example in analysis.exemplars[_EXEMPLAR].to_dict("records"):
641
+ key = example["run"], example["feature"]
642
+ if key not in members:
643
+ raise ValueError("exemplar has no matching event member")
644
+ if (
645
+ example["status"] == "selected"
646
+ and (example["run"], example["sample_id"]) not in canonical
647
+ ):
648
+ raise ValueError(
649
+ "exemplar reference does not resolve to a canonical rollout"
650
+ )
651
+ grouped_examples[key].append(example)
652
+
653
+ grouped_events = defaultdict(list)
654
+ for key, ordered in ranking.items():
655
+ event = events[key]
656
+ group = sorted(
657
+ grouped_members[event["event_id"]],
658
+ key=lambda m: (m["onset_step"], m["alert_step"], m["feature"]),
659
+ )
660
+ references = {(member["run"], member["feature"]) for member in group}
661
+ if (
662
+ event["run"] not in actual_runs
663
+ or ordered["run"] != event["run"]
664
+ or event["n_members"] != len(group)
665
+ or ordered["n_members"] != len(group)
666
+ or set(event["members"]) != references
667
+ or (ordered["run"], ordered["representative_feature"]) not in references
668
+ ):
669
+ raise ValueError("event summary conflicts with its members or ranking")
670
+ row = _fields(ordered, _EVENT)
671
+ row["ranking"] = {
672
+ **_fields(ordered, _RANKING),
673
+ **{name: _references(ordered[name], references) for name in _REFERENCES},
674
+ }
675
+ row["members"] = [
676
+ _member_report(
677
+ member,
678
+ evidence.get((member["run"], member["feature"])),
679
+ grouped_examples[(member["run"], member["feature"])],
680
+ shifts[(member["run"], member["feature"])],
681
+ policy,
682
+ )
683
+ for member in group
684
+ ]
685
+ grouped_events[event["run"]].append(row)
686
+
687
+ _require_columns(
688
+ analysis.aggregation.prompt_schedule,
689
+ ["run", "step", "prompt_id"],
690
+ "prompt schedule",
691
+ )
692
+ steps, prompts = defaultdict(set), defaultdict(set)
693
+ for row in analysis.aggregation.prompt_schedule[
694
+ ["run", "step", "prompt_id"]
695
+ ].to_dict("records"):
696
+ steps[row["run"]].add(row["step"])
697
+ prompts[row["run"]].add(row["prompt_id"])
698
+ rollout_counts = Counter(key[0] for key in canonical)
699
+ retrospective_runs = (
700
+ _retrospective_runs(analysis, policy, runs, actual_runs, evidence)
701
+ if policy is not None
702
+ else {}
703
+ )
704
+ output = []
705
+ for name in sorted(runs):
706
+ ordered = sorted(
707
+ grouped_events[name], key=lambda event: (event["rank"], event["event_id"])
708
+ )
709
+ if [event["rank"] for event in ordered] != list(range(1, len(ordered) + 1)):
710
+ raise ValueError("event ranks must be contiguous within each run")
711
+ counts = (
712
+ _fields(screening[(name,)], _SCREENING)
713
+ if (name,) in screening
714
+ else dict.fromkeys(_SCREENING, 0)
715
+ )
716
+ if counts["sequential_alerts"] != sum(key[0] == name for key in alerts):
717
+ raise ValueError("screening alert count does not match sequential alerts")
718
+ behavioral = sum(e["ranking_status"] == "behavioral" for e in ordered)
719
+ contextual = sum(e["ranking_status"] == "contextual" for e in ordered)
720
+ counts.update(
721
+ rollouts=rollout_counts[name],
722
+ observed_steps=len(steps[name]),
723
+ prompts=len(prompts[name]),
724
+ events=len(ordered),
725
+ behavioral_events=behavioral,
726
+ contextual_events=contextual,
727
+ other_events=len(ordered) - behavioral - contextual,
728
+ )
729
+ if analysis.sensitivity_metadata is not None:
730
+ counts["excluded_series"] = (
731
+ _integer(screening[(name,)]["n_excluded"])
732
+ if (name,) in screening
733
+ else 0
734
+ )
735
+ output.append(
736
+ {
737
+ "run": _identifier(name),
738
+ "status": "analyzed" if rollout_counts[name] else "empty",
739
+ "counts": counts,
740
+ "events": ordered,
741
+ **(
742
+ {"retrospective": retrospective_runs[name]}
743
+ if policy is not None
744
+ else {}
745
+ ),
746
+ }
747
+ )
748
+ report = {"schema_version": 1, "runs": output}
749
+ metadata = analysis.benchmark_metadata
750
+ if analysis.sensitivity_metadata is not None:
751
+ report["schema_version"] = 2
752
+ profile = deepcopy(analysis.sensitivity_metadata)
753
+ for rows in profile["counts"].values():
754
+ for entry in output:
755
+ if entry["status"] == "empty" and not any(
756
+ r["run"] == entry["run"] for r in rows
757
+ ):
758
+ rows.append(
759
+ {
760
+ "run": entry["run"],
761
+ "n_series": 0,
762
+ "n_screened": 0,
763
+ "n_alerted": 0,
764
+ "n_unscorable": 0,
765
+ }
766
+ )
767
+ rows.sort(key=lambda row: row["run"])
768
+ report["sensitivity"] = profile
769
+ metadata = {**(metadata or {}), "sensitivity": profile["id"]}
770
+ if policy is not None:
771
+ report.update(schema_version=3, retrospective=deepcopy(policy))
772
+ if benchmark is _BUNDLED_BENCHMARK:
773
+ from .benchmark_reference import load_bundled_benchmark_reference
774
+
775
+ benchmark = load_bundled_benchmark_reference(
776
+ metadata,
777
+ retrospective_disabled=policy is not None
778
+ and policy["status"] == "disabled",
779
+ )
780
+ report["benchmark"] = benchmark
781
+ elif benchmark is not None:
782
+ from .benchmark_reference import validate_supplied_benchmark_reference
783
+
784
+ report["benchmark"] = validate_supplied_benchmark_reference(benchmark)
785
+ if report["schema_version"] >= 2:
786
+ from ._report_validation import validate_report
787
+
788
+ report = validate_report(report)
789
+ return report
790
+
791
+
792
+ def report_json(report: dict) -> str:
793
+ """Encode a report as deterministic UTF-8-compatible strict JSON."""
794
+ return (
795
+ json.dumps(
796
+ report, ensure_ascii=False, allow_nan=False, sort_keys=True, indent=2
797
+ )
798
+ + "\n"
799
+ )
800
+
801
+
802
+ def _check_output_path(path: Path) -> None:
803
+ if path.exists() or path.is_symlink():
804
+ raise FileExistsError(f"output already exists: {path}")
805
+ if not path.parent.is_dir():
806
+ raise FileNotFoundError(f"output directory does not exist: {path.parent}")
807
+
808
+
809
+ def write_report(report: dict, path: str | Path) -> None:
810
+ """Publish a complete report atomically; never replace an existing path."""
811
+ path = Path(path)
812
+ _check_output_path(path)
813
+ _write_text(report_json(report), path)
814
+
815
+
816
+ def _write_text(text: str, path: str | Path) -> None:
817
+ path = Path(path)
818
+ _check_output_path(path)
819
+ temporary = None
820
+ try:
821
+ with tempfile.NamedTemporaryFile(
822
+ mode="w",
823
+ encoding="utf-8",
824
+ newline="\n",
825
+ dir=path.parent,
826
+ prefix=".rolloutdiff-",
827
+ suffix=".tmp",
828
+ delete=False,
829
+ ) as stream:
830
+ temporary = Path(stream.name)
831
+ stream.write(text)
832
+ stream.flush()
833
+ os.fsync(stream.fileno())
834
+ # A same-filesystem hard link publishes without an overwrite race.
835
+ os.link(temporary, path)
836
+ finally:
837
+ if temporary is not None:
838
+ temporary.unlink(missing_ok=True)