rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,758 @@
1
+ """Read analysis reports without recomputing their evidence."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from pathlib import Path
7
+ from typing import Annotated, Literal
8
+
9
+ from pydantic import (
10
+ AfterValidator,
11
+ BaseModel,
12
+ ConfigDict,
13
+ Field,
14
+ ValidationError,
15
+ field_validator,
16
+ model_validator,
17
+ )
18
+
19
+ from ._retrospective_policy import ALGORITHM, WARNING
20
+ from .benchmark_reference import (
21
+ BenchmarkReference,
22
+ _Retrospective,
23
+ _Sequential,
24
+ _Vocabulary,
25
+ validate_benchmark_reference,
26
+ )
27
+ from .sensitivity import (
28
+ _DEFAULT_EVIDENCE,
29
+ _DEFAULT_RETROSPECTIVE,
30
+ _DEFAULT_SEQUENTIAL,
31
+ _DEFAULT_VOCABULARY,
32
+ LENIENT_SEQUENTIAL,
33
+ )
34
+
35
+
36
+ def _nonempty(value: str) -> str:
37
+ if not value.strip():
38
+ raise ValueError("identifier must be nonempty")
39
+ return value
40
+
41
+
42
+ def _descriptor(value: str) -> str:
43
+ from .report import _identifier
44
+
45
+ return _identifier(value)
46
+
47
+
48
+ Identifier = Annotated[str, AfterValidator(_nonempty)]
49
+ Count = Annotated[int, Field(ge=0)]
50
+ Number = int | float
51
+
52
+
53
+ class _Model(BaseModel):
54
+ model_config = ConfigDict(strict=True, extra="ignore", allow_inf_nan=False)
55
+
56
+
57
+ class _Reference(_Model):
58
+ run: Identifier
59
+ feature: Identifier
60
+
61
+
62
+ class _Replication(_Model):
63
+ status: str | None
64
+ coverage_policy: str | None
65
+ horizon_complete: bool | None
66
+ cutoff_step: Count | None
67
+ replicates: bool | None
68
+
69
+
70
+ class _Persistence(_Model):
71
+ status: str | None
72
+ fraction: Number | None
73
+ coverage: Number | None
74
+ n_observations: Count | None
75
+ n_evaluable: Count | None
76
+ n_retained: Count | None
77
+ n_unavailable: Count | None
78
+
79
+
80
+ class _Reward(_Model):
81
+ status: str | None
82
+ mean_difference: Number | None
83
+ n_groups: Count | None
84
+ n_mixed_groups: Count | None
85
+ n_rollouts: Count | None
86
+ n_included_rollouts: Count | None
87
+ window_start: Count | None
88
+ window_end: Count | None
89
+
90
+
91
+ class _Breadth(_Model):
92
+ positive_prompts: Count | None
93
+ total_prompts: Count | None
94
+ promptmix_status: str | None
95
+
96
+
97
+ class _PromptShift(_Model):
98
+ descriptor: Annotated[Identifier, AfterValidator(_descriptor)]
99
+ statistic: Literal["ks", "absolute_prevalence", "total_variation"]
100
+ before: Number | None
101
+ after: Number | None
102
+ delta: Number | None
103
+ distance: Annotated[Number, Field(ge=0, le=1)] | None
104
+ n_before: Count
105
+ n_after: Count
106
+ missing_before: Count
107
+ missing_after: Count
108
+
109
+
110
+ class _PromptMix(_Model):
111
+ status: str | None
112
+ before_start: Count | None
113
+ before_end: Count | None
114
+ after_start: Count | None
115
+ after_end: Count | None
116
+ shared_prompts: Count | None
117
+ shared_sampling_units: Count | None
118
+ shared_change: Number | None
119
+ shared_se: Annotated[Number, Field(ge=0)] | None
120
+ prompt_shift_status: Literal["recorded", "missing"]
121
+ prompt_shift: list[_PromptShift]
122
+
123
+
124
+ class _MemberRetrospective(_Model):
125
+ status: Literal[
126
+ "disabled",
127
+ "change",
128
+ "no_change",
129
+ "insufficient_data",
130
+ "unavailable_uncertainty",
131
+ "missing",
132
+ ]
133
+ changepoint_step: Count | None
134
+
135
+
136
+ class _Evidence(_Model):
137
+ status: str
138
+ tier: str | None
139
+ scale_source: str | None
140
+ numerical_floor_dominant: bool | None
141
+ replication: _Replication
142
+ persistence: _Persistence
143
+ reward_association: _Reward
144
+ prompt_breadth: _Breadth
145
+ prompt_mix: _PromptMix | None = None
146
+ retrospective: _MemberRetrospective | None = None
147
+
148
+
149
+ class _Exemplar(_Reference):
150
+ kind: str
151
+ direction: str
152
+ onset_step: Count
153
+ alert_step: Count
154
+ status: str
155
+ n_window_rollouts: Count | None
156
+ n_candidates: Count | None
157
+ reward_selection_status: str | None
158
+ selection_order: Count | None
159
+ sample_id: Identifier | None
160
+ step: Count | None
161
+ prompt_id: Identifier | None
162
+ feature_value: Number | None
163
+ reward: Number | None
164
+ reward_group_size: Count | None
165
+ reward_group_mean: Number | None
166
+ reward_deviation: Number | None
167
+ reward_deviation_status: str | None
168
+ selection_reason: str | None
169
+
170
+
171
+ class _Member(_Reference):
172
+ kind: str
173
+ direction: str
174
+ onset_step: Count
175
+ alert_step: Count
176
+ status: str
177
+ evidence: _Evidence
178
+ exemplar_status: str
179
+ exemplars: list[_Exemplar]
180
+
181
+
182
+ class _Ranking(_Model):
183
+ n_prioritizable_members: Count
184
+ replication_availability: str
185
+ persistence_availability: str
186
+ reward_availability: str
187
+ breadth_availability: str
188
+ missing_evidence_members: list[_Reference]
189
+ numerical_floor_members: list[_Reference]
190
+ numerical_floor_unavailable_members: list[_Reference]
191
+
192
+
193
+ class _Event(_Model):
194
+ event_id: Identifier
195
+ run: Identifier
196
+ rank: Annotated[int, Field(ge=1)]
197
+ representative_feature: Identifier
198
+ ranking_status: str
199
+ onset_start: Count
200
+ onset_end: Count
201
+ onset_step: Count
202
+ alert_step: Count
203
+ n_members: Annotated[int, Field(ge=1)]
204
+ ranking: _Ranking
205
+ members: list[_Member]
206
+
207
+
208
+ class _Counts(_Model):
209
+ series: Count
210
+ screened_series: Count
211
+ sequential_alerts: Count
212
+ unscorable_series: Count
213
+ rollouts: Count
214
+ observed_steps: Count
215
+ prompts: Count
216
+ events: Count
217
+ behavioral_events: Count
218
+ contextual_events: Count
219
+ other_events: Count
220
+ excluded_series: Count | None = None
221
+
222
+
223
+ class _ProfileModel(BaseModel):
224
+ model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
225
+
226
+
227
+ class _ProfileCount(_ProfileModel):
228
+ run: Annotated[str, AfterValidator(_descriptor)]
229
+ n_series: Count
230
+ n_screened: Count
231
+ n_alerted: Count
232
+ n_unscorable: Count
233
+
234
+ @model_validator(mode="after")
235
+ def consistency(self):
236
+ if (
237
+ self.n_series != self.n_screened + self.n_unscorable
238
+ or self.n_alerted > self.n_screened
239
+ ):
240
+ raise ValueError("inconsistent sensitivity counts")
241
+ return self
242
+
243
+
244
+ class _ProfileComparisons(_ProfileModel):
245
+ default_full: list[_ProfileCount]
246
+ default_eligible: list[_ProfileCount]
247
+ experimental_eligible: list[_ProfileCount]
248
+
249
+
250
+ class _ProfileEvidence(_ProfileModel):
251
+ replication_tolerance: Count
252
+ replication_coverage_policy: Literal["contiguous_prefix"]
253
+ corroboration_radius: Count
254
+ persistence_effect_fraction: Annotated[float, Field(gt=0, le=1)]
255
+
256
+
257
+ class _ProfileSettings(_ProfileModel):
258
+ vocabulary: _Vocabulary
259
+ sequential: _Sequential
260
+ context_sequential: _Sequential
261
+ retrospective: _Retrospective | None
262
+ evidence: _ProfileEvidence
263
+ sampling_units: Literal["exact_prompt", "explicit"]
264
+
265
+ @model_validator(mode="after")
266
+ def fixed(self):
267
+ for key, expected in (
268
+ ("vocabulary", _DEFAULT_VOCABULARY),
269
+ ("sequential", LENIENT_SEQUENTIAL),
270
+ ("context_sequential", _DEFAULT_SEQUENTIAL),
271
+ ("evidence", _DEFAULT_EVIDENCE),
272
+ ):
273
+ if getattr(self, key).model_dump() != expected:
274
+ raise ValueError("sensitivity settings differ from the named profile")
275
+ return self
276
+
277
+
278
+ class _ProfileProvenance(_ProfileModel):
279
+ decision_sha256: Literal[
280
+ "3dd87c8305ac7a5606f4cf9d35b7dd97c82c9c141362b459f6c372dc8fb6098a"
281
+ ]
282
+
283
+
284
+ class _ProfileEligibility(_ProfileModel):
285
+ tier: Literal["structural"]
286
+ kind: Literal["binary"]
287
+ selection: Literal["current retained catalog intersection"]
288
+
289
+
290
+ class _Sensitivity(_ProfileModel):
291
+ id: Literal["lenient-structural-binary-v1-reconstructed"]
292
+ selector: Literal["lenient"]
293
+ identity: Literal[
294
+ "new reconstruction on current semantics; not the unsaved original policy"
295
+ ]
296
+ experimental: Literal[True]
297
+ scope: Literal["offline_full_run"]
298
+ source_checkpoint: Literal["f233cb2bd0a6a3a9c1b5f80878ebc7c1e92381f2"]
299
+ provenance: _ProfileProvenance
300
+ eligibility: _ProfileEligibility
301
+ effective_settings: _ProfileSettings
302
+ counts: _ProfileComparisons
303
+
304
+ @field_validator("experimental", mode="before")
305
+ @classmethod
306
+ def true_boolean(cls, value):
307
+ if type(value) is not bool or value is not True:
308
+ raise ValueError("sensitivity must be marked experimental")
309
+ return value
310
+
311
+
312
+ class _RetrospectivePolicy(_ProfileModel):
313
+ status: Literal["ran", "disabled"]
314
+ experimental: bool
315
+ algorithm: str | None
316
+ effective_settings: _Retrospective | None
317
+ warning: str | None
318
+
319
+ @model_validator(mode="after")
320
+ def execution(self):
321
+ ran = self.status == "ran"
322
+ if (
323
+ self.experimental != ran
324
+ or (self.effective_settings is not None) != ran
325
+ or self.algorithm != (ALGORITHM if ran else None)
326
+ or self.warning != (WARNING if ran else None)
327
+ ):
328
+ raise ValueError("inconsistent retrospective policy")
329
+ return self
330
+
331
+
332
+ class _Boundary(_ProfileModel):
333
+ feature: Annotated[Identifier, AfterValidator(_descriptor)]
334
+ changepoint_step: Count
335
+
336
+
337
+ class _RetrospectiveCounts(_ProfileModel):
338
+ n_series: Count
339
+ n_screened: Count
340
+ n_changes: Count
341
+ n_unscorable: Count
342
+
343
+
344
+ class _RunRetrospective(_ProfileModel):
345
+ status: Literal["ran", "disabled"]
346
+ screening: _RetrospectiveCounts | None
347
+ changes: list[_Boundary] | None
348
+
349
+
350
+ class _Run(_Model):
351
+ run: Identifier
352
+ status: str
353
+ counts: _Counts
354
+ events: list[_Event]
355
+ retrospective: _RunRetrospective | None = None
356
+
357
+
358
+ class _Report(_Model):
359
+ schema_version: int
360
+ runs: list[_Run]
361
+ benchmark: BenchmarkReference | None = None
362
+ sensitivity: _Sensitivity | None = None
363
+ retrospective: _RetrospectivePolicy | None = None
364
+
365
+
366
+ def _check_sensitivity(report: _Report) -> None:
367
+ reference = report.benchmark
368
+ applicability = getattr(reference, "applicability", None)
369
+ if report.sensitivity is None:
370
+ if (
371
+ report.schema_version not in (1, 3)
372
+ or any(r.counts.excluded_series is not None for r in report.runs)
373
+ or (
374
+ applicability is not None
375
+ and "experimental_profile" in applicability.reasons
376
+ )
377
+ ):
378
+ raise ValueError("experimental report requires sensitivity metadata")
379
+ return
380
+ if report.schema_version not in (2, 3):
381
+ raise ValueError("experimental sensitivity requires report version 2 or 3")
382
+ settings = report.sensitivity.effective_settings.retrospective
383
+ if (report.schema_version == 2 and settings is None) or (
384
+ settings is not None and settings.model_dump() != _DEFAULT_RETROSPECTIVE
385
+ ):
386
+ raise ValueError(
387
+ "report retrospective settings differ from the lenient profile"
388
+ )
389
+ if (
390
+ report.schema_version == 3
391
+ and settings != report.retrospective.effective_settings
392
+ ):
393
+ raise ValueError(
394
+ "sensitivity retrospective settings conflict with execution policy"
395
+ )
396
+ if (
397
+ applicability is not None
398
+ and applicability.source == "bundled"
399
+ and (
400
+ applicability.status != "different"
401
+ or "experimental_profile" not in applicability.reasons
402
+ )
403
+ ):
404
+ raise ValueError("benchmark applicability conflicts with sensitivity profile")
405
+ comparisons = []
406
+ run_names = {r.run for r in report.runs}
407
+ for name in ("default_full", "default_eligible", "experimental_eligible"):
408
+ rows = getattr(report.sensitivity.counts, name)
409
+ if (
410
+ len({r.run for r in rows}) != len(rows)
411
+ or {r.run for r in rows} != run_names
412
+ ):
413
+ raise ValueError("sensitivity count coverage differs from report runs")
414
+ comparisons.append({r.run: r for r in rows})
415
+ for run in report.runs:
416
+ full, ordinary, selected = (c[run.run] for c in comparisons)
417
+ counts = run.counts
418
+ excluded = counts.excluded_series
419
+ if (
420
+ excluded is None
421
+ or counts.series
422
+ != counts.screened_series + counts.unscorable_series + excluded
423
+ ):
424
+ raise ValueError("sensitivity excluded counts are inconsistent")
425
+ if (
426
+ full.n_series != counts.series
427
+ or ordinary.n_series != selected.n_series
428
+ or ordinary.n_screened != selected.n_screened
429
+ or ordinary.n_unscorable != selected.n_unscorable
430
+ or ordinary.n_alerted > selected.n_alerted
431
+ or ordinary.n_series > full.n_series
432
+ or ordinary.n_screened > full.n_screened
433
+ or ordinary.n_unscorable > full.n_unscorable
434
+ ):
435
+ raise ValueError("sensitivity cohort counts are inconsistent")
436
+ context_series = full.n_series - selected.n_series - excluded
437
+ context_screened = counts.screened_series - selected.n_screened
438
+ context_alerts = counts.sequential_alerts - selected.n_alerted
439
+ excluded_screened = full.n_screened - counts.screened_series
440
+ excluded_unscorable = full.n_unscorable - counts.unscorable_series
441
+ excluded_alerts = full.n_alerted - ordinary.n_alerted - context_alerts
442
+ if (
443
+ not (0 <= excluded_alerts <= excluded_screened <= excluded)
444
+ or excluded_unscorable != excluded - excluded_screened
445
+ ):
446
+ raise ValueError("sensitivity excluded default counts are inconsistent")
447
+ if (
448
+ not (0 <= context_alerts <= context_screened <= context_series)
449
+ or counts.unscorable_series - selected.n_unscorable
450
+ != context_series - context_screened
451
+ or context_alerts > full.n_alerted - ordinary.n_alerted
452
+ ):
453
+ raise ValueError("sensitivity context counts are inconsistent")
454
+ members = [m for event in run.events for m in event.members]
455
+ if sum(m.kind == "binary" for m in members) != selected.n_alerted:
456
+ raise ValueError("sensitivity alerts differ from event membership")
457
+ if any(
458
+ m.kind == "binary"
459
+ and m.evidence is not None
460
+ and m.evidence.tier not in (None, "structural")
461
+ for m in members
462
+ ):
463
+ raise ValueError("excluded feature appears in sensitivity events")
464
+
465
+
466
+ def _check_member(member: _Member) -> None:
467
+ if member.onset_step > member.alert_step:
468
+ raise ValueError("report member onset exceeds its alert step")
469
+ _check_prompt_mix(member)
470
+ orders, selected = set(), set()
471
+ for example in member.exemplars:
472
+ if any(
473
+ getattr(example, field) != getattr(member, field)
474
+ for field in [
475
+ "run",
476
+ "feature",
477
+ "kind",
478
+ "direction",
479
+ "onset_step",
480
+ "alert_step",
481
+ ]
482
+ ):
483
+ raise ValueError("report exemplar does not match its member")
484
+ if example.selection_order in orders:
485
+ raise ValueError("report exemplar selection orders must be unique")
486
+ orders.add(example.selection_order)
487
+ if example.status == "selected":
488
+ if (
489
+ example.sample_id is None
490
+ or example.prompt_id is None
491
+ or example.step is None
492
+ or not example.selection_order
493
+ or not member.onset_step <= example.step <= member.alert_step
494
+ or example.sample_id in selected
495
+ ):
496
+ raise ValueError("report selected exemplar references are invalid")
497
+ selected.add(example.sample_id)
498
+
499
+
500
+ def _check_prompt_mix(member: _Member) -> None:
501
+ mix = member.evidence.prompt_mix
502
+ if mix is None:
503
+ return
504
+ windows = (mix.before_start, mix.before_end, mix.after_start, mix.after_end)
505
+ if mix.status in {
506
+ "available",
507
+ "shared_uncertainty_unavailable",
508
+ "no_shared_prompts",
509
+ } and any(value is None for value in windows):
510
+ raise ValueError(
511
+ "report prompt-mix windows are required for evaluated evidence"
512
+ )
513
+ if any(value is not None for value in windows) and (
514
+ any(value is None for value in windows)
515
+ or not mix.before_start <= mix.before_end < mix.after_start <= mix.after_end
516
+ or (mix.after_start, mix.after_end) != (member.onset_step, member.alert_step)
517
+ ):
518
+ raise ValueError("report prompt-mix windows are inconsistent")
519
+ if mix.status != member.evidence.prompt_breadth.promptmix_status:
520
+ raise ValueError("report prompt-mix statuses disagree")
521
+ if (
522
+ mix.shared_prompts is not None
523
+ and mix.shared_sampling_units is not None
524
+ and (mix.shared_sampling_units > mix.shared_prompts)
525
+ ):
526
+ raise ValueError("report shared-prompt counts are inconsistent")
527
+ if mix.status == "no_shared_prompts" and (
528
+ mix.shared_prompts != 0
529
+ or mix.shared_sampling_units != 0
530
+ or mix.shared_change is not None
531
+ or mix.shared_se is not None
532
+ ):
533
+ raise ValueError("report no-shared-prompt evidence is inconsistent")
534
+ if mix.status in {"available", "shared_uncertainty_unavailable"} and (
535
+ not mix.shared_prompts
536
+ or not mix.shared_sampling_units
537
+ or mix.shared_change is None
538
+ or (mix.status == "available") != (mix.shared_se is not None)
539
+ ):
540
+ raise ValueError("report shared-prompt evidence is inconsistent")
541
+ if mix.status == "available" and mix.shared_sampling_units < 2:
542
+ raise ValueError(
543
+ "report shared-prompt uncertainty requires at least two sampling units"
544
+ )
545
+ descriptors = [shift.descriptor for shift in mix.prompt_shift]
546
+ if len(descriptors) != len(set(descriptors)) or (
547
+ mix.prompt_shift_status == "recorded"
548
+ ) != bool(mix.prompt_shift):
549
+ raise ValueError("report prompt-shift records are inconsistent")
550
+ before_counts = {
551
+ shift.n_before + shift.missing_before for shift in mix.prompt_shift
552
+ }
553
+ after_counts = {shift.n_after + shift.missing_after for shift in mix.prompt_shift}
554
+ if (
555
+ len(before_counts) > 1
556
+ or len(after_counts) > 1
557
+ or (
558
+ after_counts
559
+ and member.evidence.prompt_breadth.total_prompts not in after_counts
560
+ )
561
+ or (
562
+ mix.shared_prompts is not None
563
+ and before_counts
564
+ and mix.shared_prompts > min(*before_counts, *after_counts)
565
+ )
566
+ ):
567
+ raise ValueError("report prompt-shift counts are inconsistent")
568
+
569
+
570
+ def _check_event(event: _Event, run: str, seen_members: set) -> None:
571
+ members = {(member.run, member.feature): member for member in event.members}
572
+ representative = members.get((run, event.representative_feature))
573
+ if (
574
+ event.run != run
575
+ or len(members) != len(event.members)
576
+ or len(members) != event.n_members
577
+ or seen_members.intersection(members)
578
+ or any(member.run != run for member in event.members)
579
+ or representative is None
580
+ or event.ranking.n_prioritizable_members > event.n_members
581
+ ):
582
+ raise ValueError("report event membership is inconsistent")
583
+ seen_members.update(members)
584
+ if (
585
+ event.onset_start != min(member.onset_step for member in event.members)
586
+ or event.onset_end != max(member.onset_step for member in event.members)
587
+ or event.onset_step != representative.onset_step
588
+ or event.alert_step != representative.alert_step
589
+ ):
590
+ raise ValueError("report event anchors do not match its members")
591
+ for references in [
592
+ event.ranking.missing_evidence_members,
593
+ event.ranking.numerical_floor_members,
594
+ event.ranking.numerical_floor_unavailable_members,
595
+ ]:
596
+ keys = {(reference.run, reference.feature) for reference in references}
597
+ if len(keys) != len(references) or not keys.issubset(members):
598
+ raise ValueError("report ranking references do not match its members")
599
+ for member in event.members:
600
+ _check_member(member)
601
+
602
+
603
+ def _check_run_retrospective(run: _Run, policy: _RetrospectivePolicy) -> None:
604
+ summary = run.retrospective
605
+ if summary is None or summary.status != policy.status:
606
+ raise ValueError("report run conflicts with retrospective execution policy")
607
+ changed = {}
608
+ if policy.status == "disabled":
609
+ if summary.screening is not None or summary.changes is not None:
610
+ raise ValueError("disabled retrospective report contains measurements")
611
+ else:
612
+ counts = summary.screening
613
+ if counts is None or summary.changes is None:
614
+ raise ValueError("retrospective run is missing measurements")
615
+ changed = {row.feature: row.changepoint_step for row in summary.changes}
616
+ if (
617
+ counts.n_series != run.counts.series
618
+ or counts.n_series != counts.n_screened + counts.n_unscorable
619
+ or not counts.n_changes == len(summary.changes) == len(changed)
620
+ or counts.n_changes > counts.n_screened
621
+ ):
622
+ raise ValueError(
623
+ "inconsistent retrospective screening counts or boundaries"
624
+ )
625
+ unscorable = no_change = 0
626
+ for event in run.events:
627
+ for member in event.members:
628
+ detail = member.evidence.retrospective
629
+ if detail is None or (detail.status == "disabled") != (
630
+ policy.status == "disabled"
631
+ ):
632
+ raise ValueError(
633
+ "report member conflicts with retrospective execution policy"
634
+ )
635
+ if detail.status == "missing":
636
+ if (
637
+ member.evidence.status != "missing"
638
+ or detail.changepoint_step is not None
639
+ ):
640
+ raise ValueError("inconsistent missing retrospective evidence")
641
+ continue
642
+ if (detail.status == "change") != (
643
+ member.feature in changed
644
+ ) or detail.changepoint_step != changed.get(member.feature):
645
+ raise ValueError("report member conflicts with retrospective boundary")
646
+ unscorable += detail.status in {
647
+ "insufficient_data",
648
+ "unavailable_uncertainty",
649
+ }
650
+ no_change += detail.status == "no_change"
651
+ if summary.screening is not None and (
652
+ unscorable > summary.screening.n_unscorable
653
+ or no_change > summary.screening.n_screened - summary.screening.n_changes
654
+ ):
655
+ raise ValueError("report member statuses exceed retrospective screening counts")
656
+
657
+
658
+ def _check_retrospective(report: _Report) -> None:
659
+ policy = report.retrospective
660
+ if report.schema_version < 3:
661
+ if policy is not None or any(
662
+ run.retrospective is not None
663
+ or any(
664
+ member.evidence.retrospective is not None
665
+ for event in run.events
666
+ for member in event.members
667
+ )
668
+ for run in report.runs
669
+ ):
670
+ raise ValueError("retrospective execution policy requires report version 3")
671
+ return
672
+ if policy is None:
673
+ raise ValueError("report version 3 requires retrospective execution policy")
674
+ applicability = getattr(report.benchmark, "applicability", None)
675
+ if (
676
+ applicability is not None
677
+ and applicability.source == "bundled"
678
+ and (
679
+ ("retrospective_disabled" in applicability.reasons)
680
+ != (policy.status == "disabled")
681
+ )
682
+ ):
683
+ raise ValueError(
684
+ "report benchmark applicability conflicts with retrospective execution policy"
685
+ )
686
+ for run in report.runs:
687
+ _check_run_retrospective(run, policy)
688
+
689
+
690
+ def validate_report(value: object) -> dict:
691
+ """Validate report fields and links; omit unknown fields from the copy."""
692
+ if not isinstance(value, dict):
693
+ raise ValueError("report must be a JSON object")
694
+ version = value.get("schema_version")
695
+ if type(version) is not int or version not in (1, 2, 3):
696
+ raise ValueError(
697
+ "unsupported report version; expected schema_version 1, 2, or 3"
698
+ )
699
+ if version == 2 and value.get("sensitivity") is None:
700
+ raise ValueError("report version 2 requires sensitivity metadata")
701
+ try:
702
+ report = _Report.model_validate(value)
703
+ except (ValidationError, RecursionError):
704
+ raise ValueError("invalid report fields or types") from None
705
+ _check_retrospective(report)
706
+ _check_sensitivity(report)
707
+ seen_runs, seen_events, seen_members = set(), set(), set()
708
+ for run in report.runs:
709
+ if run.run in seen_runs:
710
+ raise ValueError("report run identifiers must be unique")
711
+ seen_runs.add(run.run)
712
+ counts = run.counts
713
+ behavioral = sum(event.ranking_status == "behavioral" for event in run.events)
714
+ contextual = sum(event.ranking_status == "contextual" for event in run.events)
715
+ if (
716
+ counts.events != len(run.events)
717
+ or counts.sequential_alerts != sum(event.n_members for event in run.events)
718
+ or counts.behavioral_events != behavioral
719
+ or counts.contextual_events != contextual
720
+ or counts.other_events != len(run.events) - behavioral - contextual
721
+ or sorted(event.rank for event in run.events)
722
+ != list(range(1, len(run.events) + 1))
723
+ ):
724
+ raise ValueError("report counts or event ranks are inconsistent")
725
+ for event in run.events:
726
+ if event.event_id in seen_events:
727
+ raise ValueError("report event identifiers must be unique")
728
+ seen_events.add(event.event_id)
729
+ _check_event(event, run.run, seen_members)
730
+ result = report.model_dump(exclude_unset=True)
731
+ if report.benchmark is not None:
732
+ result["benchmark"] = validate_benchmark_reference(result["benchmark"])
733
+ return result
734
+
735
+
736
+ def _object(pairs: list[tuple]) -> dict:
737
+ result = {}
738
+ for key, value in pairs:
739
+ if key in result:
740
+ raise ValueError("duplicate report JSON key")
741
+ result[key] = value
742
+ return result
743
+
744
+
745
+ def _constant(value: str) -> None:
746
+ raise ValueError("nonfinite report JSON number")
747
+
748
+
749
+ def load_report(path: str | Path) -> dict:
750
+ """Read strict UTF-8 JSON with safe errors for malformed input."""
751
+ try:
752
+ with Path(path).open(encoding="utf-8") as stream:
753
+ value = json.load(
754
+ stream, object_pairs_hook=_object, parse_constant=_constant
755
+ )
756
+ except (ValueError, RecursionError):
757
+ raise ValueError("invalid report JSON") from None
758
+ return validate_report(value)