rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,758 @@
|
|
|
1
|
+
"""Read analysis reports without recomputing their evidence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Annotated, Literal
|
|
8
|
+
|
|
9
|
+
from pydantic import (
|
|
10
|
+
AfterValidator,
|
|
11
|
+
BaseModel,
|
|
12
|
+
ConfigDict,
|
|
13
|
+
Field,
|
|
14
|
+
ValidationError,
|
|
15
|
+
field_validator,
|
|
16
|
+
model_validator,
|
|
17
|
+
)
|
|
18
|
+
|
|
19
|
+
from ._retrospective_policy import ALGORITHM, WARNING
|
|
20
|
+
from .benchmark_reference import (
|
|
21
|
+
BenchmarkReference,
|
|
22
|
+
_Retrospective,
|
|
23
|
+
_Sequential,
|
|
24
|
+
_Vocabulary,
|
|
25
|
+
validate_benchmark_reference,
|
|
26
|
+
)
|
|
27
|
+
from .sensitivity import (
|
|
28
|
+
_DEFAULT_EVIDENCE,
|
|
29
|
+
_DEFAULT_RETROSPECTIVE,
|
|
30
|
+
_DEFAULT_SEQUENTIAL,
|
|
31
|
+
_DEFAULT_VOCABULARY,
|
|
32
|
+
LENIENT_SEQUENTIAL,
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def _nonempty(value: str) -> str:
|
|
37
|
+
if not value.strip():
|
|
38
|
+
raise ValueError("identifier must be nonempty")
|
|
39
|
+
return value
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _descriptor(value: str) -> str:
|
|
43
|
+
from .report import _identifier
|
|
44
|
+
|
|
45
|
+
return _identifier(value)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
Identifier = Annotated[str, AfterValidator(_nonempty)]
|
|
49
|
+
Count = Annotated[int, Field(ge=0)]
|
|
50
|
+
Number = int | float
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class _Model(BaseModel):
|
|
54
|
+
model_config = ConfigDict(strict=True, extra="ignore", allow_inf_nan=False)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class _Reference(_Model):
|
|
58
|
+
run: Identifier
|
|
59
|
+
feature: Identifier
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class _Replication(_Model):
|
|
63
|
+
status: str | None
|
|
64
|
+
coverage_policy: str | None
|
|
65
|
+
horizon_complete: bool | None
|
|
66
|
+
cutoff_step: Count | None
|
|
67
|
+
replicates: bool | None
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class _Persistence(_Model):
|
|
71
|
+
status: str | None
|
|
72
|
+
fraction: Number | None
|
|
73
|
+
coverage: Number | None
|
|
74
|
+
n_observations: Count | None
|
|
75
|
+
n_evaluable: Count | None
|
|
76
|
+
n_retained: Count | None
|
|
77
|
+
n_unavailable: Count | None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class _Reward(_Model):
|
|
81
|
+
status: str | None
|
|
82
|
+
mean_difference: Number | None
|
|
83
|
+
n_groups: Count | None
|
|
84
|
+
n_mixed_groups: Count | None
|
|
85
|
+
n_rollouts: Count | None
|
|
86
|
+
n_included_rollouts: Count | None
|
|
87
|
+
window_start: Count | None
|
|
88
|
+
window_end: Count | None
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class _Breadth(_Model):
|
|
92
|
+
positive_prompts: Count | None
|
|
93
|
+
total_prompts: Count | None
|
|
94
|
+
promptmix_status: str | None
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class _PromptShift(_Model):
|
|
98
|
+
descriptor: Annotated[Identifier, AfterValidator(_descriptor)]
|
|
99
|
+
statistic: Literal["ks", "absolute_prevalence", "total_variation"]
|
|
100
|
+
before: Number | None
|
|
101
|
+
after: Number | None
|
|
102
|
+
delta: Number | None
|
|
103
|
+
distance: Annotated[Number, Field(ge=0, le=1)] | None
|
|
104
|
+
n_before: Count
|
|
105
|
+
n_after: Count
|
|
106
|
+
missing_before: Count
|
|
107
|
+
missing_after: Count
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
class _PromptMix(_Model):
|
|
111
|
+
status: str | None
|
|
112
|
+
before_start: Count | None
|
|
113
|
+
before_end: Count | None
|
|
114
|
+
after_start: Count | None
|
|
115
|
+
after_end: Count | None
|
|
116
|
+
shared_prompts: Count | None
|
|
117
|
+
shared_sampling_units: Count | None
|
|
118
|
+
shared_change: Number | None
|
|
119
|
+
shared_se: Annotated[Number, Field(ge=0)] | None
|
|
120
|
+
prompt_shift_status: Literal["recorded", "missing"]
|
|
121
|
+
prompt_shift: list[_PromptShift]
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _MemberRetrospective(_Model):
|
|
125
|
+
status: Literal[
|
|
126
|
+
"disabled",
|
|
127
|
+
"change",
|
|
128
|
+
"no_change",
|
|
129
|
+
"insufficient_data",
|
|
130
|
+
"unavailable_uncertainty",
|
|
131
|
+
"missing",
|
|
132
|
+
]
|
|
133
|
+
changepoint_step: Count | None
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
class _Evidence(_Model):
|
|
137
|
+
status: str
|
|
138
|
+
tier: str | None
|
|
139
|
+
scale_source: str | None
|
|
140
|
+
numerical_floor_dominant: bool | None
|
|
141
|
+
replication: _Replication
|
|
142
|
+
persistence: _Persistence
|
|
143
|
+
reward_association: _Reward
|
|
144
|
+
prompt_breadth: _Breadth
|
|
145
|
+
prompt_mix: _PromptMix | None = None
|
|
146
|
+
retrospective: _MemberRetrospective | None = None
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
class _Exemplar(_Reference):
|
|
150
|
+
kind: str
|
|
151
|
+
direction: str
|
|
152
|
+
onset_step: Count
|
|
153
|
+
alert_step: Count
|
|
154
|
+
status: str
|
|
155
|
+
n_window_rollouts: Count | None
|
|
156
|
+
n_candidates: Count | None
|
|
157
|
+
reward_selection_status: str | None
|
|
158
|
+
selection_order: Count | None
|
|
159
|
+
sample_id: Identifier | None
|
|
160
|
+
step: Count | None
|
|
161
|
+
prompt_id: Identifier | None
|
|
162
|
+
feature_value: Number | None
|
|
163
|
+
reward: Number | None
|
|
164
|
+
reward_group_size: Count | None
|
|
165
|
+
reward_group_mean: Number | None
|
|
166
|
+
reward_deviation: Number | None
|
|
167
|
+
reward_deviation_status: str | None
|
|
168
|
+
selection_reason: str | None
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
class _Member(_Reference):
|
|
172
|
+
kind: str
|
|
173
|
+
direction: str
|
|
174
|
+
onset_step: Count
|
|
175
|
+
alert_step: Count
|
|
176
|
+
status: str
|
|
177
|
+
evidence: _Evidence
|
|
178
|
+
exemplar_status: str
|
|
179
|
+
exemplars: list[_Exemplar]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
class _Ranking(_Model):
|
|
183
|
+
n_prioritizable_members: Count
|
|
184
|
+
replication_availability: str
|
|
185
|
+
persistence_availability: str
|
|
186
|
+
reward_availability: str
|
|
187
|
+
breadth_availability: str
|
|
188
|
+
missing_evidence_members: list[_Reference]
|
|
189
|
+
numerical_floor_members: list[_Reference]
|
|
190
|
+
numerical_floor_unavailable_members: list[_Reference]
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
class _Event(_Model):
|
|
194
|
+
event_id: Identifier
|
|
195
|
+
run: Identifier
|
|
196
|
+
rank: Annotated[int, Field(ge=1)]
|
|
197
|
+
representative_feature: Identifier
|
|
198
|
+
ranking_status: str
|
|
199
|
+
onset_start: Count
|
|
200
|
+
onset_end: Count
|
|
201
|
+
onset_step: Count
|
|
202
|
+
alert_step: Count
|
|
203
|
+
n_members: Annotated[int, Field(ge=1)]
|
|
204
|
+
ranking: _Ranking
|
|
205
|
+
members: list[_Member]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
class _Counts(_Model):
|
|
209
|
+
series: Count
|
|
210
|
+
screened_series: Count
|
|
211
|
+
sequential_alerts: Count
|
|
212
|
+
unscorable_series: Count
|
|
213
|
+
rollouts: Count
|
|
214
|
+
observed_steps: Count
|
|
215
|
+
prompts: Count
|
|
216
|
+
events: Count
|
|
217
|
+
behavioral_events: Count
|
|
218
|
+
contextual_events: Count
|
|
219
|
+
other_events: Count
|
|
220
|
+
excluded_series: Count | None = None
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
class _ProfileModel(BaseModel):
|
|
224
|
+
model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class _ProfileCount(_ProfileModel):
|
|
228
|
+
run: Annotated[str, AfterValidator(_descriptor)]
|
|
229
|
+
n_series: Count
|
|
230
|
+
n_screened: Count
|
|
231
|
+
n_alerted: Count
|
|
232
|
+
n_unscorable: Count
|
|
233
|
+
|
|
234
|
+
@model_validator(mode="after")
|
|
235
|
+
def consistency(self):
|
|
236
|
+
if (
|
|
237
|
+
self.n_series != self.n_screened + self.n_unscorable
|
|
238
|
+
or self.n_alerted > self.n_screened
|
|
239
|
+
):
|
|
240
|
+
raise ValueError("inconsistent sensitivity counts")
|
|
241
|
+
return self
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class _ProfileComparisons(_ProfileModel):
|
|
245
|
+
default_full: list[_ProfileCount]
|
|
246
|
+
default_eligible: list[_ProfileCount]
|
|
247
|
+
experimental_eligible: list[_ProfileCount]
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
class _ProfileEvidence(_ProfileModel):
|
|
251
|
+
replication_tolerance: Count
|
|
252
|
+
replication_coverage_policy: Literal["contiguous_prefix"]
|
|
253
|
+
corroboration_radius: Count
|
|
254
|
+
persistence_effect_fraction: Annotated[float, Field(gt=0, le=1)]
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
class _ProfileSettings(_ProfileModel):
|
|
258
|
+
vocabulary: _Vocabulary
|
|
259
|
+
sequential: _Sequential
|
|
260
|
+
context_sequential: _Sequential
|
|
261
|
+
retrospective: _Retrospective | None
|
|
262
|
+
evidence: _ProfileEvidence
|
|
263
|
+
sampling_units: Literal["exact_prompt", "explicit"]
|
|
264
|
+
|
|
265
|
+
@model_validator(mode="after")
|
|
266
|
+
def fixed(self):
|
|
267
|
+
for key, expected in (
|
|
268
|
+
("vocabulary", _DEFAULT_VOCABULARY),
|
|
269
|
+
("sequential", LENIENT_SEQUENTIAL),
|
|
270
|
+
("context_sequential", _DEFAULT_SEQUENTIAL),
|
|
271
|
+
("evidence", _DEFAULT_EVIDENCE),
|
|
272
|
+
):
|
|
273
|
+
if getattr(self, key).model_dump() != expected:
|
|
274
|
+
raise ValueError("sensitivity settings differ from the named profile")
|
|
275
|
+
return self
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
class _ProfileProvenance(_ProfileModel):
|
|
279
|
+
decision_sha256: Literal[
|
|
280
|
+
"3dd87c8305ac7a5606f4cf9d35b7dd97c82c9c141362b459f6c372dc8fb6098a"
|
|
281
|
+
]
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
class _ProfileEligibility(_ProfileModel):
|
|
285
|
+
tier: Literal["structural"]
|
|
286
|
+
kind: Literal["binary"]
|
|
287
|
+
selection: Literal["current retained catalog intersection"]
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
class _Sensitivity(_ProfileModel):
|
|
291
|
+
id: Literal["lenient-structural-binary-v1-reconstructed"]
|
|
292
|
+
selector: Literal["lenient"]
|
|
293
|
+
identity: Literal[
|
|
294
|
+
"new reconstruction on current semantics; not the unsaved original policy"
|
|
295
|
+
]
|
|
296
|
+
experimental: Literal[True]
|
|
297
|
+
scope: Literal["offline_full_run"]
|
|
298
|
+
source_checkpoint: Literal["f233cb2bd0a6a3a9c1b5f80878ebc7c1e92381f2"]
|
|
299
|
+
provenance: _ProfileProvenance
|
|
300
|
+
eligibility: _ProfileEligibility
|
|
301
|
+
effective_settings: _ProfileSettings
|
|
302
|
+
counts: _ProfileComparisons
|
|
303
|
+
|
|
304
|
+
@field_validator("experimental", mode="before")
|
|
305
|
+
@classmethod
|
|
306
|
+
def true_boolean(cls, value):
|
|
307
|
+
if type(value) is not bool or value is not True:
|
|
308
|
+
raise ValueError("sensitivity must be marked experimental")
|
|
309
|
+
return value
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
class _RetrospectivePolicy(_ProfileModel):
|
|
313
|
+
status: Literal["ran", "disabled"]
|
|
314
|
+
experimental: bool
|
|
315
|
+
algorithm: str | None
|
|
316
|
+
effective_settings: _Retrospective | None
|
|
317
|
+
warning: str | None
|
|
318
|
+
|
|
319
|
+
@model_validator(mode="after")
|
|
320
|
+
def execution(self):
|
|
321
|
+
ran = self.status == "ran"
|
|
322
|
+
if (
|
|
323
|
+
self.experimental != ran
|
|
324
|
+
or (self.effective_settings is not None) != ran
|
|
325
|
+
or self.algorithm != (ALGORITHM if ran else None)
|
|
326
|
+
or self.warning != (WARNING if ran else None)
|
|
327
|
+
):
|
|
328
|
+
raise ValueError("inconsistent retrospective policy")
|
|
329
|
+
return self
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
class _Boundary(_ProfileModel):
|
|
333
|
+
feature: Annotated[Identifier, AfterValidator(_descriptor)]
|
|
334
|
+
changepoint_step: Count
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class _RetrospectiveCounts(_ProfileModel):
|
|
338
|
+
n_series: Count
|
|
339
|
+
n_screened: Count
|
|
340
|
+
n_changes: Count
|
|
341
|
+
n_unscorable: Count
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
class _RunRetrospective(_ProfileModel):
|
|
345
|
+
status: Literal["ran", "disabled"]
|
|
346
|
+
screening: _RetrospectiveCounts | None
|
|
347
|
+
changes: list[_Boundary] | None
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class _Run(_Model):
|
|
351
|
+
run: Identifier
|
|
352
|
+
status: str
|
|
353
|
+
counts: _Counts
|
|
354
|
+
events: list[_Event]
|
|
355
|
+
retrospective: _RunRetrospective | None = None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
class _Report(_Model):
|
|
359
|
+
schema_version: int
|
|
360
|
+
runs: list[_Run]
|
|
361
|
+
benchmark: BenchmarkReference | None = None
|
|
362
|
+
sensitivity: _Sensitivity | None = None
|
|
363
|
+
retrospective: _RetrospectivePolicy | None = None
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _check_sensitivity(report: _Report) -> None:
|
|
367
|
+
reference = report.benchmark
|
|
368
|
+
applicability = getattr(reference, "applicability", None)
|
|
369
|
+
if report.sensitivity is None:
|
|
370
|
+
if (
|
|
371
|
+
report.schema_version not in (1, 3)
|
|
372
|
+
or any(r.counts.excluded_series is not None for r in report.runs)
|
|
373
|
+
or (
|
|
374
|
+
applicability is not None
|
|
375
|
+
and "experimental_profile" in applicability.reasons
|
|
376
|
+
)
|
|
377
|
+
):
|
|
378
|
+
raise ValueError("experimental report requires sensitivity metadata")
|
|
379
|
+
return
|
|
380
|
+
if report.schema_version not in (2, 3):
|
|
381
|
+
raise ValueError("experimental sensitivity requires report version 2 or 3")
|
|
382
|
+
settings = report.sensitivity.effective_settings.retrospective
|
|
383
|
+
if (report.schema_version == 2 and settings is None) or (
|
|
384
|
+
settings is not None and settings.model_dump() != _DEFAULT_RETROSPECTIVE
|
|
385
|
+
):
|
|
386
|
+
raise ValueError(
|
|
387
|
+
"report retrospective settings differ from the lenient profile"
|
|
388
|
+
)
|
|
389
|
+
if (
|
|
390
|
+
report.schema_version == 3
|
|
391
|
+
and settings != report.retrospective.effective_settings
|
|
392
|
+
):
|
|
393
|
+
raise ValueError(
|
|
394
|
+
"sensitivity retrospective settings conflict with execution policy"
|
|
395
|
+
)
|
|
396
|
+
if (
|
|
397
|
+
applicability is not None
|
|
398
|
+
and applicability.source == "bundled"
|
|
399
|
+
and (
|
|
400
|
+
applicability.status != "different"
|
|
401
|
+
or "experimental_profile" not in applicability.reasons
|
|
402
|
+
)
|
|
403
|
+
):
|
|
404
|
+
raise ValueError("benchmark applicability conflicts with sensitivity profile")
|
|
405
|
+
comparisons = []
|
|
406
|
+
run_names = {r.run for r in report.runs}
|
|
407
|
+
for name in ("default_full", "default_eligible", "experimental_eligible"):
|
|
408
|
+
rows = getattr(report.sensitivity.counts, name)
|
|
409
|
+
if (
|
|
410
|
+
len({r.run for r in rows}) != len(rows)
|
|
411
|
+
or {r.run for r in rows} != run_names
|
|
412
|
+
):
|
|
413
|
+
raise ValueError("sensitivity count coverage differs from report runs")
|
|
414
|
+
comparisons.append({r.run: r for r in rows})
|
|
415
|
+
for run in report.runs:
|
|
416
|
+
full, ordinary, selected = (c[run.run] for c in comparisons)
|
|
417
|
+
counts = run.counts
|
|
418
|
+
excluded = counts.excluded_series
|
|
419
|
+
if (
|
|
420
|
+
excluded is None
|
|
421
|
+
or counts.series
|
|
422
|
+
!= counts.screened_series + counts.unscorable_series + excluded
|
|
423
|
+
):
|
|
424
|
+
raise ValueError("sensitivity excluded counts are inconsistent")
|
|
425
|
+
if (
|
|
426
|
+
full.n_series != counts.series
|
|
427
|
+
or ordinary.n_series != selected.n_series
|
|
428
|
+
or ordinary.n_screened != selected.n_screened
|
|
429
|
+
or ordinary.n_unscorable != selected.n_unscorable
|
|
430
|
+
or ordinary.n_alerted > selected.n_alerted
|
|
431
|
+
or ordinary.n_series > full.n_series
|
|
432
|
+
or ordinary.n_screened > full.n_screened
|
|
433
|
+
or ordinary.n_unscorable > full.n_unscorable
|
|
434
|
+
):
|
|
435
|
+
raise ValueError("sensitivity cohort counts are inconsistent")
|
|
436
|
+
context_series = full.n_series - selected.n_series - excluded
|
|
437
|
+
context_screened = counts.screened_series - selected.n_screened
|
|
438
|
+
context_alerts = counts.sequential_alerts - selected.n_alerted
|
|
439
|
+
excluded_screened = full.n_screened - counts.screened_series
|
|
440
|
+
excluded_unscorable = full.n_unscorable - counts.unscorable_series
|
|
441
|
+
excluded_alerts = full.n_alerted - ordinary.n_alerted - context_alerts
|
|
442
|
+
if (
|
|
443
|
+
not (0 <= excluded_alerts <= excluded_screened <= excluded)
|
|
444
|
+
or excluded_unscorable != excluded - excluded_screened
|
|
445
|
+
):
|
|
446
|
+
raise ValueError("sensitivity excluded default counts are inconsistent")
|
|
447
|
+
if (
|
|
448
|
+
not (0 <= context_alerts <= context_screened <= context_series)
|
|
449
|
+
or counts.unscorable_series - selected.n_unscorable
|
|
450
|
+
!= context_series - context_screened
|
|
451
|
+
or context_alerts > full.n_alerted - ordinary.n_alerted
|
|
452
|
+
):
|
|
453
|
+
raise ValueError("sensitivity context counts are inconsistent")
|
|
454
|
+
members = [m for event in run.events for m in event.members]
|
|
455
|
+
if sum(m.kind == "binary" for m in members) != selected.n_alerted:
|
|
456
|
+
raise ValueError("sensitivity alerts differ from event membership")
|
|
457
|
+
if any(
|
|
458
|
+
m.kind == "binary"
|
|
459
|
+
and m.evidence is not None
|
|
460
|
+
and m.evidence.tier not in (None, "structural")
|
|
461
|
+
for m in members
|
|
462
|
+
):
|
|
463
|
+
raise ValueError("excluded feature appears in sensitivity events")
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _check_member(member: _Member) -> None:
|
|
467
|
+
if member.onset_step > member.alert_step:
|
|
468
|
+
raise ValueError("report member onset exceeds its alert step")
|
|
469
|
+
_check_prompt_mix(member)
|
|
470
|
+
orders, selected = set(), set()
|
|
471
|
+
for example in member.exemplars:
|
|
472
|
+
if any(
|
|
473
|
+
getattr(example, field) != getattr(member, field)
|
|
474
|
+
for field in [
|
|
475
|
+
"run",
|
|
476
|
+
"feature",
|
|
477
|
+
"kind",
|
|
478
|
+
"direction",
|
|
479
|
+
"onset_step",
|
|
480
|
+
"alert_step",
|
|
481
|
+
]
|
|
482
|
+
):
|
|
483
|
+
raise ValueError("report exemplar does not match its member")
|
|
484
|
+
if example.selection_order in orders:
|
|
485
|
+
raise ValueError("report exemplar selection orders must be unique")
|
|
486
|
+
orders.add(example.selection_order)
|
|
487
|
+
if example.status == "selected":
|
|
488
|
+
if (
|
|
489
|
+
example.sample_id is None
|
|
490
|
+
or example.prompt_id is None
|
|
491
|
+
or example.step is None
|
|
492
|
+
or not example.selection_order
|
|
493
|
+
or not member.onset_step <= example.step <= member.alert_step
|
|
494
|
+
or example.sample_id in selected
|
|
495
|
+
):
|
|
496
|
+
raise ValueError("report selected exemplar references are invalid")
|
|
497
|
+
selected.add(example.sample_id)
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
def _check_prompt_mix(member: _Member) -> None:
|
|
501
|
+
mix = member.evidence.prompt_mix
|
|
502
|
+
if mix is None:
|
|
503
|
+
return
|
|
504
|
+
windows = (mix.before_start, mix.before_end, mix.after_start, mix.after_end)
|
|
505
|
+
if mix.status in {
|
|
506
|
+
"available",
|
|
507
|
+
"shared_uncertainty_unavailable",
|
|
508
|
+
"no_shared_prompts",
|
|
509
|
+
} and any(value is None for value in windows):
|
|
510
|
+
raise ValueError(
|
|
511
|
+
"report prompt-mix windows are required for evaluated evidence"
|
|
512
|
+
)
|
|
513
|
+
if any(value is not None for value in windows) and (
|
|
514
|
+
any(value is None for value in windows)
|
|
515
|
+
or not mix.before_start <= mix.before_end < mix.after_start <= mix.after_end
|
|
516
|
+
or (mix.after_start, mix.after_end) != (member.onset_step, member.alert_step)
|
|
517
|
+
):
|
|
518
|
+
raise ValueError("report prompt-mix windows are inconsistent")
|
|
519
|
+
if mix.status != member.evidence.prompt_breadth.promptmix_status:
|
|
520
|
+
raise ValueError("report prompt-mix statuses disagree")
|
|
521
|
+
if (
|
|
522
|
+
mix.shared_prompts is not None
|
|
523
|
+
and mix.shared_sampling_units is not None
|
|
524
|
+
and (mix.shared_sampling_units > mix.shared_prompts)
|
|
525
|
+
):
|
|
526
|
+
raise ValueError("report shared-prompt counts are inconsistent")
|
|
527
|
+
if mix.status == "no_shared_prompts" and (
|
|
528
|
+
mix.shared_prompts != 0
|
|
529
|
+
or mix.shared_sampling_units != 0
|
|
530
|
+
or mix.shared_change is not None
|
|
531
|
+
or mix.shared_se is not None
|
|
532
|
+
):
|
|
533
|
+
raise ValueError("report no-shared-prompt evidence is inconsistent")
|
|
534
|
+
if mix.status in {"available", "shared_uncertainty_unavailable"} and (
|
|
535
|
+
not mix.shared_prompts
|
|
536
|
+
or not mix.shared_sampling_units
|
|
537
|
+
or mix.shared_change is None
|
|
538
|
+
or (mix.status == "available") != (mix.shared_se is not None)
|
|
539
|
+
):
|
|
540
|
+
raise ValueError("report shared-prompt evidence is inconsistent")
|
|
541
|
+
if mix.status == "available" and mix.shared_sampling_units < 2:
|
|
542
|
+
raise ValueError(
|
|
543
|
+
"report shared-prompt uncertainty requires at least two sampling units"
|
|
544
|
+
)
|
|
545
|
+
descriptors = [shift.descriptor for shift in mix.prompt_shift]
|
|
546
|
+
if len(descriptors) != len(set(descriptors)) or (
|
|
547
|
+
mix.prompt_shift_status == "recorded"
|
|
548
|
+
) != bool(mix.prompt_shift):
|
|
549
|
+
raise ValueError("report prompt-shift records are inconsistent")
|
|
550
|
+
before_counts = {
|
|
551
|
+
shift.n_before + shift.missing_before for shift in mix.prompt_shift
|
|
552
|
+
}
|
|
553
|
+
after_counts = {shift.n_after + shift.missing_after for shift in mix.prompt_shift}
|
|
554
|
+
if (
|
|
555
|
+
len(before_counts) > 1
|
|
556
|
+
or len(after_counts) > 1
|
|
557
|
+
or (
|
|
558
|
+
after_counts
|
|
559
|
+
and member.evidence.prompt_breadth.total_prompts not in after_counts
|
|
560
|
+
)
|
|
561
|
+
or (
|
|
562
|
+
mix.shared_prompts is not None
|
|
563
|
+
and before_counts
|
|
564
|
+
and mix.shared_prompts > min(*before_counts, *after_counts)
|
|
565
|
+
)
|
|
566
|
+
):
|
|
567
|
+
raise ValueError("report prompt-shift counts are inconsistent")
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def _check_event(event: _Event, run: str, seen_members: set) -> None:
|
|
571
|
+
members = {(member.run, member.feature): member for member in event.members}
|
|
572
|
+
representative = members.get((run, event.representative_feature))
|
|
573
|
+
if (
|
|
574
|
+
event.run != run
|
|
575
|
+
or len(members) != len(event.members)
|
|
576
|
+
or len(members) != event.n_members
|
|
577
|
+
or seen_members.intersection(members)
|
|
578
|
+
or any(member.run != run for member in event.members)
|
|
579
|
+
or representative is None
|
|
580
|
+
or event.ranking.n_prioritizable_members > event.n_members
|
|
581
|
+
):
|
|
582
|
+
raise ValueError("report event membership is inconsistent")
|
|
583
|
+
seen_members.update(members)
|
|
584
|
+
if (
|
|
585
|
+
event.onset_start != min(member.onset_step for member in event.members)
|
|
586
|
+
or event.onset_end != max(member.onset_step for member in event.members)
|
|
587
|
+
or event.onset_step != representative.onset_step
|
|
588
|
+
or event.alert_step != representative.alert_step
|
|
589
|
+
):
|
|
590
|
+
raise ValueError("report event anchors do not match its members")
|
|
591
|
+
for references in [
|
|
592
|
+
event.ranking.missing_evidence_members,
|
|
593
|
+
event.ranking.numerical_floor_members,
|
|
594
|
+
event.ranking.numerical_floor_unavailable_members,
|
|
595
|
+
]:
|
|
596
|
+
keys = {(reference.run, reference.feature) for reference in references}
|
|
597
|
+
if len(keys) != len(references) or not keys.issubset(members):
|
|
598
|
+
raise ValueError("report ranking references do not match its members")
|
|
599
|
+
for member in event.members:
|
|
600
|
+
_check_member(member)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def _check_run_retrospective(run: _Run, policy: _RetrospectivePolicy) -> None:
|
|
604
|
+
summary = run.retrospective
|
|
605
|
+
if summary is None or summary.status != policy.status:
|
|
606
|
+
raise ValueError("report run conflicts with retrospective execution policy")
|
|
607
|
+
changed = {}
|
|
608
|
+
if policy.status == "disabled":
|
|
609
|
+
if summary.screening is not None or summary.changes is not None:
|
|
610
|
+
raise ValueError("disabled retrospective report contains measurements")
|
|
611
|
+
else:
|
|
612
|
+
counts = summary.screening
|
|
613
|
+
if counts is None or summary.changes is None:
|
|
614
|
+
raise ValueError("retrospective run is missing measurements")
|
|
615
|
+
changed = {row.feature: row.changepoint_step for row in summary.changes}
|
|
616
|
+
if (
|
|
617
|
+
counts.n_series != run.counts.series
|
|
618
|
+
or counts.n_series != counts.n_screened + counts.n_unscorable
|
|
619
|
+
or not counts.n_changes == len(summary.changes) == len(changed)
|
|
620
|
+
or counts.n_changes > counts.n_screened
|
|
621
|
+
):
|
|
622
|
+
raise ValueError(
|
|
623
|
+
"inconsistent retrospective screening counts or boundaries"
|
|
624
|
+
)
|
|
625
|
+
unscorable = no_change = 0
|
|
626
|
+
for event in run.events:
|
|
627
|
+
for member in event.members:
|
|
628
|
+
detail = member.evidence.retrospective
|
|
629
|
+
if detail is None or (detail.status == "disabled") != (
|
|
630
|
+
policy.status == "disabled"
|
|
631
|
+
):
|
|
632
|
+
raise ValueError(
|
|
633
|
+
"report member conflicts with retrospective execution policy"
|
|
634
|
+
)
|
|
635
|
+
if detail.status == "missing":
|
|
636
|
+
if (
|
|
637
|
+
member.evidence.status != "missing"
|
|
638
|
+
or detail.changepoint_step is not None
|
|
639
|
+
):
|
|
640
|
+
raise ValueError("inconsistent missing retrospective evidence")
|
|
641
|
+
continue
|
|
642
|
+
if (detail.status == "change") != (
|
|
643
|
+
member.feature in changed
|
|
644
|
+
) or detail.changepoint_step != changed.get(member.feature):
|
|
645
|
+
raise ValueError("report member conflicts with retrospective boundary")
|
|
646
|
+
unscorable += detail.status in {
|
|
647
|
+
"insufficient_data",
|
|
648
|
+
"unavailable_uncertainty",
|
|
649
|
+
}
|
|
650
|
+
no_change += detail.status == "no_change"
|
|
651
|
+
if summary.screening is not None and (
|
|
652
|
+
unscorable > summary.screening.n_unscorable
|
|
653
|
+
or no_change > summary.screening.n_screened - summary.screening.n_changes
|
|
654
|
+
):
|
|
655
|
+
raise ValueError("report member statuses exceed retrospective screening counts")
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _check_retrospective(report: _Report) -> None:
|
|
659
|
+
policy = report.retrospective
|
|
660
|
+
if report.schema_version < 3:
|
|
661
|
+
if policy is not None or any(
|
|
662
|
+
run.retrospective is not None
|
|
663
|
+
or any(
|
|
664
|
+
member.evidence.retrospective is not None
|
|
665
|
+
for event in run.events
|
|
666
|
+
for member in event.members
|
|
667
|
+
)
|
|
668
|
+
for run in report.runs
|
|
669
|
+
):
|
|
670
|
+
raise ValueError("retrospective execution policy requires report version 3")
|
|
671
|
+
return
|
|
672
|
+
if policy is None:
|
|
673
|
+
raise ValueError("report version 3 requires retrospective execution policy")
|
|
674
|
+
applicability = getattr(report.benchmark, "applicability", None)
|
|
675
|
+
if (
|
|
676
|
+
applicability is not None
|
|
677
|
+
and applicability.source == "bundled"
|
|
678
|
+
and (
|
|
679
|
+
("retrospective_disabled" in applicability.reasons)
|
|
680
|
+
!= (policy.status == "disabled")
|
|
681
|
+
)
|
|
682
|
+
):
|
|
683
|
+
raise ValueError(
|
|
684
|
+
"report benchmark applicability conflicts with retrospective execution policy"
|
|
685
|
+
)
|
|
686
|
+
for run in report.runs:
|
|
687
|
+
_check_run_retrospective(run, policy)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
def validate_report(value: object) -> dict:
|
|
691
|
+
"""Validate report fields and links; omit unknown fields from the copy."""
|
|
692
|
+
if not isinstance(value, dict):
|
|
693
|
+
raise ValueError("report must be a JSON object")
|
|
694
|
+
version = value.get("schema_version")
|
|
695
|
+
if type(version) is not int or version not in (1, 2, 3):
|
|
696
|
+
raise ValueError(
|
|
697
|
+
"unsupported report version; expected schema_version 1, 2, or 3"
|
|
698
|
+
)
|
|
699
|
+
if version == 2 and value.get("sensitivity") is None:
|
|
700
|
+
raise ValueError("report version 2 requires sensitivity metadata")
|
|
701
|
+
try:
|
|
702
|
+
report = _Report.model_validate(value)
|
|
703
|
+
except (ValidationError, RecursionError):
|
|
704
|
+
raise ValueError("invalid report fields or types") from None
|
|
705
|
+
_check_retrospective(report)
|
|
706
|
+
_check_sensitivity(report)
|
|
707
|
+
seen_runs, seen_events, seen_members = set(), set(), set()
|
|
708
|
+
for run in report.runs:
|
|
709
|
+
if run.run in seen_runs:
|
|
710
|
+
raise ValueError("report run identifiers must be unique")
|
|
711
|
+
seen_runs.add(run.run)
|
|
712
|
+
counts = run.counts
|
|
713
|
+
behavioral = sum(event.ranking_status == "behavioral" for event in run.events)
|
|
714
|
+
contextual = sum(event.ranking_status == "contextual" for event in run.events)
|
|
715
|
+
if (
|
|
716
|
+
counts.events != len(run.events)
|
|
717
|
+
or counts.sequential_alerts != sum(event.n_members for event in run.events)
|
|
718
|
+
or counts.behavioral_events != behavioral
|
|
719
|
+
or counts.contextual_events != contextual
|
|
720
|
+
or counts.other_events != len(run.events) - behavioral - contextual
|
|
721
|
+
or sorted(event.rank for event in run.events)
|
|
722
|
+
!= list(range(1, len(run.events) + 1))
|
|
723
|
+
):
|
|
724
|
+
raise ValueError("report counts or event ranks are inconsistent")
|
|
725
|
+
for event in run.events:
|
|
726
|
+
if event.event_id in seen_events:
|
|
727
|
+
raise ValueError("report event identifiers must be unique")
|
|
728
|
+
seen_events.add(event.event_id)
|
|
729
|
+
_check_event(event, run.run, seen_members)
|
|
730
|
+
result = report.model_dump(exclude_unset=True)
|
|
731
|
+
if report.benchmark is not None:
|
|
732
|
+
result["benchmark"] = validate_benchmark_reference(result["benchmark"])
|
|
733
|
+
return result
|
|
734
|
+
|
|
735
|
+
|
|
736
|
+
def _object(pairs: list[tuple]) -> dict:
|
|
737
|
+
result = {}
|
|
738
|
+
for key, value in pairs:
|
|
739
|
+
if key in result:
|
|
740
|
+
raise ValueError("duplicate report JSON key")
|
|
741
|
+
result[key] = value
|
|
742
|
+
return result
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def _constant(value: str) -> None:
|
|
746
|
+
raise ValueError("nonfinite report JSON number")
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def load_report(path: str | Path) -> dict:
|
|
750
|
+
"""Read strict UTF-8 JSON with safe errors for malformed input."""
|
|
751
|
+
try:
|
|
752
|
+
with Path(path).open(encoding="utf-8") as stream:
|
|
753
|
+
value = json.load(
|
|
754
|
+
stream, object_pairs_hook=_object, parse_constant=_constant
|
|
755
|
+
)
|
|
756
|
+
except (ValueError, RecursionError):
|
|
757
|
+
raise ValueError("invalid report JSON") from None
|
|
758
|
+
return validate_report(value)
|