rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
"""Count-based benchmark references supplied separately from rollout analysis."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
from importlib.resources import files
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Annotated, Literal
|
|
10
|
+
|
|
11
|
+
from pydantic import (
|
|
12
|
+
BaseModel,
|
|
13
|
+
ConfigDict,
|
|
14
|
+
Field,
|
|
15
|
+
TypeAdapter,
|
|
16
|
+
ValidationError,
|
|
17
|
+
field_validator,
|
|
18
|
+
model_validator,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
from .sensitivity import LENIENT_ID
|
|
22
|
+
|
|
23
|
+
Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
|
|
24
|
+
Count = Annotated[int, Field(ge=0)]
|
|
25
|
+
Digests = Annotated[list[Digest], Field(min_length=1)]
|
|
26
|
+
Commit = Annotated[str, Field(pattern=r"^[0-9a-f]{40}$")]
|
|
27
|
+
Version = Annotated[str, Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+(?:[a-zA-Z0-9.+-]*)?$")]
|
|
28
|
+
Fraction = Annotated[float, Field(ge=0, le=1)]
|
|
29
|
+
Positive = Annotated[float, Field(gt=0)]
|
|
30
|
+
CodeFile = Annotated[str, Field(pattern=r"^[a-z_][a-z0-9_]*\.py$")]
|
|
31
|
+
CodeHashes = Annotated[dict[CodeFile, Digest], Field(min_length=1)]
|
|
32
|
+
|
|
33
|
+
ANALYTICAL_MODULES = (
|
|
34
|
+
"_dependence.py",
|
|
35
|
+
"_feature_values.py",
|
|
36
|
+
"_sampling.py",
|
|
37
|
+
"_series_validation.py",
|
|
38
|
+
"corroboration.py",
|
|
39
|
+
"detect.py",
|
|
40
|
+
"events.py",
|
|
41
|
+
"evidence.py",
|
|
42
|
+
"exemplars.py",
|
|
43
|
+
"features.py",
|
|
44
|
+
"pipeline.py",
|
|
45
|
+
"prompt_mix.py",
|
|
46
|
+
"ranking.py",
|
|
47
|
+
"replication.py",
|
|
48
|
+
"retrospective.py",
|
|
49
|
+
"schema.py",
|
|
50
|
+
"series.py",
|
|
51
|
+
)
|
|
52
|
+
_BUNDLED_SHA256 = "49c25bed5204846a80f5812d0e52313b1c12d5e18d32f15370169eac87ce6e26"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class _Model(BaseModel):
|
|
56
|
+
model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class _SyntheticNull(_Model):
|
|
60
|
+
detector: Literal["sequential", "retrospective"]
|
|
61
|
+
n_trials: Count
|
|
62
|
+
n_scorable: Count
|
|
63
|
+
n_alerts: Count
|
|
64
|
+
|
|
65
|
+
@model_validator(mode="after")
|
|
66
|
+
def counts(self):
|
|
67
|
+
if not self.n_alerts <= self.n_scorable <= self.n_trials:
|
|
68
|
+
raise ValueError("inconsistent benchmark trial counts")
|
|
69
|
+
return self
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class _PreOnset(_Model):
|
|
73
|
+
run: str
|
|
74
|
+
source_sha256s: Digests
|
|
75
|
+
label: Literal["label_attempt", "label_hack"]
|
|
76
|
+
reference_onset: Count | None
|
|
77
|
+
status: Literal["available", "unscorable", "no_reference_onset"]
|
|
78
|
+
n_alerts: Count | None
|
|
79
|
+
|
|
80
|
+
@field_validator("run")
|
|
81
|
+
@classmethod
|
|
82
|
+
def identifier(cls, value):
|
|
83
|
+
from .report import _identifier
|
|
84
|
+
|
|
85
|
+
return _identifier(value)
|
|
86
|
+
|
|
87
|
+
@model_validator(mode="after")
|
|
88
|
+
def coverage(self):
|
|
89
|
+
if self.status == "available":
|
|
90
|
+
if self.reference_onset is None or self.n_alerts is None:
|
|
91
|
+
raise ValueError("available benchmark counts require a reference onset")
|
|
92
|
+
elif self.n_alerts is not None:
|
|
93
|
+
raise ValueError("unavailable benchmark counts must stay null")
|
|
94
|
+
if (self.reference_onset is None) != (self.status == "no_reference_onset"):
|
|
95
|
+
raise ValueError("inconsistent benchmark onset status")
|
|
96
|
+
return self
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class _ReferenceV1(_Model):
|
|
100
|
+
schema_version: Literal[1]
|
|
101
|
+
source_commit: Commit
|
|
102
|
+
protocol_sha256: Digest
|
|
103
|
+
artifact_sha256s: Digests
|
|
104
|
+
file_sha256: Digest | None = None
|
|
105
|
+
synthetic_null: Annotated[list[_SyntheticNull], Field(min_length=2, max_length=2)]
|
|
106
|
+
pre_onset: Annotated[list[_PreOnset], Field(min_length=1)]
|
|
107
|
+
|
|
108
|
+
@field_validator("schema_version", mode="before")
|
|
109
|
+
@classmethod
|
|
110
|
+
def version(cls, value):
|
|
111
|
+
if type(value) is not int or value != 1:
|
|
112
|
+
raise ValueError("unsupported benchmark reference version")
|
|
113
|
+
return value
|
|
114
|
+
|
|
115
|
+
@model_validator(mode="after")
|
|
116
|
+
def unique_rows(self):
|
|
117
|
+
if len({row.detector for row in self.synthetic_null}) != 2:
|
|
118
|
+
raise ValueError("duplicate benchmark detector")
|
|
119
|
+
if len({row.run for row in self.pre_onset}) != len(self.pre_onset):
|
|
120
|
+
raise ValueError("duplicate benchmark run")
|
|
121
|
+
return self
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
class _Vocabulary(_Model):
|
|
125
|
+
max_sparse_per_tier: Count
|
|
126
|
+
min_prevalence: Fraction
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
class _Sequential(_Model):
|
|
130
|
+
baseline_steps: Annotated[int, Field(ge=2)]
|
|
131
|
+
z_threshold: Positive
|
|
132
|
+
sustain: Annotated[int, Field(ge=1)]
|
|
133
|
+
min_prevalence_change: Fraction
|
|
134
|
+
numerical_floor: Positive
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class _Retrospective(_Model):
|
|
138
|
+
min_segment_steps: Annotated[int, Field(ge=2)]
|
|
139
|
+
penalty_multiplier: Positive
|
|
140
|
+
min_prevalence_change: Fraction
|
|
141
|
+
min_continuous_change: Annotated[float, Field(ge=0)]
|
|
142
|
+
dependence: Literal["ar1", "independent"]
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _Settings(_Model):
|
|
146
|
+
vocabulary: _Vocabulary
|
|
147
|
+
sequential: _Sequential
|
|
148
|
+
retrospective: _Retrospective
|
|
149
|
+
sampling_units: Literal["exact_prompt", "explicit"]
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
class _Runtime(_Model):
|
|
153
|
+
python: Version
|
|
154
|
+
numpy: Version
|
|
155
|
+
pandas: Version
|
|
156
|
+
pyarrow: Version
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class _AnalysisSettings(_Settings):
|
|
160
|
+
retrospective: _Retrospective | None
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class _AnalysisMetadata(_Model):
|
|
164
|
+
settings: _AnalysisSettings
|
|
165
|
+
analytical_code_sha256s: CodeHashes
|
|
166
|
+
runtime: _Runtime
|
|
167
|
+
sensitivity: Literal["lenient-structural-binary-v1-reconstructed"] | None = None
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class _Generation(_Model):
|
|
171
|
+
n_replicates: Annotated[int, Field(ge=1)]
|
|
172
|
+
n_steps: Annotated[int, Field(ge=1)]
|
|
173
|
+
onset: Count
|
|
174
|
+
binary_effect: Fraction
|
|
175
|
+
continuous_effect_sd: Positive
|
|
176
|
+
|
|
177
|
+
@model_validator(mode="after")
|
|
178
|
+
def bounds(self):
|
|
179
|
+
if self.onset >= self.n_steps:
|
|
180
|
+
raise ValueError("benchmark onset exceeds the generated horizon")
|
|
181
|
+
return self
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class _Protocol(_Model):
|
|
185
|
+
fit_window: Annotated[list[Count], Field(min_length=2, max_length=2)]
|
|
186
|
+
n_primary_profiles: Annotated[int, Field(ge=1)]
|
|
187
|
+
generation: _Generation
|
|
188
|
+
null_seed: Count
|
|
189
|
+
plant_seed: Count
|
|
190
|
+
timing_tolerance: Count
|
|
191
|
+
null_trial_unit: Literal["generated_single_series"]
|
|
192
|
+
fit_sampling_units: Literal["aria_exact_prompts_aisi_source_problems"]
|
|
193
|
+
public_onset: Literal["three_complete_logged_observations_at_one_percent"]
|
|
194
|
+
public_onset_weighting: Literal["rollout"]
|
|
195
|
+
pre_onset_count_unit: Literal["sequential_alerted_features"]
|
|
196
|
+
scope: Literal["exposed_development_inputs_and_fitted_surrogates"]
|
|
197
|
+
|
|
198
|
+
@model_validator(mode="after")
|
|
199
|
+
def window(self):
|
|
200
|
+
if self.fit_window[0] > self.fit_window[1]:
|
|
201
|
+
raise ValueError("invalid benchmark fit window")
|
|
202
|
+
return self
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class _PlantedRecovery(_Model):
|
|
206
|
+
detector: Literal["sequential", "retrospective"]
|
|
207
|
+
n_planted: Count
|
|
208
|
+
n_within_tolerance: Count
|
|
209
|
+
|
|
210
|
+
@model_validator(mode="after")
|
|
211
|
+
def counts(self):
|
|
212
|
+
if self.n_within_tolerance > self.n_planted:
|
|
213
|
+
raise ValueError("inconsistent benchmark recovery counts")
|
|
214
|
+
return self
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class _OriginalAcceptance(_Model):
|
|
218
|
+
max_null_false_alert_rate: Fraction
|
|
219
|
+
min_planted_within_tolerance: Fraction
|
|
220
|
+
planted_recovery: Annotated[
|
|
221
|
+
list[_PlantedRecovery], Field(min_length=2, max_length=2)
|
|
222
|
+
]
|
|
223
|
+
|
|
224
|
+
@model_validator(mode="after")
|
|
225
|
+
def unique_detectors(self):
|
|
226
|
+
if len({row.detector for row in self.planted_recovery}) != 2:
|
|
227
|
+
raise ValueError("duplicate benchmark recovery detector")
|
|
228
|
+
return self
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class _Dataset(_Model):
|
|
232
|
+
run: Literal["aria", "aisi0", "aisi2", "trl"]
|
|
233
|
+
repo_id: Annotated[
|
|
234
|
+
str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$")
|
|
235
|
+
]
|
|
236
|
+
revision: Commit
|
|
237
|
+
license_status: Literal["declared", "not_asserted", "unresolved"]
|
|
238
|
+
license_identifier: Literal["apache-2.0", "other"] | None
|
|
239
|
+
license_evidence_sha256: Digest
|
|
240
|
+
restriction: Literal[
|
|
241
|
+
"local_inputs_only_unresolved_redistribution",
|
|
242
|
+
"apache_2_0_underlying_source_terms",
|
|
243
|
+
]
|
|
244
|
+
attribution: Literal[
|
|
245
|
+
"Gutenberg PBC Aria reproduction",
|
|
246
|
+
"UK AI Security Institute mt-somo",
|
|
247
|
+
"sunshineNew TRL completion log",
|
|
248
|
+
]
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
class _Provenance(_Model):
|
|
252
|
+
recorded_date: Literal["2026-11-06"]
|
|
253
|
+
compatible_source_commit: Commit
|
|
254
|
+
compatibility_basis: Literal[
|
|
255
|
+
"reviewed_analytical_policy_and_reproduced_synthetic_counts"
|
|
256
|
+
]
|
|
257
|
+
expected_settings: _Settings
|
|
258
|
+
analytical_code_sha256s: CodeHashes
|
|
259
|
+
runtime: _Runtime
|
|
260
|
+
protocol: _Protocol
|
|
261
|
+
original_acceptance: _OriginalAcceptance
|
|
262
|
+
datasets: Annotated[list[_Dataset], Field(min_length=4, max_length=4)]
|
|
263
|
+
|
|
264
|
+
@model_validator(mode="after")
|
|
265
|
+
def unique_datasets(self):
|
|
266
|
+
if len({row.run for row in self.datasets}) != 4:
|
|
267
|
+
raise ValueError("duplicate benchmark dataset")
|
|
268
|
+
if set(self.analytical_code_sha256s) != set(ANALYTICAL_MODULES):
|
|
269
|
+
raise ValueError("incomplete benchmark implementation provenance")
|
|
270
|
+
return self
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
Reason = Literal[
|
|
274
|
+
"missing_metadata",
|
|
275
|
+
"invalid_metadata",
|
|
276
|
+
"settings_differ",
|
|
277
|
+
"analytical_code_differ",
|
|
278
|
+
"runtime_differ",
|
|
279
|
+
"explicit_sampling_units",
|
|
280
|
+
"supplied_reference",
|
|
281
|
+
"experimental_profile",
|
|
282
|
+
"retrospective_disabled",
|
|
283
|
+
]
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class _Applicability(_Model):
|
|
287
|
+
source: Literal["bundled", "supplied"]
|
|
288
|
+
status: Literal["matched", "different", "unverified"]
|
|
289
|
+
reasons: list[Reason]
|
|
290
|
+
|
|
291
|
+
@model_validator(mode="after")
|
|
292
|
+
def consistency(self):
|
|
293
|
+
if len(set(self.reasons)) != len(self.reasons):
|
|
294
|
+
raise ValueError("duplicate benchmark applicability reason")
|
|
295
|
+
if (self.status == "matched") != (not self.reasons):
|
|
296
|
+
raise ValueError("inconsistent benchmark applicability")
|
|
297
|
+
if self.source == "supplied":
|
|
298
|
+
valid = self.status == "unverified" and self.reasons == [
|
|
299
|
+
"supplied_reference"
|
|
300
|
+
]
|
|
301
|
+
elif self.status == "unverified":
|
|
302
|
+
valid = self.reasons in (["missing_metadata"], ["invalid_metadata"])
|
|
303
|
+
else:
|
|
304
|
+
valid = set(self.reasons) <= {
|
|
305
|
+
"settings_differ",
|
|
306
|
+
"analytical_code_differ",
|
|
307
|
+
"runtime_differ",
|
|
308
|
+
"explicit_sampling_units",
|
|
309
|
+
"experimental_profile",
|
|
310
|
+
"retrospective_disabled",
|
|
311
|
+
} | (
|
|
312
|
+
{"missing_metadata", "invalid_metadata"}
|
|
313
|
+
if {"experimental_profile", "retrospective_disabled"}.intersection(
|
|
314
|
+
self.reasons
|
|
315
|
+
)
|
|
316
|
+
else set()
|
|
317
|
+
)
|
|
318
|
+
if not valid:
|
|
319
|
+
raise ValueError("inconsistent benchmark applicability source or reasons")
|
|
320
|
+
return self
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class _ReferenceV2(_Model):
|
|
324
|
+
schema_version: Literal[2]
|
|
325
|
+
reference_id: Literal["m0-production-2026-11-06"]
|
|
326
|
+
results: _ReferenceV1
|
|
327
|
+
provenance: _Provenance
|
|
328
|
+
file_sha256: Digest | None = None
|
|
329
|
+
applicability: _Applicability | None = None
|
|
330
|
+
|
|
331
|
+
@field_validator("schema_version", mode="before")
|
|
332
|
+
@classmethod
|
|
333
|
+
def version(cls, value):
|
|
334
|
+
if type(value) is not int or value != 2:
|
|
335
|
+
raise ValueError("unsupported benchmark reference version")
|
|
336
|
+
return value
|
|
337
|
+
|
|
338
|
+
@model_validator(mode="after")
|
|
339
|
+
def coverage(self):
|
|
340
|
+
protocol = self.provenance.protocol
|
|
341
|
+
trials = protocol.n_primary_profiles * protocol.generation.n_replicates
|
|
342
|
+
if any(row.n_trials != trials for row in self.results.synthetic_null) or any(
|
|
343
|
+
row.n_planted != trials
|
|
344
|
+
for row in self.provenance.original_acceptance.planted_recovery
|
|
345
|
+
):
|
|
346
|
+
raise ValueError("benchmark counts differ from the recorded protocol")
|
|
347
|
+
if {row.run for row in self.results.pre_onset} != {
|
|
348
|
+
row.run for row in self.provenance.datasets
|
|
349
|
+
}:
|
|
350
|
+
raise ValueError("benchmark dataset coverage differs from results")
|
|
351
|
+
return self
|
|
352
|
+
|
|
353
|
+
|
|
354
|
+
BenchmarkReference = _ReferenceV1 | _ReferenceV2
|
|
355
|
+
_REFERENCE = TypeAdapter(BenchmarkReference)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def validate_benchmark_reference(value: object) -> dict:
|
|
359
|
+
"""Validate recorded facts without authenticating or refreshing provenance."""
|
|
360
|
+
try:
|
|
361
|
+
reference = _REFERENCE.validate_python(value)
|
|
362
|
+
except (ValidationError, RecursionError):
|
|
363
|
+
raise ValueError("invalid benchmark reference") from None
|
|
364
|
+
result = reference.model_dump(exclude_unset=True)
|
|
365
|
+
counts = result if result["schema_version"] == 1 else result["results"]
|
|
366
|
+
counts["synthetic_null"].sort(key=lambda row: row["detector"])
|
|
367
|
+
counts["pre_onset"].sort(key=lambda row: row["run"])
|
|
368
|
+
return result
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _decode_reference(encoded: bytes) -> dict:
|
|
372
|
+
from ._report_validation import _constant, _object
|
|
373
|
+
|
|
374
|
+
try:
|
|
375
|
+
value = json.loads(encoded, object_pairs_hook=_object, parse_constant=_constant)
|
|
376
|
+
result = validate_benchmark_reference(value)
|
|
377
|
+
except (ValueError, RecursionError):
|
|
378
|
+
raise ValueError("invalid benchmark reference JSON") from None
|
|
379
|
+
result["file_sha256"] = hashlib.sha256(encoded).hexdigest()
|
|
380
|
+
return result
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def load_benchmark_reference(path: str | Path) -> dict:
|
|
384
|
+
"""Load a supplied file without accepting its claimed applicability."""
|
|
385
|
+
return validate_supplied_benchmark_reference(
|
|
386
|
+
_decode_reference(Path(path).read_bytes())
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def validate_supplied_benchmark_reference(value: object) -> dict:
|
|
391
|
+
"""Validate an explicit reference without trusting claims from another analysis."""
|
|
392
|
+
result = validate_benchmark_reference(value)
|
|
393
|
+
if result["schema_version"] == 2:
|
|
394
|
+
result["applicability"] = {
|
|
395
|
+
"source": "supplied",
|
|
396
|
+
"status": "unverified",
|
|
397
|
+
"reasons": ["supplied_reference"],
|
|
398
|
+
}
|
|
399
|
+
return result
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _applicability(
|
|
403
|
+
metadata: object, provenance: dict, retrospective_disabled: bool
|
|
404
|
+
) -> dict:
|
|
405
|
+
result = {"source": "bundled", "status": "unverified", "reasons": []}
|
|
406
|
+
if isinstance(metadata, dict) and metadata.get("sensitivity") == LENIENT_ID:
|
|
407
|
+
result.update(status="different", reasons=["experimental_profile"])
|
|
408
|
+
if retrospective_disabled:
|
|
409
|
+
result["status"] = "different"
|
|
410
|
+
result["reasons"].append("retrospective_disabled")
|
|
411
|
+
if metadata is None:
|
|
412
|
+
result["reasons"].append("missing_metadata")
|
|
413
|
+
return result
|
|
414
|
+
try:
|
|
415
|
+
actual = _AnalysisMetadata.model_validate(metadata).model_dump()
|
|
416
|
+
except (ValidationError, RecursionError):
|
|
417
|
+
result["reasons"].append("invalid_metadata")
|
|
418
|
+
return result
|
|
419
|
+
expected = provenance["expected_settings"]
|
|
420
|
+
reasons = result["reasons"]
|
|
421
|
+
checked = ["vocabulary", "sequential"]
|
|
422
|
+
if actual["settings"]["retrospective"] is None:
|
|
423
|
+
if "retrospective_disabled" not in reasons:
|
|
424
|
+
reasons.append("retrospective_disabled")
|
|
425
|
+
else:
|
|
426
|
+
checked.append("retrospective")
|
|
427
|
+
if any(actual["settings"][key] != expected[key] for key in checked):
|
|
428
|
+
reasons.append("settings_differ")
|
|
429
|
+
if actual["settings"]["sampling_units"] == "explicit":
|
|
430
|
+
reasons.append("explicit_sampling_units")
|
|
431
|
+
if actual["analytical_code_sha256s"] != provenance["analytical_code_sha256s"]:
|
|
432
|
+
reasons.append("analytical_code_differ")
|
|
433
|
+
if actual["runtime"] != provenance["runtime"]:
|
|
434
|
+
reasons.append("runtime_differ")
|
|
435
|
+
result["status"] = "different" if reasons else "matched"
|
|
436
|
+
return result
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
def load_bundled_benchmark_reference(
|
|
440
|
+
metadata: object = None, *, retrospective_disabled: bool = False
|
|
441
|
+
) -> dict:
|
|
442
|
+
"""Bind reviewed aggregate bytes and compare recorded implementation settings.
|
|
443
|
+
|
|
444
|
+
A match is implementation context only, never calibration of analyzed data.
|
|
445
|
+
"""
|
|
446
|
+
try:
|
|
447
|
+
encoded = (
|
|
448
|
+
files("rolloutdiff").joinpath("_benchmarks/m0-2026-11-06.json").read_bytes()
|
|
449
|
+
)
|
|
450
|
+
except OSError:
|
|
451
|
+
raise ValueError("bundled benchmark reference unavailable") from None
|
|
452
|
+
if hashlib.sha256(encoded).hexdigest() != _BUNDLED_SHA256:
|
|
453
|
+
raise ValueError("bundled benchmark reference integrity check failed")
|
|
454
|
+
result = _decode_reference(encoded)
|
|
455
|
+
result["applicability"] = _applicability(
|
|
456
|
+
metadata, result["provenance"], retrospective_disabled
|
|
457
|
+
)
|
|
458
|
+
return result
|