rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,458 @@
1
+ """Count-based benchmark references supplied separately from rollout analysis."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import json
7
+ from importlib.resources import files
8
+ from pathlib import Path
9
+ from typing import Annotated, Literal
10
+
11
+ from pydantic import (
12
+ BaseModel,
13
+ ConfigDict,
14
+ Field,
15
+ TypeAdapter,
16
+ ValidationError,
17
+ field_validator,
18
+ model_validator,
19
+ )
20
+
21
+ from .sensitivity import LENIENT_ID
22
+
23
+ Digest = Annotated[str, Field(pattern=r"^[0-9a-f]{64}$")]
24
+ Count = Annotated[int, Field(ge=0)]
25
+ Digests = Annotated[list[Digest], Field(min_length=1)]
26
+ Commit = Annotated[str, Field(pattern=r"^[0-9a-f]{40}$")]
27
+ Version = Annotated[str, Field(pattern=r"^[0-9]+\.[0-9]+\.[0-9]+(?:[a-zA-Z0-9.+-]*)?$")]
28
+ Fraction = Annotated[float, Field(ge=0, le=1)]
29
+ Positive = Annotated[float, Field(gt=0)]
30
+ CodeFile = Annotated[str, Field(pattern=r"^[a-z_][a-z0-9_]*\.py$")]
31
+ CodeHashes = Annotated[dict[CodeFile, Digest], Field(min_length=1)]
32
+
33
+ ANALYTICAL_MODULES = (
34
+ "_dependence.py",
35
+ "_feature_values.py",
36
+ "_sampling.py",
37
+ "_series_validation.py",
38
+ "corroboration.py",
39
+ "detect.py",
40
+ "events.py",
41
+ "evidence.py",
42
+ "exemplars.py",
43
+ "features.py",
44
+ "pipeline.py",
45
+ "prompt_mix.py",
46
+ "ranking.py",
47
+ "replication.py",
48
+ "retrospective.py",
49
+ "schema.py",
50
+ "series.py",
51
+ )
52
+ _BUNDLED_SHA256 = "49c25bed5204846a80f5812d0e52313b1c12d5e18d32f15370169eac87ce6e26"
53
+
54
+
55
+ class _Model(BaseModel):
56
+ model_config = ConfigDict(strict=True, extra="forbid", allow_inf_nan=False)
57
+
58
+
59
+ class _SyntheticNull(_Model):
60
+ detector: Literal["sequential", "retrospective"]
61
+ n_trials: Count
62
+ n_scorable: Count
63
+ n_alerts: Count
64
+
65
+ @model_validator(mode="after")
66
+ def counts(self):
67
+ if not self.n_alerts <= self.n_scorable <= self.n_trials:
68
+ raise ValueError("inconsistent benchmark trial counts")
69
+ return self
70
+
71
+
72
+ class _PreOnset(_Model):
73
+ run: str
74
+ source_sha256s: Digests
75
+ label: Literal["label_attempt", "label_hack"]
76
+ reference_onset: Count | None
77
+ status: Literal["available", "unscorable", "no_reference_onset"]
78
+ n_alerts: Count | None
79
+
80
+ @field_validator("run")
81
+ @classmethod
82
+ def identifier(cls, value):
83
+ from .report import _identifier
84
+
85
+ return _identifier(value)
86
+
87
+ @model_validator(mode="after")
88
+ def coverage(self):
89
+ if self.status == "available":
90
+ if self.reference_onset is None or self.n_alerts is None:
91
+ raise ValueError("available benchmark counts require a reference onset")
92
+ elif self.n_alerts is not None:
93
+ raise ValueError("unavailable benchmark counts must stay null")
94
+ if (self.reference_onset is None) != (self.status == "no_reference_onset"):
95
+ raise ValueError("inconsistent benchmark onset status")
96
+ return self
97
+
98
+
99
+ class _ReferenceV1(_Model):
100
+ schema_version: Literal[1]
101
+ source_commit: Commit
102
+ protocol_sha256: Digest
103
+ artifact_sha256s: Digests
104
+ file_sha256: Digest | None = None
105
+ synthetic_null: Annotated[list[_SyntheticNull], Field(min_length=2, max_length=2)]
106
+ pre_onset: Annotated[list[_PreOnset], Field(min_length=1)]
107
+
108
+ @field_validator("schema_version", mode="before")
109
+ @classmethod
110
+ def version(cls, value):
111
+ if type(value) is not int or value != 1:
112
+ raise ValueError("unsupported benchmark reference version")
113
+ return value
114
+
115
+ @model_validator(mode="after")
116
+ def unique_rows(self):
117
+ if len({row.detector for row in self.synthetic_null}) != 2:
118
+ raise ValueError("duplicate benchmark detector")
119
+ if len({row.run for row in self.pre_onset}) != len(self.pre_onset):
120
+ raise ValueError("duplicate benchmark run")
121
+ return self
122
+
123
+
124
+ class _Vocabulary(_Model):
125
+ max_sparse_per_tier: Count
126
+ min_prevalence: Fraction
127
+
128
+
129
+ class _Sequential(_Model):
130
+ baseline_steps: Annotated[int, Field(ge=2)]
131
+ z_threshold: Positive
132
+ sustain: Annotated[int, Field(ge=1)]
133
+ min_prevalence_change: Fraction
134
+ numerical_floor: Positive
135
+
136
+
137
+ class _Retrospective(_Model):
138
+ min_segment_steps: Annotated[int, Field(ge=2)]
139
+ penalty_multiplier: Positive
140
+ min_prevalence_change: Fraction
141
+ min_continuous_change: Annotated[float, Field(ge=0)]
142
+ dependence: Literal["ar1", "independent"]
143
+
144
+
145
+ class _Settings(_Model):
146
+ vocabulary: _Vocabulary
147
+ sequential: _Sequential
148
+ retrospective: _Retrospective
149
+ sampling_units: Literal["exact_prompt", "explicit"]
150
+
151
+
152
+ class _Runtime(_Model):
153
+ python: Version
154
+ numpy: Version
155
+ pandas: Version
156
+ pyarrow: Version
157
+
158
+
159
+ class _AnalysisSettings(_Settings):
160
+ retrospective: _Retrospective | None
161
+
162
+
163
+ class _AnalysisMetadata(_Model):
164
+ settings: _AnalysisSettings
165
+ analytical_code_sha256s: CodeHashes
166
+ runtime: _Runtime
167
+ sensitivity: Literal["lenient-structural-binary-v1-reconstructed"] | None = None
168
+
169
+
170
+ class _Generation(_Model):
171
+ n_replicates: Annotated[int, Field(ge=1)]
172
+ n_steps: Annotated[int, Field(ge=1)]
173
+ onset: Count
174
+ binary_effect: Fraction
175
+ continuous_effect_sd: Positive
176
+
177
+ @model_validator(mode="after")
178
+ def bounds(self):
179
+ if self.onset >= self.n_steps:
180
+ raise ValueError("benchmark onset exceeds the generated horizon")
181
+ return self
182
+
183
+
184
+ class _Protocol(_Model):
185
+ fit_window: Annotated[list[Count], Field(min_length=2, max_length=2)]
186
+ n_primary_profiles: Annotated[int, Field(ge=1)]
187
+ generation: _Generation
188
+ null_seed: Count
189
+ plant_seed: Count
190
+ timing_tolerance: Count
191
+ null_trial_unit: Literal["generated_single_series"]
192
+ fit_sampling_units: Literal["aria_exact_prompts_aisi_source_problems"]
193
+ public_onset: Literal["three_complete_logged_observations_at_one_percent"]
194
+ public_onset_weighting: Literal["rollout"]
195
+ pre_onset_count_unit: Literal["sequential_alerted_features"]
196
+ scope: Literal["exposed_development_inputs_and_fitted_surrogates"]
197
+
198
+ @model_validator(mode="after")
199
+ def window(self):
200
+ if self.fit_window[0] > self.fit_window[1]:
201
+ raise ValueError("invalid benchmark fit window")
202
+ return self
203
+
204
+
205
+ class _PlantedRecovery(_Model):
206
+ detector: Literal["sequential", "retrospective"]
207
+ n_planted: Count
208
+ n_within_tolerance: Count
209
+
210
+ @model_validator(mode="after")
211
+ def counts(self):
212
+ if self.n_within_tolerance > self.n_planted:
213
+ raise ValueError("inconsistent benchmark recovery counts")
214
+ return self
215
+
216
+
217
+ class _OriginalAcceptance(_Model):
218
+ max_null_false_alert_rate: Fraction
219
+ min_planted_within_tolerance: Fraction
220
+ planted_recovery: Annotated[
221
+ list[_PlantedRecovery], Field(min_length=2, max_length=2)
222
+ ]
223
+
224
+ @model_validator(mode="after")
225
+ def unique_detectors(self):
226
+ if len({row.detector for row in self.planted_recovery}) != 2:
227
+ raise ValueError("duplicate benchmark recovery detector")
228
+ return self
229
+
230
+
231
+ class _Dataset(_Model):
232
+ run: Literal["aria", "aisi0", "aisi2", "trl"]
233
+ repo_id: Annotated[
234
+ str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$")
235
+ ]
236
+ revision: Commit
237
+ license_status: Literal["declared", "not_asserted", "unresolved"]
238
+ license_identifier: Literal["apache-2.0", "other"] | None
239
+ license_evidence_sha256: Digest
240
+ restriction: Literal[
241
+ "local_inputs_only_unresolved_redistribution",
242
+ "apache_2_0_underlying_source_terms",
243
+ ]
244
+ attribution: Literal[
245
+ "Gutenberg PBC Aria reproduction",
246
+ "UK AI Security Institute mt-somo",
247
+ "sunshineNew TRL completion log",
248
+ ]
249
+
250
+
251
+ class _Provenance(_Model):
252
+ recorded_date: Literal["2026-11-06"]
253
+ compatible_source_commit: Commit
254
+ compatibility_basis: Literal[
255
+ "reviewed_analytical_policy_and_reproduced_synthetic_counts"
256
+ ]
257
+ expected_settings: _Settings
258
+ analytical_code_sha256s: CodeHashes
259
+ runtime: _Runtime
260
+ protocol: _Protocol
261
+ original_acceptance: _OriginalAcceptance
262
+ datasets: Annotated[list[_Dataset], Field(min_length=4, max_length=4)]
263
+
264
+ @model_validator(mode="after")
265
+ def unique_datasets(self):
266
+ if len({row.run for row in self.datasets}) != 4:
267
+ raise ValueError("duplicate benchmark dataset")
268
+ if set(self.analytical_code_sha256s) != set(ANALYTICAL_MODULES):
269
+ raise ValueError("incomplete benchmark implementation provenance")
270
+ return self
271
+
272
+
273
+ Reason = Literal[
274
+ "missing_metadata",
275
+ "invalid_metadata",
276
+ "settings_differ",
277
+ "analytical_code_differ",
278
+ "runtime_differ",
279
+ "explicit_sampling_units",
280
+ "supplied_reference",
281
+ "experimental_profile",
282
+ "retrospective_disabled",
283
+ ]
284
+
285
+
286
+ class _Applicability(_Model):
287
+ source: Literal["bundled", "supplied"]
288
+ status: Literal["matched", "different", "unverified"]
289
+ reasons: list[Reason]
290
+
291
+ @model_validator(mode="after")
292
+ def consistency(self):
293
+ if len(set(self.reasons)) != len(self.reasons):
294
+ raise ValueError("duplicate benchmark applicability reason")
295
+ if (self.status == "matched") != (not self.reasons):
296
+ raise ValueError("inconsistent benchmark applicability")
297
+ if self.source == "supplied":
298
+ valid = self.status == "unverified" and self.reasons == [
299
+ "supplied_reference"
300
+ ]
301
+ elif self.status == "unverified":
302
+ valid = self.reasons in (["missing_metadata"], ["invalid_metadata"])
303
+ else:
304
+ valid = set(self.reasons) <= {
305
+ "settings_differ",
306
+ "analytical_code_differ",
307
+ "runtime_differ",
308
+ "explicit_sampling_units",
309
+ "experimental_profile",
310
+ "retrospective_disabled",
311
+ } | (
312
+ {"missing_metadata", "invalid_metadata"}
313
+ if {"experimental_profile", "retrospective_disabled"}.intersection(
314
+ self.reasons
315
+ )
316
+ else set()
317
+ )
318
+ if not valid:
319
+ raise ValueError("inconsistent benchmark applicability source or reasons")
320
+ return self
321
+
322
+
323
+ class _ReferenceV2(_Model):
324
+ schema_version: Literal[2]
325
+ reference_id: Literal["m0-production-2026-11-06"]
326
+ results: _ReferenceV1
327
+ provenance: _Provenance
328
+ file_sha256: Digest | None = None
329
+ applicability: _Applicability | None = None
330
+
331
+ @field_validator("schema_version", mode="before")
332
+ @classmethod
333
+ def version(cls, value):
334
+ if type(value) is not int or value != 2:
335
+ raise ValueError("unsupported benchmark reference version")
336
+ return value
337
+
338
+ @model_validator(mode="after")
339
+ def coverage(self):
340
+ protocol = self.provenance.protocol
341
+ trials = protocol.n_primary_profiles * protocol.generation.n_replicates
342
+ if any(row.n_trials != trials for row in self.results.synthetic_null) or any(
343
+ row.n_planted != trials
344
+ for row in self.provenance.original_acceptance.planted_recovery
345
+ ):
346
+ raise ValueError("benchmark counts differ from the recorded protocol")
347
+ if {row.run for row in self.results.pre_onset} != {
348
+ row.run for row in self.provenance.datasets
349
+ }:
350
+ raise ValueError("benchmark dataset coverage differs from results")
351
+ return self
352
+
353
+
354
+ BenchmarkReference = _ReferenceV1 | _ReferenceV2
355
+ _REFERENCE = TypeAdapter(BenchmarkReference)
356
+
357
+
358
+ def validate_benchmark_reference(value: object) -> dict:
359
+ """Validate recorded facts without authenticating or refreshing provenance."""
360
+ try:
361
+ reference = _REFERENCE.validate_python(value)
362
+ except (ValidationError, RecursionError):
363
+ raise ValueError("invalid benchmark reference") from None
364
+ result = reference.model_dump(exclude_unset=True)
365
+ counts = result if result["schema_version"] == 1 else result["results"]
366
+ counts["synthetic_null"].sort(key=lambda row: row["detector"])
367
+ counts["pre_onset"].sort(key=lambda row: row["run"])
368
+ return result
369
+
370
+
371
+ def _decode_reference(encoded: bytes) -> dict:
372
+ from ._report_validation import _constant, _object
373
+
374
+ try:
375
+ value = json.loads(encoded, object_pairs_hook=_object, parse_constant=_constant)
376
+ result = validate_benchmark_reference(value)
377
+ except (ValueError, RecursionError):
378
+ raise ValueError("invalid benchmark reference JSON") from None
379
+ result["file_sha256"] = hashlib.sha256(encoded).hexdigest()
380
+ return result
381
+
382
+
383
+ def load_benchmark_reference(path: str | Path) -> dict:
384
+ """Load a supplied file without accepting its claimed applicability."""
385
+ return validate_supplied_benchmark_reference(
386
+ _decode_reference(Path(path).read_bytes())
387
+ )
388
+
389
+
390
+ def validate_supplied_benchmark_reference(value: object) -> dict:
391
+ """Validate an explicit reference without trusting claims from another analysis."""
392
+ result = validate_benchmark_reference(value)
393
+ if result["schema_version"] == 2:
394
+ result["applicability"] = {
395
+ "source": "supplied",
396
+ "status": "unverified",
397
+ "reasons": ["supplied_reference"],
398
+ }
399
+ return result
400
+
401
+
402
+ def _applicability(
403
+ metadata: object, provenance: dict, retrospective_disabled: bool
404
+ ) -> dict:
405
+ result = {"source": "bundled", "status": "unverified", "reasons": []}
406
+ if isinstance(metadata, dict) and metadata.get("sensitivity") == LENIENT_ID:
407
+ result.update(status="different", reasons=["experimental_profile"])
408
+ if retrospective_disabled:
409
+ result["status"] = "different"
410
+ result["reasons"].append("retrospective_disabled")
411
+ if metadata is None:
412
+ result["reasons"].append("missing_metadata")
413
+ return result
414
+ try:
415
+ actual = _AnalysisMetadata.model_validate(metadata).model_dump()
416
+ except (ValidationError, RecursionError):
417
+ result["reasons"].append("invalid_metadata")
418
+ return result
419
+ expected = provenance["expected_settings"]
420
+ reasons = result["reasons"]
421
+ checked = ["vocabulary", "sequential"]
422
+ if actual["settings"]["retrospective"] is None:
423
+ if "retrospective_disabled" not in reasons:
424
+ reasons.append("retrospective_disabled")
425
+ else:
426
+ checked.append("retrospective")
427
+ if any(actual["settings"][key] != expected[key] for key in checked):
428
+ reasons.append("settings_differ")
429
+ if actual["settings"]["sampling_units"] == "explicit":
430
+ reasons.append("explicit_sampling_units")
431
+ if actual["analytical_code_sha256s"] != provenance["analytical_code_sha256s"]:
432
+ reasons.append("analytical_code_differ")
433
+ if actual["runtime"] != provenance["runtime"]:
434
+ reasons.append("runtime_differ")
435
+ result["status"] = "different" if reasons else "matched"
436
+ return result
437
+
438
+
439
+ def load_bundled_benchmark_reference(
440
+ metadata: object = None, *, retrospective_disabled: bool = False
441
+ ) -> dict:
442
+ """Bind reviewed aggregate bytes and compare recorded implementation settings.
443
+
444
+ A match is implementation context only, never calibration of analyzed data.
445
+ """
446
+ try:
447
+ encoded = (
448
+ files("rolloutdiff").joinpath("_benchmarks/m0-2026-11-06.json").read_bytes()
449
+ )
450
+ except OSError:
451
+ raise ValueError("bundled benchmark reference unavailable") from None
452
+ if hashlib.sha256(encoded).hexdigest() != _BUNDLED_SHA256:
453
+ raise ValueError("bundled benchmark reference integrity check failed")
454
+ result = _decode_reference(encoded)
455
+ result["applicability"] = _applicability(
456
+ metadata, result["provenance"], retrospective_disabled
457
+ )
458
+ return result