rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,1146 @@
1
+ """Frozen, single-variant Aria investigations with locally consumed approvals."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import difflib
6
+ import hashlib
7
+ import html
8
+ import json
9
+ import os
10
+ import re
11
+ import stat
12
+ from copy import deepcopy
13
+ from datetime import UTC, datetime
14
+ from pathlib import Path
15
+ from urllib.parse import urlsplit
16
+
17
+ from . import _aria_replay as aria
18
+ from . import _replay_container as boundary
19
+ from . import replay
20
+ from .report import _write_text
21
+
22
+ _LIMITS = {
23
+ "memory_bytes": 1073741824,
24
+ "cpus": 1,
25
+ "pids": 16,
26
+ "wall_seconds": 6,
27
+ "output_bytes": 1048576,
28
+ "stdin_bytes": 2097152,
29
+ "tmp_bytes": 16777216,
30
+ }
31
+ _PREDICTIONS = {
32
+ "historical_passed",
33
+ "historical_evaluated",
34
+ "exception_types",
35
+ "task_matched",
36
+ "task_mismatched",
37
+ "task_stdout_sha256",
38
+ "task_status",
39
+ }
40
+
41
+
42
+ def _require(condition, reason):
43
+ if not condition:
44
+ raise ValueError(reason)
45
+
46
+
47
+ def _sha(content):
48
+ return hashlib.sha256(content).hexdigest()
49
+
50
+
51
+ def _json(value):
52
+ return json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + "\n"
53
+
54
+
55
+ def _read(path):
56
+ path = Path(path)
57
+ _require(
58
+ not path.is_symlink()
59
+ and stat.S_ISREG(path.stat().st_mode)
60
+ and path.stat().st_size <= 16 * 1024 * 1024,
61
+ "investigation input must be a regular bounded file",
62
+ )
63
+ return path.read_bytes()
64
+
65
+
66
+ def _load(path):
67
+ value = replay._json_output(_read(path))
68
+ _require(type(value) is dict, "invalid investigation JSON object")
69
+ return value
70
+
71
+
72
+ def _save(directory, name, value):
73
+ _write_text(_json(value), directory / name)
74
+ # File publication is atomic; sync the directory before spending execution authority.
75
+ descriptor = os.open(directory, os.O_RDONLY)
76
+ try:
77
+ os.fsync(descriptor)
78
+ finally:
79
+ os.close(descriptor)
80
+
81
+
82
+ def _hash(value):
83
+ return type(value) is str and re.fullmatch(r"[a-f0-9]{64}", value) is not None
84
+
85
+
86
+ def _integer(value):
87
+ return type(value) is int and value >= 0
88
+
89
+
90
+ def _implementation():
91
+ return {
92
+ name: _sha(Path(replay.__file__).with_name(name).read_bytes())
93
+ for name in ("replay.py", "_aria_replay.py", "_replay_container.py")
94
+ }
95
+
96
+
97
+ def _sandbox(config):
98
+ endpoint = urlsplit(config.docker_host)
99
+ _require(
100
+ config.image == boundary.DEFAULT_IMAGE
101
+ and endpoint.scheme == "unix"
102
+ and not endpoint.netloc
103
+ and endpoint.path.startswith("/")
104
+ and not endpoint.query
105
+ and not endpoint.fragment
106
+ and "%" not in config.docker_host
107
+ and all(ord(char) >= 33 for char in config.docker_host),
108
+ "unsupported investigation sandbox",
109
+ )
110
+ return deepcopy(
111
+ {
112
+ "image": config.image,
113
+ "limits": _LIMITS,
114
+ "host_config": boundary._HOST_CONFIG,
115
+ "ulimits": boundary._ULIMITS,
116
+ "uid_gid": "65534:65534",
117
+ "cpu_limit_seconds": 3,
118
+ "python_version": "3.12.13",
119
+ }
120
+ )
121
+
122
+
123
+ def _extraction(response):
124
+ result = aria.historical_extract(response)
125
+ program = result.pop("program")
126
+ result["program_sha256"] = _sha(program.encode()) if program is not None else None
127
+ return program, result
128
+
129
+
130
+ def _validate_runtime(runtime):
131
+ _require(
132
+ type(runtime) is dict
133
+ and runtime.get("config_verified") is True
134
+ and runtime.get("cleanup_verified") is True
135
+ and runtime.get("error") is None,
136
+ "incomplete runtime or unverified cleanup",
137
+ )
138
+ status, exit_code = runtime.get("status"), runtime.get("exit_code")
139
+ _require(
140
+ status in {"completed", "nonzero_exit", "timeout", "output_limit"},
141
+ "incompatible runtime status",
142
+ )
143
+ if status in {"completed", "nonzero_exit"}:
144
+ _require(
145
+ type(exit_code) is int and (exit_code == 0) == (status == "completed"),
146
+ "inconsistent runtime exit status",
147
+ )
148
+ metadata = runtime.get("metadata", {})
149
+ _require(type(metadata) is dict, "invalid runtime metadata")
150
+ _require(
151
+ metadata.get("image_id") == boundary.DEFAULT_IMAGE
152
+ and metadata.get("python_version") == "3.12.13"
153
+ and metadata.get("architecture") in {"arm64", "amd64"}
154
+ and metadata.get("runtime") == "runc"
155
+ and metadata.get("cgroup_version") == "2"
156
+ and metadata.get("limits") == _LIMITS
157
+ and _hash(metadata.get("container_id")),
158
+ "incompatible runtime configuration",
159
+ )
160
+ for stream in ("stdout", "stderr"):
161
+ _require(
162
+ _hash(runtime.get(stream + "_sha256"))
163
+ and _integer(runtime.get(stream + "_bytes")),
164
+ "missing stream evidence",
165
+ )
166
+ _require(
167
+ runtime["stdout_bytes"] + runtime["stderr_bytes"] <= _LIMITS["output_bytes"],
168
+ "invalid stream sizes",
169
+ )
170
+
171
+
172
+ def _validate_architecture(modes, expected=None):
173
+ for mode in modes:
174
+ architecture = mode.get("runtime", {}).get("metadata", {}).get("architecture")
175
+ if architecture is not None:
176
+ expected = architecture if expected is None else expected
177
+ _require(
178
+ architecture == expected, "runtime architecture differs from control"
179
+ )
180
+
181
+
182
+ def _validate_outcomes(record, checks, *, partial=False, architecture=None):
183
+ _require(_integer(checks) and checks > 0, "invalid applicable task count")
184
+ historical, task = record["historical"], record["task"]
185
+ for mode in (historical, task):
186
+ if partial and "runtime" not in mode:
187
+ _require(
188
+ mode.get("status") == "unavailable" and type(mode.get("reason")) is str,
189
+ "missing partial mode status",
190
+ )
191
+ elif partial and (
192
+ mode["runtime"].get("cleanup_verified") is not True
193
+ or mode["runtime"].get("config_verified") is not True
194
+ or mode["runtime"].get("status") == "infrastructure_error"
195
+ ):
196
+ runtime = mode["runtime"]
197
+ _require(
198
+ runtime.get("status")
199
+ in {
200
+ "completed",
201
+ "nonzero_exit",
202
+ "timeout",
203
+ "output_limit",
204
+ "infrastructure_error",
205
+ }
206
+ and type(runtime.get("cleanup_verified")) is bool
207
+ and type(runtime.get("config_verified")) is bool
208
+ and all(
209
+ _hash(runtime.get(stream + "_sha256"))
210
+ and _integer(runtime.get(stream + "_bytes"))
211
+ for stream in ("stdout", "stderr")
212
+ ),
213
+ "invalid partial runtime",
214
+ )
215
+ else:
216
+ _validate_runtime(mode.get("runtime"))
217
+ _validate_architecture((historical, task), architecture)
218
+ if partial and "runtime" not in historical:
219
+ _require(
220
+ historical == {"status": "unavailable", "reason": "not_attempted"},
221
+ "invalid unattempted historical mode",
222
+ )
223
+ else:
224
+ _validate_historical(historical)
225
+ if not partial:
226
+ _require(
227
+ replay._historical_allows_protected(historical),
228
+ "completed assessment requires admissible historical evidence",
229
+ )
230
+ if partial and "runtime" not in task:
231
+ _require(
232
+ task
233
+ == {
234
+ "status": "unavailable",
235
+ "reason": "not_attempted",
236
+ "determinate": False,
237
+ },
238
+ "invalid unattempted task mode",
239
+ )
240
+ else:
241
+ _validate_task(task, checks)
242
+ identities = [
243
+ mode.get("runtime", {}).get("metadata", {}).get("container_id")
244
+ for mode in (historical, task)
245
+ ]
246
+ _require(
247
+ identities[0] is None
248
+ or identities[1] is None
249
+ or identities[0] != identities[1],
250
+ "evaluation modes require distinct containers",
251
+ )
252
+ _require(
253
+ record.get("mechanism_status") == "unresolved", "unsupported mechanism claim"
254
+ )
255
+
256
+
257
+ def _validate_historical(historical):
258
+ _require(
259
+ historical.get("authority") == "candidate_controlled_historical_report"
260
+ and historical.get("semantic_completion_authenticated") is False,
261
+ "historical claims must remain untrusted",
262
+ )
263
+ reason = historical.get("reason")
264
+ if reason in replay._HISTORICAL_UNAVAILABLE_FLAGS:
265
+ flags = historical.get("reported_flags")
266
+ expected = dict(
267
+ zip(
268
+ replay._FLAG_NAMES,
269
+ replay._HISTORICAL_UNAVAILABLE_FLAGS[reason],
270
+ strict=True,
271
+ )
272
+ )
273
+ counters = historical.get("counter_observations")
274
+ _require(
275
+ historical["status"] == "unavailable"
276
+ and historical["runtime"]["status"] == "completed"
277
+ and historical.get("reported_result") is None
278
+ and "rejected_report" not in historical
279
+ and type(flags) is dict
280
+ and set(flags) == set(expected)
281
+ and all(flags[name] is value for name, value in expected.items())
282
+ and type(counters) is dict
283
+ and set(counters) == set(replay._COUNTER_NAMES)
284
+ and all(
285
+ type(item) is dict
286
+ and set(item) == {"present", "value"}
287
+ and item["present"] is False
288
+ and item["value"] is None
289
+ for item in counters.values()
290
+ ),
291
+ "invalid unavailable historical evidence",
292
+ )
293
+ if reason == "reported_runtime_error":
294
+ raw = historical.get("reported_raw")
295
+ _require(
296
+ type(raw) is dict
297
+ and set(raw) == {"bytes", "sha256"}
298
+ and _integer(raw["bytes"])
299
+ and raw["bytes"] >= 2
300
+ and _hash(raw["sha256"]),
301
+ "invalid historical error text fingerprint",
302
+ )
303
+ else:
304
+ _require(
305
+ "reported_raw" not in historical, "unexpected historical error text"
306
+ )
307
+ if historical["status"] == "reported":
308
+ _require(
309
+ historical["runtime"]["status"] == "completed",
310
+ "claim from incomplete runtime",
311
+ )
312
+ claim = historical.get("reported_result")
313
+ _require(type(claim) is dict, "missing historical claim")
314
+ for key in ("success", "compiled", "timeout", "oom"):
315
+ _require(type(claim.get(key)) is bool, "invalid historical claim")
316
+ for key in ("tests_evaluated", "tests_passed"):
317
+ _require(
318
+ type(claim.get(key)) is int and claim[key] in (0, 1),
319
+ "invalid historical counters",
320
+ )
321
+ _require(
322
+ type(claim.get("exception_types")) is list
323
+ and all(
324
+ type(value) is str
325
+ and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", value)
326
+ for value in claim["exception_types"]
327
+ ),
328
+ "invalid exception evidence",
329
+ )
330
+ else:
331
+ _require(
332
+ historical["status"] in {"unavailable", "unresolved"}
333
+ and historical.get("reported_result") is None,
334
+ "invalid unavailable historical evidence",
335
+ )
336
+ if historical.get("reason") == "invalid_historical_counters":
337
+ counters = historical.get("counter_observations")
338
+ _require(
339
+ type(counters) is dict
340
+ and set(counters) == {"tests_evaluated", "tests_passed"},
341
+ "missing rejected counter observations",
342
+ )
343
+ for item in counters.values():
344
+ _require(
345
+ type(item) is dict
346
+ and set(item) == {"present", "value"}
347
+ and type(item["present"]) is bool
348
+ and (item["present"] or item["value"] is None),
349
+ "invalid rejected counter observation",
350
+ )
351
+ _require(
352
+ not all(
353
+ item["present"] and _integer(item["value"]) and item["value"] <= 1
354
+ for item in counters.values()
355
+ ),
356
+ "counter rejection lacks an invalid counter",
357
+ )
358
+
359
+
360
+ _historical_observation = replay._historical_report
361
+
362
+
363
+ def _validate_historical_stream(observation, content):
364
+ runtime = observation["runtime"]
365
+ _require(
366
+ _sha(content) == runtime["stdout_sha256"]
367
+ and len(content) == runtime["stdout_bytes"],
368
+ "historical stream differs from runtime evidence",
369
+ )
370
+ parsed = replay._historical_report(
371
+ boundary.ContainerResult(runtime["status"], stdout=content)
372
+ )
373
+ parsed["runtime"] = runtime
374
+ _require(
375
+ _json(parsed) == _json(observation),
376
+ "historical stream differs from observation",
377
+ )
378
+
379
+
380
+ def _validate_task(task, checks):
381
+ counts = [
382
+ task.get(key)
383
+ for key in (
384
+ "tests_applicable",
385
+ "tests_completed",
386
+ "tests_passed",
387
+ "tests_failed",
388
+ )
389
+ ]
390
+ _require(all(_integer(value) for value in counts), "missing task counts")
391
+ total, completed, passed, failed = counts
392
+ _require(
393
+ total == checks and passed + failed == completed <= total,
394
+ "task counts disagree with the task contract",
395
+ )
396
+ _require(
397
+ type(task.get("determinate")) is bool
398
+ and type(task.get("test_coverage_complete")) is bool
399
+ and task.get("comparison_authority") == "host",
400
+ "invalid task authority",
401
+ )
402
+ if task["determinate"]:
403
+ _require(
404
+ completed == total
405
+ and task["test_coverage_complete"]
406
+ and task["status"] == ("failed" if failed else "passed")
407
+ and task.get("reason") is None
408
+ and task["runtime"]["status"] == "completed",
409
+ "inconsistent determinate task evidence",
410
+ )
411
+ _require(
412
+ task["runtime"].get("config_verified") is True
413
+ and task["runtime"].get("cleanup_verified") is True
414
+ and task["runtime"].get("error") is None,
415
+ "determinate task requires verified runtime and cleanup",
416
+ )
417
+ else:
418
+ _require(
419
+ task["status"] == "unresolved"
420
+ and not task["test_coverage_complete"]
421
+ and type(task.get("reason")) is str,
422
+ "inconsistent unresolved task evidence",
423
+ )
424
+
425
+
426
+ def _inputs(control, source, artifacts):
427
+ try:
428
+ _require(
429
+ type(control.get("replay_schema_version")) is int
430
+ and control["replay_schema_version"] == 1,
431
+ "unsupported replay version",
432
+ )
433
+ identity = control["identity"]
434
+ _require(_integer(identity.get("step")), "invalid canonical step type")
435
+ row = replay._selected_source(
436
+ source,
437
+ identity["run"],
438
+ identity["sample_id"],
439
+ control["source"]["response_sha256"],
440
+ )
441
+ _require(
442
+ identity
443
+ == {key: row[key] for key in ("run", "sample_id", "step", "prompt_id")},
444
+ "canonical identity mismatch",
445
+ )
446
+ _require(
447
+ control["source"]["sha256"] == aria.ARIA_SHA256, "source hash mismatch"
448
+ )
449
+ _require(
450
+ control["implementation_sha256"] == _implementation(),
451
+ "replay implementation hash mismatch",
452
+ )
453
+ _require(
454
+ control["protected_worker_sha256"] == _sha(replay._ANSWER_WORKER.encode()),
455
+ "protected worker hash mismatch",
456
+ )
457
+ paths = aria.verify_artifacts(artifacts)
458
+ evaluator = control["evaluator"]
459
+ _require(
460
+ evaluator["commit"] == aria.UPSTREAM_REVISION
461
+ and evaluator["component"] == "hinted_test_call"
462
+ and evaluator["source_sha256"]
463
+ == {key: _sha(path.read_bytes()) for key, path in paths.items()},
464
+ "evaluator provenance mismatch",
465
+ )
466
+ task = aria.load_task(paths["fixture"], row["prompt_id"], row["prompt"])
467
+ cases = aria.protected_cases(task)
468
+ _require(
469
+ _integer(control["task_source"].get("checks")),
470
+ "invalid task check count type",
471
+ )
472
+ _require(
473
+ control["task_source"]
474
+ == {
475
+ "prompt_id": row["prompt_id"],
476
+ "task_sha256": _sha(json.dumps(task, sort_keys=True).encode()),
477
+ "checks": len(cases),
478
+ },
479
+ "task contract mismatch",
480
+ )
481
+ program, extraction = _extraction(row["response"])
482
+ _require(
483
+ program is not None and control["extraction"] == extraction,
484
+ "control extraction mismatch",
485
+ )
486
+ _require(
487
+ control["configuration"] == {"image": boundary.DEFAULT_IMAGE},
488
+ "incompatible control sandbox",
489
+ )
490
+ _validate_outcomes(control, len(cases))
491
+ except (KeyError, TypeError, AttributeError) as exc:
492
+ raise ValueError("incomplete or incompatible replay evidence") from exc
493
+ return row, task, cases, paths
494
+
495
+
496
+ def _predictions(values):
497
+ _require(
498
+ type(values) is dict and values and set(values) <= _PREDICTIONS,
499
+ "predictions must use supported outcome fields",
500
+ )
501
+ for name, value in values.items():
502
+ if name == "exception_types":
503
+ valid = type(value) is list and all(
504
+ type(v) is str and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", v)
505
+ for v in value
506
+ )
507
+ elif name == "task_stdout_sha256":
508
+ valid = _hash(value)
509
+ elif name == "task_status":
510
+ valid = value in ("passed", "failed", "unresolved", "unavailable")
511
+ else:
512
+ valid = _integer(value) and (
513
+ not name.startswith("historical_") or value <= 1
514
+ )
515
+ _require(valid, "invalid predicted outcome")
516
+
517
+
518
+ def _workers(response, task, paths):
519
+ program, extraction = _extraction(response)
520
+ _require(
521
+ program is not None and extraction["parse_status"] != "lookup_error",
522
+ "intervention requires an extractable historical program",
523
+ )
524
+ historical = aria.historical_script(
525
+ paths, task["setup_code"], program, extraction["requested_call"]
526
+ )
527
+ cases = aria.protected_cases(task)
528
+ protected = (
529
+ replay._ANSWER_WORKER,
530
+ json.dumps(
531
+ {
532
+ "setup": task["setup_code"],
533
+ "program": program,
534
+ "cases": [
535
+ {"method": c["method"], "kwargs": c["kwargs"]} for c in cases
536
+ ],
537
+ },
538
+ allow_nan=False,
539
+ ).encode(),
540
+ )
541
+ return [historical, protected], extraction
542
+
543
+
544
+ def prepare_aria_intervention(
545
+ control, source, *, artifacts, edit, hypothesis, predictions, directory, config=None
546
+ ):
547
+ """Freeze a control and one unique exact text replacement without execution.
548
+
549
+ Case files contain private derived text. Use a fresh local directory outside
550
+ source control. An operationally recorded unresolved outcome remains unresolved.
551
+ """
552
+ content = _read(control)
553
+ frozen = replay._json_output(content)
554
+ _require(type(frozen) is dict, "invalid investigation JSON object")
555
+ row, task, _cases, paths = _inputs(frozen, source, artifacts)
556
+ _require(
557
+ type(edit) is dict
558
+ and set(edit) == {"old", "new"}
559
+ and all(type(value) is str for value in edit.values())
560
+ and edit["old"]
561
+ and edit["old"] != edit["new"]
562
+ and row["response"].count(edit["old"]) == 1,
563
+ "edit must replace exactly one unique nonempty occurrence",
564
+ )
565
+ _require(
566
+ type(hypothesis) is str and hypothesis.strip() and len(hypothesis) <= 4096,
567
+ "hypothesis must be nonempty and at most 4096 characters",
568
+ )
569
+ _predictions(predictions)
570
+ config = config or boundary.SandboxConfig()
571
+ configuration = _sandbox(config)
572
+ variant = row["response"].replace(edit["old"], edit["new"], 1)
573
+ workers, extraction = _workers(variant, task, paths)
574
+ diff = "".join(
575
+ difflib.unified_diff(
576
+ row["response"].splitlines(keepends=True),
577
+ variant.splitlines(keepends=True),
578
+ fromfile="canonical-response",
579
+ tofile="derived-response",
580
+ )
581
+ )
582
+ manifest = {
583
+ "case_schema_version": 1,
584
+ "parent_identity": frozen["identity"],
585
+ "parent_response_sha256": frozen["source"]["response_sha256"],
586
+ "source_sha256": frozen["source"]["sha256"],
587
+ "control_result_sha256": _sha(content),
588
+ "variant_response_sha256": _sha(variant.encode()),
589
+ "source_diff_sha256": _sha(diff.encode()),
590
+ "variant_program_sha256": extraction["program_sha256"],
591
+ "extraction": extraction,
592
+ "hypothesis": hypothesis,
593
+ "predictions": predictions,
594
+ "edit": edit,
595
+ "evaluator": frozen["evaluator"],
596
+ "task_source": frozen["task_source"],
597
+ "implementation_sha256": _implementation(),
598
+ "workflow_sha256": _sha(Path(__file__).read_bytes()),
599
+ "configuration": configuration,
600
+ "docker_host": config.docker_host,
601
+ "worker_inputs": [
602
+ {"script_sha256": _sha(script.encode()), "payload_sha256": _sha(payload)}
603
+ for script, payload in workers
604
+ ],
605
+ }
606
+ manifest["case_id"] = "aria-" + _sha(_json(manifest).encode())
607
+ manifest["variant_id"] = manifest["case_id"] + "-variant"
608
+ directory = Path(directory)
609
+ directory.mkdir(mode=0o700, exist_ok=False)
610
+ _write_text(content.decode("utf-8"), directory / "control.json")
611
+ _write_text(variant, directory / "variant.txt")
612
+ _write_text(diff, directory / "source.diff")
613
+ _save(directory, "case.json", manifest)
614
+ return {**manifest, "case_sha256": _sha(_read(directory / "case.json"))}
615
+
616
+
617
+ def _case(directory):
618
+ directory = Path(directory).resolve(strict=True)
619
+ manifest = _load(directory / "case.json")
620
+ try:
621
+ _require(
622
+ type(manifest["case_schema_version"]) is int
623
+ and manifest["case_schema_version"] == 1,
624
+ "unsupported case version",
625
+ )
626
+ identity_fields = {
627
+ key: value
628
+ for key, value in manifest.items()
629
+ if key not in {"case_id", "variant_id"}
630
+ }
631
+ case_id = "aria-" + _sha(_json(identity_fields).encode())
632
+ _require(
633
+ manifest["case_id"] == case_id
634
+ and manifest["variant_id"] == case_id + "-variant",
635
+ "case identity hash mismatch",
636
+ )
637
+ for name, key in [
638
+ ("control.json", "control_result_sha256"),
639
+ ("variant.txt", "variant_response_sha256"),
640
+ ("source.diff", "source_diff_sha256"),
641
+ ]:
642
+ _require(
643
+ _sha(_read(directory / name)) == manifest[key],
644
+ "case file hash mismatch",
645
+ )
646
+ _require(
647
+ manifest["implementation_sha256"] == _implementation()
648
+ and manifest["workflow_sha256"] == _sha(Path(__file__).read_bytes()),
649
+ "case implementation hash mismatch",
650
+ )
651
+ _require(
652
+ manifest["configuration"]
653
+ == _sandbox(boundary.SandboxConfig(docker_host=manifest["docker_host"])),
654
+ "case sandbox mismatch",
655
+ )
656
+ _predictions(manifest["predictions"])
657
+ control = _load(directory / "control.json")
658
+ for name, expected in {
659
+ "parent_identity": control["identity"],
660
+ "parent_response_sha256": control["source"]["response_sha256"],
661
+ "source_sha256": control["source"]["sha256"],
662
+ "evaluator": control["evaluator"],
663
+ "task_source": control["task_source"],
664
+ "implementation_sha256": control["implementation_sha256"],
665
+ }.items():
666
+ _require(
667
+ manifest[name] == expected,
668
+ "case metadata disagrees with frozen control",
669
+ )
670
+ _validate_outcomes(control, manifest["task_source"]["checks"])
671
+ except (KeyError, TypeError, AttributeError) as exc:
672
+ raise ValueError("incomplete case manifest") from exc
673
+ return directory, manifest, _sha(_read(directory / "case.json"))
674
+
675
+
676
+ def approve_aria_intervention(
677
+ directory,
678
+ *,
679
+ case_sha256,
680
+ variant_sha256,
681
+ max_containers,
682
+ execution_authorized=False,
683
+ ):
684
+ """Write one path-bound approval; approval alone never starts a container."""
685
+ _require(execution_authorized is True, "explicit execution authorization required")
686
+ directory, manifest, actual = _case(directory)
687
+ _require(not (directory / "consumed.json").exists(), "authorization is spent")
688
+ _require(
689
+ case_sha256 == actual and variant_sha256 == manifest["variant_response_sha256"],
690
+ "approval hash mismatch",
691
+ )
692
+ _require(
693
+ type(max_containers) is int and max_containers == 2,
694
+ "this workflow requires a budget of exactly two evaluation containers",
695
+ )
696
+ approval = {
697
+ "approval_schema_version": 1,
698
+ "case_sha256": actual,
699
+ "variant_response_sha256": variant_sha256,
700
+ "case_directory": str(directory),
701
+ "max_containers": max_containers,
702
+ "execution_authorized": True,
703
+ }
704
+ _save(directory, "approval.json", approval)
705
+ return approval
706
+
707
+
708
+ def _comparison(manifest, control, variant, *, status, origin):
709
+ claim = variant.get("historical", {}).get("reported_result") or {}
710
+ task = variant.get("task", {})
711
+ actual = {
712
+ "historical_passed": claim.get("tests_passed"),
713
+ "historical_evaluated": claim.get("tests_evaluated"),
714
+ "exception_types": claim.get("exception_types"),
715
+ "task_matched": task.get("tests_passed"),
716
+ "task_mismatched": task.get("tests_failed"),
717
+ "task_status": task.get("status"),
718
+ "task_stdout_sha256": task.get("runtime", {}).get("stdout_sha256"),
719
+ }
720
+ history_runtime = variant.get("historical", {}).get("runtime", {})
721
+ history_available = (
722
+ variant.get("historical", {}).get("status") == "reported"
723
+ and history_runtime.get("status") == "completed"
724
+ and history_runtime.get("config_verified") is True
725
+ and history_runtime.get("cleanup_verified") is True
726
+ )
727
+ checks = [
728
+ {
729
+ "field": name,
730
+ "expected": expected,
731
+ "observed": actual[name],
732
+ "status": "unavailable"
733
+ if actual[name] is None
734
+ or (
735
+ name in {"historical_passed", "historical_evaluated", "exception_types"}
736
+ and not history_available
737
+ )
738
+ or (
739
+ name in {"task_matched", "task_mismatched", "task_stdout_sha256"}
740
+ and not task.get("test_coverage_complete", False)
741
+ )
742
+ else "matched"
743
+ if type(expected) is type(actual[name]) and expected == actual[name]
744
+ else "violated",
745
+ }
746
+ for name, expected in sorted(manifest["predictions"].items())
747
+ ]
748
+ a = control["task"]["runtime"]["stdout_sha256"]
749
+ b = actual["task_stdout_sha256"]
750
+ return {
751
+ "comparison_schema_version": 1,
752
+ "case_id": manifest["case_id"],
753
+ "variant_id": manifest["variant_id"],
754
+ "parent_identity": manifest["parent_identity"],
755
+ "hypothesis": manifest["hypothesis"],
756
+ "status": status,
757
+ "origin": origin,
758
+ "predictions": checks,
759
+ "control": {name: control[name] for name in ("historical", "task")},
760
+ "variant": {name: variant[name] for name in ("historical", "task")},
761
+ "answer_stream_equal": a == b
762
+ if b is not None
763
+ and control["task"]["test_coverage_complete"]
764
+ and task.get("test_coverage_complete")
765
+ else None,
766
+ "mechanism_status": "unresolved",
767
+ "historical_claims_authenticated": False,
768
+ "interpretation": "Predictions are compared literally with recorded observations; incomplete comparisons remain unavailable. Historical counters remain candidate-controlled. No internal call trace or reward-hacking mechanism is authenticated.",
769
+ "control_result_sha256": manifest["control_result_sha256"],
770
+ "variant_response_sha256": manifest["variant_response_sha256"],
771
+ "provenance": {
772
+ key: manifest[key]
773
+ for key in (
774
+ "source_sha256",
775
+ "parent_response_sha256",
776
+ "variant_program_sha256",
777
+ "evaluator",
778
+ "task_source",
779
+ "configuration",
780
+ )
781
+ },
782
+ }
783
+
784
+
785
+ def compare_aria_intervention(directory, result, *, provenance=None):
786
+ """Compare saved evidence without execution, optionally importing a pinned protocol.
787
+
788
+ External derived-replay records require their original hash-bound protocol.
789
+ Imported evidence confers no new execution authorization or authenticity.
790
+ """
791
+ directory, manifest, case_sha = _case(directory)
792
+ variant = _load(result)
793
+ control = _load(directory / "control.json")
794
+ try:
795
+ _require(
796
+ variant["variant_response_sha256"] == manifest["variant_response_sha256"],
797
+ "variant hash mismatch",
798
+ )
799
+ if provenance is None:
800
+ _require(
801
+ variant["case_sha256"] == case_sha
802
+ and variant["case_id"] == manifest["case_id"],
803
+ "result case hash mismatch",
804
+ )
805
+ _require(
806
+ type(variant.get("investigation_result_version")) is int
807
+ and variant["investigation_result_version"] == 1,
808
+ "unsupported investigation result",
809
+ )
810
+ status = variant["status"]
811
+ else:
812
+ protocol = _load(provenance)
813
+ _require(
814
+ type(variant.get("derived_replay_schema_version")) is int
815
+ and variant["derived_replay_schema_version"] == 1,
816
+ "unsupported derived replay evidence",
817
+ )
818
+ _require(
819
+ variant["protocol_sha256"] == _sha(_read(provenance)),
820
+ "provenance hash mismatch",
821
+ )
822
+ for key in (
823
+ "control_result_sha256",
824
+ "parent_identity",
825
+ "parent_response_sha256",
826
+ "source_sha256",
827
+ "variant_response_sha256",
828
+ "variant_program_sha256",
829
+ "implementation_sha256",
830
+ "evaluator",
831
+ "task_source",
832
+ "configuration",
833
+ "worker_inputs",
834
+ ):
835
+ _require(
836
+ protocol[key] == manifest[key],
837
+ "incompatible saved intervention provenance",
838
+ )
839
+ _require(
840
+ variant["parent_identity"] == manifest["parent_identity"],
841
+ "derived parent mismatch",
842
+ )
843
+ status = "complete"
844
+ _require(
845
+ variant["extraction"] == manifest["extraction"],
846
+ "derived extraction mismatch",
847
+ )
848
+ _require(status in {"complete", "stopped"}, "invalid attempt status")
849
+ _validate_outcomes(
850
+ variant,
851
+ manifest["task_source"]["checks"],
852
+ partial=status == "stopped",
853
+ architecture=control["historical"]["runtime"]["metadata"]["architecture"],
854
+ )
855
+ if provenance is None and "runtime" in variant["historical"]:
856
+ stream_path = directory / "mode-1.stdout.hex"
857
+ if status == "complete" or stream_path.exists() or stream_path.is_symlink():
858
+ _require(stream_path.is_file(), "historical stream unavailable")
859
+ _validate_historical_stream(
860
+ variant["historical"],
861
+ bytes.fromhex(_read(stream_path).decode("ascii")),
862
+ )
863
+ except (KeyError, TypeError, AttributeError) as exc:
864
+ raise ValueError("incomplete or incompatible derived evidence") from exc
865
+ return _comparison(
866
+ manifest, control, variant, status=status, origin="saved_evidence"
867
+ )
868
+
869
+
870
+ def execute_aria_intervention(
871
+ directory, source, *, artifacts, execution_authorized=False
872
+ ):
873
+ """Consume one approval durably, then attempt each mode at most once.
874
+
875
+ Failures stay spent. Local filesystem ownership is trusted: deleting records
876
+ or bypassing this API is outside the approval mechanism's threat model.
877
+ """
878
+ _require(execution_authorized is True, "explicit execution authorization required")
879
+ directory, manifest, case_sha = _case(directory)
880
+ _require(not (directory / "consumed.json").exists(), "authorization is spent")
881
+ approval = _load(directory / "approval.json")
882
+ _require(
883
+ type(approval.get("approval_schema_version")) is int
884
+ and type(approval.get("max_containers")) is int
885
+ and approval.get("execution_authorized") is True,
886
+ "invalid approval field types",
887
+ )
888
+ _require(
889
+ approval
890
+ == {
891
+ "approval_schema_version": 1,
892
+ "case_sha256": case_sha,
893
+ "variant_response_sha256": manifest["variant_response_sha256"],
894
+ "case_directory": str(directory),
895
+ "max_containers": 2,
896
+ "execution_authorized": True,
897
+ },
898
+ "approval hashes, directory, or budget do not match this case",
899
+ )
900
+ control = _load(directory / "control.json")
901
+ row, task, cases, paths = _inputs(control, source, artifacts)
902
+ response = _read(directory / "variant.txt").decode("utf-8")
903
+ _require(
904
+ response
905
+ == row["response"].replace(manifest["edit"]["old"], manifest["edit"]["new"], 1),
906
+ "canonical edit mismatch",
907
+ )
908
+ workers, extraction = _workers(response, task, paths)
909
+ _require(
910
+ extraction == manifest["extraction"]
911
+ and manifest["worker_inputs"]
912
+ == [
913
+ {"script_sha256": _sha(script.encode()), "payload_sha256": _sha(payload)}
914
+ for script, payload in workers
915
+ ],
916
+ "worker input hash mismatch",
917
+ )
918
+ for name in (
919
+ "result.json",
920
+ "comparison.json",
921
+ "comparison.html",
922
+ "mode-1.json",
923
+ "mode-1.stdout.hex",
924
+ "mode-2.json",
925
+ ):
926
+ _require(not (directory / name).exists(), "execution output already exists")
927
+ claim = {
928
+ "approval_sha256": _sha(_read(directory / "approval.json")),
929
+ "case_sha256": case_sha,
930
+ "reserved_containers": 2,
931
+ "consumed_at": datetime.now(UTC).isoformat(),
932
+ "status": "spent",
933
+ }
934
+ _save(directory, "consumed.json", claim)
935
+ result = {
936
+ "investigation_result_version": 1,
937
+ "case_id": manifest["case_id"],
938
+ "case_sha256": case_sha,
939
+ "variant_id": manifest["variant_id"],
940
+ "variant_response_sha256": manifest["variant_response_sha256"],
941
+ "extraction": extraction,
942
+ "status": "stopped",
943
+ "mechanism_status": "unresolved",
944
+ "historical": {"status": "unavailable", "reason": "not_attempted"},
945
+ "task": {
946
+ "status": "unavailable",
947
+ "reason": "not_attempted",
948
+ "determinate": False,
949
+ },
950
+ }
951
+ config = boundary.SandboxConfig(docker_host=manifest["docker_host"])
952
+ interruption = None
953
+ try:
954
+ for index, (script, payload) in enumerate(workers):
955
+ _require(index < approval["max_containers"], "container budget exhausted")
956
+ _require(
957
+ _read(directory / "consumed.json") == _json(claim).encode()
958
+ and _sha(_read(directory / "approval.json"))
959
+ == claim["approval_sha256"],
960
+ "execution authorization accounting changed",
961
+ )
962
+ if index == 1:
963
+ _require(
964
+ _read(directory / "mode-1-started.json")
965
+ == _json(manifest["worker_inputs"][0]).encode()
966
+ and _read(directory / "mode-1.json")
967
+ == _json(result["historical"]).encode(),
968
+ "historical mode accounting changed",
969
+ )
970
+ _validate_historical_stream(
971
+ result["historical"],
972
+ bytes.fromhex(
973
+ _read(directory / "mode-1.stdout.hex").decode("ascii")
974
+ ),
975
+ )
976
+ _save(
977
+ directory,
978
+ f"mode-{index + 1}-started.json",
979
+ manifest["worker_inputs"][index],
980
+ )
981
+ raw = None
982
+ try:
983
+ raw = boundary.run_container(
984
+ script, payload, config, execution_authorized=True
985
+ )
986
+ except BaseException as error:
987
+ raw = getattr(error, "replay_result", None)
988
+ if raw is None:
989
+ raise
990
+ interruption = error
991
+ if index == 0:
992
+ observation = _historical_observation(raw)
993
+ else:
994
+ observation = replay.assess_answers(
995
+ replay._json_output(raw.stdout)
996
+ if raw.status == "completed"
997
+ else None,
998
+ cases,
999
+ )
1000
+ if raw.status != "completed":
1001
+ observation["reason"] = raw.status
1002
+ observation["runtime"] = replay._runtime(raw)
1003
+ if not raw.config_verified or not raw.cleanup_verified:
1004
+ observation.update(
1005
+ status="unresolved",
1006
+ determinate=False,
1007
+ test_coverage_complete=False,
1008
+ reason="unverified_runtime_or_cleanup",
1009
+ )
1010
+ observation["contract"] = (
1011
+ "all original checks; fresh Solution per call; host equality"
1012
+ )
1013
+ result["historical" if index == 0 else "task"] = observation
1014
+ if index == 0:
1015
+ _write_text(raw.stdout.hex(), directory / "mode-1.stdout.hex")
1016
+ _save(directory, f"mode-{index + 1}.json", observation)
1017
+ if interruption is not None:
1018
+ raise interruption
1019
+ _validate_runtime(observation["runtime"])
1020
+ _validate_architecture(
1021
+ (observation,),
1022
+ control["historical"]["runtime"]["metadata"]["architecture"],
1023
+ )
1024
+ if index == 0:
1025
+ _validate_historical(observation)
1026
+ _require(
1027
+ replay._historical_allows_protected(observation),
1028
+ observation.get("reason", "historical protocol unresolved"),
1029
+ )
1030
+ _validate_outcomes(result, len(cases))
1031
+ result["status"] = "complete"
1032
+ except BaseException as error: # noqa: BLE001 - preserve partial evidence and spent approval
1033
+ result["stop_reason"] = (
1034
+ str(error)
1035
+ if isinstance(error, ValueError)
1036
+ else "execution interrupted or infrastructure unavailable"
1037
+ )
1038
+ result["exception_type"] = type(error).__name__
1039
+ if isinstance(error, (KeyboardInterrupt, SystemExit)):
1040
+ interruption = error
1041
+ _save(directory, "result.json", result)
1042
+ comparison = _comparison(
1043
+ manifest,
1044
+ control,
1045
+ result,
1046
+ status=result["status"],
1047
+ origin="authorized_execution",
1048
+ )
1049
+ _save(directory, "comparison.json", comparison)
1050
+ _write_text(render_aria_comparison(comparison), directory / "comparison.html")
1051
+ if interruption is not None:
1052
+ raise interruption
1053
+ return comparison
1054
+
1055
+
1056
+ def render_aria_comparison(comparison):
1057
+ """Render claims and host comparisons without embedding response text."""
1058
+ _require(
1059
+ type(comparison) is dict and comparison.get("comparison_schema_version") == 1,
1060
+ "unsupported Aria comparison version",
1061
+ )
1062
+
1063
+ def escaped(value):
1064
+ return html.escape("Unavailable" if value is None else str(value))
1065
+
1066
+ rows = []
1067
+ for label, mode, field in [
1068
+ ("Historical claimed passes", "historical", "tests_passed"),
1069
+ ("Reported exception types", "historical", "exception_types"),
1070
+ ("Protected status", "task", "status"),
1071
+ ("Protected matched", "task", "tests_passed"),
1072
+ ("Protected mismatched", "task", "tests_failed"),
1073
+ ("Protected completed", "task", "tests_completed"),
1074
+ ("Protected applicable", "task", "tests_applicable"),
1075
+ ("Protected unavailable reason", "task", "reason"),
1076
+ ]:
1077
+ values = []
1078
+ for condition in ("control", "variant"):
1079
+ item = comparison[condition][mode]
1080
+ value = (
1081
+ (item.get("reported_result") or {}).get(field)
1082
+ if mode == "historical"
1083
+ else item.get(field)
1084
+ )
1085
+ if field == "reason" and item.get("determinate") is True and value is None:
1086
+ value = "Not applicable"
1087
+ values.append(value)
1088
+ rows.append(
1089
+ "<tr><th>"
1090
+ + escaped(label)
1091
+ + "</th>"
1092
+ + "".join("<td>" + escaped(v) + "</td>" for v in values)
1093
+ + "</tr>"
1094
+ )
1095
+ for mode in ("historical", "task"):
1096
+ for field in ("status", "stdout_sha256", "cleanup_verified"):
1097
+ values = [
1098
+ comparison[condition][mode].get("runtime", {}).get(field)
1099
+ for condition in ("control", "variant")
1100
+ ]
1101
+ rows.append(
1102
+ "<tr><th>"
1103
+ + escaped(mode + " runtime " + field)
1104
+ + "</th>"
1105
+ + "".join("<td>" + escaped(value) + "</td>" for value in values)
1106
+ + "</tr>"
1107
+ )
1108
+ predictions = "".join(
1109
+ "<tr>"
1110
+ + "".join(
1111
+ "<td>" + escaped(row[key]) + "</td>"
1112
+ for key in ("field", "expected", "observed", "status")
1113
+ )
1114
+ + "</tr>"
1115
+ for row in comparison["predictions"]
1116
+ )
1117
+ return (
1118
+ '<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">'
1119
+ "<title>Aria intervention comparison</title><style>*{box-sizing:border-box}"
1120
+ "body{font:16px/1.5 system-ui;max-width:1000px;margin:32px auto;padding:0 20px;color:#182128;overflow-wrap:anywhere}"
1121
+ ".table-scroll{max-width:100%;overflow-x:auto}"
1122
+ "table{border-collapse:collapse;width:100%;min-width:600px}td,th{text-align:left;min-width:100px;padding:8px;border-bottom:1px solid #ddd;overflow-wrap:anywhere}"
1123
+ "th:first-child,td:first-child{min-width:180px}"
1124
+ "code{overflow-wrap:anywhere}</style><h1>Aria intervention comparison</h1><p>Case: "
1125
+ + escaped(comparison["case_id"])
1126
+ + "</p>"
1127
+ "<p>Canonical parent: " + escaped(comparison["parent_identity"]) + "</p>"
1128
+ "<p>Status: "
1129
+ + escaped(comparison["status"])
1130
+ + "; evidence: "
1131
+ + escaped(comparison["origin"])
1132
+ + "</p>"
1133
+ "<p>Hypothesis: "
1134
+ + escaped(comparison["hypothesis"])
1135
+ + '</p><div class="table-scroll" tabindex="0" role="region" aria-label="Measurements"><table><tr><th>Measurement</th><th>Frozen control</th><th>Derived variant</th></tr>'
1136
+ + "".join(rows)
1137
+ + '</table></div><h2>Predictions</h2><div class="table-scroll" tabindex="0" role="region" aria-label="Predictions"><table><tr><th>Field</th><th>Expected</th><th>Observed</th><th>Result</th></tr>'
1138
+ + predictions
1139
+ + "</table></div>"
1140
+ "<p>Protected answer-stream hashes equal: "
1141
+ + escaped(comparison["answer_stream_equal"])
1142
+ + "</p>"
1143
+ "<p>"
1144
+ + escaped(comparison["interpretation"])
1145
+ + "</p><p>Mechanism status: unresolved. The control was not rerun.</p></html>"
1146
+ )