rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/replay.py ADDED
@@ -0,0 +1,589 @@
1
+ """Explicit Aria replay with historical reports and external task comparisons."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import io
7
+ import json
8
+ import math
9
+ import re
10
+ from dataclasses import asdict
11
+ from pathlib import Path
12
+
13
+ import pandas as pd
14
+
15
+ from ._aria_replay import (
16
+ ARIA_SHA256,
17
+ historical_extract,
18
+ historical_script,
19
+ load_task,
20
+ protected_cases,
21
+ verify_artifacts,
22
+ )
23
+ from ._replay_container import SandboxConfig, run_container
24
+
25
+ EVALUATOR_COMMIT = "73695ff5533b566f7cc99b02bfeb9168936e740d"
26
+
27
+ # Only candidate code and public inputs enter this interpreter. No expected values.
28
+ _ANSWER_WORKER = r"""
29
+ import contextlib
30
+ import json
31
+ import math
32
+ import sys
33
+
34
+ def encode(value, depth=0):
35
+ if depth > 64:
36
+ raise ValueError("unsupported result depth")
37
+ kind = type(value)
38
+ if value is None:
39
+ return ["none", None]
40
+ if kind is bool:
41
+ return ["bool", value]
42
+ if kind is int:
43
+ return ["int", value]
44
+ if kind is str:
45
+ return ["str", value]
46
+ if kind is float:
47
+ return ["float", value if math.isfinite(value) else
48
+ ("nan" if math.isnan(value) else "inf" if value > 0 else "-inf")]
49
+ if kind is list:
50
+ return ["list", [encode(item, depth + 1) for item in value]]
51
+ raise ValueError("unsupported result type")
52
+
53
+ data = json.loads(sys.stdin.buffer.read())
54
+ scope = {}
55
+ results = []
56
+ try:
57
+ with contextlib.redirect_stdout(sys.stderr):
58
+ exec(data["setup"] + "\n" + data["program"], scope)
59
+ except BaseException as error:
60
+ result = {"setup_exception": type(error).__name__}
61
+ else:
62
+ for case in data["cases"]:
63
+ try:
64
+ with contextlib.redirect_stdout(sys.stderr):
65
+ value = getattr(scope["Solution"](), case["method"])(**case["kwargs"])
66
+ results.append({"value": encode(value)})
67
+ except BaseException as error:
68
+ results.append({"exception": type(error).__name__})
69
+ result = {"results": results}
70
+ sys.stdout.write(json.dumps(result, allow_nan=False))
71
+ """
72
+
73
+
74
+ def _sha(value: bytes) -> str:
75
+ return hashlib.sha256(value).hexdigest()
76
+
77
+
78
+ def _decode(value, depth=0):
79
+ if depth > 64 or type(value) is not list or len(value) != 2:
80
+ raise ValueError("invalid candidate answer encoding")
81
+ tag, item = value
82
+ if type(tag) is not str:
83
+ raise ValueError("invalid candidate answer tag")
84
+ if tag == "none" and item is None:
85
+ return None
86
+ if tag in {"bool", "int", "str"}:
87
+ expected = {"bool": bool, "int": int, "str": str}[tag]
88
+ if type(item) is expected:
89
+ return item
90
+ if tag == "float":
91
+ if type(item) is float and math.isfinite(item):
92
+ return item
93
+ if type(item) is str and item in {"inf", "-inf", "nan"}:
94
+ return float(item)
95
+ if tag == "list" and type(item) is list:
96
+ return [_decode(child, depth + 1) for child in item]
97
+ raise ValueError("invalid candidate answer encoding")
98
+
99
+
100
+ def _exception(value):
101
+ return (
102
+ value
103
+ if isinstance(value, str)
104
+ and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", value)
105
+ else "UnrecognizedException"
106
+ )
107
+
108
+
109
+ def assess_answers(payload: object, cases: list[dict]) -> dict:
110
+ """Compare primitive proposed answers on the host; worker verdicts have no authority."""
111
+ result = {
112
+ "status": "unresolved",
113
+ "reason": "invalid_answer_protocol",
114
+ "determinate": False,
115
+ "tests_applicable": len(cases),
116
+ "tests_completed": 0,
117
+ "tests_passed": 0,
118
+ "tests_failed": 0,
119
+ "reported_exceptions": [],
120
+ "comparison_authority": "host",
121
+ "test_coverage_complete": False,
122
+ }
123
+ if not cases or type(payload) is not dict:
124
+ return result
125
+ if set(payload) == {"setup_exception"}:
126
+ result["reason"] = "candidate_reported_setup_failure"
127
+ result["reported_exceptions"] = [_exception(payload["setup_exception"])]
128
+ return result
129
+ if set(payload) != {"results"} or type(payload["results"]) is not list:
130
+ return result
131
+ exact_count = len(payload["results"]) == len(cases)
132
+ for proposed, case in zip(payload["results"], cases, strict=False):
133
+ if type(proposed) is not dict:
134
+ continue
135
+ if set(proposed) == {"exception"}:
136
+ result["reported_exceptions"].append(_exception(proposed["exception"]))
137
+ elif set(proposed) == {"value"}:
138
+ try:
139
+ observed = _decode(proposed["value"])
140
+ passed = observed == case["expected"]
141
+ except (ValueError, RecursionError):
142
+ continue
143
+ result["tests_completed"] += 1
144
+ result["tests_passed" if passed else "tests_failed"] += 1
145
+ complete = exact_count and result["tests_completed"] == len(cases)
146
+ result["test_coverage_complete"] = complete
147
+ if not complete:
148
+ result["reason"] = (
149
+ "incomplete_or_unsupported_answers"
150
+ if exact_count
151
+ else "incomplete_or_extra_answers"
152
+ )
153
+ else:
154
+ result.update(
155
+ status="failed" if result["tests_failed"] else "passed",
156
+ determinate=True,
157
+ reason=None,
158
+ )
159
+ return result
160
+
161
+
162
+ def _json_output(content):
163
+ def invalid_constant(_):
164
+ raise ValueError("nonfinite JSON constant")
165
+
166
+ def unique_members(pairs):
167
+ result = {}
168
+ for key, value in pairs:
169
+ if key in result:
170
+ raise ValueError("duplicate JSON member")
171
+ result[key] = value
172
+ return result
173
+
174
+ try:
175
+ return json.loads(
176
+ content, parse_constant=invalid_constant, object_pairs_hook=unique_members
177
+ )
178
+ except (ValueError, UnicodeError, RecursionError):
179
+ return None
180
+
181
+
182
+ def _runtime(result):
183
+ fields = asdict(result)
184
+ for stream in ("stdout", "stderr"):
185
+ content = fields.pop(stream)
186
+ fields[stream + "_sha256"] = _sha(content)
187
+ fields[stream + "_bytes"] = len(content)
188
+ fields["peak_memory_bytes"] = None
189
+ fields["memory_measurement_status"] = (
190
+ "unavailable; daemon OOM flag and limit recorded"
191
+ )
192
+ return fields
193
+
194
+
195
+ _FLAG_NAMES = ("success", "compiled", "timeout", "oom")
196
+ _COUNTER_NAMES = ("tests_evaluated", "tests_passed")
197
+ _HISTORICAL_UNAVAILABLE_FLAGS = {
198
+ "reported_compile_failure": (False, False, False, False),
199
+ "reported_inner_timeout": (False, True, True, False),
200
+ "reported_inner_oom": (False, True, False, True),
201
+ "reported_runtime_error": (False, True, False, False),
202
+ }
203
+
204
+
205
+ def _counter_diagnostic(value):
206
+ # Leave headroom to serialize the diagnostic inside a complete case record.
207
+ nodes = 1
208
+ pending = [(value, 0)]
209
+ while pending:
210
+ item, depth = pending.pop()
211
+ if isinstance(item, (list, dict)):
212
+ if depth >= 64:
213
+ return {
214
+ "invalid_value": "nesting_limit",
215
+ "value_type": type(value).__name__,
216
+ }
217
+ children = item.values() if isinstance(item, dict) else item
218
+ nodes += len(item)
219
+ if nodes > 256:
220
+ return {
221
+ "invalid_value": "size_limit",
222
+ "value_type": type(value).__name__,
223
+ }
224
+ pending.extend((child, depth + 1) for child in children)
225
+ try:
226
+ if len(json.dumps(value, allow_nan=False)) > 4096:
227
+ return {"invalid_value": "size_limit", "value_type": type(value).__name__}
228
+ except ValueError:
229
+ diagnostic = {
230
+ "invalid_value": "non_finite_number",
231
+ "value_type": type(value).__name__,
232
+ }
233
+ if type(value) is float:
234
+ diagnostic["representation"] = repr(value)
235
+ return diagnostic
236
+ except RecursionError:
237
+ return {"invalid_value": "nesting_limit", "value_type": type(value).__name__}
238
+ return value
239
+
240
+
241
+ def _historical_allows_protected(observation):
242
+ """Apply the protocol gate after host-side runtime checks."""
243
+ if observation["runtime"]["status"] != "completed":
244
+ return (
245
+ observation["runtime"]["status"]
246
+ in {"nonzero_exit", "timeout", "output_limit"}
247
+ and observation["status"] == "unresolved"
248
+ )
249
+ if observation["status"] == "reported":
250
+ claim = observation.get("reported_result")
251
+ if type(claim) is not dict:
252
+ return False
253
+ passed, errors = claim.get("tests_passed"), claim.get("exception_types")
254
+ return (
255
+ type(claim.get("success")) is bool
256
+ and claim.get("compiled") is True
257
+ and claim.get("timeout") is False
258
+ and claim.get("oom") is False
259
+ and type(claim.get("tests_evaluated")) is int
260
+ and claim["tests_evaluated"] == 1
261
+ and type(passed) is int
262
+ and passed in (0, 1)
263
+ and type(errors) is list
264
+ and len(errors) == 1 - passed
265
+ and all(
266
+ type(e) is str and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", e)
267
+ for e in errors
268
+ )
269
+ )
270
+ return (
271
+ observation["status"] == "unavailable"
272
+ and type(observation.get("reason")) is str
273
+ and observation["reason"] in _HISTORICAL_UNAVAILABLE_FLAGS
274
+ )
275
+
276
+
277
+ def _historical_report(result):
278
+ record = {
279
+ "status": "unresolved",
280
+ "authority": "candidate_controlled_historical_report",
281
+ "semantic_completion_authenticated": False,
282
+ "trace_status": "reported exception types only; authenticated stack unavailable",
283
+ "reported_result": None,
284
+ "runtime": _runtime(result),
285
+ }
286
+ if result.status != "completed":
287
+ return {**record, "reason": result.status}
288
+ payload = _json_output(result.stdout)
289
+ if type(payload) is not dict:
290
+ return {**record, "reason": "invalid_historical_result_envelope"}
291
+ if (
292
+ set(payload) == {"historical_unavailable", "exception_type"}
293
+ and payload["historical_unavailable"]
294
+ == "unsupported_result_validation_or_supervisor_error"
295
+ and type(payload["exception_type"]) is str
296
+ and payload["exception_type"].isidentifier()
297
+ ):
298
+ return {
299
+ **record,
300
+ "status": "unavailable",
301
+ "reason": "unsupported_historical_result",
302
+ }
303
+ if (
304
+ set(payload) != {*_FLAG_NAMES, "stdout"}
305
+ or any(type(payload[key]) is not bool for key in _FLAG_NAMES)
306
+ or type(payload["stdout"]) is not dict
307
+ ):
308
+ return {**record, "reason": "invalid_historical_result_envelope"}
309
+ counts = payload["stdout"]
310
+ flags = tuple(payload[k] for k in _FLAG_NAMES)
311
+ is_raw_text = set(counts) == {"raw"} and type(counts["raw"]) is str
312
+ reason = None
313
+ if is_raw_text:
314
+ if flags == _HISTORICAL_UNAVAILABLE_FLAGS["reported_runtime_error"]:
315
+ reason = "reported_runtime_error"
316
+ elif flags == _HISTORICAL_UNAVAILABLE_FLAGS["reported_compile_failure"]:
317
+ # The parent uses this shape for both OSError and invalid child JSON.
318
+ reason = "reported_child_output_or_setup_failure"
319
+ elif not counts:
320
+ reason = next(
321
+ (
322
+ name
323
+ for name, expected in _HISTORICAL_UNAVAILABLE_FLAGS.items()
324
+ if name != "reported_runtime_error" and flags == expected
325
+ ),
326
+ None,
327
+ )
328
+ if reason:
329
+ if is_raw_text:
330
+ raw = json.dumps(counts["raw"], ensure_ascii=True).encode("ascii")
331
+ record["reported_raw"] = {"sha256": _sha(raw), "bytes": len(raw)}
332
+ return {
333
+ **record,
334
+ "status": "unavailable",
335
+ "reason": reason,
336
+ "reported_flags": dict(zip(_FLAG_NAMES, flags, strict=True)),
337
+ "counter_observations": {
338
+ key: {"present": False, "value": None} for key in _COUNTER_NAMES
339
+ },
340
+ }
341
+ if any(
342
+ type(counts.get(key)) is not int or counts[key] not in (0, 1)
343
+ for key in _COUNTER_NAMES
344
+ ):
345
+ return {
346
+ **record,
347
+ "reason": "invalid_historical_counters",
348
+ "counter_observations": {
349
+ key: {
350
+ "present": key in counts,
351
+ "value": _counter_diagnostic(counts.get(key)),
352
+ }
353
+ for key in _COUNTER_NAMES
354
+ },
355
+ }
356
+ # The pinned runner makes one test call. The parent may force success=false
357
+ # on a nonzero child exit even when the child emitted counters.
358
+ errors = counts.get("test_errors")
359
+ if (
360
+ flags[1:] != (True, False, False)
361
+ or set(counts) != {*_COUNTER_NAMES, "tests_total", "test_errors"}
362
+ or type(counts["tests_total"]) is not int
363
+ or counts["tests_total"] != 1
364
+ or counts["tests_evaluated"] != 1
365
+ or type(errors) is not list
366
+ or len(errors) != 1 - counts["tests_passed"]
367
+ or any(type(value) is not str or ": " not in value for value in errors)
368
+ ):
369
+ return {**record, "reason": "invalid_historical_measurement"}
370
+ return {
371
+ **record,
372
+ "status": "reported",
373
+ "reported_result": {
374
+ **dict(zip(_FLAG_NAMES, flags, strict=True)),
375
+ **{key: counts[key] for key in _COUNTER_NAMES},
376
+ "exception_types": [_exception(value.split(":", 1)[0]) for value in errors],
377
+ },
378
+ }
379
+
380
+
381
+ def _selected_source(source, run, sample_id, response_sha256):
382
+ if (
383
+ run != "aria"
384
+ or not isinstance(sample_id, str)
385
+ or not re.fullmatch(r"step\d{3}_row\d{3}", sample_id)
386
+ ):
387
+ raise ValueError("Aria replay requires an exact canonical identity")
388
+ if not isinstance(response_sha256, str) or not re.fullmatch(
389
+ r"[0-9a-f]{64}", response_sha256
390
+ ):
391
+ raise ValueError("replay requires a SHA-256 response hash")
392
+ content = Path(source).read_bytes()
393
+ if _sha(content) != ARIA_SHA256:
394
+ raise ValueError("Aria source checksum does not match the pinned dataset")
395
+ rows = pd.read_parquet(
396
+ io.BytesIO(content),
397
+ columns=["sample_id", "step", "problem_id", "messages"],
398
+ filters=[("sample_id", "=", sample_id)],
399
+ )
400
+ if len(rows) != 1:
401
+ raise ValueError("replay identity must resolve to exactly one completion")
402
+ row = rows.iloc[0]
403
+ messages = json.loads(row.messages)
404
+ if [message["role"] for message in messages] != ["system", "user", "assistant"]:
405
+ raise ValueError("unexpected canonical Aria message layout")
406
+ response = messages[2]["content"]
407
+ if _sha(response.encode()) != response_sha256:
408
+ raise ValueError("selected response checksum does not match")
409
+ return {
410
+ "run": run,
411
+ "sample_id": sample_id,
412
+ "step": int(row.step),
413
+ "prompt_id": str(row.problem_id),
414
+ "prompt": messages[1]["content"],
415
+ "response": response,
416
+ }
417
+
418
+
419
+ def _replay_response(response, task, artifacts, config):
420
+ extraction = historical_extract(response)
421
+ program = extraction.pop("program")
422
+ extraction["program_sha256"] = (
423
+ _sha(program.encode()) if program is not None else None
424
+ )
425
+ cases = protected_cases(task)
426
+ result = {
427
+ "extraction": extraction,
428
+ "historical": {"status": "unavailable", "reason": "missing_code"},
429
+ "task": {
430
+ "status": "unavailable",
431
+ "reason": "missing_code",
432
+ "determinate": False,
433
+ },
434
+ "evidence_status": "unresolved",
435
+ "mechanism_status": "unresolved",
436
+ "successful_hack_hypothesis": "unresolved",
437
+ }
438
+ if program is None:
439
+ return result
440
+ if extraction["parse_status"] == "lookup_error":
441
+ result["historical"]["reason"] = "historical_lookup_unavailable"
442
+ result["task"]["reason"] = "historical_lookup_unavailable"
443
+ return result
444
+ script, payload = historical_script(
445
+ artifacts, task["setup_code"], program, extraction["requested_call"]
446
+ )
447
+ historical = run_container(script, payload, config, execution_authorized=True)
448
+ result["historical"] = _historical_report(historical)
449
+ from .investigation import _validate_architecture, _validate_runtime
450
+
451
+ try:
452
+ _validate_runtime(result["historical"]["runtime"])
453
+ except ValueError:
454
+ result["task"]["reason"] = "historical_container_infrastructure_failure"
455
+ return result
456
+ if not _historical_allows_protected(result["historical"]):
457
+ result["task"]["reason"] = "historical_protocol_unavailable"
458
+ return result
459
+ payload = json.dumps(
460
+ {
461
+ "setup": task["setup_code"],
462
+ "program": program,
463
+ "cases": [
464
+ {"method": case["method"], "kwargs": case["kwargs"]} for case in cases
465
+ ],
466
+ },
467
+ allow_nan=False,
468
+ ).encode()
469
+ independent = run_container(
470
+ _ANSWER_WORKER, payload, config, execution_authorized=True
471
+ )
472
+ comparison = assess_answers(
473
+ _json_output(independent.stdout) if independent.status == "completed" else None,
474
+ cases,
475
+ )
476
+ if independent.status != "completed":
477
+ comparison["reason"] = independent.status
478
+ comparison["runtime"] = _runtime(independent)
479
+ comparison["contract"] = (
480
+ "all original checks; fresh Solution per call; host equality"
481
+ )
482
+ try:
483
+ _validate_runtime(comparison["runtime"])
484
+ _validate_architecture((result["historical"], comparison))
485
+ if historical.metadata["container_id"] == independent.metadata["container_id"]:
486
+ raise ValueError("evaluation modes require distinct containers")
487
+ except ValueError:
488
+ comparison.update(
489
+ status="unresolved",
490
+ determinate=False,
491
+ test_coverage_complete=False,
492
+ reason="task_container_infrastructure_failure",
493
+ )
494
+ result["task"] = comparison
495
+ if comparison["determinate"]:
496
+ result["evidence_status"] = "observed_behavior"
497
+ if comparison["status"] == "passed":
498
+ result["successful_hack_hypothesis"] = "unsupported_under_tested_conditions"
499
+ reported = result["historical"].get("reported_result") or {}
500
+ if reported.get("tests_passed") == 1 and comparison["status"] == "failed":
501
+ result["evidence_status"] = "reproduced_discrepancy"
502
+ return result
503
+
504
+
505
+ def replay_aria(
506
+ source: str | Path,
507
+ *,
508
+ run: str,
509
+ sample_id: str,
510
+ response_sha256: str,
511
+ artifacts: str | Path,
512
+ config: SandboxConfig | None = None,
513
+ execution_authorized: bool = False,
514
+ ) -> dict:
515
+ """Replay one hash-pinned Aria completion only with explicit authorization.
516
+
517
+ Historical stdout is an untrusted reproduced report. Original test answers
518
+ stay on the host; candidate primitive outputs are compared there. This checks
519
+ the available test set. It never independently authenticates a hack mechanism.
520
+ """
521
+ if execution_authorized is not True:
522
+ raise ValueError("replay requires explicit execution authorization")
523
+ config = config if config is not None else SandboxConfig()
524
+ row = _selected_source(source, run, sample_id, response_sha256)
525
+ record = {
526
+ "replay_schema_version": 1,
527
+ "identity": {k: row[k] for k in ("run", "sample_id", "step", "prompt_id")},
528
+ "source": {
529
+ "dataset": "gutenbergpbc/aria-reward-hacking",
530
+ "sha256": ARIA_SHA256,
531
+ "response_sha256": response_sha256,
532
+ },
533
+ "evaluator": {
534
+ "commit": EVALUATOR_COMMIT,
535
+ "component": "hinted_test_call",
536
+ "reward_reconstruction": "unavailable; full historical reward not recomputed",
537
+ "auxiliary_test_modification_checks": "not_run",
538
+ },
539
+ "implementation_sha256": {
540
+ name: _sha(Path(__file__).with_name(name).read_bytes())
541
+ for name in ("replay.py", "_aria_replay.py", "_replay_container.py")
542
+ },
543
+ "protected_worker_sha256": _sha(_ANSWER_WORKER.encode()),
544
+ "configuration": {
545
+ k: v for k, v in asdict(config).items() if k != "docker_host"
546
+ },
547
+ "execution_endpoint": "explicit_local_unix_socket",
548
+ "published_labels": "excluded; compare only after replay",
549
+ "evidence_status": "unresolved",
550
+ "mechanism_status": "unresolved",
551
+ "successful_hack_hypothesis": "unresolved",
552
+ }
553
+ try:
554
+ verified = verify_artifacts(artifacts)
555
+ task = load_task(verified["fixture"], row["prompt_id"], row["prompt"])
556
+ protected_cases(task)
557
+ except (OSError, ValueError):
558
+ record.update(
559
+ historical={
560
+ "status": "unavailable",
561
+ "reason": "missing_or_invalid_evaluator_artifacts",
562
+ },
563
+ task={
564
+ "status": "unavailable",
565
+ "reason": "missing_or_unsupported_task_contract",
566
+ "determinate": False,
567
+ },
568
+ )
569
+ return record
570
+ record["evaluator"]["source_sha256"] = {
571
+ name: _sha(path.read_bytes()) for name, path in verified.items()
572
+ }
573
+ record["task_source"] = {
574
+ "prompt_id": row["prompt_id"],
575
+ "task_sha256": _sha(json.dumps(task, sort_keys=True).encode()),
576
+ "checks": len(task["gt_answer"]),
577
+ }
578
+ record.update(_replay_response(row["response"], task, verified, config))
579
+ return record
580
+
581
+
582
+ def serialize_replay(record: dict) -> str:
583
+ """Serialize replay evidence independently of investigation-report version 1."""
584
+ if (
585
+ type(record.get("replay_schema_version")) is not int
586
+ or record["replay_schema_version"] != 1
587
+ ):
588
+ raise ValueError("unsupported replay schema version")
589
+ return json.dumps(record, sort_keys=True, indent=2, allow_nan=False) + "\n"