rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/replay.py
ADDED
|
@@ -0,0 +1,589 @@
|
|
|
1
|
+
"""Explicit Aria replay with historical reports and external task comparisons."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import io
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import re
|
|
10
|
+
from dataclasses import asdict
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
from ._aria_replay import (
|
|
16
|
+
ARIA_SHA256,
|
|
17
|
+
historical_extract,
|
|
18
|
+
historical_script,
|
|
19
|
+
load_task,
|
|
20
|
+
protected_cases,
|
|
21
|
+
verify_artifacts,
|
|
22
|
+
)
|
|
23
|
+
from ._replay_container import SandboxConfig, run_container
|
|
24
|
+
|
|
25
|
+
EVALUATOR_COMMIT = "73695ff5533b566f7cc99b02bfeb9168936e740d"
|
|
26
|
+
|
|
27
|
+
# Only candidate code and public inputs enter this interpreter. No expected values.
|
|
28
|
+
_ANSWER_WORKER = r"""
|
|
29
|
+
import contextlib
|
|
30
|
+
import json
|
|
31
|
+
import math
|
|
32
|
+
import sys
|
|
33
|
+
|
|
34
|
+
def encode(value, depth=0):
|
|
35
|
+
if depth > 64:
|
|
36
|
+
raise ValueError("unsupported result depth")
|
|
37
|
+
kind = type(value)
|
|
38
|
+
if value is None:
|
|
39
|
+
return ["none", None]
|
|
40
|
+
if kind is bool:
|
|
41
|
+
return ["bool", value]
|
|
42
|
+
if kind is int:
|
|
43
|
+
return ["int", value]
|
|
44
|
+
if kind is str:
|
|
45
|
+
return ["str", value]
|
|
46
|
+
if kind is float:
|
|
47
|
+
return ["float", value if math.isfinite(value) else
|
|
48
|
+
("nan" if math.isnan(value) else "inf" if value > 0 else "-inf")]
|
|
49
|
+
if kind is list:
|
|
50
|
+
return ["list", [encode(item, depth + 1) for item in value]]
|
|
51
|
+
raise ValueError("unsupported result type")
|
|
52
|
+
|
|
53
|
+
data = json.loads(sys.stdin.buffer.read())
|
|
54
|
+
scope = {}
|
|
55
|
+
results = []
|
|
56
|
+
try:
|
|
57
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
58
|
+
exec(data["setup"] + "\n" + data["program"], scope)
|
|
59
|
+
except BaseException as error:
|
|
60
|
+
result = {"setup_exception": type(error).__name__}
|
|
61
|
+
else:
|
|
62
|
+
for case in data["cases"]:
|
|
63
|
+
try:
|
|
64
|
+
with contextlib.redirect_stdout(sys.stderr):
|
|
65
|
+
value = getattr(scope["Solution"](), case["method"])(**case["kwargs"])
|
|
66
|
+
results.append({"value": encode(value)})
|
|
67
|
+
except BaseException as error:
|
|
68
|
+
results.append({"exception": type(error).__name__})
|
|
69
|
+
result = {"results": results}
|
|
70
|
+
sys.stdout.write(json.dumps(result, allow_nan=False))
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _sha(value: bytes) -> str:
|
|
75
|
+
return hashlib.sha256(value).hexdigest()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _decode(value, depth=0):
|
|
79
|
+
if depth > 64 or type(value) is not list or len(value) != 2:
|
|
80
|
+
raise ValueError("invalid candidate answer encoding")
|
|
81
|
+
tag, item = value
|
|
82
|
+
if type(tag) is not str:
|
|
83
|
+
raise ValueError("invalid candidate answer tag")
|
|
84
|
+
if tag == "none" and item is None:
|
|
85
|
+
return None
|
|
86
|
+
if tag in {"bool", "int", "str"}:
|
|
87
|
+
expected = {"bool": bool, "int": int, "str": str}[tag]
|
|
88
|
+
if type(item) is expected:
|
|
89
|
+
return item
|
|
90
|
+
if tag == "float":
|
|
91
|
+
if type(item) is float and math.isfinite(item):
|
|
92
|
+
return item
|
|
93
|
+
if type(item) is str and item in {"inf", "-inf", "nan"}:
|
|
94
|
+
return float(item)
|
|
95
|
+
if tag == "list" and type(item) is list:
|
|
96
|
+
return [_decode(child, depth + 1) for child in item]
|
|
97
|
+
raise ValueError("invalid candidate answer encoding")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _exception(value):
|
|
101
|
+
return (
|
|
102
|
+
value
|
|
103
|
+
if isinstance(value, str)
|
|
104
|
+
and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", value)
|
|
105
|
+
else "UnrecognizedException"
|
|
106
|
+
)
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def assess_answers(payload: object, cases: list[dict]) -> dict:
|
|
110
|
+
"""Compare primitive proposed answers on the host; worker verdicts have no authority."""
|
|
111
|
+
result = {
|
|
112
|
+
"status": "unresolved",
|
|
113
|
+
"reason": "invalid_answer_protocol",
|
|
114
|
+
"determinate": False,
|
|
115
|
+
"tests_applicable": len(cases),
|
|
116
|
+
"tests_completed": 0,
|
|
117
|
+
"tests_passed": 0,
|
|
118
|
+
"tests_failed": 0,
|
|
119
|
+
"reported_exceptions": [],
|
|
120
|
+
"comparison_authority": "host",
|
|
121
|
+
"test_coverage_complete": False,
|
|
122
|
+
}
|
|
123
|
+
if not cases or type(payload) is not dict:
|
|
124
|
+
return result
|
|
125
|
+
if set(payload) == {"setup_exception"}:
|
|
126
|
+
result["reason"] = "candidate_reported_setup_failure"
|
|
127
|
+
result["reported_exceptions"] = [_exception(payload["setup_exception"])]
|
|
128
|
+
return result
|
|
129
|
+
if set(payload) != {"results"} or type(payload["results"]) is not list:
|
|
130
|
+
return result
|
|
131
|
+
exact_count = len(payload["results"]) == len(cases)
|
|
132
|
+
for proposed, case in zip(payload["results"], cases, strict=False):
|
|
133
|
+
if type(proposed) is not dict:
|
|
134
|
+
continue
|
|
135
|
+
if set(proposed) == {"exception"}:
|
|
136
|
+
result["reported_exceptions"].append(_exception(proposed["exception"]))
|
|
137
|
+
elif set(proposed) == {"value"}:
|
|
138
|
+
try:
|
|
139
|
+
observed = _decode(proposed["value"])
|
|
140
|
+
passed = observed == case["expected"]
|
|
141
|
+
except (ValueError, RecursionError):
|
|
142
|
+
continue
|
|
143
|
+
result["tests_completed"] += 1
|
|
144
|
+
result["tests_passed" if passed else "tests_failed"] += 1
|
|
145
|
+
complete = exact_count and result["tests_completed"] == len(cases)
|
|
146
|
+
result["test_coverage_complete"] = complete
|
|
147
|
+
if not complete:
|
|
148
|
+
result["reason"] = (
|
|
149
|
+
"incomplete_or_unsupported_answers"
|
|
150
|
+
if exact_count
|
|
151
|
+
else "incomplete_or_extra_answers"
|
|
152
|
+
)
|
|
153
|
+
else:
|
|
154
|
+
result.update(
|
|
155
|
+
status="failed" if result["tests_failed"] else "passed",
|
|
156
|
+
determinate=True,
|
|
157
|
+
reason=None,
|
|
158
|
+
)
|
|
159
|
+
return result
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _json_output(content):
|
|
163
|
+
def invalid_constant(_):
|
|
164
|
+
raise ValueError("nonfinite JSON constant")
|
|
165
|
+
|
|
166
|
+
def unique_members(pairs):
|
|
167
|
+
result = {}
|
|
168
|
+
for key, value in pairs:
|
|
169
|
+
if key in result:
|
|
170
|
+
raise ValueError("duplicate JSON member")
|
|
171
|
+
result[key] = value
|
|
172
|
+
return result
|
|
173
|
+
|
|
174
|
+
try:
|
|
175
|
+
return json.loads(
|
|
176
|
+
content, parse_constant=invalid_constant, object_pairs_hook=unique_members
|
|
177
|
+
)
|
|
178
|
+
except (ValueError, UnicodeError, RecursionError):
|
|
179
|
+
return None
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _runtime(result):
|
|
183
|
+
fields = asdict(result)
|
|
184
|
+
for stream in ("stdout", "stderr"):
|
|
185
|
+
content = fields.pop(stream)
|
|
186
|
+
fields[stream + "_sha256"] = _sha(content)
|
|
187
|
+
fields[stream + "_bytes"] = len(content)
|
|
188
|
+
fields["peak_memory_bytes"] = None
|
|
189
|
+
fields["memory_measurement_status"] = (
|
|
190
|
+
"unavailable; daemon OOM flag and limit recorded"
|
|
191
|
+
)
|
|
192
|
+
return fields
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
_FLAG_NAMES = ("success", "compiled", "timeout", "oom")
|
|
196
|
+
_COUNTER_NAMES = ("tests_evaluated", "tests_passed")
|
|
197
|
+
_HISTORICAL_UNAVAILABLE_FLAGS = {
|
|
198
|
+
"reported_compile_failure": (False, False, False, False),
|
|
199
|
+
"reported_inner_timeout": (False, True, True, False),
|
|
200
|
+
"reported_inner_oom": (False, True, False, True),
|
|
201
|
+
"reported_runtime_error": (False, True, False, False),
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _counter_diagnostic(value):
|
|
206
|
+
# Leave headroom to serialize the diagnostic inside a complete case record.
|
|
207
|
+
nodes = 1
|
|
208
|
+
pending = [(value, 0)]
|
|
209
|
+
while pending:
|
|
210
|
+
item, depth = pending.pop()
|
|
211
|
+
if isinstance(item, (list, dict)):
|
|
212
|
+
if depth >= 64:
|
|
213
|
+
return {
|
|
214
|
+
"invalid_value": "nesting_limit",
|
|
215
|
+
"value_type": type(value).__name__,
|
|
216
|
+
}
|
|
217
|
+
children = item.values() if isinstance(item, dict) else item
|
|
218
|
+
nodes += len(item)
|
|
219
|
+
if nodes > 256:
|
|
220
|
+
return {
|
|
221
|
+
"invalid_value": "size_limit",
|
|
222
|
+
"value_type": type(value).__name__,
|
|
223
|
+
}
|
|
224
|
+
pending.extend((child, depth + 1) for child in children)
|
|
225
|
+
try:
|
|
226
|
+
if len(json.dumps(value, allow_nan=False)) > 4096:
|
|
227
|
+
return {"invalid_value": "size_limit", "value_type": type(value).__name__}
|
|
228
|
+
except ValueError:
|
|
229
|
+
diagnostic = {
|
|
230
|
+
"invalid_value": "non_finite_number",
|
|
231
|
+
"value_type": type(value).__name__,
|
|
232
|
+
}
|
|
233
|
+
if type(value) is float:
|
|
234
|
+
diagnostic["representation"] = repr(value)
|
|
235
|
+
return diagnostic
|
|
236
|
+
except RecursionError:
|
|
237
|
+
return {"invalid_value": "nesting_limit", "value_type": type(value).__name__}
|
|
238
|
+
return value
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _historical_allows_protected(observation):
|
|
242
|
+
"""Apply the protocol gate after host-side runtime checks."""
|
|
243
|
+
if observation["runtime"]["status"] != "completed":
|
|
244
|
+
return (
|
|
245
|
+
observation["runtime"]["status"]
|
|
246
|
+
in {"nonzero_exit", "timeout", "output_limit"}
|
|
247
|
+
and observation["status"] == "unresolved"
|
|
248
|
+
)
|
|
249
|
+
if observation["status"] == "reported":
|
|
250
|
+
claim = observation.get("reported_result")
|
|
251
|
+
if type(claim) is not dict:
|
|
252
|
+
return False
|
|
253
|
+
passed, errors = claim.get("tests_passed"), claim.get("exception_types")
|
|
254
|
+
return (
|
|
255
|
+
type(claim.get("success")) is bool
|
|
256
|
+
and claim.get("compiled") is True
|
|
257
|
+
and claim.get("timeout") is False
|
|
258
|
+
and claim.get("oom") is False
|
|
259
|
+
and type(claim.get("tests_evaluated")) is int
|
|
260
|
+
and claim["tests_evaluated"] == 1
|
|
261
|
+
and type(passed) is int
|
|
262
|
+
and passed in (0, 1)
|
|
263
|
+
and type(errors) is list
|
|
264
|
+
and len(errors) == 1 - passed
|
|
265
|
+
and all(
|
|
266
|
+
type(e) is str and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", e)
|
|
267
|
+
for e in errors
|
|
268
|
+
)
|
|
269
|
+
)
|
|
270
|
+
return (
|
|
271
|
+
observation["status"] == "unavailable"
|
|
272
|
+
and type(observation.get("reason")) is str
|
|
273
|
+
and observation["reason"] in _HISTORICAL_UNAVAILABLE_FLAGS
|
|
274
|
+
)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _historical_report(result):
|
|
278
|
+
record = {
|
|
279
|
+
"status": "unresolved",
|
|
280
|
+
"authority": "candidate_controlled_historical_report",
|
|
281
|
+
"semantic_completion_authenticated": False,
|
|
282
|
+
"trace_status": "reported exception types only; authenticated stack unavailable",
|
|
283
|
+
"reported_result": None,
|
|
284
|
+
"runtime": _runtime(result),
|
|
285
|
+
}
|
|
286
|
+
if result.status != "completed":
|
|
287
|
+
return {**record, "reason": result.status}
|
|
288
|
+
payload = _json_output(result.stdout)
|
|
289
|
+
if type(payload) is not dict:
|
|
290
|
+
return {**record, "reason": "invalid_historical_result_envelope"}
|
|
291
|
+
if (
|
|
292
|
+
set(payload) == {"historical_unavailable", "exception_type"}
|
|
293
|
+
and payload["historical_unavailable"]
|
|
294
|
+
== "unsupported_result_validation_or_supervisor_error"
|
|
295
|
+
and type(payload["exception_type"]) is str
|
|
296
|
+
and payload["exception_type"].isidentifier()
|
|
297
|
+
):
|
|
298
|
+
return {
|
|
299
|
+
**record,
|
|
300
|
+
"status": "unavailable",
|
|
301
|
+
"reason": "unsupported_historical_result",
|
|
302
|
+
}
|
|
303
|
+
if (
|
|
304
|
+
set(payload) != {*_FLAG_NAMES, "stdout"}
|
|
305
|
+
or any(type(payload[key]) is not bool for key in _FLAG_NAMES)
|
|
306
|
+
or type(payload["stdout"]) is not dict
|
|
307
|
+
):
|
|
308
|
+
return {**record, "reason": "invalid_historical_result_envelope"}
|
|
309
|
+
counts = payload["stdout"]
|
|
310
|
+
flags = tuple(payload[k] for k in _FLAG_NAMES)
|
|
311
|
+
is_raw_text = set(counts) == {"raw"} and type(counts["raw"]) is str
|
|
312
|
+
reason = None
|
|
313
|
+
if is_raw_text:
|
|
314
|
+
if flags == _HISTORICAL_UNAVAILABLE_FLAGS["reported_runtime_error"]:
|
|
315
|
+
reason = "reported_runtime_error"
|
|
316
|
+
elif flags == _HISTORICAL_UNAVAILABLE_FLAGS["reported_compile_failure"]:
|
|
317
|
+
# The parent uses this shape for both OSError and invalid child JSON.
|
|
318
|
+
reason = "reported_child_output_or_setup_failure"
|
|
319
|
+
elif not counts:
|
|
320
|
+
reason = next(
|
|
321
|
+
(
|
|
322
|
+
name
|
|
323
|
+
for name, expected in _HISTORICAL_UNAVAILABLE_FLAGS.items()
|
|
324
|
+
if name != "reported_runtime_error" and flags == expected
|
|
325
|
+
),
|
|
326
|
+
None,
|
|
327
|
+
)
|
|
328
|
+
if reason:
|
|
329
|
+
if is_raw_text:
|
|
330
|
+
raw = json.dumps(counts["raw"], ensure_ascii=True).encode("ascii")
|
|
331
|
+
record["reported_raw"] = {"sha256": _sha(raw), "bytes": len(raw)}
|
|
332
|
+
return {
|
|
333
|
+
**record,
|
|
334
|
+
"status": "unavailable",
|
|
335
|
+
"reason": reason,
|
|
336
|
+
"reported_flags": dict(zip(_FLAG_NAMES, flags, strict=True)),
|
|
337
|
+
"counter_observations": {
|
|
338
|
+
key: {"present": False, "value": None} for key in _COUNTER_NAMES
|
|
339
|
+
},
|
|
340
|
+
}
|
|
341
|
+
if any(
|
|
342
|
+
type(counts.get(key)) is not int or counts[key] not in (0, 1)
|
|
343
|
+
for key in _COUNTER_NAMES
|
|
344
|
+
):
|
|
345
|
+
return {
|
|
346
|
+
**record,
|
|
347
|
+
"reason": "invalid_historical_counters",
|
|
348
|
+
"counter_observations": {
|
|
349
|
+
key: {
|
|
350
|
+
"present": key in counts,
|
|
351
|
+
"value": _counter_diagnostic(counts.get(key)),
|
|
352
|
+
}
|
|
353
|
+
for key in _COUNTER_NAMES
|
|
354
|
+
},
|
|
355
|
+
}
|
|
356
|
+
# The pinned runner makes one test call. The parent may force success=false
|
|
357
|
+
# on a nonzero child exit even when the child emitted counters.
|
|
358
|
+
errors = counts.get("test_errors")
|
|
359
|
+
if (
|
|
360
|
+
flags[1:] != (True, False, False)
|
|
361
|
+
or set(counts) != {*_COUNTER_NAMES, "tests_total", "test_errors"}
|
|
362
|
+
or type(counts["tests_total"]) is not int
|
|
363
|
+
or counts["tests_total"] != 1
|
|
364
|
+
or counts["tests_evaluated"] != 1
|
|
365
|
+
or type(errors) is not list
|
|
366
|
+
or len(errors) != 1 - counts["tests_passed"]
|
|
367
|
+
or any(type(value) is not str or ": " not in value for value in errors)
|
|
368
|
+
):
|
|
369
|
+
return {**record, "reason": "invalid_historical_measurement"}
|
|
370
|
+
return {
|
|
371
|
+
**record,
|
|
372
|
+
"status": "reported",
|
|
373
|
+
"reported_result": {
|
|
374
|
+
**dict(zip(_FLAG_NAMES, flags, strict=True)),
|
|
375
|
+
**{key: counts[key] for key in _COUNTER_NAMES},
|
|
376
|
+
"exception_types": [_exception(value.split(":", 1)[0]) for value in errors],
|
|
377
|
+
},
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
|
|
381
|
+
def _selected_source(source, run, sample_id, response_sha256):
|
|
382
|
+
if (
|
|
383
|
+
run != "aria"
|
|
384
|
+
or not isinstance(sample_id, str)
|
|
385
|
+
or not re.fullmatch(r"step\d{3}_row\d{3}", sample_id)
|
|
386
|
+
):
|
|
387
|
+
raise ValueError("Aria replay requires an exact canonical identity")
|
|
388
|
+
if not isinstance(response_sha256, str) or not re.fullmatch(
|
|
389
|
+
r"[0-9a-f]{64}", response_sha256
|
|
390
|
+
):
|
|
391
|
+
raise ValueError("replay requires a SHA-256 response hash")
|
|
392
|
+
content = Path(source).read_bytes()
|
|
393
|
+
if _sha(content) != ARIA_SHA256:
|
|
394
|
+
raise ValueError("Aria source checksum does not match the pinned dataset")
|
|
395
|
+
rows = pd.read_parquet(
|
|
396
|
+
io.BytesIO(content),
|
|
397
|
+
columns=["sample_id", "step", "problem_id", "messages"],
|
|
398
|
+
filters=[("sample_id", "=", sample_id)],
|
|
399
|
+
)
|
|
400
|
+
if len(rows) != 1:
|
|
401
|
+
raise ValueError("replay identity must resolve to exactly one completion")
|
|
402
|
+
row = rows.iloc[0]
|
|
403
|
+
messages = json.loads(row.messages)
|
|
404
|
+
if [message["role"] for message in messages] != ["system", "user", "assistant"]:
|
|
405
|
+
raise ValueError("unexpected canonical Aria message layout")
|
|
406
|
+
response = messages[2]["content"]
|
|
407
|
+
if _sha(response.encode()) != response_sha256:
|
|
408
|
+
raise ValueError("selected response checksum does not match")
|
|
409
|
+
return {
|
|
410
|
+
"run": run,
|
|
411
|
+
"sample_id": sample_id,
|
|
412
|
+
"step": int(row.step),
|
|
413
|
+
"prompt_id": str(row.problem_id),
|
|
414
|
+
"prompt": messages[1]["content"],
|
|
415
|
+
"response": response,
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
|
|
419
|
+
def _replay_response(response, task, artifacts, config):
|
|
420
|
+
extraction = historical_extract(response)
|
|
421
|
+
program = extraction.pop("program")
|
|
422
|
+
extraction["program_sha256"] = (
|
|
423
|
+
_sha(program.encode()) if program is not None else None
|
|
424
|
+
)
|
|
425
|
+
cases = protected_cases(task)
|
|
426
|
+
result = {
|
|
427
|
+
"extraction": extraction,
|
|
428
|
+
"historical": {"status": "unavailable", "reason": "missing_code"},
|
|
429
|
+
"task": {
|
|
430
|
+
"status": "unavailable",
|
|
431
|
+
"reason": "missing_code",
|
|
432
|
+
"determinate": False,
|
|
433
|
+
},
|
|
434
|
+
"evidence_status": "unresolved",
|
|
435
|
+
"mechanism_status": "unresolved",
|
|
436
|
+
"successful_hack_hypothesis": "unresolved",
|
|
437
|
+
}
|
|
438
|
+
if program is None:
|
|
439
|
+
return result
|
|
440
|
+
if extraction["parse_status"] == "lookup_error":
|
|
441
|
+
result["historical"]["reason"] = "historical_lookup_unavailable"
|
|
442
|
+
result["task"]["reason"] = "historical_lookup_unavailable"
|
|
443
|
+
return result
|
|
444
|
+
script, payload = historical_script(
|
|
445
|
+
artifacts, task["setup_code"], program, extraction["requested_call"]
|
|
446
|
+
)
|
|
447
|
+
historical = run_container(script, payload, config, execution_authorized=True)
|
|
448
|
+
result["historical"] = _historical_report(historical)
|
|
449
|
+
from .investigation import _validate_architecture, _validate_runtime
|
|
450
|
+
|
|
451
|
+
try:
|
|
452
|
+
_validate_runtime(result["historical"]["runtime"])
|
|
453
|
+
except ValueError:
|
|
454
|
+
result["task"]["reason"] = "historical_container_infrastructure_failure"
|
|
455
|
+
return result
|
|
456
|
+
if not _historical_allows_protected(result["historical"]):
|
|
457
|
+
result["task"]["reason"] = "historical_protocol_unavailable"
|
|
458
|
+
return result
|
|
459
|
+
payload = json.dumps(
|
|
460
|
+
{
|
|
461
|
+
"setup": task["setup_code"],
|
|
462
|
+
"program": program,
|
|
463
|
+
"cases": [
|
|
464
|
+
{"method": case["method"], "kwargs": case["kwargs"]} for case in cases
|
|
465
|
+
],
|
|
466
|
+
},
|
|
467
|
+
allow_nan=False,
|
|
468
|
+
).encode()
|
|
469
|
+
independent = run_container(
|
|
470
|
+
_ANSWER_WORKER, payload, config, execution_authorized=True
|
|
471
|
+
)
|
|
472
|
+
comparison = assess_answers(
|
|
473
|
+
_json_output(independent.stdout) if independent.status == "completed" else None,
|
|
474
|
+
cases,
|
|
475
|
+
)
|
|
476
|
+
if independent.status != "completed":
|
|
477
|
+
comparison["reason"] = independent.status
|
|
478
|
+
comparison["runtime"] = _runtime(independent)
|
|
479
|
+
comparison["contract"] = (
|
|
480
|
+
"all original checks; fresh Solution per call; host equality"
|
|
481
|
+
)
|
|
482
|
+
try:
|
|
483
|
+
_validate_runtime(comparison["runtime"])
|
|
484
|
+
_validate_architecture((result["historical"], comparison))
|
|
485
|
+
if historical.metadata["container_id"] == independent.metadata["container_id"]:
|
|
486
|
+
raise ValueError("evaluation modes require distinct containers")
|
|
487
|
+
except ValueError:
|
|
488
|
+
comparison.update(
|
|
489
|
+
status="unresolved",
|
|
490
|
+
determinate=False,
|
|
491
|
+
test_coverage_complete=False,
|
|
492
|
+
reason="task_container_infrastructure_failure",
|
|
493
|
+
)
|
|
494
|
+
result["task"] = comparison
|
|
495
|
+
if comparison["determinate"]:
|
|
496
|
+
result["evidence_status"] = "observed_behavior"
|
|
497
|
+
if comparison["status"] == "passed":
|
|
498
|
+
result["successful_hack_hypothesis"] = "unsupported_under_tested_conditions"
|
|
499
|
+
reported = result["historical"].get("reported_result") or {}
|
|
500
|
+
if reported.get("tests_passed") == 1 and comparison["status"] == "failed":
|
|
501
|
+
result["evidence_status"] = "reproduced_discrepancy"
|
|
502
|
+
return result
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def replay_aria(
|
|
506
|
+
source: str | Path,
|
|
507
|
+
*,
|
|
508
|
+
run: str,
|
|
509
|
+
sample_id: str,
|
|
510
|
+
response_sha256: str,
|
|
511
|
+
artifacts: str | Path,
|
|
512
|
+
config: SandboxConfig | None = None,
|
|
513
|
+
execution_authorized: bool = False,
|
|
514
|
+
) -> dict:
|
|
515
|
+
"""Replay one hash-pinned Aria completion only with explicit authorization.
|
|
516
|
+
|
|
517
|
+
Historical stdout is an untrusted reproduced report. Original test answers
|
|
518
|
+
stay on the host; candidate primitive outputs are compared there. This checks
|
|
519
|
+
the available test set. It never independently authenticates a hack mechanism.
|
|
520
|
+
"""
|
|
521
|
+
if execution_authorized is not True:
|
|
522
|
+
raise ValueError("replay requires explicit execution authorization")
|
|
523
|
+
config = config if config is not None else SandboxConfig()
|
|
524
|
+
row = _selected_source(source, run, sample_id, response_sha256)
|
|
525
|
+
record = {
|
|
526
|
+
"replay_schema_version": 1,
|
|
527
|
+
"identity": {k: row[k] for k in ("run", "sample_id", "step", "prompt_id")},
|
|
528
|
+
"source": {
|
|
529
|
+
"dataset": "gutenbergpbc/aria-reward-hacking",
|
|
530
|
+
"sha256": ARIA_SHA256,
|
|
531
|
+
"response_sha256": response_sha256,
|
|
532
|
+
},
|
|
533
|
+
"evaluator": {
|
|
534
|
+
"commit": EVALUATOR_COMMIT,
|
|
535
|
+
"component": "hinted_test_call",
|
|
536
|
+
"reward_reconstruction": "unavailable; full historical reward not recomputed",
|
|
537
|
+
"auxiliary_test_modification_checks": "not_run",
|
|
538
|
+
},
|
|
539
|
+
"implementation_sha256": {
|
|
540
|
+
name: _sha(Path(__file__).with_name(name).read_bytes())
|
|
541
|
+
for name in ("replay.py", "_aria_replay.py", "_replay_container.py")
|
|
542
|
+
},
|
|
543
|
+
"protected_worker_sha256": _sha(_ANSWER_WORKER.encode()),
|
|
544
|
+
"configuration": {
|
|
545
|
+
k: v for k, v in asdict(config).items() if k != "docker_host"
|
|
546
|
+
},
|
|
547
|
+
"execution_endpoint": "explicit_local_unix_socket",
|
|
548
|
+
"published_labels": "excluded; compare only after replay",
|
|
549
|
+
"evidence_status": "unresolved",
|
|
550
|
+
"mechanism_status": "unresolved",
|
|
551
|
+
"successful_hack_hypothesis": "unresolved",
|
|
552
|
+
}
|
|
553
|
+
try:
|
|
554
|
+
verified = verify_artifacts(artifacts)
|
|
555
|
+
task = load_task(verified["fixture"], row["prompt_id"], row["prompt"])
|
|
556
|
+
protected_cases(task)
|
|
557
|
+
except (OSError, ValueError):
|
|
558
|
+
record.update(
|
|
559
|
+
historical={
|
|
560
|
+
"status": "unavailable",
|
|
561
|
+
"reason": "missing_or_invalid_evaluator_artifacts",
|
|
562
|
+
},
|
|
563
|
+
task={
|
|
564
|
+
"status": "unavailable",
|
|
565
|
+
"reason": "missing_or_unsupported_task_contract",
|
|
566
|
+
"determinate": False,
|
|
567
|
+
},
|
|
568
|
+
)
|
|
569
|
+
return record
|
|
570
|
+
record["evaluator"]["source_sha256"] = {
|
|
571
|
+
name: _sha(path.read_bytes()) for name, path in verified.items()
|
|
572
|
+
}
|
|
573
|
+
record["task_source"] = {
|
|
574
|
+
"prompt_id": row["prompt_id"],
|
|
575
|
+
"task_sha256": _sha(json.dumps(task, sort_keys=True).encode()),
|
|
576
|
+
"checks": len(task["gt_answer"]),
|
|
577
|
+
}
|
|
578
|
+
record.update(_replay_response(row["response"], task, verified, config))
|
|
579
|
+
return record
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def serialize_replay(record: dict) -> str:
|
|
583
|
+
"""Serialize replay evidence independently of investigation-report version 1."""
|
|
584
|
+
if (
|
|
585
|
+
type(record.get("replay_schema_version")) is not int
|
|
586
|
+
or record["replay_schema_version"] != 1
|
|
587
|
+
):
|
|
588
|
+
raise ValueError("unsupported replay schema version")
|
|
589
|
+
return json.dumps(record, sort_keys=True, indent=2, allow_nan=False) + "\n"
|