rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,1146 @@
|
|
|
1
|
+
"""Frozen, single-variant Aria investigations with locally consumed approvals."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import difflib
|
|
6
|
+
import hashlib
|
|
7
|
+
import html
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import stat
|
|
12
|
+
from copy import deepcopy
|
|
13
|
+
from datetime import UTC, datetime
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from urllib.parse import urlsplit
|
|
16
|
+
|
|
17
|
+
from . import _aria_replay as aria
|
|
18
|
+
from . import _replay_container as boundary
|
|
19
|
+
from . import replay
|
|
20
|
+
from .report import _write_text
|
|
21
|
+
|
|
22
|
+
_LIMITS = {
|
|
23
|
+
"memory_bytes": 1073741824,
|
|
24
|
+
"cpus": 1,
|
|
25
|
+
"pids": 16,
|
|
26
|
+
"wall_seconds": 6,
|
|
27
|
+
"output_bytes": 1048576,
|
|
28
|
+
"stdin_bytes": 2097152,
|
|
29
|
+
"tmp_bytes": 16777216,
|
|
30
|
+
}
|
|
31
|
+
_PREDICTIONS = {
|
|
32
|
+
"historical_passed",
|
|
33
|
+
"historical_evaluated",
|
|
34
|
+
"exception_types",
|
|
35
|
+
"task_matched",
|
|
36
|
+
"task_mismatched",
|
|
37
|
+
"task_stdout_sha256",
|
|
38
|
+
"task_status",
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def _require(condition, reason):
|
|
43
|
+
if not condition:
|
|
44
|
+
raise ValueError(reason)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _sha(content):
|
|
48
|
+
return hashlib.sha256(content).hexdigest()
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _json(value):
|
|
52
|
+
return json.dumps(value, sort_keys=True, indent=2, allow_nan=False) + "\n"
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _read(path):
|
|
56
|
+
path = Path(path)
|
|
57
|
+
_require(
|
|
58
|
+
not path.is_symlink()
|
|
59
|
+
and stat.S_ISREG(path.stat().st_mode)
|
|
60
|
+
and path.stat().st_size <= 16 * 1024 * 1024,
|
|
61
|
+
"investigation input must be a regular bounded file",
|
|
62
|
+
)
|
|
63
|
+
return path.read_bytes()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _load(path):
|
|
67
|
+
value = replay._json_output(_read(path))
|
|
68
|
+
_require(type(value) is dict, "invalid investigation JSON object")
|
|
69
|
+
return value
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _save(directory, name, value):
|
|
73
|
+
_write_text(_json(value), directory / name)
|
|
74
|
+
# File publication is atomic; sync the directory before spending execution authority.
|
|
75
|
+
descriptor = os.open(directory, os.O_RDONLY)
|
|
76
|
+
try:
|
|
77
|
+
os.fsync(descriptor)
|
|
78
|
+
finally:
|
|
79
|
+
os.close(descriptor)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _hash(value):
|
|
83
|
+
return type(value) is str and re.fullmatch(r"[a-f0-9]{64}", value) is not None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _integer(value):
|
|
87
|
+
return type(value) is int and value >= 0
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _implementation():
|
|
91
|
+
return {
|
|
92
|
+
name: _sha(Path(replay.__file__).with_name(name).read_bytes())
|
|
93
|
+
for name in ("replay.py", "_aria_replay.py", "_replay_container.py")
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
def _sandbox(config):
|
|
98
|
+
endpoint = urlsplit(config.docker_host)
|
|
99
|
+
_require(
|
|
100
|
+
config.image == boundary.DEFAULT_IMAGE
|
|
101
|
+
and endpoint.scheme == "unix"
|
|
102
|
+
and not endpoint.netloc
|
|
103
|
+
and endpoint.path.startswith("/")
|
|
104
|
+
and not endpoint.query
|
|
105
|
+
and not endpoint.fragment
|
|
106
|
+
and "%" not in config.docker_host
|
|
107
|
+
and all(ord(char) >= 33 for char in config.docker_host),
|
|
108
|
+
"unsupported investigation sandbox",
|
|
109
|
+
)
|
|
110
|
+
return deepcopy(
|
|
111
|
+
{
|
|
112
|
+
"image": config.image,
|
|
113
|
+
"limits": _LIMITS,
|
|
114
|
+
"host_config": boundary._HOST_CONFIG,
|
|
115
|
+
"ulimits": boundary._ULIMITS,
|
|
116
|
+
"uid_gid": "65534:65534",
|
|
117
|
+
"cpu_limit_seconds": 3,
|
|
118
|
+
"python_version": "3.12.13",
|
|
119
|
+
}
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _extraction(response):
|
|
124
|
+
result = aria.historical_extract(response)
|
|
125
|
+
program = result.pop("program")
|
|
126
|
+
result["program_sha256"] = _sha(program.encode()) if program is not None else None
|
|
127
|
+
return program, result
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def _validate_runtime(runtime):
|
|
131
|
+
_require(
|
|
132
|
+
type(runtime) is dict
|
|
133
|
+
and runtime.get("config_verified") is True
|
|
134
|
+
and runtime.get("cleanup_verified") is True
|
|
135
|
+
and runtime.get("error") is None,
|
|
136
|
+
"incomplete runtime or unverified cleanup",
|
|
137
|
+
)
|
|
138
|
+
status, exit_code = runtime.get("status"), runtime.get("exit_code")
|
|
139
|
+
_require(
|
|
140
|
+
status in {"completed", "nonzero_exit", "timeout", "output_limit"},
|
|
141
|
+
"incompatible runtime status",
|
|
142
|
+
)
|
|
143
|
+
if status in {"completed", "nonzero_exit"}:
|
|
144
|
+
_require(
|
|
145
|
+
type(exit_code) is int and (exit_code == 0) == (status == "completed"),
|
|
146
|
+
"inconsistent runtime exit status",
|
|
147
|
+
)
|
|
148
|
+
metadata = runtime.get("metadata", {})
|
|
149
|
+
_require(type(metadata) is dict, "invalid runtime metadata")
|
|
150
|
+
_require(
|
|
151
|
+
metadata.get("image_id") == boundary.DEFAULT_IMAGE
|
|
152
|
+
and metadata.get("python_version") == "3.12.13"
|
|
153
|
+
and metadata.get("architecture") in {"arm64", "amd64"}
|
|
154
|
+
and metadata.get("runtime") == "runc"
|
|
155
|
+
and metadata.get("cgroup_version") == "2"
|
|
156
|
+
and metadata.get("limits") == _LIMITS
|
|
157
|
+
and _hash(metadata.get("container_id")),
|
|
158
|
+
"incompatible runtime configuration",
|
|
159
|
+
)
|
|
160
|
+
for stream in ("stdout", "stderr"):
|
|
161
|
+
_require(
|
|
162
|
+
_hash(runtime.get(stream + "_sha256"))
|
|
163
|
+
and _integer(runtime.get(stream + "_bytes")),
|
|
164
|
+
"missing stream evidence",
|
|
165
|
+
)
|
|
166
|
+
_require(
|
|
167
|
+
runtime["stdout_bytes"] + runtime["stderr_bytes"] <= _LIMITS["output_bytes"],
|
|
168
|
+
"invalid stream sizes",
|
|
169
|
+
)
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _validate_architecture(modes, expected=None):
|
|
173
|
+
for mode in modes:
|
|
174
|
+
architecture = mode.get("runtime", {}).get("metadata", {}).get("architecture")
|
|
175
|
+
if architecture is not None:
|
|
176
|
+
expected = architecture if expected is None else expected
|
|
177
|
+
_require(
|
|
178
|
+
architecture == expected, "runtime architecture differs from control"
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _validate_outcomes(record, checks, *, partial=False, architecture=None):
|
|
183
|
+
_require(_integer(checks) and checks > 0, "invalid applicable task count")
|
|
184
|
+
historical, task = record["historical"], record["task"]
|
|
185
|
+
for mode in (historical, task):
|
|
186
|
+
if partial and "runtime" not in mode:
|
|
187
|
+
_require(
|
|
188
|
+
mode.get("status") == "unavailable" and type(mode.get("reason")) is str,
|
|
189
|
+
"missing partial mode status",
|
|
190
|
+
)
|
|
191
|
+
elif partial and (
|
|
192
|
+
mode["runtime"].get("cleanup_verified") is not True
|
|
193
|
+
or mode["runtime"].get("config_verified") is not True
|
|
194
|
+
or mode["runtime"].get("status") == "infrastructure_error"
|
|
195
|
+
):
|
|
196
|
+
runtime = mode["runtime"]
|
|
197
|
+
_require(
|
|
198
|
+
runtime.get("status")
|
|
199
|
+
in {
|
|
200
|
+
"completed",
|
|
201
|
+
"nonzero_exit",
|
|
202
|
+
"timeout",
|
|
203
|
+
"output_limit",
|
|
204
|
+
"infrastructure_error",
|
|
205
|
+
}
|
|
206
|
+
and type(runtime.get("cleanup_verified")) is bool
|
|
207
|
+
and type(runtime.get("config_verified")) is bool
|
|
208
|
+
and all(
|
|
209
|
+
_hash(runtime.get(stream + "_sha256"))
|
|
210
|
+
and _integer(runtime.get(stream + "_bytes"))
|
|
211
|
+
for stream in ("stdout", "stderr")
|
|
212
|
+
),
|
|
213
|
+
"invalid partial runtime",
|
|
214
|
+
)
|
|
215
|
+
else:
|
|
216
|
+
_validate_runtime(mode.get("runtime"))
|
|
217
|
+
_validate_architecture((historical, task), architecture)
|
|
218
|
+
if partial and "runtime" not in historical:
|
|
219
|
+
_require(
|
|
220
|
+
historical == {"status": "unavailable", "reason": "not_attempted"},
|
|
221
|
+
"invalid unattempted historical mode",
|
|
222
|
+
)
|
|
223
|
+
else:
|
|
224
|
+
_validate_historical(historical)
|
|
225
|
+
if not partial:
|
|
226
|
+
_require(
|
|
227
|
+
replay._historical_allows_protected(historical),
|
|
228
|
+
"completed assessment requires admissible historical evidence",
|
|
229
|
+
)
|
|
230
|
+
if partial and "runtime" not in task:
|
|
231
|
+
_require(
|
|
232
|
+
task
|
|
233
|
+
== {
|
|
234
|
+
"status": "unavailable",
|
|
235
|
+
"reason": "not_attempted",
|
|
236
|
+
"determinate": False,
|
|
237
|
+
},
|
|
238
|
+
"invalid unattempted task mode",
|
|
239
|
+
)
|
|
240
|
+
else:
|
|
241
|
+
_validate_task(task, checks)
|
|
242
|
+
identities = [
|
|
243
|
+
mode.get("runtime", {}).get("metadata", {}).get("container_id")
|
|
244
|
+
for mode in (historical, task)
|
|
245
|
+
]
|
|
246
|
+
_require(
|
|
247
|
+
identities[0] is None
|
|
248
|
+
or identities[1] is None
|
|
249
|
+
or identities[0] != identities[1],
|
|
250
|
+
"evaluation modes require distinct containers",
|
|
251
|
+
)
|
|
252
|
+
_require(
|
|
253
|
+
record.get("mechanism_status") == "unresolved", "unsupported mechanism claim"
|
|
254
|
+
)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _validate_historical(historical):
|
|
258
|
+
_require(
|
|
259
|
+
historical.get("authority") == "candidate_controlled_historical_report"
|
|
260
|
+
and historical.get("semantic_completion_authenticated") is False,
|
|
261
|
+
"historical claims must remain untrusted",
|
|
262
|
+
)
|
|
263
|
+
reason = historical.get("reason")
|
|
264
|
+
if reason in replay._HISTORICAL_UNAVAILABLE_FLAGS:
|
|
265
|
+
flags = historical.get("reported_flags")
|
|
266
|
+
expected = dict(
|
|
267
|
+
zip(
|
|
268
|
+
replay._FLAG_NAMES,
|
|
269
|
+
replay._HISTORICAL_UNAVAILABLE_FLAGS[reason],
|
|
270
|
+
strict=True,
|
|
271
|
+
)
|
|
272
|
+
)
|
|
273
|
+
counters = historical.get("counter_observations")
|
|
274
|
+
_require(
|
|
275
|
+
historical["status"] == "unavailable"
|
|
276
|
+
and historical["runtime"]["status"] == "completed"
|
|
277
|
+
and historical.get("reported_result") is None
|
|
278
|
+
and "rejected_report" not in historical
|
|
279
|
+
and type(flags) is dict
|
|
280
|
+
and set(flags) == set(expected)
|
|
281
|
+
and all(flags[name] is value for name, value in expected.items())
|
|
282
|
+
and type(counters) is dict
|
|
283
|
+
and set(counters) == set(replay._COUNTER_NAMES)
|
|
284
|
+
and all(
|
|
285
|
+
type(item) is dict
|
|
286
|
+
and set(item) == {"present", "value"}
|
|
287
|
+
and item["present"] is False
|
|
288
|
+
and item["value"] is None
|
|
289
|
+
for item in counters.values()
|
|
290
|
+
),
|
|
291
|
+
"invalid unavailable historical evidence",
|
|
292
|
+
)
|
|
293
|
+
if reason == "reported_runtime_error":
|
|
294
|
+
raw = historical.get("reported_raw")
|
|
295
|
+
_require(
|
|
296
|
+
type(raw) is dict
|
|
297
|
+
and set(raw) == {"bytes", "sha256"}
|
|
298
|
+
and _integer(raw["bytes"])
|
|
299
|
+
and raw["bytes"] >= 2
|
|
300
|
+
and _hash(raw["sha256"]),
|
|
301
|
+
"invalid historical error text fingerprint",
|
|
302
|
+
)
|
|
303
|
+
else:
|
|
304
|
+
_require(
|
|
305
|
+
"reported_raw" not in historical, "unexpected historical error text"
|
|
306
|
+
)
|
|
307
|
+
if historical["status"] == "reported":
|
|
308
|
+
_require(
|
|
309
|
+
historical["runtime"]["status"] == "completed",
|
|
310
|
+
"claim from incomplete runtime",
|
|
311
|
+
)
|
|
312
|
+
claim = historical.get("reported_result")
|
|
313
|
+
_require(type(claim) is dict, "missing historical claim")
|
|
314
|
+
for key in ("success", "compiled", "timeout", "oom"):
|
|
315
|
+
_require(type(claim.get(key)) is bool, "invalid historical claim")
|
|
316
|
+
for key in ("tests_evaluated", "tests_passed"):
|
|
317
|
+
_require(
|
|
318
|
+
type(claim.get(key)) is int and claim[key] in (0, 1),
|
|
319
|
+
"invalid historical counters",
|
|
320
|
+
)
|
|
321
|
+
_require(
|
|
322
|
+
type(claim.get("exception_types")) is list
|
|
323
|
+
and all(
|
|
324
|
+
type(value) is str
|
|
325
|
+
and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", value)
|
|
326
|
+
for value in claim["exception_types"]
|
|
327
|
+
),
|
|
328
|
+
"invalid exception evidence",
|
|
329
|
+
)
|
|
330
|
+
else:
|
|
331
|
+
_require(
|
|
332
|
+
historical["status"] in {"unavailable", "unresolved"}
|
|
333
|
+
and historical.get("reported_result") is None,
|
|
334
|
+
"invalid unavailable historical evidence",
|
|
335
|
+
)
|
|
336
|
+
if historical.get("reason") == "invalid_historical_counters":
|
|
337
|
+
counters = historical.get("counter_observations")
|
|
338
|
+
_require(
|
|
339
|
+
type(counters) is dict
|
|
340
|
+
and set(counters) == {"tests_evaluated", "tests_passed"},
|
|
341
|
+
"missing rejected counter observations",
|
|
342
|
+
)
|
|
343
|
+
for item in counters.values():
|
|
344
|
+
_require(
|
|
345
|
+
type(item) is dict
|
|
346
|
+
and set(item) == {"present", "value"}
|
|
347
|
+
and type(item["present"]) is bool
|
|
348
|
+
and (item["present"] or item["value"] is None),
|
|
349
|
+
"invalid rejected counter observation",
|
|
350
|
+
)
|
|
351
|
+
_require(
|
|
352
|
+
not all(
|
|
353
|
+
item["present"] and _integer(item["value"]) and item["value"] <= 1
|
|
354
|
+
for item in counters.values()
|
|
355
|
+
),
|
|
356
|
+
"counter rejection lacks an invalid counter",
|
|
357
|
+
)
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
_historical_observation = replay._historical_report
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _validate_historical_stream(observation, content):
|
|
364
|
+
runtime = observation["runtime"]
|
|
365
|
+
_require(
|
|
366
|
+
_sha(content) == runtime["stdout_sha256"]
|
|
367
|
+
and len(content) == runtime["stdout_bytes"],
|
|
368
|
+
"historical stream differs from runtime evidence",
|
|
369
|
+
)
|
|
370
|
+
parsed = replay._historical_report(
|
|
371
|
+
boundary.ContainerResult(runtime["status"], stdout=content)
|
|
372
|
+
)
|
|
373
|
+
parsed["runtime"] = runtime
|
|
374
|
+
_require(
|
|
375
|
+
_json(parsed) == _json(observation),
|
|
376
|
+
"historical stream differs from observation",
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _validate_task(task, checks):
|
|
381
|
+
counts = [
|
|
382
|
+
task.get(key)
|
|
383
|
+
for key in (
|
|
384
|
+
"tests_applicable",
|
|
385
|
+
"tests_completed",
|
|
386
|
+
"tests_passed",
|
|
387
|
+
"tests_failed",
|
|
388
|
+
)
|
|
389
|
+
]
|
|
390
|
+
_require(all(_integer(value) for value in counts), "missing task counts")
|
|
391
|
+
total, completed, passed, failed = counts
|
|
392
|
+
_require(
|
|
393
|
+
total == checks and passed + failed == completed <= total,
|
|
394
|
+
"task counts disagree with the task contract",
|
|
395
|
+
)
|
|
396
|
+
_require(
|
|
397
|
+
type(task.get("determinate")) is bool
|
|
398
|
+
and type(task.get("test_coverage_complete")) is bool
|
|
399
|
+
and task.get("comparison_authority") == "host",
|
|
400
|
+
"invalid task authority",
|
|
401
|
+
)
|
|
402
|
+
if task["determinate"]:
|
|
403
|
+
_require(
|
|
404
|
+
completed == total
|
|
405
|
+
and task["test_coverage_complete"]
|
|
406
|
+
and task["status"] == ("failed" if failed else "passed")
|
|
407
|
+
and task.get("reason") is None
|
|
408
|
+
and task["runtime"]["status"] == "completed",
|
|
409
|
+
"inconsistent determinate task evidence",
|
|
410
|
+
)
|
|
411
|
+
_require(
|
|
412
|
+
task["runtime"].get("config_verified") is True
|
|
413
|
+
and task["runtime"].get("cleanup_verified") is True
|
|
414
|
+
and task["runtime"].get("error") is None,
|
|
415
|
+
"determinate task requires verified runtime and cleanup",
|
|
416
|
+
)
|
|
417
|
+
else:
|
|
418
|
+
_require(
|
|
419
|
+
task["status"] == "unresolved"
|
|
420
|
+
and not task["test_coverage_complete"]
|
|
421
|
+
and type(task.get("reason")) is str,
|
|
422
|
+
"inconsistent unresolved task evidence",
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def _inputs(control, source, artifacts):
|
|
427
|
+
try:
|
|
428
|
+
_require(
|
|
429
|
+
type(control.get("replay_schema_version")) is int
|
|
430
|
+
and control["replay_schema_version"] == 1,
|
|
431
|
+
"unsupported replay version",
|
|
432
|
+
)
|
|
433
|
+
identity = control["identity"]
|
|
434
|
+
_require(_integer(identity.get("step")), "invalid canonical step type")
|
|
435
|
+
row = replay._selected_source(
|
|
436
|
+
source,
|
|
437
|
+
identity["run"],
|
|
438
|
+
identity["sample_id"],
|
|
439
|
+
control["source"]["response_sha256"],
|
|
440
|
+
)
|
|
441
|
+
_require(
|
|
442
|
+
identity
|
|
443
|
+
== {key: row[key] for key in ("run", "sample_id", "step", "prompt_id")},
|
|
444
|
+
"canonical identity mismatch",
|
|
445
|
+
)
|
|
446
|
+
_require(
|
|
447
|
+
control["source"]["sha256"] == aria.ARIA_SHA256, "source hash mismatch"
|
|
448
|
+
)
|
|
449
|
+
_require(
|
|
450
|
+
control["implementation_sha256"] == _implementation(),
|
|
451
|
+
"replay implementation hash mismatch",
|
|
452
|
+
)
|
|
453
|
+
_require(
|
|
454
|
+
control["protected_worker_sha256"] == _sha(replay._ANSWER_WORKER.encode()),
|
|
455
|
+
"protected worker hash mismatch",
|
|
456
|
+
)
|
|
457
|
+
paths = aria.verify_artifacts(artifacts)
|
|
458
|
+
evaluator = control["evaluator"]
|
|
459
|
+
_require(
|
|
460
|
+
evaluator["commit"] == aria.UPSTREAM_REVISION
|
|
461
|
+
and evaluator["component"] == "hinted_test_call"
|
|
462
|
+
and evaluator["source_sha256"]
|
|
463
|
+
== {key: _sha(path.read_bytes()) for key, path in paths.items()},
|
|
464
|
+
"evaluator provenance mismatch",
|
|
465
|
+
)
|
|
466
|
+
task = aria.load_task(paths["fixture"], row["prompt_id"], row["prompt"])
|
|
467
|
+
cases = aria.protected_cases(task)
|
|
468
|
+
_require(
|
|
469
|
+
_integer(control["task_source"].get("checks")),
|
|
470
|
+
"invalid task check count type",
|
|
471
|
+
)
|
|
472
|
+
_require(
|
|
473
|
+
control["task_source"]
|
|
474
|
+
== {
|
|
475
|
+
"prompt_id": row["prompt_id"],
|
|
476
|
+
"task_sha256": _sha(json.dumps(task, sort_keys=True).encode()),
|
|
477
|
+
"checks": len(cases),
|
|
478
|
+
},
|
|
479
|
+
"task contract mismatch",
|
|
480
|
+
)
|
|
481
|
+
program, extraction = _extraction(row["response"])
|
|
482
|
+
_require(
|
|
483
|
+
program is not None and control["extraction"] == extraction,
|
|
484
|
+
"control extraction mismatch",
|
|
485
|
+
)
|
|
486
|
+
_require(
|
|
487
|
+
control["configuration"] == {"image": boundary.DEFAULT_IMAGE},
|
|
488
|
+
"incompatible control sandbox",
|
|
489
|
+
)
|
|
490
|
+
_validate_outcomes(control, len(cases))
|
|
491
|
+
except (KeyError, TypeError, AttributeError) as exc:
|
|
492
|
+
raise ValueError("incomplete or incompatible replay evidence") from exc
|
|
493
|
+
return row, task, cases, paths
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def _predictions(values):
|
|
497
|
+
_require(
|
|
498
|
+
type(values) is dict and values and set(values) <= _PREDICTIONS,
|
|
499
|
+
"predictions must use supported outcome fields",
|
|
500
|
+
)
|
|
501
|
+
for name, value in values.items():
|
|
502
|
+
if name == "exception_types":
|
|
503
|
+
valid = type(value) is list and all(
|
|
504
|
+
type(v) is str and re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]{0,63}", v)
|
|
505
|
+
for v in value
|
|
506
|
+
)
|
|
507
|
+
elif name == "task_stdout_sha256":
|
|
508
|
+
valid = _hash(value)
|
|
509
|
+
elif name == "task_status":
|
|
510
|
+
valid = value in ("passed", "failed", "unresolved", "unavailable")
|
|
511
|
+
else:
|
|
512
|
+
valid = _integer(value) and (
|
|
513
|
+
not name.startswith("historical_") or value <= 1
|
|
514
|
+
)
|
|
515
|
+
_require(valid, "invalid predicted outcome")
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _workers(response, task, paths):
|
|
519
|
+
program, extraction = _extraction(response)
|
|
520
|
+
_require(
|
|
521
|
+
program is not None and extraction["parse_status"] != "lookup_error",
|
|
522
|
+
"intervention requires an extractable historical program",
|
|
523
|
+
)
|
|
524
|
+
historical = aria.historical_script(
|
|
525
|
+
paths, task["setup_code"], program, extraction["requested_call"]
|
|
526
|
+
)
|
|
527
|
+
cases = aria.protected_cases(task)
|
|
528
|
+
protected = (
|
|
529
|
+
replay._ANSWER_WORKER,
|
|
530
|
+
json.dumps(
|
|
531
|
+
{
|
|
532
|
+
"setup": task["setup_code"],
|
|
533
|
+
"program": program,
|
|
534
|
+
"cases": [
|
|
535
|
+
{"method": c["method"], "kwargs": c["kwargs"]} for c in cases
|
|
536
|
+
],
|
|
537
|
+
},
|
|
538
|
+
allow_nan=False,
|
|
539
|
+
).encode(),
|
|
540
|
+
)
|
|
541
|
+
return [historical, protected], extraction
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def prepare_aria_intervention(
|
|
545
|
+
control, source, *, artifacts, edit, hypothesis, predictions, directory, config=None
|
|
546
|
+
):
|
|
547
|
+
"""Freeze a control and one unique exact text replacement without execution.
|
|
548
|
+
|
|
549
|
+
Case files contain private derived text. Use a fresh local directory outside
|
|
550
|
+
source control. An operationally recorded unresolved outcome remains unresolved.
|
|
551
|
+
"""
|
|
552
|
+
content = _read(control)
|
|
553
|
+
frozen = replay._json_output(content)
|
|
554
|
+
_require(type(frozen) is dict, "invalid investigation JSON object")
|
|
555
|
+
row, task, _cases, paths = _inputs(frozen, source, artifacts)
|
|
556
|
+
_require(
|
|
557
|
+
type(edit) is dict
|
|
558
|
+
and set(edit) == {"old", "new"}
|
|
559
|
+
and all(type(value) is str for value in edit.values())
|
|
560
|
+
and edit["old"]
|
|
561
|
+
and edit["old"] != edit["new"]
|
|
562
|
+
and row["response"].count(edit["old"]) == 1,
|
|
563
|
+
"edit must replace exactly one unique nonempty occurrence",
|
|
564
|
+
)
|
|
565
|
+
_require(
|
|
566
|
+
type(hypothesis) is str and hypothesis.strip() and len(hypothesis) <= 4096,
|
|
567
|
+
"hypothesis must be nonempty and at most 4096 characters",
|
|
568
|
+
)
|
|
569
|
+
_predictions(predictions)
|
|
570
|
+
config = config or boundary.SandboxConfig()
|
|
571
|
+
configuration = _sandbox(config)
|
|
572
|
+
variant = row["response"].replace(edit["old"], edit["new"], 1)
|
|
573
|
+
workers, extraction = _workers(variant, task, paths)
|
|
574
|
+
diff = "".join(
|
|
575
|
+
difflib.unified_diff(
|
|
576
|
+
row["response"].splitlines(keepends=True),
|
|
577
|
+
variant.splitlines(keepends=True),
|
|
578
|
+
fromfile="canonical-response",
|
|
579
|
+
tofile="derived-response",
|
|
580
|
+
)
|
|
581
|
+
)
|
|
582
|
+
manifest = {
|
|
583
|
+
"case_schema_version": 1,
|
|
584
|
+
"parent_identity": frozen["identity"],
|
|
585
|
+
"parent_response_sha256": frozen["source"]["response_sha256"],
|
|
586
|
+
"source_sha256": frozen["source"]["sha256"],
|
|
587
|
+
"control_result_sha256": _sha(content),
|
|
588
|
+
"variant_response_sha256": _sha(variant.encode()),
|
|
589
|
+
"source_diff_sha256": _sha(diff.encode()),
|
|
590
|
+
"variant_program_sha256": extraction["program_sha256"],
|
|
591
|
+
"extraction": extraction,
|
|
592
|
+
"hypothesis": hypothesis,
|
|
593
|
+
"predictions": predictions,
|
|
594
|
+
"edit": edit,
|
|
595
|
+
"evaluator": frozen["evaluator"],
|
|
596
|
+
"task_source": frozen["task_source"],
|
|
597
|
+
"implementation_sha256": _implementation(),
|
|
598
|
+
"workflow_sha256": _sha(Path(__file__).read_bytes()),
|
|
599
|
+
"configuration": configuration,
|
|
600
|
+
"docker_host": config.docker_host,
|
|
601
|
+
"worker_inputs": [
|
|
602
|
+
{"script_sha256": _sha(script.encode()), "payload_sha256": _sha(payload)}
|
|
603
|
+
for script, payload in workers
|
|
604
|
+
],
|
|
605
|
+
}
|
|
606
|
+
manifest["case_id"] = "aria-" + _sha(_json(manifest).encode())
|
|
607
|
+
manifest["variant_id"] = manifest["case_id"] + "-variant"
|
|
608
|
+
directory = Path(directory)
|
|
609
|
+
directory.mkdir(mode=0o700, exist_ok=False)
|
|
610
|
+
_write_text(content.decode("utf-8"), directory / "control.json")
|
|
611
|
+
_write_text(variant, directory / "variant.txt")
|
|
612
|
+
_write_text(diff, directory / "source.diff")
|
|
613
|
+
_save(directory, "case.json", manifest)
|
|
614
|
+
return {**manifest, "case_sha256": _sha(_read(directory / "case.json"))}
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _case(directory):
|
|
618
|
+
directory = Path(directory).resolve(strict=True)
|
|
619
|
+
manifest = _load(directory / "case.json")
|
|
620
|
+
try:
|
|
621
|
+
_require(
|
|
622
|
+
type(manifest["case_schema_version"]) is int
|
|
623
|
+
and manifest["case_schema_version"] == 1,
|
|
624
|
+
"unsupported case version",
|
|
625
|
+
)
|
|
626
|
+
identity_fields = {
|
|
627
|
+
key: value
|
|
628
|
+
for key, value in manifest.items()
|
|
629
|
+
if key not in {"case_id", "variant_id"}
|
|
630
|
+
}
|
|
631
|
+
case_id = "aria-" + _sha(_json(identity_fields).encode())
|
|
632
|
+
_require(
|
|
633
|
+
manifest["case_id"] == case_id
|
|
634
|
+
and manifest["variant_id"] == case_id + "-variant",
|
|
635
|
+
"case identity hash mismatch",
|
|
636
|
+
)
|
|
637
|
+
for name, key in [
|
|
638
|
+
("control.json", "control_result_sha256"),
|
|
639
|
+
("variant.txt", "variant_response_sha256"),
|
|
640
|
+
("source.diff", "source_diff_sha256"),
|
|
641
|
+
]:
|
|
642
|
+
_require(
|
|
643
|
+
_sha(_read(directory / name)) == manifest[key],
|
|
644
|
+
"case file hash mismatch",
|
|
645
|
+
)
|
|
646
|
+
_require(
|
|
647
|
+
manifest["implementation_sha256"] == _implementation()
|
|
648
|
+
and manifest["workflow_sha256"] == _sha(Path(__file__).read_bytes()),
|
|
649
|
+
"case implementation hash mismatch",
|
|
650
|
+
)
|
|
651
|
+
_require(
|
|
652
|
+
manifest["configuration"]
|
|
653
|
+
== _sandbox(boundary.SandboxConfig(docker_host=manifest["docker_host"])),
|
|
654
|
+
"case sandbox mismatch",
|
|
655
|
+
)
|
|
656
|
+
_predictions(manifest["predictions"])
|
|
657
|
+
control = _load(directory / "control.json")
|
|
658
|
+
for name, expected in {
|
|
659
|
+
"parent_identity": control["identity"],
|
|
660
|
+
"parent_response_sha256": control["source"]["response_sha256"],
|
|
661
|
+
"source_sha256": control["source"]["sha256"],
|
|
662
|
+
"evaluator": control["evaluator"],
|
|
663
|
+
"task_source": control["task_source"],
|
|
664
|
+
"implementation_sha256": control["implementation_sha256"],
|
|
665
|
+
}.items():
|
|
666
|
+
_require(
|
|
667
|
+
manifest[name] == expected,
|
|
668
|
+
"case metadata disagrees with frozen control",
|
|
669
|
+
)
|
|
670
|
+
_validate_outcomes(control, manifest["task_source"]["checks"])
|
|
671
|
+
except (KeyError, TypeError, AttributeError) as exc:
|
|
672
|
+
raise ValueError("incomplete case manifest") from exc
|
|
673
|
+
return directory, manifest, _sha(_read(directory / "case.json"))
|
|
674
|
+
|
|
675
|
+
|
|
676
|
+
def approve_aria_intervention(
|
|
677
|
+
directory,
|
|
678
|
+
*,
|
|
679
|
+
case_sha256,
|
|
680
|
+
variant_sha256,
|
|
681
|
+
max_containers,
|
|
682
|
+
execution_authorized=False,
|
|
683
|
+
):
|
|
684
|
+
"""Write one path-bound approval; approval alone never starts a container."""
|
|
685
|
+
_require(execution_authorized is True, "explicit execution authorization required")
|
|
686
|
+
directory, manifest, actual = _case(directory)
|
|
687
|
+
_require(not (directory / "consumed.json").exists(), "authorization is spent")
|
|
688
|
+
_require(
|
|
689
|
+
case_sha256 == actual and variant_sha256 == manifest["variant_response_sha256"],
|
|
690
|
+
"approval hash mismatch",
|
|
691
|
+
)
|
|
692
|
+
_require(
|
|
693
|
+
type(max_containers) is int and max_containers == 2,
|
|
694
|
+
"this workflow requires a budget of exactly two evaluation containers",
|
|
695
|
+
)
|
|
696
|
+
approval = {
|
|
697
|
+
"approval_schema_version": 1,
|
|
698
|
+
"case_sha256": actual,
|
|
699
|
+
"variant_response_sha256": variant_sha256,
|
|
700
|
+
"case_directory": str(directory),
|
|
701
|
+
"max_containers": max_containers,
|
|
702
|
+
"execution_authorized": True,
|
|
703
|
+
}
|
|
704
|
+
_save(directory, "approval.json", approval)
|
|
705
|
+
return approval
|
|
706
|
+
|
|
707
|
+
|
|
708
|
+
def _comparison(manifest, control, variant, *, status, origin):
|
|
709
|
+
claim = variant.get("historical", {}).get("reported_result") or {}
|
|
710
|
+
task = variant.get("task", {})
|
|
711
|
+
actual = {
|
|
712
|
+
"historical_passed": claim.get("tests_passed"),
|
|
713
|
+
"historical_evaluated": claim.get("tests_evaluated"),
|
|
714
|
+
"exception_types": claim.get("exception_types"),
|
|
715
|
+
"task_matched": task.get("tests_passed"),
|
|
716
|
+
"task_mismatched": task.get("tests_failed"),
|
|
717
|
+
"task_status": task.get("status"),
|
|
718
|
+
"task_stdout_sha256": task.get("runtime", {}).get("stdout_sha256"),
|
|
719
|
+
}
|
|
720
|
+
history_runtime = variant.get("historical", {}).get("runtime", {})
|
|
721
|
+
history_available = (
|
|
722
|
+
variant.get("historical", {}).get("status") == "reported"
|
|
723
|
+
and history_runtime.get("status") == "completed"
|
|
724
|
+
and history_runtime.get("config_verified") is True
|
|
725
|
+
and history_runtime.get("cleanup_verified") is True
|
|
726
|
+
)
|
|
727
|
+
checks = [
|
|
728
|
+
{
|
|
729
|
+
"field": name,
|
|
730
|
+
"expected": expected,
|
|
731
|
+
"observed": actual[name],
|
|
732
|
+
"status": "unavailable"
|
|
733
|
+
if actual[name] is None
|
|
734
|
+
or (
|
|
735
|
+
name in {"historical_passed", "historical_evaluated", "exception_types"}
|
|
736
|
+
and not history_available
|
|
737
|
+
)
|
|
738
|
+
or (
|
|
739
|
+
name in {"task_matched", "task_mismatched", "task_stdout_sha256"}
|
|
740
|
+
and not task.get("test_coverage_complete", False)
|
|
741
|
+
)
|
|
742
|
+
else "matched"
|
|
743
|
+
if type(expected) is type(actual[name]) and expected == actual[name]
|
|
744
|
+
else "violated",
|
|
745
|
+
}
|
|
746
|
+
for name, expected in sorted(manifest["predictions"].items())
|
|
747
|
+
]
|
|
748
|
+
a = control["task"]["runtime"]["stdout_sha256"]
|
|
749
|
+
b = actual["task_stdout_sha256"]
|
|
750
|
+
return {
|
|
751
|
+
"comparison_schema_version": 1,
|
|
752
|
+
"case_id": manifest["case_id"],
|
|
753
|
+
"variant_id": manifest["variant_id"],
|
|
754
|
+
"parent_identity": manifest["parent_identity"],
|
|
755
|
+
"hypothesis": manifest["hypothesis"],
|
|
756
|
+
"status": status,
|
|
757
|
+
"origin": origin,
|
|
758
|
+
"predictions": checks,
|
|
759
|
+
"control": {name: control[name] for name in ("historical", "task")},
|
|
760
|
+
"variant": {name: variant[name] for name in ("historical", "task")},
|
|
761
|
+
"answer_stream_equal": a == b
|
|
762
|
+
if b is not None
|
|
763
|
+
and control["task"]["test_coverage_complete"]
|
|
764
|
+
and task.get("test_coverage_complete")
|
|
765
|
+
else None,
|
|
766
|
+
"mechanism_status": "unresolved",
|
|
767
|
+
"historical_claims_authenticated": False,
|
|
768
|
+
"interpretation": "Predictions are compared literally with recorded observations; incomplete comparisons remain unavailable. Historical counters remain candidate-controlled. No internal call trace or reward-hacking mechanism is authenticated.",
|
|
769
|
+
"control_result_sha256": manifest["control_result_sha256"],
|
|
770
|
+
"variant_response_sha256": manifest["variant_response_sha256"],
|
|
771
|
+
"provenance": {
|
|
772
|
+
key: manifest[key]
|
|
773
|
+
for key in (
|
|
774
|
+
"source_sha256",
|
|
775
|
+
"parent_response_sha256",
|
|
776
|
+
"variant_program_sha256",
|
|
777
|
+
"evaluator",
|
|
778
|
+
"task_source",
|
|
779
|
+
"configuration",
|
|
780
|
+
)
|
|
781
|
+
},
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
|
|
785
|
+
def compare_aria_intervention(directory, result, *, provenance=None):
|
|
786
|
+
"""Compare saved evidence without execution, optionally importing a pinned protocol.
|
|
787
|
+
|
|
788
|
+
External derived-replay records require their original hash-bound protocol.
|
|
789
|
+
Imported evidence confers no new execution authorization or authenticity.
|
|
790
|
+
"""
|
|
791
|
+
directory, manifest, case_sha = _case(directory)
|
|
792
|
+
variant = _load(result)
|
|
793
|
+
control = _load(directory / "control.json")
|
|
794
|
+
try:
|
|
795
|
+
_require(
|
|
796
|
+
variant["variant_response_sha256"] == manifest["variant_response_sha256"],
|
|
797
|
+
"variant hash mismatch",
|
|
798
|
+
)
|
|
799
|
+
if provenance is None:
|
|
800
|
+
_require(
|
|
801
|
+
variant["case_sha256"] == case_sha
|
|
802
|
+
and variant["case_id"] == manifest["case_id"],
|
|
803
|
+
"result case hash mismatch",
|
|
804
|
+
)
|
|
805
|
+
_require(
|
|
806
|
+
type(variant.get("investigation_result_version")) is int
|
|
807
|
+
and variant["investigation_result_version"] == 1,
|
|
808
|
+
"unsupported investigation result",
|
|
809
|
+
)
|
|
810
|
+
status = variant["status"]
|
|
811
|
+
else:
|
|
812
|
+
protocol = _load(provenance)
|
|
813
|
+
_require(
|
|
814
|
+
type(variant.get("derived_replay_schema_version")) is int
|
|
815
|
+
and variant["derived_replay_schema_version"] == 1,
|
|
816
|
+
"unsupported derived replay evidence",
|
|
817
|
+
)
|
|
818
|
+
_require(
|
|
819
|
+
variant["protocol_sha256"] == _sha(_read(provenance)),
|
|
820
|
+
"provenance hash mismatch",
|
|
821
|
+
)
|
|
822
|
+
for key in (
|
|
823
|
+
"control_result_sha256",
|
|
824
|
+
"parent_identity",
|
|
825
|
+
"parent_response_sha256",
|
|
826
|
+
"source_sha256",
|
|
827
|
+
"variant_response_sha256",
|
|
828
|
+
"variant_program_sha256",
|
|
829
|
+
"implementation_sha256",
|
|
830
|
+
"evaluator",
|
|
831
|
+
"task_source",
|
|
832
|
+
"configuration",
|
|
833
|
+
"worker_inputs",
|
|
834
|
+
):
|
|
835
|
+
_require(
|
|
836
|
+
protocol[key] == manifest[key],
|
|
837
|
+
"incompatible saved intervention provenance",
|
|
838
|
+
)
|
|
839
|
+
_require(
|
|
840
|
+
variant["parent_identity"] == manifest["parent_identity"],
|
|
841
|
+
"derived parent mismatch",
|
|
842
|
+
)
|
|
843
|
+
status = "complete"
|
|
844
|
+
_require(
|
|
845
|
+
variant["extraction"] == manifest["extraction"],
|
|
846
|
+
"derived extraction mismatch",
|
|
847
|
+
)
|
|
848
|
+
_require(status in {"complete", "stopped"}, "invalid attempt status")
|
|
849
|
+
_validate_outcomes(
|
|
850
|
+
variant,
|
|
851
|
+
manifest["task_source"]["checks"],
|
|
852
|
+
partial=status == "stopped",
|
|
853
|
+
architecture=control["historical"]["runtime"]["metadata"]["architecture"],
|
|
854
|
+
)
|
|
855
|
+
if provenance is None and "runtime" in variant["historical"]:
|
|
856
|
+
stream_path = directory / "mode-1.stdout.hex"
|
|
857
|
+
if status == "complete" or stream_path.exists() or stream_path.is_symlink():
|
|
858
|
+
_require(stream_path.is_file(), "historical stream unavailable")
|
|
859
|
+
_validate_historical_stream(
|
|
860
|
+
variant["historical"],
|
|
861
|
+
bytes.fromhex(_read(stream_path).decode("ascii")),
|
|
862
|
+
)
|
|
863
|
+
except (KeyError, TypeError, AttributeError) as exc:
|
|
864
|
+
raise ValueError("incomplete or incompatible derived evidence") from exc
|
|
865
|
+
return _comparison(
|
|
866
|
+
manifest, control, variant, status=status, origin="saved_evidence"
|
|
867
|
+
)
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
def execute_aria_intervention(
|
|
871
|
+
directory, source, *, artifacts, execution_authorized=False
|
|
872
|
+
):
|
|
873
|
+
"""Consume one approval durably, then attempt each mode at most once.
|
|
874
|
+
|
|
875
|
+
Failures stay spent. Local filesystem ownership is trusted: deleting records
|
|
876
|
+
or bypassing this API is outside the approval mechanism's threat model.
|
|
877
|
+
"""
|
|
878
|
+
_require(execution_authorized is True, "explicit execution authorization required")
|
|
879
|
+
directory, manifest, case_sha = _case(directory)
|
|
880
|
+
_require(not (directory / "consumed.json").exists(), "authorization is spent")
|
|
881
|
+
approval = _load(directory / "approval.json")
|
|
882
|
+
_require(
|
|
883
|
+
type(approval.get("approval_schema_version")) is int
|
|
884
|
+
and type(approval.get("max_containers")) is int
|
|
885
|
+
and approval.get("execution_authorized") is True,
|
|
886
|
+
"invalid approval field types",
|
|
887
|
+
)
|
|
888
|
+
_require(
|
|
889
|
+
approval
|
|
890
|
+
== {
|
|
891
|
+
"approval_schema_version": 1,
|
|
892
|
+
"case_sha256": case_sha,
|
|
893
|
+
"variant_response_sha256": manifest["variant_response_sha256"],
|
|
894
|
+
"case_directory": str(directory),
|
|
895
|
+
"max_containers": 2,
|
|
896
|
+
"execution_authorized": True,
|
|
897
|
+
},
|
|
898
|
+
"approval hashes, directory, or budget do not match this case",
|
|
899
|
+
)
|
|
900
|
+
control = _load(directory / "control.json")
|
|
901
|
+
row, task, cases, paths = _inputs(control, source, artifacts)
|
|
902
|
+
response = _read(directory / "variant.txt").decode("utf-8")
|
|
903
|
+
_require(
|
|
904
|
+
response
|
|
905
|
+
== row["response"].replace(manifest["edit"]["old"], manifest["edit"]["new"], 1),
|
|
906
|
+
"canonical edit mismatch",
|
|
907
|
+
)
|
|
908
|
+
workers, extraction = _workers(response, task, paths)
|
|
909
|
+
_require(
|
|
910
|
+
extraction == manifest["extraction"]
|
|
911
|
+
and manifest["worker_inputs"]
|
|
912
|
+
== [
|
|
913
|
+
{"script_sha256": _sha(script.encode()), "payload_sha256": _sha(payload)}
|
|
914
|
+
for script, payload in workers
|
|
915
|
+
],
|
|
916
|
+
"worker input hash mismatch",
|
|
917
|
+
)
|
|
918
|
+
for name in (
|
|
919
|
+
"result.json",
|
|
920
|
+
"comparison.json",
|
|
921
|
+
"comparison.html",
|
|
922
|
+
"mode-1.json",
|
|
923
|
+
"mode-1.stdout.hex",
|
|
924
|
+
"mode-2.json",
|
|
925
|
+
):
|
|
926
|
+
_require(not (directory / name).exists(), "execution output already exists")
|
|
927
|
+
claim = {
|
|
928
|
+
"approval_sha256": _sha(_read(directory / "approval.json")),
|
|
929
|
+
"case_sha256": case_sha,
|
|
930
|
+
"reserved_containers": 2,
|
|
931
|
+
"consumed_at": datetime.now(UTC).isoformat(),
|
|
932
|
+
"status": "spent",
|
|
933
|
+
}
|
|
934
|
+
_save(directory, "consumed.json", claim)
|
|
935
|
+
result = {
|
|
936
|
+
"investigation_result_version": 1,
|
|
937
|
+
"case_id": manifest["case_id"],
|
|
938
|
+
"case_sha256": case_sha,
|
|
939
|
+
"variant_id": manifest["variant_id"],
|
|
940
|
+
"variant_response_sha256": manifest["variant_response_sha256"],
|
|
941
|
+
"extraction": extraction,
|
|
942
|
+
"status": "stopped",
|
|
943
|
+
"mechanism_status": "unresolved",
|
|
944
|
+
"historical": {"status": "unavailable", "reason": "not_attempted"},
|
|
945
|
+
"task": {
|
|
946
|
+
"status": "unavailable",
|
|
947
|
+
"reason": "not_attempted",
|
|
948
|
+
"determinate": False,
|
|
949
|
+
},
|
|
950
|
+
}
|
|
951
|
+
config = boundary.SandboxConfig(docker_host=manifest["docker_host"])
|
|
952
|
+
interruption = None
|
|
953
|
+
try:
|
|
954
|
+
for index, (script, payload) in enumerate(workers):
|
|
955
|
+
_require(index < approval["max_containers"], "container budget exhausted")
|
|
956
|
+
_require(
|
|
957
|
+
_read(directory / "consumed.json") == _json(claim).encode()
|
|
958
|
+
and _sha(_read(directory / "approval.json"))
|
|
959
|
+
== claim["approval_sha256"],
|
|
960
|
+
"execution authorization accounting changed",
|
|
961
|
+
)
|
|
962
|
+
if index == 1:
|
|
963
|
+
_require(
|
|
964
|
+
_read(directory / "mode-1-started.json")
|
|
965
|
+
== _json(manifest["worker_inputs"][0]).encode()
|
|
966
|
+
and _read(directory / "mode-1.json")
|
|
967
|
+
== _json(result["historical"]).encode(),
|
|
968
|
+
"historical mode accounting changed",
|
|
969
|
+
)
|
|
970
|
+
_validate_historical_stream(
|
|
971
|
+
result["historical"],
|
|
972
|
+
bytes.fromhex(
|
|
973
|
+
_read(directory / "mode-1.stdout.hex").decode("ascii")
|
|
974
|
+
),
|
|
975
|
+
)
|
|
976
|
+
_save(
|
|
977
|
+
directory,
|
|
978
|
+
f"mode-{index + 1}-started.json",
|
|
979
|
+
manifest["worker_inputs"][index],
|
|
980
|
+
)
|
|
981
|
+
raw = None
|
|
982
|
+
try:
|
|
983
|
+
raw = boundary.run_container(
|
|
984
|
+
script, payload, config, execution_authorized=True
|
|
985
|
+
)
|
|
986
|
+
except BaseException as error:
|
|
987
|
+
raw = getattr(error, "replay_result", None)
|
|
988
|
+
if raw is None:
|
|
989
|
+
raise
|
|
990
|
+
interruption = error
|
|
991
|
+
if index == 0:
|
|
992
|
+
observation = _historical_observation(raw)
|
|
993
|
+
else:
|
|
994
|
+
observation = replay.assess_answers(
|
|
995
|
+
replay._json_output(raw.stdout)
|
|
996
|
+
if raw.status == "completed"
|
|
997
|
+
else None,
|
|
998
|
+
cases,
|
|
999
|
+
)
|
|
1000
|
+
if raw.status != "completed":
|
|
1001
|
+
observation["reason"] = raw.status
|
|
1002
|
+
observation["runtime"] = replay._runtime(raw)
|
|
1003
|
+
if not raw.config_verified or not raw.cleanup_verified:
|
|
1004
|
+
observation.update(
|
|
1005
|
+
status="unresolved",
|
|
1006
|
+
determinate=False,
|
|
1007
|
+
test_coverage_complete=False,
|
|
1008
|
+
reason="unverified_runtime_or_cleanup",
|
|
1009
|
+
)
|
|
1010
|
+
observation["contract"] = (
|
|
1011
|
+
"all original checks; fresh Solution per call; host equality"
|
|
1012
|
+
)
|
|
1013
|
+
result["historical" if index == 0 else "task"] = observation
|
|
1014
|
+
if index == 0:
|
|
1015
|
+
_write_text(raw.stdout.hex(), directory / "mode-1.stdout.hex")
|
|
1016
|
+
_save(directory, f"mode-{index + 1}.json", observation)
|
|
1017
|
+
if interruption is not None:
|
|
1018
|
+
raise interruption
|
|
1019
|
+
_validate_runtime(observation["runtime"])
|
|
1020
|
+
_validate_architecture(
|
|
1021
|
+
(observation,),
|
|
1022
|
+
control["historical"]["runtime"]["metadata"]["architecture"],
|
|
1023
|
+
)
|
|
1024
|
+
if index == 0:
|
|
1025
|
+
_validate_historical(observation)
|
|
1026
|
+
_require(
|
|
1027
|
+
replay._historical_allows_protected(observation),
|
|
1028
|
+
observation.get("reason", "historical protocol unresolved"),
|
|
1029
|
+
)
|
|
1030
|
+
_validate_outcomes(result, len(cases))
|
|
1031
|
+
result["status"] = "complete"
|
|
1032
|
+
except BaseException as error: # noqa: BLE001 - preserve partial evidence and spent approval
|
|
1033
|
+
result["stop_reason"] = (
|
|
1034
|
+
str(error)
|
|
1035
|
+
if isinstance(error, ValueError)
|
|
1036
|
+
else "execution interrupted or infrastructure unavailable"
|
|
1037
|
+
)
|
|
1038
|
+
result["exception_type"] = type(error).__name__
|
|
1039
|
+
if isinstance(error, (KeyboardInterrupt, SystemExit)):
|
|
1040
|
+
interruption = error
|
|
1041
|
+
_save(directory, "result.json", result)
|
|
1042
|
+
comparison = _comparison(
|
|
1043
|
+
manifest,
|
|
1044
|
+
control,
|
|
1045
|
+
result,
|
|
1046
|
+
status=result["status"],
|
|
1047
|
+
origin="authorized_execution",
|
|
1048
|
+
)
|
|
1049
|
+
_save(directory, "comparison.json", comparison)
|
|
1050
|
+
_write_text(render_aria_comparison(comparison), directory / "comparison.html")
|
|
1051
|
+
if interruption is not None:
|
|
1052
|
+
raise interruption
|
|
1053
|
+
return comparison
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def render_aria_comparison(comparison):
|
|
1057
|
+
"""Render claims and host comparisons without embedding response text."""
|
|
1058
|
+
_require(
|
|
1059
|
+
type(comparison) is dict and comparison.get("comparison_schema_version") == 1,
|
|
1060
|
+
"unsupported Aria comparison version",
|
|
1061
|
+
)
|
|
1062
|
+
|
|
1063
|
+
def escaped(value):
|
|
1064
|
+
return html.escape("Unavailable" if value is None else str(value))
|
|
1065
|
+
|
|
1066
|
+
rows = []
|
|
1067
|
+
for label, mode, field in [
|
|
1068
|
+
("Historical claimed passes", "historical", "tests_passed"),
|
|
1069
|
+
("Reported exception types", "historical", "exception_types"),
|
|
1070
|
+
("Protected status", "task", "status"),
|
|
1071
|
+
("Protected matched", "task", "tests_passed"),
|
|
1072
|
+
("Protected mismatched", "task", "tests_failed"),
|
|
1073
|
+
("Protected completed", "task", "tests_completed"),
|
|
1074
|
+
("Protected applicable", "task", "tests_applicable"),
|
|
1075
|
+
("Protected unavailable reason", "task", "reason"),
|
|
1076
|
+
]:
|
|
1077
|
+
values = []
|
|
1078
|
+
for condition in ("control", "variant"):
|
|
1079
|
+
item = comparison[condition][mode]
|
|
1080
|
+
value = (
|
|
1081
|
+
(item.get("reported_result") or {}).get(field)
|
|
1082
|
+
if mode == "historical"
|
|
1083
|
+
else item.get(field)
|
|
1084
|
+
)
|
|
1085
|
+
if field == "reason" and item.get("determinate") is True and value is None:
|
|
1086
|
+
value = "Not applicable"
|
|
1087
|
+
values.append(value)
|
|
1088
|
+
rows.append(
|
|
1089
|
+
"<tr><th>"
|
|
1090
|
+
+ escaped(label)
|
|
1091
|
+
+ "</th>"
|
|
1092
|
+
+ "".join("<td>" + escaped(v) + "</td>" for v in values)
|
|
1093
|
+
+ "</tr>"
|
|
1094
|
+
)
|
|
1095
|
+
for mode in ("historical", "task"):
|
|
1096
|
+
for field in ("status", "stdout_sha256", "cleanup_verified"):
|
|
1097
|
+
values = [
|
|
1098
|
+
comparison[condition][mode].get("runtime", {}).get(field)
|
|
1099
|
+
for condition in ("control", "variant")
|
|
1100
|
+
]
|
|
1101
|
+
rows.append(
|
|
1102
|
+
"<tr><th>"
|
|
1103
|
+
+ escaped(mode + " runtime " + field)
|
|
1104
|
+
+ "</th>"
|
|
1105
|
+
+ "".join("<td>" + escaped(value) + "</td>" for value in values)
|
|
1106
|
+
+ "</tr>"
|
|
1107
|
+
)
|
|
1108
|
+
predictions = "".join(
|
|
1109
|
+
"<tr>"
|
|
1110
|
+
+ "".join(
|
|
1111
|
+
"<td>" + escaped(row[key]) + "</td>"
|
|
1112
|
+
for key in ("field", "expected", "observed", "status")
|
|
1113
|
+
)
|
|
1114
|
+
+ "</tr>"
|
|
1115
|
+
for row in comparison["predictions"]
|
|
1116
|
+
)
|
|
1117
|
+
return (
|
|
1118
|
+
'<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">'
|
|
1119
|
+
"<title>Aria intervention comparison</title><style>*{box-sizing:border-box}"
|
|
1120
|
+
"body{font:16px/1.5 system-ui;max-width:1000px;margin:32px auto;padding:0 20px;color:#182128;overflow-wrap:anywhere}"
|
|
1121
|
+
".table-scroll{max-width:100%;overflow-x:auto}"
|
|
1122
|
+
"table{border-collapse:collapse;width:100%;min-width:600px}td,th{text-align:left;min-width:100px;padding:8px;border-bottom:1px solid #ddd;overflow-wrap:anywhere}"
|
|
1123
|
+
"th:first-child,td:first-child{min-width:180px}"
|
|
1124
|
+
"code{overflow-wrap:anywhere}</style><h1>Aria intervention comparison</h1><p>Case: "
|
|
1125
|
+
+ escaped(comparison["case_id"])
|
|
1126
|
+
+ "</p>"
|
|
1127
|
+
"<p>Canonical parent: " + escaped(comparison["parent_identity"]) + "</p>"
|
|
1128
|
+
"<p>Status: "
|
|
1129
|
+
+ escaped(comparison["status"])
|
|
1130
|
+
+ "; evidence: "
|
|
1131
|
+
+ escaped(comparison["origin"])
|
|
1132
|
+
+ "</p>"
|
|
1133
|
+
"<p>Hypothesis: "
|
|
1134
|
+
+ escaped(comparison["hypothesis"])
|
|
1135
|
+
+ '</p><div class="table-scroll" tabindex="0" role="region" aria-label="Measurements"><table><tr><th>Measurement</th><th>Frozen control</th><th>Derived variant</th></tr>'
|
|
1136
|
+
+ "".join(rows)
|
|
1137
|
+
+ '</table></div><h2>Predictions</h2><div class="table-scroll" tabindex="0" role="region" aria-label="Predictions"><table><tr><th>Field</th><th>Expected</th><th>Observed</th><th>Result</th></tr>'
|
|
1138
|
+
+ predictions
|
|
1139
|
+
+ "</table></div>"
|
|
1140
|
+
"<p>Protected answer-stream hashes equal: "
|
|
1141
|
+
+ escaped(comparison["answer_stream_equal"])
|
|
1142
|
+
+ "</p>"
|
|
1143
|
+
"<p>"
|
|
1144
|
+
+ escaped(comparison["interpretation"])
|
|
1145
|
+
+ "</p><p>Mechanism status: unresolved. The control was not rerun.</p></html>"
|
|
1146
|
+
)
|