rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,678 @@
|
|
|
1
|
+
"""Bounded, read-only views of recorded investigation evidence."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import stat
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
import pandas as pd
|
|
14
|
+
|
|
15
|
+
from . import investigation
|
|
16
|
+
from ._report_validation import _constant, _object, validate_report
|
|
17
|
+
from ._watch_report import validate_watch_report
|
|
18
|
+
from .comparison_report import validate_comparison_report
|
|
19
|
+
|
|
20
|
+
_MAX_BYTES = 16 * 1024 * 1024
|
|
21
|
+
_IDENTITY = ("run", "sample_id", "step", "prompt_id")
|
|
22
|
+
_TASK_COUNTS = ("tests_applicable", "tests_completed", "tests_passed", "tests_failed")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _require(condition, message):
|
|
26
|
+
if not condition:
|
|
27
|
+
raise ValueError(message)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _digest(content):
|
|
31
|
+
return hashlib.sha256(content).hexdigest()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _read(path, root, *, optional=False):
|
|
35
|
+
"""Require a canonical root spelling and refuse symlinks at every component."""
|
|
36
|
+
root_path = Path(root).absolute()
|
|
37
|
+
_require(".." not in root_path.parts, "evidence root must use a canonical path")
|
|
38
|
+
path = Path(path)
|
|
39
|
+
if path.is_absolute():
|
|
40
|
+
try:
|
|
41
|
+
relative = path.relative_to(root_path)
|
|
42
|
+
except ValueError:
|
|
43
|
+
raise ValueError("evidence path must remain inside the root") from None
|
|
44
|
+
else:
|
|
45
|
+
relative = path
|
|
46
|
+
_require(
|
|
47
|
+
relative.parts and all(part not in {"..", "."} for part in relative.parts),
|
|
48
|
+
"evidence path must remain inside the root",
|
|
49
|
+
)
|
|
50
|
+
descriptors = []
|
|
51
|
+
try:
|
|
52
|
+
directory = os.open(root_path.anchor, os.O_RDONLY | os.O_DIRECTORY)
|
|
53
|
+
descriptors.append(directory)
|
|
54
|
+
parts = (*root_path.parts[1:], *relative.parts)
|
|
55
|
+
for part in parts[:-1]:
|
|
56
|
+
directory = os.open(
|
|
57
|
+
part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory
|
|
58
|
+
)
|
|
59
|
+
descriptors.append(directory)
|
|
60
|
+
descriptor = os.open(
|
|
61
|
+
relative.name,
|
|
62
|
+
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
63
|
+
dir_fd=directory,
|
|
64
|
+
)
|
|
65
|
+
descriptors.append(descriptor)
|
|
66
|
+
before = os.fstat(descriptor)
|
|
67
|
+
_require(
|
|
68
|
+
stat.S_ISREG(before.st_mode) and before.st_size <= _MAX_BYTES,
|
|
69
|
+
"evidence input must be a regular bounded file",
|
|
70
|
+
)
|
|
71
|
+
with os.fdopen(os.dup(descriptor), "rb") as stream:
|
|
72
|
+
content = stream.read(_MAX_BYTES + 1)
|
|
73
|
+
after = os.fstat(descriptor)
|
|
74
|
+
_require(
|
|
75
|
+
len(content) <= _MAX_BYTES
|
|
76
|
+
and (before.st_size, before.st_mtime_ns, before.st_ctime_ns)
|
|
77
|
+
== (after.st_size, after.st_mtime_ns, after.st_ctime_ns),
|
|
78
|
+
"evidence input changed during reading",
|
|
79
|
+
)
|
|
80
|
+
return content, relative
|
|
81
|
+
except FileNotFoundError:
|
|
82
|
+
if optional:
|
|
83
|
+
return None
|
|
84
|
+
raise ValueError("evidence file is unavailable inside the root") from None
|
|
85
|
+
except OSError:
|
|
86
|
+
raise ValueError(
|
|
87
|
+
"evidence input must be a regular file without symlinks"
|
|
88
|
+
) from None
|
|
89
|
+
finally:
|
|
90
|
+
for descriptor in reversed(descriptors):
|
|
91
|
+
os.close(descriptor)
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _parse(content):
|
|
95
|
+
def finite(number):
|
|
96
|
+
value = float(number)
|
|
97
|
+
_require(math.isfinite(value), "nonfinite JSON number")
|
|
98
|
+
return value
|
|
99
|
+
|
|
100
|
+
try:
|
|
101
|
+
value = json.loads(
|
|
102
|
+
content.decode("utf-8"),
|
|
103
|
+
object_pairs_hook=_object,
|
|
104
|
+
parse_constant=_constant,
|
|
105
|
+
parse_float=finite,
|
|
106
|
+
)
|
|
107
|
+
except (ValueError, UnicodeError, RecursionError):
|
|
108
|
+
raise ValueError("invalid evidence JSON") from None
|
|
109
|
+
_require(type(value) is dict, "evidence JSON must be an object")
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _identity(value):
|
|
114
|
+
_require(type(value) is dict, "missing canonical identity")
|
|
115
|
+
_require(
|
|
116
|
+
all(
|
|
117
|
+
type(value.get(key)) is str and 0 < len(value[key]) <= 4096
|
|
118
|
+
for key in ("run", "sample_id", "prompt_id")
|
|
119
|
+
)
|
|
120
|
+
and investigation._integer(value.get("step")),
|
|
121
|
+
"invalid canonical identity",
|
|
122
|
+
)
|
|
123
|
+
return {key: value[key] for key in _IDENTITY}
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _reason(value):
|
|
127
|
+
_require(
|
|
128
|
+
value is None
|
|
129
|
+
or (type(value) is str and re.fullmatch(r"[a-z][a-z0-9_]{0,127}", value)),
|
|
130
|
+
"unsupported recorded reason",
|
|
131
|
+
)
|
|
132
|
+
return value
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _runtime(value):
|
|
136
|
+
if value is None:
|
|
137
|
+
return {}
|
|
138
|
+
_require(type(value) is dict, "invalid recorded runtime")
|
|
139
|
+
if (
|
|
140
|
+
value.get("config_verified") is True
|
|
141
|
+
and value.get("cleanup_verified") is True
|
|
142
|
+
and value.get("status") != "infrastructure_error"
|
|
143
|
+
):
|
|
144
|
+
investigation._validate_runtime(value)
|
|
145
|
+
else:
|
|
146
|
+
_require(
|
|
147
|
+
value.get("status")
|
|
148
|
+
in {
|
|
149
|
+
"completed",
|
|
150
|
+
"nonzero_exit",
|
|
151
|
+
"timeout",
|
|
152
|
+
"output_limit",
|
|
153
|
+
"infrastructure_error",
|
|
154
|
+
}
|
|
155
|
+
and type(value.get("config_verified")) is bool
|
|
156
|
+
and type(value.get("cleanup_verified")) is bool
|
|
157
|
+
and all(
|
|
158
|
+
investigation._hash(value.get(stream + "_sha256"))
|
|
159
|
+
and investigation._integer(value.get(stream + "_bytes"))
|
|
160
|
+
for stream in ("stdout", "stderr")
|
|
161
|
+
),
|
|
162
|
+
"invalid partial runtime evidence",
|
|
163
|
+
)
|
|
164
|
+
return value
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _condition(name, record, checks):
|
|
168
|
+
historical, task = record.get("historical"), record.get("task")
|
|
169
|
+
_require(
|
|
170
|
+
type(historical) is dict and type(task) is dict, "missing recorded outcomes"
|
|
171
|
+
)
|
|
172
|
+
_require(
|
|
173
|
+
historical.get("status") in {"reported", "unavailable", "unresolved"}
|
|
174
|
+
and task.get("status") in {"passed", "failed", "unavailable", "unresolved"},
|
|
175
|
+
"unsupported recorded outcome status",
|
|
176
|
+
)
|
|
177
|
+
history_runtime = _runtime(historical.get("runtime"))
|
|
178
|
+
task_runtime = _runtime(task.get("runtime"))
|
|
179
|
+
investigation._validate_architecture((historical, task))
|
|
180
|
+
identifiers = [
|
|
181
|
+
runtime.get("metadata", {}).get("container_id")
|
|
182
|
+
for runtime in (history_runtime, task_runtime)
|
|
183
|
+
]
|
|
184
|
+
_require(
|
|
185
|
+
not all(identifiers) or identifiers[0] != identifiers[1],
|
|
186
|
+
"evaluation modes require distinct containers",
|
|
187
|
+
)
|
|
188
|
+
history_status = historical["status"]
|
|
189
|
+
history_reason = _reason(historical.get("reason"))
|
|
190
|
+
claim = historical.get("reported_result")
|
|
191
|
+
_require(claim is None or type(claim) is dict, "invalid historical claim")
|
|
192
|
+
if claim:
|
|
193
|
+
supplied = [claim.get(key) for key in ("tests_passed", "tests_evaluated")]
|
|
194
|
+
_require(
|
|
195
|
+
all(
|
|
196
|
+
value is None or (type(value) is int and value in (0, 1))
|
|
197
|
+
for value in supplied
|
|
198
|
+
)
|
|
199
|
+
and (None in supplied or supplied[0] <= supplied[1]),
|
|
200
|
+
"contradictory historical counts",
|
|
201
|
+
)
|
|
202
|
+
legacy = history_status == "reported" and (
|
|
203
|
+
not claim
|
|
204
|
+
or any(key not in claim for key in ("tests_passed", "tests_evaluated"))
|
|
205
|
+
or claim.get("tests_evaluated") == 0
|
|
206
|
+
or "authority" not in historical
|
|
207
|
+
)
|
|
208
|
+
if legacy:
|
|
209
|
+
history_status = "unavailable"
|
|
210
|
+
history_reason = "legacy_historical_counters_unavailable"
|
|
211
|
+
elif history_runtime:
|
|
212
|
+
investigation._validate_historical(historical)
|
|
213
|
+
else:
|
|
214
|
+
_require(
|
|
215
|
+
history_status != "reported", "historical claim requires runtime evidence"
|
|
216
|
+
)
|
|
217
|
+
passed = evaluated = None
|
|
218
|
+
if history_status == "reported":
|
|
219
|
+
passed, evaluated = claim["tests_passed"], claim["tests_evaluated"]
|
|
220
|
+
_require(passed <= evaluated == 1, "contradictory historical counts")
|
|
221
|
+
|
|
222
|
+
determinate = task.get("determinate")
|
|
223
|
+
_require(type(determinate) is bool, "missing task determinate status")
|
|
224
|
+
task_reason = _reason(task.get("reason"))
|
|
225
|
+
counts = [task.get(key) for key in _TASK_COUNTS]
|
|
226
|
+
_require(
|
|
227
|
+
all(value is None or investigation._integer(value) for value in counts),
|
|
228
|
+
"invalid task counts",
|
|
229
|
+
)
|
|
230
|
+
if all(value is not None for value in counts):
|
|
231
|
+
_require(
|
|
232
|
+
investigation._integer(checks) and checks > 0, "unsupported task contract"
|
|
233
|
+
)
|
|
234
|
+
investigation._validate_task(task, checks)
|
|
235
|
+
else:
|
|
236
|
+
_require(
|
|
237
|
+
not determinate and task["status"] in {"unavailable", "unresolved"},
|
|
238
|
+
"determinate task requires complete counts and coverage",
|
|
239
|
+
)
|
|
240
|
+
discrepancy = None
|
|
241
|
+
if (
|
|
242
|
+
determinate
|
|
243
|
+
and history_status == "reported"
|
|
244
|
+
and history_runtime.get("config_verified") is True
|
|
245
|
+
and history_runtime.get("cleanup_verified") is True
|
|
246
|
+
):
|
|
247
|
+
discrepancy = passed == 1 and task["status"] == "failed"
|
|
248
|
+
return {
|
|
249
|
+
"name": name,
|
|
250
|
+
"historical_status": history_status,
|
|
251
|
+
"historical_passed": passed,
|
|
252
|
+
"historical_evaluated": evaluated,
|
|
253
|
+
"historical_reason": history_reason,
|
|
254
|
+
"task_status": task["status"],
|
|
255
|
+
"task_determinate": determinate,
|
|
256
|
+
"task_applicable": counts[0],
|
|
257
|
+
"task_completed": counts[1],
|
|
258
|
+
"task_passed": counts[2],
|
|
259
|
+
"task_failed": counts[3],
|
|
260
|
+
"task_reason": task_reason,
|
|
261
|
+
"discrepancy": discrepancy,
|
|
262
|
+
"answer_stream_sha256": task_runtime.get("stdout_sha256"),
|
|
263
|
+
"cleanup_verified": task_runtime.get("cleanup_verified"),
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _provenance(value, identity):
|
|
268
|
+
_require(type(value) is dict, "missing source provenance")
|
|
269
|
+
for key in ("source_sha256", "parent_response_sha256"):
|
|
270
|
+
_require(investigation._hash(value.get(key)), "invalid source provenance hash")
|
|
271
|
+
task = value.get("task_source", {})
|
|
272
|
+
_require(type(task) is dict, "unsupported task source")
|
|
273
|
+
if task:
|
|
274
|
+
_require(
|
|
275
|
+
task.get("prompt_id") == identity["prompt_id"]
|
|
276
|
+
and investigation._hash(task.get("task_sha256"))
|
|
277
|
+
and investigation._integer(task.get("checks"))
|
|
278
|
+
and task["checks"] > 0,
|
|
279
|
+
"unsupported task source contract",
|
|
280
|
+
)
|
|
281
|
+
evaluator = value.get("evaluator", {})
|
|
282
|
+
_require(
|
|
283
|
+
type(evaluator) is dict
|
|
284
|
+
and (not task or evaluator.get("component") == "hinted_test_call"),
|
|
285
|
+
"unsupported recorded evaluator",
|
|
286
|
+
)
|
|
287
|
+
return task.get("checks")
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _replay(value):
|
|
291
|
+
_require(
|
|
292
|
+
type(value.get("replay_schema_version")) is int
|
|
293
|
+
and value["replay_schema_version"] == 1,
|
|
294
|
+
"unsupported replay schema version",
|
|
295
|
+
)
|
|
296
|
+
identity = _identity(value.get("identity"))
|
|
297
|
+
source = value.get("source")
|
|
298
|
+
_require(type(source) is dict, "missing replay source")
|
|
299
|
+
provenance = {
|
|
300
|
+
"source_sha256": source.get("sha256"),
|
|
301
|
+
"parent_response_sha256": source.get("response_sha256"),
|
|
302
|
+
"task_source": value.get("task_source", {}),
|
|
303
|
+
"evaluator": value.get("evaluator", {}),
|
|
304
|
+
}
|
|
305
|
+
checks = _provenance(provenance, identity)
|
|
306
|
+
_require(
|
|
307
|
+
value.get("mechanism_status") == "unresolved", "unsupported mechanism claim"
|
|
308
|
+
)
|
|
309
|
+
return identity, provenance, _condition("recorded", value, checks)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _case(value, path, root):
|
|
313
|
+
_require(
|
|
314
|
+
type(value.get("case_schema_version")) is int
|
|
315
|
+
and value["case_schema_version"] == 1,
|
|
316
|
+
"unsupported case schema version",
|
|
317
|
+
)
|
|
318
|
+
fields = {
|
|
319
|
+
key: item for key, item in value.items() if key not in {"case_id", "variant_id"}
|
|
320
|
+
}
|
|
321
|
+
case_id = "aria-" + _digest(investigation._json(fields).encode())
|
|
322
|
+
_require(
|
|
323
|
+
value.get("case_id") == case_id
|
|
324
|
+
and value.get("variant_id") == case_id + "-variant",
|
|
325
|
+
"case identity hash mismatch",
|
|
326
|
+
)
|
|
327
|
+
files = {}
|
|
328
|
+
for name, field in [
|
|
329
|
+
("control.json", "control_result_sha256"),
|
|
330
|
+
("variant.txt", "variant_response_sha256"),
|
|
331
|
+
("source.diff", "source_diff_sha256"),
|
|
332
|
+
]:
|
|
333
|
+
content, _ = _read(path.parent / name, root)
|
|
334
|
+
_require(_digest(content) == value.get(field), "case file hash mismatch")
|
|
335
|
+
files[name] = content
|
|
336
|
+
control = _parse(files["control.json"])
|
|
337
|
+
identity, provenance, condition = _replay(control)
|
|
338
|
+
for key, expected in {
|
|
339
|
+
"parent_identity": control["identity"],
|
|
340
|
+
**provenance,
|
|
341
|
+
"implementation_sha256": control.get("implementation_sha256"),
|
|
342
|
+
}.items():
|
|
343
|
+
_require(
|
|
344
|
+
value.get(key) == expected, "case metadata disagrees with frozen control"
|
|
345
|
+
)
|
|
346
|
+
edit = value.get("edit")
|
|
347
|
+
_require(
|
|
348
|
+
type(edit) is dict
|
|
349
|
+
and set(edit) == {"old", "new"}
|
|
350
|
+
and all(type(item) is str and len(item) <= 65536 for item in edit.values())
|
|
351
|
+
and edit["old"]
|
|
352
|
+
and edit["old"] != edit["new"],
|
|
353
|
+
"unsupported recorded edit",
|
|
354
|
+
)
|
|
355
|
+
investigation._predictions(value.get("predictions"))
|
|
356
|
+
_require(len(files["source.diff"]) <= 262144, "source diff exceeds the view limit")
|
|
357
|
+
try:
|
|
358
|
+
source_diff = files["source.diff"].decode("utf-8")
|
|
359
|
+
except UnicodeError:
|
|
360
|
+
raise ValueError("invalid source diff encoding") from None
|
|
361
|
+
condition["name"] = "control"
|
|
362
|
+
return (
|
|
363
|
+
identity,
|
|
364
|
+
provenance,
|
|
365
|
+
condition,
|
|
366
|
+
{**edit, "occurrence": 1},
|
|
367
|
+
source_diff,
|
|
368
|
+
control,
|
|
369
|
+
)
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def _hypothesis(value):
|
|
373
|
+
_require(
|
|
374
|
+
type(value) is str and value.strip() and len(value) <= 4096,
|
|
375
|
+
"invalid recorded hypothesis",
|
|
376
|
+
)
|
|
377
|
+
return value
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _predictions(rows, record, condition):
|
|
381
|
+
_require(type(rows) is list, "invalid recorded predictions")
|
|
382
|
+
claim = record["historical"].get("reported_result") or {}
|
|
383
|
+
actual = {
|
|
384
|
+
"historical_passed": condition["historical_passed"],
|
|
385
|
+
"historical_evaluated": condition["historical_evaluated"],
|
|
386
|
+
"exception_types": claim.get("exception_types"),
|
|
387
|
+
"task_matched": condition["task_passed"],
|
|
388
|
+
"task_mismatched": condition["task_failed"],
|
|
389
|
+
"task_status": condition["task_status"],
|
|
390
|
+
"task_stdout_sha256": condition["answer_stream_sha256"],
|
|
391
|
+
}
|
|
392
|
+
runtime = record["historical"].get("runtime", {})
|
|
393
|
+
historical_available = (
|
|
394
|
+
condition["historical_status"] == "reported"
|
|
395
|
+
and runtime.get("status") == "completed"
|
|
396
|
+
and runtime.get("config_verified") is True
|
|
397
|
+
and runtime.get("cleanup_verified") is True
|
|
398
|
+
)
|
|
399
|
+
output, seen = [], set()
|
|
400
|
+
for row in rows:
|
|
401
|
+
_require(type(row) is dict, "invalid recorded prediction")
|
|
402
|
+
name = row.get("field")
|
|
403
|
+
_require(name not in seen, "duplicate recorded prediction")
|
|
404
|
+
investigation._predictions({name: row.get("expected")})
|
|
405
|
+
observed = row.get("observed")
|
|
406
|
+
if observed is not None:
|
|
407
|
+
investigation._predictions({name: observed})
|
|
408
|
+
_require(
|
|
409
|
+
row.get("status") in {"matched", "violated", "unavailable"},
|
|
410
|
+
"invalid prediction status",
|
|
411
|
+
)
|
|
412
|
+
_require(
|
|
413
|
+
type(observed) is type(actual[name]) and observed == actual[name],
|
|
414
|
+
"recorded prediction differs from observed outcomes",
|
|
415
|
+
)
|
|
416
|
+
unavailable = (
|
|
417
|
+
observed is None
|
|
418
|
+
or (
|
|
419
|
+
name in {"historical_passed", "historical_evaluated", "exception_types"}
|
|
420
|
+
and not historical_available
|
|
421
|
+
)
|
|
422
|
+
or (
|
|
423
|
+
name in {"task_matched", "task_mismatched", "task_stdout_sha256"}
|
|
424
|
+
and not condition["task_determinate"]
|
|
425
|
+
)
|
|
426
|
+
)
|
|
427
|
+
status = (
|
|
428
|
+
"unavailable"
|
|
429
|
+
if unavailable
|
|
430
|
+
else "matched"
|
|
431
|
+
if type(row["expected"]) is type(observed) and row["expected"] == observed
|
|
432
|
+
else "violated"
|
|
433
|
+
)
|
|
434
|
+
_require(row["status"] == status, "contradictory recorded prediction status")
|
|
435
|
+
seen.add(name)
|
|
436
|
+
output.append(
|
|
437
|
+
{key: row.get(key) for key in ("field", "expected", "observed", "status")}
|
|
438
|
+
)
|
|
439
|
+
return output
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _source_link(identity, provenance, rollouts, source):
|
|
443
|
+
if rollouts is None:
|
|
444
|
+
return {
|
|
445
|
+
"status": "not_loaded",
|
|
446
|
+
"reason": "No canonical rollout source is loaded.",
|
|
447
|
+
}
|
|
448
|
+
if identity is None or provenance is None:
|
|
449
|
+
return {
|
|
450
|
+
"status": "mismatch",
|
|
451
|
+
"reason": "Saved report lacks a verifiable response binding.",
|
|
452
|
+
}
|
|
453
|
+
rows = rollouts.loc[
|
|
454
|
+
(rollouts["run"] == identity["run"])
|
|
455
|
+
& (rollouts["sample_id"] == identity["sample_id"])
|
|
456
|
+
]
|
|
457
|
+
if len(rows) != 1:
|
|
458
|
+
return {
|
|
459
|
+
"status": "mismatch",
|
|
460
|
+
"reason": "Canonical identity is absent or ambiguous.",
|
|
461
|
+
}
|
|
462
|
+
row = rows.iloc[0]
|
|
463
|
+
if (
|
|
464
|
+
row["step"] != identity["step"]
|
|
465
|
+
or row["prompt_id"] != identity["prompt_id"]
|
|
466
|
+
or type(row["response"]) is not str
|
|
467
|
+
or _digest(row["response"].encode()) != provenance["parent_response_sha256"]
|
|
468
|
+
):
|
|
469
|
+
return {
|
|
470
|
+
"status": "mismatch",
|
|
471
|
+
"reason": "Canonical metadata or response hash differs.",
|
|
472
|
+
}
|
|
473
|
+
original = any(
|
|
474
|
+
file.get("sha256") == provenance["source_sha256"]
|
|
475
|
+
for file in (source or {}).get("files", [])
|
|
476
|
+
)
|
|
477
|
+
return {
|
|
478
|
+
"status": "matched" if original else "response_only",
|
|
479
|
+
"reason": "Identity, metadata, response and original file hash match."
|
|
480
|
+
if original
|
|
481
|
+
else "Identity, metadata and response match; original file hash is unverified.",
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def load_evidence(
|
|
486
|
+
path: Path,
|
|
487
|
+
*,
|
|
488
|
+
root: Path,
|
|
489
|
+
rollouts: pd.DataFrame | None = None,
|
|
490
|
+
source: dict | None = None,
|
|
491
|
+
) -> dict:
|
|
492
|
+
"""Validate saved claims without preparing, approving or executing candidates."""
|
|
493
|
+
content, relative = _read(path, root)
|
|
494
|
+
value = _parse(content)
|
|
495
|
+
view = {
|
|
496
|
+
"kind": None,
|
|
497
|
+
"path": relative.as_posix(),
|
|
498
|
+
"sha256": _digest(content),
|
|
499
|
+
"verification": "Validated recorded structure; runtime and source claims are recorded, not re-executed or authenticated.",
|
|
500
|
+
"source_link": None,
|
|
501
|
+
"identity": None,
|
|
502
|
+
"summary": {},
|
|
503
|
+
"conditions": [],
|
|
504
|
+
"hypothesis": None,
|
|
505
|
+
"predictions": [],
|
|
506
|
+
"edit": None,
|
|
507
|
+
"source_diff": None,
|
|
508
|
+
"mechanism_status": "unresolved",
|
|
509
|
+
}
|
|
510
|
+
provenance = None
|
|
511
|
+
try:
|
|
512
|
+
if "replay_schema_version" in value:
|
|
513
|
+
view["kind"] = "replay"
|
|
514
|
+
view["identity"], provenance, condition = _replay(value)
|
|
515
|
+
view["conditions"] = [condition]
|
|
516
|
+
view["summary"] = {"historical_claims_authenticated": False}
|
|
517
|
+
elif "case_schema_version" in value:
|
|
518
|
+
view["kind"] = "case"
|
|
519
|
+
identity, provenance, condition, edit, diff, _ = _case(
|
|
520
|
+
value, relative, root
|
|
521
|
+
)
|
|
522
|
+
view.update(
|
|
523
|
+
identity=identity, conditions=[condition], edit=edit, source_diff=diff
|
|
524
|
+
)
|
|
525
|
+
view["hypothesis"] = _hypothesis(value.get("hypothesis"))
|
|
526
|
+
view["summary"] = {
|
|
527
|
+
"case_id": value["case_id"],
|
|
528
|
+
"variant_id": value["variant_id"],
|
|
529
|
+
}
|
|
530
|
+
view["predictions"] = [
|
|
531
|
+
{
|
|
532
|
+
"field": key,
|
|
533
|
+
"expected": item,
|
|
534
|
+
"observed": None,
|
|
535
|
+
"status": "unavailable",
|
|
536
|
+
}
|
|
537
|
+
for key, item in sorted(value["predictions"].items())
|
|
538
|
+
]
|
|
539
|
+
elif "comparison_schema_version" in value:
|
|
540
|
+
_require(
|
|
541
|
+
type(value["comparison_schema_version"]) is int
|
|
542
|
+
and value["comparison_schema_version"] == 1,
|
|
543
|
+
"unsupported intervention schema version",
|
|
544
|
+
)
|
|
545
|
+
view["kind"] = "intervention"
|
|
546
|
+
identity = view["identity"] = _identity(value.get("parent_identity"))
|
|
547
|
+
provenance = value.get("provenance")
|
|
548
|
+
checks = _provenance(provenance, identity)
|
|
549
|
+
_require(
|
|
550
|
+
value.get("mechanism_status") == "unresolved"
|
|
551
|
+
and value.get("historical_claims_authenticated") is False,
|
|
552
|
+
"unsupported intervention authority",
|
|
553
|
+
)
|
|
554
|
+
view["conditions"] = [
|
|
555
|
+
_condition(name, value[name], checks) for name in ("control", "variant")
|
|
556
|
+
]
|
|
557
|
+
view["hypothesis"] = _hypothesis(value.get("hypothesis"))
|
|
558
|
+
view["predictions"] = _predictions(
|
|
559
|
+
value.get("predictions"), value["variant"], view["conditions"][1]
|
|
560
|
+
)
|
|
561
|
+
equal = value.get("answer_stream_equal")
|
|
562
|
+
_require(
|
|
563
|
+
equal is None or type(equal) is bool,
|
|
564
|
+
"invalid recorded answer comparison",
|
|
565
|
+
)
|
|
566
|
+
if equal is not None:
|
|
567
|
+
a, b = view["conditions"]
|
|
568
|
+
_require(
|
|
569
|
+
a["task_determinate"]
|
|
570
|
+
and b["task_determinate"]
|
|
571
|
+
and equal
|
|
572
|
+
== (a["answer_stream_sha256"] == b["answer_stream_sha256"]),
|
|
573
|
+
"contradictory recorded answer comparison",
|
|
574
|
+
)
|
|
575
|
+
_require(
|
|
576
|
+
type(value.get("case_id")) is str
|
|
577
|
+
and re.fullmatch(r"aria-[a-f0-9]{64}", value["case_id"]),
|
|
578
|
+
"invalid intervention identity",
|
|
579
|
+
)
|
|
580
|
+
_require(
|
|
581
|
+
value.get("variant_id") == value["case_id"] + "-variant",
|
|
582
|
+
"invalid intervention variant",
|
|
583
|
+
)
|
|
584
|
+
view["summary"] = {
|
|
585
|
+
"case_id": value["case_id"],
|
|
586
|
+
"answer_stream_equal": equal,
|
|
587
|
+
"historical_claims_authenticated": False,
|
|
588
|
+
}
|
|
589
|
+
sibling = _read(relative.parent / "case.json", root, optional=True)
|
|
590
|
+
if sibling is not None:
|
|
591
|
+
manifest = _parse(sibling[0])
|
|
592
|
+
_, _, _, edit, diff, control = _case(
|
|
593
|
+
manifest, relative.parent / "case.json", root
|
|
594
|
+
)
|
|
595
|
+
_require(
|
|
596
|
+
value["control"]
|
|
597
|
+
== {key: control[key] for key in ("historical", "task")},
|
|
598
|
+
"intervention control differs from frozen control",
|
|
599
|
+
)
|
|
600
|
+
_require(
|
|
601
|
+
{row["field"]: row["expected"] for row in view["predictions"]}
|
|
602
|
+
== manifest["predictions"],
|
|
603
|
+
"intervention predictions differ from frozen case",
|
|
604
|
+
)
|
|
605
|
+
for key in (
|
|
606
|
+
"case_id",
|
|
607
|
+
"variant_id",
|
|
608
|
+
"hypothesis",
|
|
609
|
+
"parent_identity",
|
|
610
|
+
"control_result_sha256",
|
|
611
|
+
"variant_response_sha256",
|
|
612
|
+
):
|
|
613
|
+
_require(
|
|
614
|
+
value.get(key) == manifest.get(key),
|
|
615
|
+
"intervention metadata differs from case",
|
|
616
|
+
)
|
|
617
|
+
for key, item in provenance.items():
|
|
618
|
+
_require(
|
|
619
|
+
item == manifest.get(key),
|
|
620
|
+
"intervention provenance differs from case",
|
|
621
|
+
)
|
|
622
|
+
view.update(edit=edit, source_diff=diff)
|
|
623
|
+
elif value.get("report_type") == "comparison":
|
|
624
|
+
report = validate_comparison_report(value)
|
|
625
|
+
view["kind"] = "comparison"
|
|
626
|
+
view["summary"] = {
|
|
627
|
+
"mode": report["mode"],
|
|
628
|
+
"sides": report["sides"],
|
|
629
|
+
"feature_count": len(report["features"]),
|
|
630
|
+
"prompt_mix": report["prompt_mix"],
|
|
631
|
+
}
|
|
632
|
+
if report.get("tool_calls") is not None:
|
|
633
|
+
view["summary"]["tool_calls"] = report["tool_calls"]
|
|
634
|
+
else:
|
|
635
|
+
watch = value.get("report_type") == "watch"
|
|
636
|
+
if watch:
|
|
637
|
+
snapshot = validate_watch_report(value)
|
|
638
|
+
report = snapshot["analysis"]
|
|
639
|
+
else:
|
|
640
|
+
report = validate_report(value)
|
|
641
|
+
view["kind"] = "watch" if watch else "analysis"
|
|
642
|
+
view["summary"] = {
|
|
643
|
+
"schema_version": report["schema_version"],
|
|
644
|
+
"runs": [
|
|
645
|
+
{
|
|
646
|
+
key: run[key]
|
|
647
|
+
for key in ("run", "status", "counts", "retrospective")
|
|
648
|
+
if key in run
|
|
649
|
+
}
|
|
650
|
+
for run in report["runs"]
|
|
651
|
+
],
|
|
652
|
+
"benchmark": report.get("benchmark"),
|
|
653
|
+
}
|
|
654
|
+
for key in ("sensitivity", "retrospective"):
|
|
655
|
+
if key in report:
|
|
656
|
+
view["summary"][key] = report[key]
|
|
657
|
+
if watch:
|
|
658
|
+
view["summary"]["watch"] = {
|
|
659
|
+
key: snapshot[key]
|
|
660
|
+
for key in ("sequence", "as_of_step", "emissions", "retrospective")
|
|
661
|
+
}
|
|
662
|
+
view["summary"]["watch"]["policy_id"] = snapshot["policy"]["id"]
|
|
663
|
+
if (
|
|
664
|
+
view["kind"] == "comparison"
|
|
665
|
+
and report["schema_version"] == 2
|
|
666
|
+
and rollouts is not None
|
|
667
|
+
):
|
|
668
|
+
view["source_link"] = {
|
|
669
|
+
"status": "unverified",
|
|
670
|
+
"reason": "This comparison has side-specific source hashes. Use rollout inspect with --side and --feature to verify its citations against the original files.",
|
|
671
|
+
}
|
|
672
|
+
else:
|
|
673
|
+
view["source_link"] = _source_link(
|
|
674
|
+
view["identity"], provenance, rollouts, source
|
|
675
|
+
)
|
|
676
|
+
except (KeyError, TypeError, AttributeError, RecursionError):
|
|
677
|
+
raise ValueError("incomplete or unsupported saved evidence") from None
|
|
678
|
+
return view
|