rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/app.py
ADDED
|
@@ -0,0 +1,956 @@
|
|
|
1
|
+
"""Authenticated loopback investigation over an explicitly selected local root."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import hmac
|
|
7
|
+
import ipaddress
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import stat
|
|
12
|
+
import threading
|
|
13
|
+
import time
|
|
14
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
15
|
+
from contextlib import ExitStack, asynccontextmanager, contextmanager
|
|
16
|
+
from itertools import islice
|
|
17
|
+
from numbers import Integral, Real
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from tempfile import TemporaryDirectory
|
|
20
|
+
from typing import Annotated, Literal
|
|
21
|
+
from urllib.parse import urlsplit
|
|
22
|
+
from uuid import uuid4
|
|
23
|
+
|
|
24
|
+
import numpy as np
|
|
25
|
+
import pandas as pd
|
|
26
|
+
from fastapi import FastAPI, HTTPException, Query, Request
|
|
27
|
+
from fastapi.exceptions import RequestValidationError
|
|
28
|
+
from fastapi.responses import FileResponse, JSONResponse, Response
|
|
29
|
+
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
|
30
|
+
|
|
31
|
+
from . import adapters
|
|
32
|
+
from .comparison import (
|
|
33
|
+
canonical_fingerprint,
|
|
34
|
+
compare_rollouts,
|
|
35
|
+
resolve_comparison_citations,
|
|
36
|
+
)
|
|
37
|
+
from .comparison_report import (
|
|
38
|
+
comparison_json,
|
|
39
|
+
render_comparison_report,
|
|
40
|
+
validate_comparison_report,
|
|
41
|
+
)
|
|
42
|
+
from .html_report import render_report
|
|
43
|
+
from .inspection import resolve_exemplars
|
|
44
|
+
from .pipeline import analyze_rollouts
|
|
45
|
+
from .report import _identifier, build_report, report_json
|
|
46
|
+
|
|
47
|
+
_SAFE_INTEGER = 2**53 - 1
|
|
48
|
+
_MAX_FILES = 10000
|
|
49
|
+
_MAX_LISTING = 1000
|
|
50
|
+
_MAX_JOBS = 32
|
|
51
|
+
_CSP = (
|
|
52
|
+
"default-src 'none'; script-src 'self'; style-src 'self'; "
|
|
53
|
+
"img-src 'self' data:; connect-src 'self'; font-src 'self'; "
|
|
54
|
+
"base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'"
|
|
55
|
+
)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _require_local_platform():
|
|
59
|
+
directory_functions = {
|
|
60
|
+
function.__name__ for function in getattr(os, "supports_dir_fd", ())
|
|
61
|
+
}
|
|
62
|
+
descriptor_functions = {
|
|
63
|
+
function.__name__ for function in getattr(os, "supports_fd", ())
|
|
64
|
+
}
|
|
65
|
+
if (
|
|
66
|
+
os.name != "posix"
|
|
67
|
+
or not all(
|
|
68
|
+
hasattr(os, name) for name in ("O_DIRECTORY", "O_NOFOLLOW", "O_NONBLOCK")
|
|
69
|
+
)
|
|
70
|
+
or not {"open", "stat"}.issubset(directory_functions)
|
|
71
|
+
or "scandir" not in descriptor_functions
|
|
72
|
+
):
|
|
73
|
+
raise ValueError(
|
|
74
|
+
"Local investigation requires macOS or Linux with descriptor-relative "
|
|
75
|
+
"and no-follow file access."
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def ui_value(value):
|
|
80
|
+
"""Preserve missing data and integers that JavaScript cannot represent exactly."""
|
|
81
|
+
if isinstance(value, dict):
|
|
82
|
+
return {key: ui_value(item) for key, item in value.items()}
|
|
83
|
+
if isinstance(value, (list, tuple)):
|
|
84
|
+
return [ui_value(item) for item in value]
|
|
85
|
+
if value is None or value is pd.NA:
|
|
86
|
+
return None
|
|
87
|
+
if isinstance(value, (bool, np.bool_)):
|
|
88
|
+
return bool(value)
|
|
89
|
+
if isinstance(value, Integral):
|
|
90
|
+
number = int(value)
|
|
91
|
+
return str(number) if abs(number) > _SAFE_INTEGER else number
|
|
92
|
+
if isinstance(value, Real):
|
|
93
|
+
number = float(value)
|
|
94
|
+
return number if math.isfinite(number) else None
|
|
95
|
+
if isinstance(value, str):
|
|
96
|
+
return value
|
|
97
|
+
raise TypeError("unsupported view value")
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _view(value, *, status_code=200):
|
|
101
|
+
return JSONResponse(ui_value(value), status_code=status_code)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _rooted(root: Path, value: str) -> Path:
|
|
105
|
+
path = Path(value)
|
|
106
|
+
if (
|
|
107
|
+
not value
|
|
108
|
+
or path.is_absolute()
|
|
109
|
+
or "\\" in value
|
|
110
|
+
or any(part == ".." or part.startswith(".") for part in path.parts)
|
|
111
|
+
):
|
|
112
|
+
raise HTTPException(400, "Select a visible path within the selected root.")
|
|
113
|
+
try:
|
|
114
|
+
selected = (root / path).resolve(strict=True)
|
|
115
|
+
relative = selected.relative_to(root)
|
|
116
|
+
if any(part.startswith(".") for part in relative.parts):
|
|
117
|
+
raise ValueError
|
|
118
|
+
except (OSError, RuntimeError, ValueError):
|
|
119
|
+
raise HTTPException(
|
|
120
|
+
400, "Path is unavailable within the selected root."
|
|
121
|
+
) from None
|
|
122
|
+
return selected
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@contextmanager
|
|
126
|
+
def _open_source(root: Path, value: str):
|
|
127
|
+
"""Open without following links, including links introduced during traversal."""
|
|
128
|
+
relative = Path(value)
|
|
129
|
+
if (
|
|
130
|
+
not value
|
|
131
|
+
or relative.is_absolute()
|
|
132
|
+
or "\\" in value
|
|
133
|
+
or any(part == ".." or part.startswith(".") for part in relative.parts)
|
|
134
|
+
):
|
|
135
|
+
raise HTTPException(400, "Select a visible path within the selected root.")
|
|
136
|
+
descriptors = []
|
|
137
|
+
try:
|
|
138
|
+
directory = os.open(root.anchor, os.O_RDONLY | os.O_DIRECTORY)
|
|
139
|
+
descriptors.append(directory)
|
|
140
|
+
parts = (*root.parts[1:], *relative.parts)
|
|
141
|
+
for part in parts[:-1]:
|
|
142
|
+
directory = os.open(
|
|
143
|
+
part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory
|
|
144
|
+
)
|
|
145
|
+
descriptors.append(directory)
|
|
146
|
+
descriptor = (
|
|
147
|
+
os.open(
|
|
148
|
+
parts[-1],
|
|
149
|
+
os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK,
|
|
150
|
+
dir_fd=directory,
|
|
151
|
+
)
|
|
152
|
+
if parts
|
|
153
|
+
else os.dup(directory)
|
|
154
|
+
)
|
|
155
|
+
descriptors.append(descriptor)
|
|
156
|
+
mode = os.fstat(descriptor).st_mode
|
|
157
|
+
if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode)):
|
|
158
|
+
raise HTTPException(400, "Select a regular file or directory.")
|
|
159
|
+
yield descriptor
|
|
160
|
+
except (OSError, ValueError):
|
|
161
|
+
raise HTTPException(
|
|
162
|
+
400, "Source is unavailable or contains a symbolic link."
|
|
163
|
+
) from None
|
|
164
|
+
finally:
|
|
165
|
+
for descriptor in reversed(descriptors):
|
|
166
|
+
os.close(descriptor)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _files(root: Path, path: str, format: str) -> list[Path]:
|
|
170
|
+
relative = Path(path)
|
|
171
|
+
pattern = adapters._TRL_FILE if format == "trl" else adapters._VERL_FILE
|
|
172
|
+
with ExitStack() as opened:
|
|
173
|
+
descriptor = opened.enter_context(_open_source(root, path))
|
|
174
|
+
if stat.S_ISREG(os.fstat(descriptor).st_mode):
|
|
175
|
+
names = [relative.name] if pattern.fullmatch(relative.name) else []
|
|
176
|
+
parent = relative.parent
|
|
177
|
+
else:
|
|
178
|
+
if format == "trl":
|
|
179
|
+
try:
|
|
180
|
+
child = os.stat(
|
|
181
|
+
"completions", dir_fd=descriptor, follow_symlinks=False
|
|
182
|
+
)
|
|
183
|
+
except FileNotFoundError:
|
|
184
|
+
child = None
|
|
185
|
+
if child is not None and stat.S_ISLNK(child.st_mode):
|
|
186
|
+
raise HTTPException(400, "Source contains a symbolic link.")
|
|
187
|
+
if child is not None and stat.S_ISDIR(child.st_mode):
|
|
188
|
+
relative = relative / "completions"
|
|
189
|
+
descriptor = opened.enter_context(
|
|
190
|
+
_open_source(root, relative.as_posix())
|
|
191
|
+
)
|
|
192
|
+
with os.scandir(descriptor) as entries:
|
|
193
|
+
names = [entry.name for entry in islice(entries, _MAX_FILES + 1)]
|
|
194
|
+
if len(names) > _MAX_FILES:
|
|
195
|
+
raise HTTPException(400, "Too many files in this selection.")
|
|
196
|
+
names = [name for name in names if pattern.fullmatch(name)]
|
|
197
|
+
for name in names:
|
|
198
|
+
mode = os.stat(name, dir_fd=descriptor, follow_symlinks=False).st_mode
|
|
199
|
+
if not stat.S_ISREG(mode):
|
|
200
|
+
raise HTTPException(
|
|
201
|
+
400, "Source must contain regular files without symbolic links."
|
|
202
|
+
)
|
|
203
|
+
parent = relative
|
|
204
|
+
if not names:
|
|
205
|
+
raise HTTPException(400, "No matching rollout files are available.")
|
|
206
|
+
names.sort(key=lambda name: (int(pattern.fullmatch(name).group(1)), name))
|
|
207
|
+
return [root / parent / name for name in names]
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _hash_file(root: Path, path: Path, *, destination=None) -> dict:
|
|
211
|
+
"""Hash the exact bytes copied, refusing nonregular and concurrently edited files."""
|
|
212
|
+
relative = path.relative_to(root).as_posix()
|
|
213
|
+
digest, size = hashlib.sha256(), 0
|
|
214
|
+
with _open_source(root, relative) as descriptor:
|
|
215
|
+
before = os.fstat(descriptor)
|
|
216
|
+
if not stat.S_ISREG(before.st_mode):
|
|
217
|
+
raise HTTPException(400, "Selected source must be a regular file.")
|
|
218
|
+
while chunk := os.read(descriptor, 1024 * 1024):
|
|
219
|
+
digest.update(chunk)
|
|
220
|
+
size += len(chunk)
|
|
221
|
+
if destination is not None:
|
|
222
|
+
destination.write(chunk)
|
|
223
|
+
after = os.fstat(descriptor)
|
|
224
|
+
if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (
|
|
225
|
+
after.st_size,
|
|
226
|
+
after.st_mtime_ns,
|
|
227
|
+
after.st_ctime_ns,
|
|
228
|
+
) or size != after.st_size:
|
|
229
|
+
raise HTTPException(409, "Source changed while it was being read.")
|
|
230
|
+
return {"path": relative, "sha256": digest.hexdigest(), "size_bytes": size}
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _snapshot(root: Path, path: str, format: str) -> list[dict]:
|
|
234
|
+
return [_hash_file(root, file) for file in _files(root, path, format)]
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
@contextmanager
|
|
238
|
+
def _captured_source(root: Path, path: str, format: str):
|
|
239
|
+
"""Adapters read private copies, never mutable paths in the selected root."""
|
|
240
|
+
originals = _files(root, path, format)
|
|
241
|
+
with TemporaryDirectory(prefix="rollout-investigation-") as temporary:
|
|
242
|
+
directory = Path(temporary)
|
|
243
|
+
files = []
|
|
244
|
+
for original in originals:
|
|
245
|
+
descriptor = os.open(
|
|
246
|
+
directory / original.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600
|
|
247
|
+
)
|
|
248
|
+
with os.fdopen(descriptor, "wb") as destination:
|
|
249
|
+
files.append(_hash_file(root, original, destination=destination))
|
|
250
|
+
if files != _snapshot(root, path, format):
|
|
251
|
+
raise HTTPException(409, "Source changed while it was being copied.")
|
|
252
|
+
yield directory, files
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
class _Input(BaseModel):
|
|
256
|
+
model_config = ConfigDict(extra="forbid", strict=True)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class RunInput(_Input):
|
|
260
|
+
path: Annotated[str, Field(min_length=1, max_length=4096)]
|
|
261
|
+
format: Literal["trl", "verl"]
|
|
262
|
+
run: Annotated[str, Field(min_length=1, max_length=512)]
|
|
263
|
+
reward_column: Annotated[str, Field(min_length=1, max_length=512)] | None = None
|
|
264
|
+
sensitivity: Literal["default", "lenient"] = "default"
|
|
265
|
+
retrospective: bool = False
|
|
266
|
+
|
|
267
|
+
@field_validator("run", "reward_column")
|
|
268
|
+
@classmethod
|
|
269
|
+
def identifier(cls, value):
|
|
270
|
+
return _identifier(value) if value is not None else None
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
class InspectInput(_Input):
|
|
274
|
+
event_rank: Annotated[int, Field(ge=1)]
|
|
275
|
+
feature: Annotated[str, Field(min_length=1, max_length=4096)]
|
|
276
|
+
sample_id: Annotated[str, Field(min_length=1, max_length=4096)]
|
|
277
|
+
with_absent: bool = False
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
class ComparisonInput(_Input):
|
|
281
|
+
run_a: str
|
|
282
|
+
run_b: str | None = None
|
|
283
|
+
a: Annotated[list[str], Field(min_length=2, max_length=2)]
|
|
284
|
+
b: Annotated[list[str], Field(min_length=2, max_length=2)]
|
|
285
|
+
|
|
286
|
+
@field_validator("a", "b")
|
|
287
|
+
@classmethod
|
|
288
|
+
def window(cls, value):
|
|
289
|
+
if any(not re.fullmatch(r"[0-9]{1,20}", item) for item in value):
|
|
290
|
+
raise ValueError("ranges require nonnegative decimal integers")
|
|
291
|
+
if int(value[0]) > int(value[1]):
|
|
292
|
+
raise ValueError("range endpoints must be ordered")
|
|
293
|
+
return value
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class ComparisonInspectInput(_Input):
|
|
297
|
+
side: Literal["a", "b"]
|
|
298
|
+
feature: Annotated[str, Field(min_length=1, max_length=4096)]
|
|
299
|
+
sample_id: Annotated[str, Field(min_length=1, max_length=4096)]
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
class EvidenceInput(_Input):
|
|
303
|
+
path: Annotated[str, Field(min_length=1, max_length=4096)]
|
|
304
|
+
run_id: str | None = None
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
class _Store:
|
|
308
|
+
def __init__(self, root):
|
|
309
|
+
self.root = root
|
|
310
|
+
self.runs = {}
|
|
311
|
+
self.comparisons = {}
|
|
312
|
+
self.lock = threading.RLock()
|
|
313
|
+
self.executor = None
|
|
314
|
+
|
|
315
|
+
def run(self, identity, *, ready=True):
|
|
316
|
+
with self.lock:
|
|
317
|
+
item = self.runs.get(identity)
|
|
318
|
+
if item is None:
|
|
319
|
+
raise HTTPException(404, "Run was not found.")
|
|
320
|
+
if ready and item["view"]["status"] != "ready":
|
|
321
|
+
raise HTTPException(409, "Run analysis is not ready.")
|
|
322
|
+
return item
|
|
323
|
+
|
|
324
|
+
def comparison(self, identity, *, ready=True):
|
|
325
|
+
with self.lock:
|
|
326
|
+
item = self.comparisons.get(identity)
|
|
327
|
+
if item is None:
|
|
328
|
+
raise HTTPException(404, "Comparison was not found.")
|
|
329
|
+
if ready and item["view"]["status"] != "ready":
|
|
330
|
+
raise HTTPException(409, "Comparison is not ready.")
|
|
331
|
+
return item
|
|
332
|
+
|
|
333
|
+
def verify(self, item):
|
|
334
|
+
try:
|
|
335
|
+
source = item["view"]["source"]
|
|
336
|
+
if (
|
|
337
|
+
_snapshot(self.root, source["path"], source["format"])
|
|
338
|
+
!= source["files"]
|
|
339
|
+
):
|
|
340
|
+
raise ValueError
|
|
341
|
+
if canonical_fingerprint(item["rollouts"]) != source["canonical_sha256"]:
|
|
342
|
+
raise ValueError
|
|
343
|
+
for row in item["rollouts"][["sample_id", "response"]].itertuples(
|
|
344
|
+
index=False
|
|
345
|
+
):
|
|
346
|
+
if (
|
|
347
|
+
hashlib.sha256(row.response.encode()).hexdigest()
|
|
348
|
+
!= item["responses"][row.sample_id]
|
|
349
|
+
):
|
|
350
|
+
raise ValueError
|
|
351
|
+
except (HTTPException, OSError, ValueError, KeyError):
|
|
352
|
+
raise HTTPException(
|
|
353
|
+
409, "Source changed or is unavailable. Reload the run."
|
|
354
|
+
) from None
|
|
355
|
+
|
|
356
|
+
def verify_comparison(self, item):
|
|
357
|
+
for identity in dict.fromkeys(item["run_ids"]):
|
|
358
|
+
self.verify(self.run(identity))
|
|
359
|
+
|
|
360
|
+
def load(self, identity, request):
|
|
361
|
+
item = self.runs[identity]
|
|
362
|
+
started = time.monotonic()
|
|
363
|
+
try:
|
|
364
|
+
with self.lock:
|
|
365
|
+
item["view"]["status"] = "loading"
|
|
366
|
+
with _captured_source(self.root, request.path, request.format) as (
|
|
367
|
+
path,
|
|
368
|
+
before,
|
|
369
|
+
):
|
|
370
|
+
rollouts = (
|
|
371
|
+
adapters.load_trl(
|
|
372
|
+
path, run=request.run, reward_column=request.reward_column
|
|
373
|
+
)
|
|
374
|
+
if request.format == "trl"
|
|
375
|
+
else adapters.load_verl(path, run=request.run)
|
|
376
|
+
)
|
|
377
|
+
if before != _snapshot(self.root, request.path, request.format):
|
|
378
|
+
raise ValueError("source changed")
|
|
379
|
+
source = {
|
|
380
|
+
"path": request.path,
|
|
381
|
+
"format": request.format,
|
|
382
|
+
"canonical_sha256": canonical_fingerprint(rollouts),
|
|
383
|
+
"files": before,
|
|
384
|
+
}
|
|
385
|
+
responses = {
|
|
386
|
+
row.sample_id: hashlib.sha256(row.response.encode()).hexdigest()
|
|
387
|
+
for row in rollouts[["sample_id", "response"]].itertuples(index=False)
|
|
388
|
+
}
|
|
389
|
+
with self.lock:
|
|
390
|
+
item["view"]["status"] = "analyzing"
|
|
391
|
+
analysis = analyze_rollouts(
|
|
392
|
+
rollouts,
|
|
393
|
+
sensitivity=request.sensitivity,
|
|
394
|
+
retrospective=request.retrospective,
|
|
395
|
+
)
|
|
396
|
+
report = build_report(analysis, run=request.run)
|
|
397
|
+
with self.lock:
|
|
398
|
+
item.update(
|
|
399
|
+
rollouts=rollouts,
|
|
400
|
+
analysis=analysis,
|
|
401
|
+
report=report,
|
|
402
|
+
responses=responses,
|
|
403
|
+
)
|
|
404
|
+
item["view"].update(source=source)
|
|
405
|
+
self.verify(item)
|
|
406
|
+
with self.lock:
|
|
407
|
+
item["view"].update(
|
|
408
|
+
status="ready",
|
|
409
|
+
counts=report["runs"][0]["counts"],
|
|
410
|
+
benchmark=report.get("benchmark"),
|
|
411
|
+
retrospective=report["retrospective"],
|
|
412
|
+
retrospective_results=report["runs"][0]["retrospective"],
|
|
413
|
+
)
|
|
414
|
+
if "sensitivity" in report:
|
|
415
|
+
item["view"]["sensitivity"] = report["sensitivity"]
|
|
416
|
+
except Exception: # noqa: BLE001 -- terminate jobs without exposing source text
|
|
417
|
+
with self.lock:
|
|
418
|
+
item["view"].update(
|
|
419
|
+
status="failed",
|
|
420
|
+
error="Could not analyze this source. Check its format, reward column, and unchanged files.",
|
|
421
|
+
)
|
|
422
|
+
finally:
|
|
423
|
+
with self.lock:
|
|
424
|
+
item["view"]["elapsed_seconds"] = time.monotonic() - started
|
|
425
|
+
|
|
426
|
+
def compare(self, identity, request):
|
|
427
|
+
item = self.comparisons[identity]
|
|
428
|
+
try:
|
|
429
|
+
with self.lock:
|
|
430
|
+
item["view"]["status"] = "comparing"
|
|
431
|
+
left, right = (self.run(run_id) for run_id in item["run_ids"])
|
|
432
|
+
self.verify_comparison(item)
|
|
433
|
+
same = request.run_b is None or request.run_a == request.run_b
|
|
434
|
+
report = compare_rollouts(
|
|
435
|
+
left["rollouts"],
|
|
436
|
+
None if same else right["rollouts"],
|
|
437
|
+
a=tuple(map(int, request.a)),
|
|
438
|
+
b=tuple(map(int, request.b)),
|
|
439
|
+
)
|
|
440
|
+
for side, run in zip(report["sides"], (left, right), strict=True):
|
|
441
|
+
source = run["view"]["source"]
|
|
442
|
+
side["source"].update(
|
|
443
|
+
format=source["format"],
|
|
444
|
+
files=[
|
|
445
|
+
{key: file[key] for key in ("sha256", "size_bytes")}
|
|
446
|
+
for file in source["files"]
|
|
447
|
+
],
|
|
448
|
+
reward_column=run["request"].reward_column,
|
|
449
|
+
)
|
|
450
|
+
report = validate_comparison_report(report)
|
|
451
|
+
self.verify_comparison(item)
|
|
452
|
+
with self.lock:
|
|
453
|
+
item["view"].update(status="ready", report=report)
|
|
454
|
+
except Exception: # noqa: BLE001 -- terminate jobs without exposing source text
|
|
455
|
+
with self.lock:
|
|
456
|
+
item["view"].update(
|
|
457
|
+
status="failed",
|
|
458
|
+
error="Could not compare these ranges. Check coverage, range order, and unchanged sources.",
|
|
459
|
+
)
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def _rows(frame, features):
|
|
463
|
+
if "feature" not in frame:
|
|
464
|
+
return []
|
|
465
|
+
return frame.loc[frame.feature.isin(features)].to_dict("records")
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
def _headlines(analysis, features):
|
|
469
|
+
"""Equal-weight shared-prompt means; omit numerically misleading pairs."""
|
|
470
|
+
values = analysis.prompt_values
|
|
471
|
+
if "change" not in values:
|
|
472
|
+
return {}
|
|
473
|
+
shared = values.loc[values.feature.isin(features) & values.change.notna()]
|
|
474
|
+
result = {}
|
|
475
|
+
for feature, rows in shared.groupby("feature", sort=False):
|
|
476
|
+
with np.errstate(over="ignore", invalid="ignore"):
|
|
477
|
+
before, after, change = (
|
|
478
|
+
float(rows[column].mean())
|
|
479
|
+
for column in ("before_mean", "after_mean", "change")
|
|
480
|
+
)
|
|
481
|
+
# Rounded endpoints can hide an available per-prompt difference.
|
|
482
|
+
if all(
|
|
483
|
+
math.isfinite(value) for value in (before, after, change)
|
|
484
|
+
) and math.isclose(after - before, change, rel_tol=1e-10, abs_tol=1e-12):
|
|
485
|
+
result[feature] = {
|
|
486
|
+
"before": before,
|
|
487
|
+
"after": after,
|
|
488
|
+
"n_prompts": len(rows),
|
|
489
|
+
}
|
|
490
|
+
return result
|
|
491
|
+
|
|
492
|
+
|
|
493
|
+
def _citation(item, citation):
|
|
494
|
+
response = citation["response"]
|
|
495
|
+
digest = hashlib.sha256(response.encode()).hexdigest()
|
|
496
|
+
if digest != item["responses"].get(citation["sample_id"]):
|
|
497
|
+
raise HTTPException(409, "Citation response changed. Reload the run.")
|
|
498
|
+
counterpart = (citation.get("comparison") or {}).get("counterpart")
|
|
499
|
+
if counterpart is not None:
|
|
500
|
+
counterpart_digest = hashlib.sha256(
|
|
501
|
+
counterpart["response"].encode()
|
|
502
|
+
).hexdigest()
|
|
503
|
+
if counterpart_digest != item["responses"].get(counterpart["sample_id"]):
|
|
504
|
+
raise HTTPException(409, "Comparison response changed. Reload the run.")
|
|
505
|
+
return _view(
|
|
506
|
+
{
|
|
507
|
+
"citation": citation,
|
|
508
|
+
"source": item["view"]["source"],
|
|
509
|
+
"response_sha256": digest,
|
|
510
|
+
}
|
|
511
|
+
)
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _download(report, format, *, comparison=False):
|
|
515
|
+
if format == "json":
|
|
516
|
+
content = comparison_json(report) if comparison else report_json(report)
|
|
517
|
+
media_type = "application/json"
|
|
518
|
+
else:
|
|
519
|
+
content = (
|
|
520
|
+
render_comparison_report(report) if comparison else render_report(report)
|
|
521
|
+
)
|
|
522
|
+
media_type = "text/html"
|
|
523
|
+
return Response(
|
|
524
|
+
content,
|
|
525
|
+
media_type=media_type,
|
|
526
|
+
headers={
|
|
527
|
+
"Content-Disposition": f'attachment; filename="rollout-{"comparison" if comparison else "analysis"}.{format}"'
|
|
528
|
+
},
|
|
529
|
+
)
|
|
530
|
+
|
|
531
|
+
|
|
532
|
+
def create_app(root, *, token: str, origin="http://127.0.0.1:8765") -> FastAPI:
|
|
533
|
+
"""Create a read-only local investigation session with serialized analysis jobs."""
|
|
534
|
+
_require_local_platform()
|
|
535
|
+
root = Path(root).resolve(strict=True)
|
|
536
|
+
if not root.is_dir() or not isinstance(token, str) or not token:
|
|
537
|
+
raise ValueError("a directory root and nonempty session token are required")
|
|
538
|
+
parsed = urlsplit(origin)
|
|
539
|
+
if (
|
|
540
|
+
parsed.scheme != "http"
|
|
541
|
+
or parsed.hostname != "127.0.0.1"
|
|
542
|
+
or parsed.path
|
|
543
|
+
or parsed.query
|
|
544
|
+
or parsed.fragment
|
|
545
|
+
or parsed.username
|
|
546
|
+
):
|
|
547
|
+
raise ValueError("origin must be a loopback HTTP origin")
|
|
548
|
+
store = _Store(root)
|
|
549
|
+
|
|
550
|
+
@asynccontextmanager
|
|
551
|
+
async def lifespan(app):
|
|
552
|
+
store.executor = ThreadPoolExecutor(
|
|
553
|
+
max_workers=1, thread_name_prefix="rollout-investigation"
|
|
554
|
+
)
|
|
555
|
+
try:
|
|
556
|
+
yield
|
|
557
|
+
finally:
|
|
558
|
+
store.executor.shutdown(wait=True, cancel_futures=True)
|
|
559
|
+
store.executor = None
|
|
560
|
+
|
|
561
|
+
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
|
562
|
+
app.state.store = store
|
|
563
|
+
|
|
564
|
+
@app.middleware("http")
|
|
565
|
+
async def local_session(request: Request, call_next):
|
|
566
|
+
try:
|
|
567
|
+
peer = request.client.host if request.client else ""
|
|
568
|
+
local = ipaddress.ip_address(peer).is_loopback
|
|
569
|
+
except ValueError:
|
|
570
|
+
local = False
|
|
571
|
+
hosts = request.headers.getlist("host")
|
|
572
|
+
origins = request.headers.getlist("origin")
|
|
573
|
+
accepted = (
|
|
574
|
+
local and hosts == [parsed.netloc] and (not origins or origins == [origin])
|
|
575
|
+
)
|
|
576
|
+
if request.headers.get("sec-fetch-site") == "cross-site":
|
|
577
|
+
accepted = False
|
|
578
|
+
if request.url.path.startswith("/api/"):
|
|
579
|
+
authorization = request.headers.getlist("authorization")
|
|
580
|
+
accepted = (
|
|
581
|
+
accepted
|
|
582
|
+
and len(authorization) == 1
|
|
583
|
+
and hmac.compare_digest(
|
|
584
|
+
authorization[0].encode(), f"Bearer {token}".encode()
|
|
585
|
+
)
|
|
586
|
+
)
|
|
587
|
+
if accepted:
|
|
588
|
+
try:
|
|
589
|
+
response = await call_next(request)
|
|
590
|
+
except Exception: # noqa: BLE001 -- API errors must not expose local data
|
|
591
|
+
response = JSONResponse(
|
|
592
|
+
{"detail": "Local request could not be completed."}, status_code=500
|
|
593
|
+
)
|
|
594
|
+
else:
|
|
595
|
+
response = JSONResponse(
|
|
596
|
+
{"detail": "Local session authorization required."}, status_code=403
|
|
597
|
+
)
|
|
598
|
+
response.headers.update(
|
|
599
|
+
{
|
|
600
|
+
"Content-Security-Policy": _CSP,
|
|
601
|
+
"X-Frame-Options": "DENY",
|
|
602
|
+
"X-Content-Type-Options": "nosniff",
|
|
603
|
+
"Referrer-Policy": "no-referrer",
|
|
604
|
+
"Cache-Control": "no-store",
|
|
605
|
+
}
|
|
606
|
+
)
|
|
607
|
+
return response
|
|
608
|
+
|
|
609
|
+
@app.exception_handler(RequestValidationError)
|
|
610
|
+
async def invalid_request(request, exc):
|
|
611
|
+
return JSONResponse({"detail": "Invalid request fields."}, status_code=422)
|
|
612
|
+
|
|
613
|
+
@app.get("/api/session")
|
|
614
|
+
def session():
|
|
615
|
+
return {
|
|
616
|
+
"root_label": root.name,
|
|
617
|
+
"formats": ["trl", "verl"],
|
|
618
|
+
"read_only_execution": True,
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
@app.get("/api/files")
|
|
622
|
+
def files(path: Annotated[str, Query(max_length=4096)] = "."):
|
|
623
|
+
selected = _rooted(root, path)
|
|
624
|
+
if not selected.is_dir():
|
|
625
|
+
raise HTTPException(400, "Select a directory within the root.")
|
|
626
|
+
entries = []
|
|
627
|
+
try:
|
|
628
|
+
children = list(islice(selected.iterdir(), _MAX_FILES + 1))
|
|
629
|
+
except OSError:
|
|
630
|
+
raise HTTPException(400, "Directory is unavailable.") from None
|
|
631
|
+
if len(children) > _MAX_FILES:
|
|
632
|
+
raise HTTPException(
|
|
633
|
+
400,
|
|
634
|
+
"Directory has too many entries; select a smaller source directory.",
|
|
635
|
+
)
|
|
636
|
+
for child in sorted(children, key=lambda item: item.name):
|
|
637
|
+
if child.name.startswith("."):
|
|
638
|
+
continue
|
|
639
|
+
relative = child.relative_to(root).as_posix()
|
|
640
|
+
try:
|
|
641
|
+
resolved = _rooted(root, relative)
|
|
642
|
+
except HTTPException:
|
|
643
|
+
continue
|
|
644
|
+
if resolved.is_dir():
|
|
645
|
+
kind = "directory"
|
|
646
|
+
elif adapters._TRL_FILE.fullmatch(child.name):
|
|
647
|
+
kind = "trl"
|
|
648
|
+
elif adapters._VERL_FILE.fullmatch(child.name):
|
|
649
|
+
kind = "verl"
|
|
650
|
+
elif child.suffix == ".json" and resolved.is_file():
|
|
651
|
+
kind = "evidence"
|
|
652
|
+
else:
|
|
653
|
+
continue
|
|
654
|
+
entries.append({"name": child.name, "path": relative, "kind": kind})
|
|
655
|
+
if len(entries) == _MAX_LISTING:
|
|
656
|
+
break
|
|
657
|
+
return {"path": selected.relative_to(root).as_posix(), "entries": entries}
|
|
658
|
+
|
|
659
|
+
@app.get("/api/runs")
|
|
660
|
+
def runs():
|
|
661
|
+
with store.lock:
|
|
662
|
+
return _view({"runs": [item["view"] for item in store.runs.values()]})
|
|
663
|
+
|
|
664
|
+
@app.post("/api/runs", status_code=202)
|
|
665
|
+
def add_run(request: RunInput):
|
|
666
|
+
_files(root, request.path, request.format)
|
|
667
|
+
if request.format == "verl" and request.reward_column is not None:
|
|
668
|
+
raise HTTPException(
|
|
669
|
+
422, "Reward column selection is only available for TRL."
|
|
670
|
+
)
|
|
671
|
+
with store.lock:
|
|
672
|
+
if len(store.runs) >= _MAX_JOBS:
|
|
673
|
+
raise HTTPException(
|
|
674
|
+
409, "Remove a finished run before loading another."
|
|
675
|
+
)
|
|
676
|
+
identity = uuid4().hex
|
|
677
|
+
view = {
|
|
678
|
+
"id": identity,
|
|
679
|
+
"run": request.run,
|
|
680
|
+
"status": "queued",
|
|
681
|
+
"error": None,
|
|
682
|
+
"counts": None,
|
|
683
|
+
"source": None,
|
|
684
|
+
"elapsed_seconds": None,
|
|
685
|
+
}
|
|
686
|
+
store.runs[identity] = {"view": view, "request": request}
|
|
687
|
+
if store.executor is None:
|
|
688
|
+
raise HTTPException(503, "Investigation session is not running.")
|
|
689
|
+
store.executor.submit(store.load, identity, request)
|
|
690
|
+
return _view(view, status_code=202)
|
|
691
|
+
|
|
692
|
+
@app.get("/api/runs/{identity}")
|
|
693
|
+
def run(identity: str):
|
|
694
|
+
with store.lock:
|
|
695
|
+
return _view(store.run(identity, ready=False)["view"])
|
|
696
|
+
|
|
697
|
+
@app.delete("/api/runs/{identity}", status_code=204)
|
|
698
|
+
def delete_run(identity: str):
|
|
699
|
+
with store.lock:
|
|
700
|
+
item = store.run(identity, ready=False)
|
|
701
|
+
dependent = [
|
|
702
|
+
key
|
|
703
|
+
for key, value in store.comparisons.items()
|
|
704
|
+
if identity in value["run_ids"]
|
|
705
|
+
]
|
|
706
|
+
if item["view"]["status"] not in {"ready", "failed"} or any(
|
|
707
|
+
store.comparisons[key]["view"]["status"] not in {"ready", "failed"}
|
|
708
|
+
for key in dependent
|
|
709
|
+
):
|
|
710
|
+
raise HTTPException(
|
|
711
|
+
409, "Wait for active jobs before removing this run."
|
|
712
|
+
)
|
|
713
|
+
for key in dependent:
|
|
714
|
+
del store.comparisons[key]
|
|
715
|
+
del store.runs[identity]
|
|
716
|
+
return Response(status_code=204)
|
|
717
|
+
|
|
718
|
+
@app.get("/api/runs/{identity}/events")
|
|
719
|
+
def events(
|
|
720
|
+
identity: str,
|
|
721
|
+
q: Annotated[str, Query(max_length=4096)] = "",
|
|
722
|
+
kind: Literal["all", "behavioral", "contextual", "unavailable"] = "all",
|
|
723
|
+
tier: Literal["all", "structural", "prose"] = "all",
|
|
724
|
+
floor: Literal["all", "yes", "no"] = "all",
|
|
725
|
+
offset: Annotated[int, Query(ge=0)] = 0,
|
|
726
|
+
limit: Annotated[int, Query(ge=1, le=200)] = 50,
|
|
727
|
+
):
|
|
728
|
+
item = store.run(identity)
|
|
729
|
+
selected = []
|
|
730
|
+
for event in item["report"]["runs"][0]["events"]:
|
|
731
|
+
if kind != "all" and event["ranking_status"] != kind:
|
|
732
|
+
continue
|
|
733
|
+
if q and not any(
|
|
734
|
+
q.casefold() in member["feature"].casefold()
|
|
735
|
+
for member in event["members"]
|
|
736
|
+
):
|
|
737
|
+
continue
|
|
738
|
+
if tier != "all" and not any(
|
|
739
|
+
member["evidence"]["tier"] == tier for member in event["members"]
|
|
740
|
+
):
|
|
741
|
+
continue
|
|
742
|
+
if floor != "all" and bool(event["ranking"]["numerical_floor_members"]) != (
|
|
743
|
+
floor == "yes"
|
|
744
|
+
):
|
|
745
|
+
continue
|
|
746
|
+
selected.append(event)
|
|
747
|
+
page = selected[offset : offset + limit]
|
|
748
|
+
return _view(
|
|
749
|
+
{
|
|
750
|
+
"total": len(selected),
|
|
751
|
+
"events": page,
|
|
752
|
+
"headlines": _headlines(
|
|
753
|
+
item["analysis"],
|
|
754
|
+
[event["representative_feature"] for event in page],
|
|
755
|
+
),
|
|
756
|
+
}
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
@app.get("/api/runs/{identity}/events/{rank}")
|
|
760
|
+
def event(identity: str, rank: int):
|
|
761
|
+
item = store.run(identity)
|
|
762
|
+
selected = next(
|
|
763
|
+
(row for row in item["report"]["runs"][0]["events"] if row["rank"] == rank),
|
|
764
|
+
None,
|
|
765
|
+
)
|
|
766
|
+
if selected is None:
|
|
767
|
+
raise HTTPException(404, "Event was not found.")
|
|
768
|
+
names = [member["feature"] for member in selected["members"]]
|
|
769
|
+
return _view(
|
|
770
|
+
{
|
|
771
|
+
"event": selected,
|
|
772
|
+
"evidence": _rows(item["analysis"].evidence, names),
|
|
773
|
+
"prompt_shifts": _rows(item["analysis"].prompt_shifts, names),
|
|
774
|
+
"headlines": _headlines(item["analysis"], names),
|
|
775
|
+
}
|
|
776
|
+
)
|
|
777
|
+
|
|
778
|
+
@app.get("/api/runs/{identity}/steps")
|
|
779
|
+
def steps(identity: str):
|
|
780
|
+
rollouts = store.run(identity)["rollouts"]
|
|
781
|
+
return _view({"steps": sorted(rollouts.step.unique().tolist())})
|
|
782
|
+
|
|
783
|
+
@app.get("/api/runs/{identity}/features")
|
|
784
|
+
def features(
|
|
785
|
+
identity: str,
|
|
786
|
+
q: Annotated[str, Query(max_length=4096)] = "",
|
|
787
|
+
limit: Annotated[int, Query(ge=1, le=5000)] = 100,
|
|
788
|
+
):
|
|
789
|
+
analysis = store.run(identity)["analysis"]
|
|
790
|
+
catalog = analysis.evidence[["feature", "tier", "kind"]]
|
|
791
|
+
catalog = catalog.loc[
|
|
792
|
+
catalog.feature.str.contains(q, case=False, regex=False)
|
|
793
|
+
].sort_values("feature")
|
|
794
|
+
return _view(
|
|
795
|
+
{"total": len(catalog), "features": catalog.head(limit).to_dict("records")}
|
|
796
|
+
)
|
|
797
|
+
|
|
798
|
+
@app.get("/api/runs/{identity}/series")
|
|
799
|
+
def series(
|
|
800
|
+
identity: str, feature: Annotated[str, Query(min_length=1, max_length=4096)]
|
|
801
|
+
):
|
|
802
|
+
item = store.run(identity)
|
|
803
|
+
analysis = item["analysis"]
|
|
804
|
+
evidence = _rows(analysis.evidence, [feature])
|
|
805
|
+
if not evidence:
|
|
806
|
+
raise HTTPException(404, "Feature was not found.")
|
|
807
|
+
return _view(
|
|
808
|
+
{
|
|
809
|
+
"feature": feature,
|
|
810
|
+
"kind": evidence[0]["kind"],
|
|
811
|
+
"tier": evidence[0]["tier"],
|
|
812
|
+
"direction": next(
|
|
813
|
+
(
|
|
814
|
+
member["direction"]
|
|
815
|
+
for event in item["report"]["runs"][0]["events"]
|
|
816
|
+
for member in event["members"]
|
|
817
|
+
if member["feature"] == feature
|
|
818
|
+
),
|
|
819
|
+
None,
|
|
820
|
+
),
|
|
821
|
+
"points": _rows(analysis.aggregation.series, [feature]),
|
|
822
|
+
"evidence": evidence[0],
|
|
823
|
+
"persistence": _rows(analysis.persistence_steps, [feature]),
|
|
824
|
+
"prompt_values": _rows(analysis.prompt_values, [feature]),
|
|
825
|
+
"prompt_shifts": _rows(analysis.prompt_shifts, [feature]),
|
|
826
|
+
"headline": _headlines(analysis, [feature]).get(feature),
|
|
827
|
+
}
|
|
828
|
+
)
|
|
829
|
+
|
|
830
|
+
@app.post("/api/runs/{identity}/inspect")
|
|
831
|
+
def inspect(identity: str, request: InspectInput):
|
|
832
|
+
item = store.run(identity)
|
|
833
|
+
store.verify(item)
|
|
834
|
+
try:
|
|
835
|
+
event = resolve_exemplars(
|
|
836
|
+
item["report"],
|
|
837
|
+
item["rollouts"],
|
|
838
|
+
run=item["view"]["run"],
|
|
839
|
+
event_rank=request.event_rank,
|
|
840
|
+
with_absent=request.with_absent,
|
|
841
|
+
)
|
|
842
|
+
citation = next(
|
|
843
|
+
example
|
|
844
|
+
for member in event["members"]
|
|
845
|
+
if member["feature"] == request.feature
|
|
846
|
+
for example in member["exemplars"]
|
|
847
|
+
if example["sample_id"] == request.sample_id
|
|
848
|
+
and example["status"] == "selected"
|
|
849
|
+
)
|
|
850
|
+
except (ValueError, StopIteration):
|
|
851
|
+
raise HTTPException(
|
|
852
|
+
400, "Requested citation is unavailable or does not match the source."
|
|
853
|
+
) from None
|
|
854
|
+
store.verify(item)
|
|
855
|
+
return _citation(item, citation)
|
|
856
|
+
|
|
857
|
+
@app.get("/api/runs/{identity}/export")
|
|
858
|
+
def export(identity: str, format: Literal["json", "html"] = "json"):
|
|
859
|
+
item = store.run(identity)
|
|
860
|
+
store.verify(item)
|
|
861
|
+
return _download(item["report"], format)
|
|
862
|
+
|
|
863
|
+
@app.post("/api/comparisons", status_code=202)
|
|
864
|
+
def add_comparison(request: ComparisonInput):
|
|
865
|
+
identities = [request.run_a, request.run_b or request.run_a]
|
|
866
|
+
for identity in dict.fromkeys(identities):
|
|
867
|
+
store.verify(store.run(identity))
|
|
868
|
+
with store.lock:
|
|
869
|
+
if len(store.comparisons) >= _MAX_JOBS:
|
|
870
|
+
raise HTTPException(409, "Remove finished runs to free comparisons.")
|
|
871
|
+
identity = uuid4().hex
|
|
872
|
+
view = {"id": identity, "status": "queued", "error": None, "report": None}
|
|
873
|
+
store.comparisons[identity] = {"view": view, "run_ids": identities}
|
|
874
|
+
if store.executor is None:
|
|
875
|
+
raise HTTPException(503, "Investigation session is not running.")
|
|
876
|
+
store.executor.submit(store.compare, identity, request)
|
|
877
|
+
return _view(view, status_code=202)
|
|
878
|
+
|
|
879
|
+
@app.get("/api/comparisons/{identity}")
|
|
880
|
+
def comparison(identity: str):
|
|
881
|
+
with store.lock:
|
|
882
|
+
return _view(store.comparison(identity, ready=False)["view"])
|
|
883
|
+
|
|
884
|
+
@app.post("/api/comparisons/{identity}/inspect")
|
|
885
|
+
def inspect_comparison(identity: str, request: ComparisonInspectInput):
|
|
886
|
+
item = store.comparison(identity)
|
|
887
|
+
store.verify_comparison(item)
|
|
888
|
+
run = store.run(item["run_ids"][0 if request.side == "a" else 1])
|
|
889
|
+
try:
|
|
890
|
+
resolved = resolve_comparison_citations(
|
|
891
|
+
item["view"]["report"],
|
|
892
|
+
run["rollouts"],
|
|
893
|
+
side=request.side,
|
|
894
|
+
feature=request.feature,
|
|
895
|
+
)
|
|
896
|
+
citation = next(
|
|
897
|
+
row
|
|
898
|
+
for row in resolved["items"]
|
|
899
|
+
if row["sample_id"] == request.sample_id
|
|
900
|
+
)
|
|
901
|
+
except (ValueError, StopIteration):
|
|
902
|
+
raise HTTPException(
|
|
903
|
+
400,
|
|
904
|
+
"Requested comparison citation is unavailable or does not match the source.",
|
|
905
|
+
) from None
|
|
906
|
+
store.verify_comparison(item)
|
|
907
|
+
return _citation(
|
|
908
|
+
run, {**citation, "side": resolved["side"], "feature": resolved["feature"]}
|
|
909
|
+
)
|
|
910
|
+
|
|
911
|
+
@app.get("/api/comparisons/{identity}/export")
|
|
912
|
+
def export_comparison(identity: str, format: Literal["json", "html"] = "json"):
|
|
913
|
+
item = store.comparison(identity)
|
|
914
|
+
store.verify_comparison(item)
|
|
915
|
+
return _download(item["view"]["report"], format, comparison=True)
|
|
916
|
+
|
|
917
|
+
@app.post("/api/evidence")
|
|
918
|
+
def evidence(request: EvidenceInput):
|
|
919
|
+
from .app_evidence import load_evidence
|
|
920
|
+
|
|
921
|
+
path = _rooted(root, request.path)
|
|
922
|
+
run = store.run(request.run_id) if request.run_id else None
|
|
923
|
+
if run is not None:
|
|
924
|
+
store.verify(run)
|
|
925
|
+
try:
|
|
926
|
+
result = load_evidence(
|
|
927
|
+
path,
|
|
928
|
+
root=root,
|
|
929
|
+
rollouts=run["rollouts"] if run else None,
|
|
930
|
+
source=run["view"]["source"] if run else None,
|
|
931
|
+
)
|
|
932
|
+
except (OSError, ValueError):
|
|
933
|
+
raise HTTPException(
|
|
934
|
+
400, "Saved evidence is unavailable, unsupported, or invalid."
|
|
935
|
+
) from None
|
|
936
|
+
if run is not None:
|
|
937
|
+
store.verify(run)
|
|
938
|
+
return _view(result)
|
|
939
|
+
|
|
940
|
+
web = Path(__file__).parent / "_web"
|
|
941
|
+
|
|
942
|
+
@app.get("/")
|
|
943
|
+
def index():
|
|
944
|
+
path = web / "index.html"
|
|
945
|
+
if not path.is_file():
|
|
946
|
+
raise HTTPException(503, "Packaged investigation assets are unavailable.")
|
|
947
|
+
return FileResponse(path)
|
|
948
|
+
|
|
949
|
+
@app.get("/assets/{asset_path:path}")
|
|
950
|
+
def asset(asset_path: str):
|
|
951
|
+
path = _rooted(web, "assets/" + asset_path)
|
|
952
|
+
if not path.is_file() or path.suffix not in {".js", ".css", ".svg", ".woff2"}:
|
|
953
|
+
raise HTTPException(404, "Asset was not found.")
|
|
954
|
+
return FileResponse(path)
|
|
955
|
+
|
|
956
|
+
return app
|