rolloutdiff 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. rolloutdiff/__init__.py +130 -0
  2. rolloutdiff/_aria_replay.py +346 -0
  3. rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
  4. rolloutdiff/_dependence.py +195 -0
  5. rolloutdiff/_feature_values.py +79 -0
  6. rolloutdiff/_replay_container.py +568 -0
  7. rolloutdiff/_report_validation.py +758 -0
  8. rolloutdiff/_retrospective_policy.py +33 -0
  9. rolloutdiff/_sampling.py +127 -0
  10. rolloutdiff/_series_validation.py +67 -0
  11. rolloutdiff/_watch_report.py +436 -0
  12. rolloutdiff/_watch_storage.py +800 -0
  13. rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
  14. rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
  15. rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
  16. rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
  17. rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
  18. rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
  19. rolloutdiff/_web/assets/third-party-notices.txt +112 -0
  20. rolloutdiff/_web/index.html +14 -0
  21. rolloutdiff/adapters.py +264 -0
  22. rolloutdiff/app.py +956 -0
  23. rolloutdiff/app_evidence.py +678 -0
  24. rolloutdiff/benchmark_reference.py +458 -0
  25. rolloutdiff/cli.py +1292 -0
  26. rolloutdiff/comparison.py +572 -0
  27. rolloutdiff/comparison_report.py +982 -0
  28. rolloutdiff/corroboration.py +287 -0
  29. rolloutdiff/detect.py +307 -0
  30. rolloutdiff/events.py +173 -0
  31. rolloutdiff/evidence.py +285 -0
  32. rolloutdiff/exemplars.py +240 -0
  33. rolloutdiff/features.py +823 -0
  34. rolloutdiff/html_report.py +726 -0
  35. rolloutdiff/inspect_logs.py +383 -0
  36. rolloutdiff/inspection.py +207 -0
  37. rolloutdiff/integrations.py +552 -0
  38. rolloutdiff/investigation.py +1146 -0
  39. rolloutdiff/notifications.py +406 -0
  40. rolloutdiff/pipeline.py +809 -0
  41. rolloutdiff/producer.py +663 -0
  42. rolloutdiff/prompt_mix.py +258 -0
  43. rolloutdiff/ranking.py +388 -0
  44. rolloutdiff/replay.py +589 -0
  45. rolloutdiff/replication.py +572 -0
  46. rolloutdiff/report.py +838 -0
  47. rolloutdiff/retrospective.py +386 -0
  48. rolloutdiff/schema.py +109 -0
  49. rolloutdiff/sensitivity.py +202 -0
  50. rolloutdiff/series.py +430 -0
  51. rolloutdiff/tool_features.py +422 -0
  52. rolloutdiff/trajectories.py +236 -0
  53. rolloutdiff/watch-compatibility.json +13 -0
  54. rolloutdiff/watch.py +742 -0
  55. rolloutdiff-0.1.0.dist-info/METADATA +158 -0
  56. rolloutdiff-0.1.0.dist-info/RECORD +62 -0
  57. rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
  58. rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
  59. rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
  60. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
  61. rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
  62. rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/app.py ADDED
@@ -0,0 +1,956 @@
1
+ """Authenticated loopback investigation over an explicitly selected local root."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import hashlib
6
+ import hmac
7
+ import ipaddress
8
+ import math
9
+ import os
10
+ import re
11
+ import stat
12
+ import threading
13
+ import time
14
+ from concurrent.futures import ThreadPoolExecutor
15
+ from contextlib import ExitStack, asynccontextmanager, contextmanager
16
+ from itertools import islice
17
+ from numbers import Integral, Real
18
+ from pathlib import Path
19
+ from tempfile import TemporaryDirectory
20
+ from typing import Annotated, Literal
21
+ from urllib.parse import urlsplit
22
+ from uuid import uuid4
23
+
24
+ import numpy as np
25
+ import pandas as pd
26
+ from fastapi import FastAPI, HTTPException, Query, Request
27
+ from fastapi.exceptions import RequestValidationError
28
+ from fastapi.responses import FileResponse, JSONResponse, Response
29
+ from pydantic import BaseModel, ConfigDict, Field, field_validator
30
+
31
+ from . import adapters
32
+ from .comparison import (
33
+ canonical_fingerprint,
34
+ compare_rollouts,
35
+ resolve_comparison_citations,
36
+ )
37
+ from .comparison_report import (
38
+ comparison_json,
39
+ render_comparison_report,
40
+ validate_comparison_report,
41
+ )
42
+ from .html_report import render_report
43
+ from .inspection import resolve_exemplars
44
+ from .pipeline import analyze_rollouts
45
+ from .report import _identifier, build_report, report_json
46
+
47
+ _SAFE_INTEGER = 2**53 - 1
48
+ _MAX_FILES = 10000
49
+ _MAX_LISTING = 1000
50
+ _MAX_JOBS = 32
51
+ _CSP = (
52
+ "default-src 'none'; script-src 'self'; style-src 'self'; "
53
+ "img-src 'self' data:; connect-src 'self'; font-src 'self'; "
54
+ "base-uri 'none'; form-action 'none'; frame-ancestors 'none'; object-src 'none'"
55
+ )
56
+
57
+
58
+ def _require_local_platform():
59
+ directory_functions = {
60
+ function.__name__ for function in getattr(os, "supports_dir_fd", ())
61
+ }
62
+ descriptor_functions = {
63
+ function.__name__ for function in getattr(os, "supports_fd", ())
64
+ }
65
+ if (
66
+ os.name != "posix"
67
+ or not all(
68
+ hasattr(os, name) for name in ("O_DIRECTORY", "O_NOFOLLOW", "O_NONBLOCK")
69
+ )
70
+ or not {"open", "stat"}.issubset(directory_functions)
71
+ or "scandir" not in descriptor_functions
72
+ ):
73
+ raise ValueError(
74
+ "Local investigation requires macOS or Linux with descriptor-relative "
75
+ "and no-follow file access."
76
+ )
77
+
78
+
79
+ def ui_value(value):
80
+ """Preserve missing data and integers that JavaScript cannot represent exactly."""
81
+ if isinstance(value, dict):
82
+ return {key: ui_value(item) for key, item in value.items()}
83
+ if isinstance(value, (list, tuple)):
84
+ return [ui_value(item) for item in value]
85
+ if value is None or value is pd.NA:
86
+ return None
87
+ if isinstance(value, (bool, np.bool_)):
88
+ return bool(value)
89
+ if isinstance(value, Integral):
90
+ number = int(value)
91
+ return str(number) if abs(number) > _SAFE_INTEGER else number
92
+ if isinstance(value, Real):
93
+ number = float(value)
94
+ return number if math.isfinite(number) else None
95
+ if isinstance(value, str):
96
+ return value
97
+ raise TypeError("unsupported view value")
98
+
99
+
100
+ def _view(value, *, status_code=200):
101
+ return JSONResponse(ui_value(value), status_code=status_code)
102
+
103
+
104
+ def _rooted(root: Path, value: str) -> Path:
105
+ path = Path(value)
106
+ if (
107
+ not value
108
+ or path.is_absolute()
109
+ or "\\" in value
110
+ or any(part == ".." or part.startswith(".") for part in path.parts)
111
+ ):
112
+ raise HTTPException(400, "Select a visible path within the selected root.")
113
+ try:
114
+ selected = (root / path).resolve(strict=True)
115
+ relative = selected.relative_to(root)
116
+ if any(part.startswith(".") for part in relative.parts):
117
+ raise ValueError
118
+ except (OSError, RuntimeError, ValueError):
119
+ raise HTTPException(
120
+ 400, "Path is unavailable within the selected root."
121
+ ) from None
122
+ return selected
123
+
124
+
125
+ @contextmanager
126
+ def _open_source(root: Path, value: str):
127
+ """Open without following links, including links introduced during traversal."""
128
+ relative = Path(value)
129
+ if (
130
+ not value
131
+ or relative.is_absolute()
132
+ or "\\" in value
133
+ or any(part == ".." or part.startswith(".") for part in relative.parts)
134
+ ):
135
+ raise HTTPException(400, "Select a visible path within the selected root.")
136
+ descriptors = []
137
+ try:
138
+ directory = os.open(root.anchor, os.O_RDONLY | os.O_DIRECTORY)
139
+ descriptors.append(directory)
140
+ parts = (*root.parts[1:], *relative.parts)
141
+ for part in parts[:-1]:
142
+ directory = os.open(
143
+ part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory
144
+ )
145
+ descriptors.append(directory)
146
+ descriptor = (
147
+ os.open(
148
+ parts[-1],
149
+ os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK,
150
+ dir_fd=directory,
151
+ )
152
+ if parts
153
+ else os.dup(directory)
154
+ )
155
+ descriptors.append(descriptor)
156
+ mode = os.fstat(descriptor).st_mode
157
+ if not (stat.S_ISDIR(mode) or stat.S_ISREG(mode)):
158
+ raise HTTPException(400, "Select a regular file or directory.")
159
+ yield descriptor
160
+ except (OSError, ValueError):
161
+ raise HTTPException(
162
+ 400, "Source is unavailable or contains a symbolic link."
163
+ ) from None
164
+ finally:
165
+ for descriptor in reversed(descriptors):
166
+ os.close(descriptor)
167
+
168
+
169
+ def _files(root: Path, path: str, format: str) -> list[Path]:
170
+ relative = Path(path)
171
+ pattern = adapters._TRL_FILE if format == "trl" else adapters._VERL_FILE
172
+ with ExitStack() as opened:
173
+ descriptor = opened.enter_context(_open_source(root, path))
174
+ if stat.S_ISREG(os.fstat(descriptor).st_mode):
175
+ names = [relative.name] if pattern.fullmatch(relative.name) else []
176
+ parent = relative.parent
177
+ else:
178
+ if format == "trl":
179
+ try:
180
+ child = os.stat(
181
+ "completions", dir_fd=descriptor, follow_symlinks=False
182
+ )
183
+ except FileNotFoundError:
184
+ child = None
185
+ if child is not None and stat.S_ISLNK(child.st_mode):
186
+ raise HTTPException(400, "Source contains a symbolic link.")
187
+ if child is not None and stat.S_ISDIR(child.st_mode):
188
+ relative = relative / "completions"
189
+ descriptor = opened.enter_context(
190
+ _open_source(root, relative.as_posix())
191
+ )
192
+ with os.scandir(descriptor) as entries:
193
+ names = [entry.name for entry in islice(entries, _MAX_FILES + 1)]
194
+ if len(names) > _MAX_FILES:
195
+ raise HTTPException(400, "Too many files in this selection.")
196
+ names = [name for name in names if pattern.fullmatch(name)]
197
+ for name in names:
198
+ mode = os.stat(name, dir_fd=descriptor, follow_symlinks=False).st_mode
199
+ if not stat.S_ISREG(mode):
200
+ raise HTTPException(
201
+ 400, "Source must contain regular files without symbolic links."
202
+ )
203
+ parent = relative
204
+ if not names:
205
+ raise HTTPException(400, "No matching rollout files are available.")
206
+ names.sort(key=lambda name: (int(pattern.fullmatch(name).group(1)), name))
207
+ return [root / parent / name for name in names]
208
+
209
+
210
+ def _hash_file(root: Path, path: Path, *, destination=None) -> dict:
211
+ """Hash the exact bytes copied, refusing nonregular and concurrently edited files."""
212
+ relative = path.relative_to(root).as_posix()
213
+ digest, size = hashlib.sha256(), 0
214
+ with _open_source(root, relative) as descriptor:
215
+ before = os.fstat(descriptor)
216
+ if not stat.S_ISREG(before.st_mode):
217
+ raise HTTPException(400, "Selected source must be a regular file.")
218
+ while chunk := os.read(descriptor, 1024 * 1024):
219
+ digest.update(chunk)
220
+ size += len(chunk)
221
+ if destination is not None:
222
+ destination.write(chunk)
223
+ after = os.fstat(descriptor)
224
+ if (before.st_size, before.st_mtime_ns, before.st_ctime_ns) != (
225
+ after.st_size,
226
+ after.st_mtime_ns,
227
+ after.st_ctime_ns,
228
+ ) or size != after.st_size:
229
+ raise HTTPException(409, "Source changed while it was being read.")
230
+ return {"path": relative, "sha256": digest.hexdigest(), "size_bytes": size}
231
+
232
+
233
+ def _snapshot(root: Path, path: str, format: str) -> list[dict]:
234
+ return [_hash_file(root, file) for file in _files(root, path, format)]
235
+
236
+
237
+ @contextmanager
238
+ def _captured_source(root: Path, path: str, format: str):
239
+ """Adapters read private copies, never mutable paths in the selected root."""
240
+ originals = _files(root, path, format)
241
+ with TemporaryDirectory(prefix="rollout-investigation-") as temporary:
242
+ directory = Path(temporary)
243
+ files = []
244
+ for original in originals:
245
+ descriptor = os.open(
246
+ directory / original.name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600
247
+ )
248
+ with os.fdopen(descriptor, "wb") as destination:
249
+ files.append(_hash_file(root, original, destination=destination))
250
+ if files != _snapshot(root, path, format):
251
+ raise HTTPException(409, "Source changed while it was being copied.")
252
+ yield directory, files
253
+
254
+
255
+ class _Input(BaseModel):
256
+ model_config = ConfigDict(extra="forbid", strict=True)
257
+
258
+
259
+ class RunInput(_Input):
260
+ path: Annotated[str, Field(min_length=1, max_length=4096)]
261
+ format: Literal["trl", "verl"]
262
+ run: Annotated[str, Field(min_length=1, max_length=512)]
263
+ reward_column: Annotated[str, Field(min_length=1, max_length=512)] | None = None
264
+ sensitivity: Literal["default", "lenient"] = "default"
265
+ retrospective: bool = False
266
+
267
+ @field_validator("run", "reward_column")
268
+ @classmethod
269
+ def identifier(cls, value):
270
+ return _identifier(value) if value is not None else None
271
+
272
+
273
+ class InspectInput(_Input):
274
+ event_rank: Annotated[int, Field(ge=1)]
275
+ feature: Annotated[str, Field(min_length=1, max_length=4096)]
276
+ sample_id: Annotated[str, Field(min_length=1, max_length=4096)]
277
+ with_absent: bool = False
278
+
279
+
280
+ class ComparisonInput(_Input):
281
+ run_a: str
282
+ run_b: str | None = None
283
+ a: Annotated[list[str], Field(min_length=2, max_length=2)]
284
+ b: Annotated[list[str], Field(min_length=2, max_length=2)]
285
+
286
+ @field_validator("a", "b")
287
+ @classmethod
288
+ def window(cls, value):
289
+ if any(not re.fullmatch(r"[0-9]{1,20}", item) for item in value):
290
+ raise ValueError("ranges require nonnegative decimal integers")
291
+ if int(value[0]) > int(value[1]):
292
+ raise ValueError("range endpoints must be ordered")
293
+ return value
294
+
295
+
296
+ class ComparisonInspectInput(_Input):
297
+ side: Literal["a", "b"]
298
+ feature: Annotated[str, Field(min_length=1, max_length=4096)]
299
+ sample_id: Annotated[str, Field(min_length=1, max_length=4096)]
300
+
301
+
302
+ class EvidenceInput(_Input):
303
+ path: Annotated[str, Field(min_length=1, max_length=4096)]
304
+ run_id: str | None = None
305
+
306
+
307
+ class _Store:
308
+ def __init__(self, root):
309
+ self.root = root
310
+ self.runs = {}
311
+ self.comparisons = {}
312
+ self.lock = threading.RLock()
313
+ self.executor = None
314
+
315
+ def run(self, identity, *, ready=True):
316
+ with self.lock:
317
+ item = self.runs.get(identity)
318
+ if item is None:
319
+ raise HTTPException(404, "Run was not found.")
320
+ if ready and item["view"]["status"] != "ready":
321
+ raise HTTPException(409, "Run analysis is not ready.")
322
+ return item
323
+
324
+ def comparison(self, identity, *, ready=True):
325
+ with self.lock:
326
+ item = self.comparisons.get(identity)
327
+ if item is None:
328
+ raise HTTPException(404, "Comparison was not found.")
329
+ if ready and item["view"]["status"] != "ready":
330
+ raise HTTPException(409, "Comparison is not ready.")
331
+ return item
332
+
333
+ def verify(self, item):
334
+ try:
335
+ source = item["view"]["source"]
336
+ if (
337
+ _snapshot(self.root, source["path"], source["format"])
338
+ != source["files"]
339
+ ):
340
+ raise ValueError
341
+ if canonical_fingerprint(item["rollouts"]) != source["canonical_sha256"]:
342
+ raise ValueError
343
+ for row in item["rollouts"][["sample_id", "response"]].itertuples(
344
+ index=False
345
+ ):
346
+ if (
347
+ hashlib.sha256(row.response.encode()).hexdigest()
348
+ != item["responses"][row.sample_id]
349
+ ):
350
+ raise ValueError
351
+ except (HTTPException, OSError, ValueError, KeyError):
352
+ raise HTTPException(
353
+ 409, "Source changed or is unavailable. Reload the run."
354
+ ) from None
355
+
356
+ def verify_comparison(self, item):
357
+ for identity in dict.fromkeys(item["run_ids"]):
358
+ self.verify(self.run(identity))
359
+
360
+ def load(self, identity, request):
361
+ item = self.runs[identity]
362
+ started = time.monotonic()
363
+ try:
364
+ with self.lock:
365
+ item["view"]["status"] = "loading"
366
+ with _captured_source(self.root, request.path, request.format) as (
367
+ path,
368
+ before,
369
+ ):
370
+ rollouts = (
371
+ adapters.load_trl(
372
+ path, run=request.run, reward_column=request.reward_column
373
+ )
374
+ if request.format == "trl"
375
+ else adapters.load_verl(path, run=request.run)
376
+ )
377
+ if before != _snapshot(self.root, request.path, request.format):
378
+ raise ValueError("source changed")
379
+ source = {
380
+ "path": request.path,
381
+ "format": request.format,
382
+ "canonical_sha256": canonical_fingerprint(rollouts),
383
+ "files": before,
384
+ }
385
+ responses = {
386
+ row.sample_id: hashlib.sha256(row.response.encode()).hexdigest()
387
+ for row in rollouts[["sample_id", "response"]].itertuples(index=False)
388
+ }
389
+ with self.lock:
390
+ item["view"]["status"] = "analyzing"
391
+ analysis = analyze_rollouts(
392
+ rollouts,
393
+ sensitivity=request.sensitivity,
394
+ retrospective=request.retrospective,
395
+ )
396
+ report = build_report(analysis, run=request.run)
397
+ with self.lock:
398
+ item.update(
399
+ rollouts=rollouts,
400
+ analysis=analysis,
401
+ report=report,
402
+ responses=responses,
403
+ )
404
+ item["view"].update(source=source)
405
+ self.verify(item)
406
+ with self.lock:
407
+ item["view"].update(
408
+ status="ready",
409
+ counts=report["runs"][0]["counts"],
410
+ benchmark=report.get("benchmark"),
411
+ retrospective=report["retrospective"],
412
+ retrospective_results=report["runs"][0]["retrospective"],
413
+ )
414
+ if "sensitivity" in report:
415
+ item["view"]["sensitivity"] = report["sensitivity"]
416
+ except Exception: # noqa: BLE001 -- terminate jobs without exposing source text
417
+ with self.lock:
418
+ item["view"].update(
419
+ status="failed",
420
+ error="Could not analyze this source. Check its format, reward column, and unchanged files.",
421
+ )
422
+ finally:
423
+ with self.lock:
424
+ item["view"]["elapsed_seconds"] = time.monotonic() - started
425
+
426
+ def compare(self, identity, request):
427
+ item = self.comparisons[identity]
428
+ try:
429
+ with self.lock:
430
+ item["view"]["status"] = "comparing"
431
+ left, right = (self.run(run_id) for run_id in item["run_ids"])
432
+ self.verify_comparison(item)
433
+ same = request.run_b is None or request.run_a == request.run_b
434
+ report = compare_rollouts(
435
+ left["rollouts"],
436
+ None if same else right["rollouts"],
437
+ a=tuple(map(int, request.a)),
438
+ b=tuple(map(int, request.b)),
439
+ )
440
+ for side, run in zip(report["sides"], (left, right), strict=True):
441
+ source = run["view"]["source"]
442
+ side["source"].update(
443
+ format=source["format"],
444
+ files=[
445
+ {key: file[key] for key in ("sha256", "size_bytes")}
446
+ for file in source["files"]
447
+ ],
448
+ reward_column=run["request"].reward_column,
449
+ )
450
+ report = validate_comparison_report(report)
451
+ self.verify_comparison(item)
452
+ with self.lock:
453
+ item["view"].update(status="ready", report=report)
454
+ except Exception: # noqa: BLE001 -- terminate jobs without exposing source text
455
+ with self.lock:
456
+ item["view"].update(
457
+ status="failed",
458
+ error="Could not compare these ranges. Check coverage, range order, and unchanged sources.",
459
+ )
460
+
461
+
462
+ def _rows(frame, features):
463
+ if "feature" not in frame:
464
+ return []
465
+ return frame.loc[frame.feature.isin(features)].to_dict("records")
466
+
467
+
468
+ def _headlines(analysis, features):
469
+ """Equal-weight shared-prompt means; omit numerically misleading pairs."""
470
+ values = analysis.prompt_values
471
+ if "change" not in values:
472
+ return {}
473
+ shared = values.loc[values.feature.isin(features) & values.change.notna()]
474
+ result = {}
475
+ for feature, rows in shared.groupby("feature", sort=False):
476
+ with np.errstate(over="ignore", invalid="ignore"):
477
+ before, after, change = (
478
+ float(rows[column].mean())
479
+ for column in ("before_mean", "after_mean", "change")
480
+ )
481
+ # Rounded endpoints can hide an available per-prompt difference.
482
+ if all(
483
+ math.isfinite(value) for value in (before, after, change)
484
+ ) and math.isclose(after - before, change, rel_tol=1e-10, abs_tol=1e-12):
485
+ result[feature] = {
486
+ "before": before,
487
+ "after": after,
488
+ "n_prompts": len(rows),
489
+ }
490
+ return result
491
+
492
+
493
+ def _citation(item, citation):
494
+ response = citation["response"]
495
+ digest = hashlib.sha256(response.encode()).hexdigest()
496
+ if digest != item["responses"].get(citation["sample_id"]):
497
+ raise HTTPException(409, "Citation response changed. Reload the run.")
498
+ counterpart = (citation.get("comparison") or {}).get("counterpart")
499
+ if counterpart is not None:
500
+ counterpart_digest = hashlib.sha256(
501
+ counterpart["response"].encode()
502
+ ).hexdigest()
503
+ if counterpart_digest != item["responses"].get(counterpart["sample_id"]):
504
+ raise HTTPException(409, "Comparison response changed. Reload the run.")
505
+ return _view(
506
+ {
507
+ "citation": citation,
508
+ "source": item["view"]["source"],
509
+ "response_sha256": digest,
510
+ }
511
+ )
512
+
513
+
514
+ def _download(report, format, *, comparison=False):
515
+ if format == "json":
516
+ content = comparison_json(report) if comparison else report_json(report)
517
+ media_type = "application/json"
518
+ else:
519
+ content = (
520
+ render_comparison_report(report) if comparison else render_report(report)
521
+ )
522
+ media_type = "text/html"
523
+ return Response(
524
+ content,
525
+ media_type=media_type,
526
+ headers={
527
+ "Content-Disposition": f'attachment; filename="rollout-{"comparison" if comparison else "analysis"}.{format}"'
528
+ },
529
+ )
530
+
531
+
532
+ def create_app(root, *, token: str, origin="http://127.0.0.1:8765") -> FastAPI:
533
+ """Create a read-only local investigation session with serialized analysis jobs."""
534
+ _require_local_platform()
535
+ root = Path(root).resolve(strict=True)
536
+ if not root.is_dir() or not isinstance(token, str) or not token:
537
+ raise ValueError("a directory root and nonempty session token are required")
538
+ parsed = urlsplit(origin)
539
+ if (
540
+ parsed.scheme != "http"
541
+ or parsed.hostname != "127.0.0.1"
542
+ or parsed.path
543
+ or parsed.query
544
+ or parsed.fragment
545
+ or parsed.username
546
+ ):
547
+ raise ValueError("origin must be a loopback HTTP origin")
548
+ store = _Store(root)
549
+
550
+ @asynccontextmanager
551
+ async def lifespan(app):
552
+ store.executor = ThreadPoolExecutor(
553
+ max_workers=1, thread_name_prefix="rollout-investigation"
554
+ )
555
+ try:
556
+ yield
557
+ finally:
558
+ store.executor.shutdown(wait=True, cancel_futures=True)
559
+ store.executor = None
560
+
561
+ app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
562
+ app.state.store = store
563
+
564
+ @app.middleware("http")
565
+ async def local_session(request: Request, call_next):
566
+ try:
567
+ peer = request.client.host if request.client else ""
568
+ local = ipaddress.ip_address(peer).is_loopback
569
+ except ValueError:
570
+ local = False
571
+ hosts = request.headers.getlist("host")
572
+ origins = request.headers.getlist("origin")
573
+ accepted = (
574
+ local and hosts == [parsed.netloc] and (not origins or origins == [origin])
575
+ )
576
+ if request.headers.get("sec-fetch-site") == "cross-site":
577
+ accepted = False
578
+ if request.url.path.startswith("/api/"):
579
+ authorization = request.headers.getlist("authorization")
580
+ accepted = (
581
+ accepted
582
+ and len(authorization) == 1
583
+ and hmac.compare_digest(
584
+ authorization[0].encode(), f"Bearer {token}".encode()
585
+ )
586
+ )
587
+ if accepted:
588
+ try:
589
+ response = await call_next(request)
590
+ except Exception: # noqa: BLE001 -- API errors must not expose local data
591
+ response = JSONResponse(
592
+ {"detail": "Local request could not be completed."}, status_code=500
593
+ )
594
+ else:
595
+ response = JSONResponse(
596
+ {"detail": "Local session authorization required."}, status_code=403
597
+ )
598
+ response.headers.update(
599
+ {
600
+ "Content-Security-Policy": _CSP,
601
+ "X-Frame-Options": "DENY",
602
+ "X-Content-Type-Options": "nosniff",
603
+ "Referrer-Policy": "no-referrer",
604
+ "Cache-Control": "no-store",
605
+ }
606
+ )
607
+ return response
608
+
609
+ @app.exception_handler(RequestValidationError)
610
+ async def invalid_request(request, exc):
611
+ return JSONResponse({"detail": "Invalid request fields."}, status_code=422)
612
+
613
+ @app.get("/api/session")
614
+ def session():
615
+ return {
616
+ "root_label": root.name,
617
+ "formats": ["trl", "verl"],
618
+ "read_only_execution": True,
619
+ }
620
+
621
+ @app.get("/api/files")
622
+ def files(path: Annotated[str, Query(max_length=4096)] = "."):
623
+ selected = _rooted(root, path)
624
+ if not selected.is_dir():
625
+ raise HTTPException(400, "Select a directory within the root.")
626
+ entries = []
627
+ try:
628
+ children = list(islice(selected.iterdir(), _MAX_FILES + 1))
629
+ except OSError:
630
+ raise HTTPException(400, "Directory is unavailable.") from None
631
+ if len(children) > _MAX_FILES:
632
+ raise HTTPException(
633
+ 400,
634
+ "Directory has too many entries; select a smaller source directory.",
635
+ )
636
+ for child in sorted(children, key=lambda item: item.name):
637
+ if child.name.startswith("."):
638
+ continue
639
+ relative = child.relative_to(root).as_posix()
640
+ try:
641
+ resolved = _rooted(root, relative)
642
+ except HTTPException:
643
+ continue
644
+ if resolved.is_dir():
645
+ kind = "directory"
646
+ elif adapters._TRL_FILE.fullmatch(child.name):
647
+ kind = "trl"
648
+ elif adapters._VERL_FILE.fullmatch(child.name):
649
+ kind = "verl"
650
+ elif child.suffix == ".json" and resolved.is_file():
651
+ kind = "evidence"
652
+ else:
653
+ continue
654
+ entries.append({"name": child.name, "path": relative, "kind": kind})
655
+ if len(entries) == _MAX_LISTING:
656
+ break
657
+ return {"path": selected.relative_to(root).as_posix(), "entries": entries}
658
+
659
+ @app.get("/api/runs")
660
+ def runs():
661
+ with store.lock:
662
+ return _view({"runs": [item["view"] for item in store.runs.values()]})
663
+
664
+ @app.post("/api/runs", status_code=202)
665
+ def add_run(request: RunInput):
666
+ _files(root, request.path, request.format)
667
+ if request.format == "verl" and request.reward_column is not None:
668
+ raise HTTPException(
669
+ 422, "Reward column selection is only available for TRL."
670
+ )
671
+ with store.lock:
672
+ if len(store.runs) >= _MAX_JOBS:
673
+ raise HTTPException(
674
+ 409, "Remove a finished run before loading another."
675
+ )
676
+ identity = uuid4().hex
677
+ view = {
678
+ "id": identity,
679
+ "run": request.run,
680
+ "status": "queued",
681
+ "error": None,
682
+ "counts": None,
683
+ "source": None,
684
+ "elapsed_seconds": None,
685
+ }
686
+ store.runs[identity] = {"view": view, "request": request}
687
+ if store.executor is None:
688
+ raise HTTPException(503, "Investigation session is not running.")
689
+ store.executor.submit(store.load, identity, request)
690
+ return _view(view, status_code=202)
691
+
692
+ @app.get("/api/runs/{identity}")
693
+ def run(identity: str):
694
+ with store.lock:
695
+ return _view(store.run(identity, ready=False)["view"])
696
+
697
+ @app.delete("/api/runs/{identity}", status_code=204)
698
+ def delete_run(identity: str):
699
+ with store.lock:
700
+ item = store.run(identity, ready=False)
701
+ dependent = [
702
+ key
703
+ for key, value in store.comparisons.items()
704
+ if identity in value["run_ids"]
705
+ ]
706
+ if item["view"]["status"] not in {"ready", "failed"} or any(
707
+ store.comparisons[key]["view"]["status"] not in {"ready", "failed"}
708
+ for key in dependent
709
+ ):
710
+ raise HTTPException(
711
+ 409, "Wait for active jobs before removing this run."
712
+ )
713
+ for key in dependent:
714
+ del store.comparisons[key]
715
+ del store.runs[identity]
716
+ return Response(status_code=204)
717
+
718
+ @app.get("/api/runs/{identity}/events")
719
+ def events(
720
+ identity: str,
721
+ q: Annotated[str, Query(max_length=4096)] = "",
722
+ kind: Literal["all", "behavioral", "contextual", "unavailable"] = "all",
723
+ tier: Literal["all", "structural", "prose"] = "all",
724
+ floor: Literal["all", "yes", "no"] = "all",
725
+ offset: Annotated[int, Query(ge=0)] = 0,
726
+ limit: Annotated[int, Query(ge=1, le=200)] = 50,
727
+ ):
728
+ item = store.run(identity)
729
+ selected = []
730
+ for event in item["report"]["runs"][0]["events"]:
731
+ if kind != "all" and event["ranking_status"] != kind:
732
+ continue
733
+ if q and not any(
734
+ q.casefold() in member["feature"].casefold()
735
+ for member in event["members"]
736
+ ):
737
+ continue
738
+ if tier != "all" and not any(
739
+ member["evidence"]["tier"] == tier for member in event["members"]
740
+ ):
741
+ continue
742
+ if floor != "all" and bool(event["ranking"]["numerical_floor_members"]) != (
743
+ floor == "yes"
744
+ ):
745
+ continue
746
+ selected.append(event)
747
+ page = selected[offset : offset + limit]
748
+ return _view(
749
+ {
750
+ "total": len(selected),
751
+ "events": page,
752
+ "headlines": _headlines(
753
+ item["analysis"],
754
+ [event["representative_feature"] for event in page],
755
+ ),
756
+ }
757
+ )
758
+
759
+ @app.get("/api/runs/{identity}/events/{rank}")
760
+ def event(identity: str, rank: int):
761
+ item = store.run(identity)
762
+ selected = next(
763
+ (row for row in item["report"]["runs"][0]["events"] if row["rank"] == rank),
764
+ None,
765
+ )
766
+ if selected is None:
767
+ raise HTTPException(404, "Event was not found.")
768
+ names = [member["feature"] for member in selected["members"]]
769
+ return _view(
770
+ {
771
+ "event": selected,
772
+ "evidence": _rows(item["analysis"].evidence, names),
773
+ "prompt_shifts": _rows(item["analysis"].prompt_shifts, names),
774
+ "headlines": _headlines(item["analysis"], names),
775
+ }
776
+ )
777
+
778
+ @app.get("/api/runs/{identity}/steps")
779
+ def steps(identity: str):
780
+ rollouts = store.run(identity)["rollouts"]
781
+ return _view({"steps": sorted(rollouts.step.unique().tolist())})
782
+
783
+ @app.get("/api/runs/{identity}/features")
784
+ def features(
785
+ identity: str,
786
+ q: Annotated[str, Query(max_length=4096)] = "",
787
+ limit: Annotated[int, Query(ge=1, le=5000)] = 100,
788
+ ):
789
+ analysis = store.run(identity)["analysis"]
790
+ catalog = analysis.evidence[["feature", "tier", "kind"]]
791
+ catalog = catalog.loc[
792
+ catalog.feature.str.contains(q, case=False, regex=False)
793
+ ].sort_values("feature")
794
+ return _view(
795
+ {"total": len(catalog), "features": catalog.head(limit).to_dict("records")}
796
+ )
797
+
798
+ @app.get("/api/runs/{identity}/series")
799
+ def series(
800
+ identity: str, feature: Annotated[str, Query(min_length=1, max_length=4096)]
801
+ ):
802
+ item = store.run(identity)
803
+ analysis = item["analysis"]
804
+ evidence = _rows(analysis.evidence, [feature])
805
+ if not evidence:
806
+ raise HTTPException(404, "Feature was not found.")
807
+ return _view(
808
+ {
809
+ "feature": feature,
810
+ "kind": evidence[0]["kind"],
811
+ "tier": evidence[0]["tier"],
812
+ "direction": next(
813
+ (
814
+ member["direction"]
815
+ for event in item["report"]["runs"][0]["events"]
816
+ for member in event["members"]
817
+ if member["feature"] == feature
818
+ ),
819
+ None,
820
+ ),
821
+ "points": _rows(analysis.aggregation.series, [feature]),
822
+ "evidence": evidence[0],
823
+ "persistence": _rows(analysis.persistence_steps, [feature]),
824
+ "prompt_values": _rows(analysis.prompt_values, [feature]),
825
+ "prompt_shifts": _rows(analysis.prompt_shifts, [feature]),
826
+ "headline": _headlines(analysis, [feature]).get(feature),
827
+ }
828
+ )
829
+
830
+ @app.post("/api/runs/{identity}/inspect")
831
+ def inspect(identity: str, request: InspectInput):
832
+ item = store.run(identity)
833
+ store.verify(item)
834
+ try:
835
+ event = resolve_exemplars(
836
+ item["report"],
837
+ item["rollouts"],
838
+ run=item["view"]["run"],
839
+ event_rank=request.event_rank,
840
+ with_absent=request.with_absent,
841
+ )
842
+ citation = next(
843
+ example
844
+ for member in event["members"]
845
+ if member["feature"] == request.feature
846
+ for example in member["exemplars"]
847
+ if example["sample_id"] == request.sample_id
848
+ and example["status"] == "selected"
849
+ )
850
+ except (ValueError, StopIteration):
851
+ raise HTTPException(
852
+ 400, "Requested citation is unavailable or does not match the source."
853
+ ) from None
854
+ store.verify(item)
855
+ return _citation(item, citation)
856
+
857
+ @app.get("/api/runs/{identity}/export")
858
+ def export(identity: str, format: Literal["json", "html"] = "json"):
859
+ item = store.run(identity)
860
+ store.verify(item)
861
+ return _download(item["report"], format)
862
+
863
+ @app.post("/api/comparisons", status_code=202)
864
+ def add_comparison(request: ComparisonInput):
865
+ identities = [request.run_a, request.run_b or request.run_a]
866
+ for identity in dict.fromkeys(identities):
867
+ store.verify(store.run(identity))
868
+ with store.lock:
869
+ if len(store.comparisons) >= _MAX_JOBS:
870
+ raise HTTPException(409, "Remove finished runs to free comparisons.")
871
+ identity = uuid4().hex
872
+ view = {"id": identity, "status": "queued", "error": None, "report": None}
873
+ store.comparisons[identity] = {"view": view, "run_ids": identities}
874
+ if store.executor is None:
875
+ raise HTTPException(503, "Investigation session is not running.")
876
+ store.executor.submit(store.compare, identity, request)
877
+ return _view(view, status_code=202)
878
+
879
+ @app.get("/api/comparisons/{identity}")
880
+ def comparison(identity: str):
881
+ with store.lock:
882
+ return _view(store.comparison(identity, ready=False)["view"])
883
+
884
+ @app.post("/api/comparisons/{identity}/inspect")
885
+ def inspect_comparison(identity: str, request: ComparisonInspectInput):
886
+ item = store.comparison(identity)
887
+ store.verify_comparison(item)
888
+ run = store.run(item["run_ids"][0 if request.side == "a" else 1])
889
+ try:
890
+ resolved = resolve_comparison_citations(
891
+ item["view"]["report"],
892
+ run["rollouts"],
893
+ side=request.side,
894
+ feature=request.feature,
895
+ )
896
+ citation = next(
897
+ row
898
+ for row in resolved["items"]
899
+ if row["sample_id"] == request.sample_id
900
+ )
901
+ except (ValueError, StopIteration):
902
+ raise HTTPException(
903
+ 400,
904
+ "Requested comparison citation is unavailable or does not match the source.",
905
+ ) from None
906
+ store.verify_comparison(item)
907
+ return _citation(
908
+ run, {**citation, "side": resolved["side"], "feature": resolved["feature"]}
909
+ )
910
+
911
+ @app.get("/api/comparisons/{identity}/export")
912
+ def export_comparison(identity: str, format: Literal["json", "html"] = "json"):
913
+ item = store.comparison(identity)
914
+ store.verify_comparison(item)
915
+ return _download(item["view"]["report"], format, comparison=True)
916
+
917
+ @app.post("/api/evidence")
918
+ def evidence(request: EvidenceInput):
919
+ from .app_evidence import load_evidence
920
+
921
+ path = _rooted(root, request.path)
922
+ run = store.run(request.run_id) if request.run_id else None
923
+ if run is not None:
924
+ store.verify(run)
925
+ try:
926
+ result = load_evidence(
927
+ path,
928
+ root=root,
929
+ rollouts=run["rollouts"] if run else None,
930
+ source=run["view"]["source"] if run else None,
931
+ )
932
+ except (OSError, ValueError):
933
+ raise HTTPException(
934
+ 400, "Saved evidence is unavailable, unsupported, or invalid."
935
+ ) from None
936
+ if run is not None:
937
+ store.verify(run)
938
+ return _view(result)
939
+
940
+ web = Path(__file__).parent / "_web"
941
+
942
+ @app.get("/")
943
+ def index():
944
+ path = web / "index.html"
945
+ if not path.is_file():
946
+ raise HTTPException(503, "Packaged investigation assets are unavailable.")
947
+ return FileResponse(path)
948
+
949
+ @app.get("/assets/{asset_path:path}")
950
+ def asset(asset_path: str):
951
+ path = _rooted(web, "assets/" + asset_path)
952
+ if not path.is_file() or path.suffix not in {".js", ".css", ".svg", ".woff2"}:
953
+ raise HTTPException(404, "Asset was not found.")
954
+ return FileResponse(path)
955
+
956
+ return app