rolloutdiff 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rolloutdiff/__init__.py +130 -0
- rolloutdiff/_aria_replay.py +346 -0
- rolloutdiff/_benchmarks/m0-2026-11-06.json +318 -0
- rolloutdiff/_dependence.py +195 -0
- rolloutdiff/_feature_values.py +79 -0
- rolloutdiff/_replay_container.py +568 -0
- rolloutdiff/_report_validation.py +758 -0
- rolloutdiff/_retrospective_policy.py +33 -0
- rolloutdiff/_sampling.py +127 -0
- rolloutdiff/_series_validation.py +67 -0
- rolloutdiff/_watch_report.py +436 -0
- rolloutdiff/_watch_storage.py +800 -0
- rolloutdiff/_web/assets/index-D8uGEQ9w.js +12 -0
- rolloutdiff/_web/assets/index-Dvo9nx9b.css +1 -0
- rolloutdiff/_web/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff/_web/assets/mona-sans-latin-ext-wght-normal-BqSk0Z3x.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-latin-wght-normal-Pz49MTQZ.woff2 +0 -0
- rolloutdiff/_web/assets/mona-sans-vietnamese-wght-normal-DsUdksa4.woff2 +0 -0
- rolloutdiff/_web/assets/third-party-notices.txt +112 -0
- rolloutdiff/_web/index.html +14 -0
- rolloutdiff/adapters.py +264 -0
- rolloutdiff/app.py +956 -0
- rolloutdiff/app_evidence.py +678 -0
- rolloutdiff/benchmark_reference.py +458 -0
- rolloutdiff/cli.py +1292 -0
- rolloutdiff/comparison.py +572 -0
- rolloutdiff/comparison_report.py +982 -0
- rolloutdiff/corroboration.py +287 -0
- rolloutdiff/detect.py +307 -0
- rolloutdiff/events.py +173 -0
- rolloutdiff/evidence.py +285 -0
- rolloutdiff/exemplars.py +240 -0
- rolloutdiff/features.py +823 -0
- rolloutdiff/html_report.py +726 -0
- rolloutdiff/inspect_logs.py +383 -0
- rolloutdiff/inspection.py +207 -0
- rolloutdiff/integrations.py +552 -0
- rolloutdiff/investigation.py +1146 -0
- rolloutdiff/notifications.py +406 -0
- rolloutdiff/pipeline.py +809 -0
- rolloutdiff/producer.py +663 -0
- rolloutdiff/prompt_mix.py +258 -0
- rolloutdiff/ranking.py +388 -0
- rolloutdiff/replay.py +589 -0
- rolloutdiff/replication.py +572 -0
- rolloutdiff/report.py +838 -0
- rolloutdiff/retrospective.py +386 -0
- rolloutdiff/schema.py +109 -0
- rolloutdiff/sensitivity.py +202 -0
- rolloutdiff/series.py +430 -0
- rolloutdiff/tool_features.py +422 -0
- rolloutdiff/trajectories.py +236 -0
- rolloutdiff/watch-compatibility.json +13 -0
- rolloutdiff/watch.py +742 -0
- rolloutdiff-0.1.0.dist-info/METADATA +158 -0
- rolloutdiff-0.1.0.dist-info/RECORD +62 -0
- rolloutdiff-0.1.0.dist-info/WHEEL +5 -0
- rolloutdiff-0.1.0.dist-info/entry_points.txt +2 -0
- rolloutdiff-0.1.0.dist-info/licenses/LICENSE +202 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/mona-sans-OFL.txt +93 -0
- rolloutdiff-0.1.0.dist-info/licenses/frontend/public/assets/third-party-notices.txt +112 -0
- rolloutdiff-0.1.0.dist-info/top_level.txt +1 -0
rolloutdiff/cli.py
ADDED
|
@@ -0,0 +1,1292 @@
|
|
|
1
|
+
"""Analyze and compare dumps, render reports, and inspect selected rollouts."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import os
|
|
10
|
+
import re
|
|
11
|
+
import secrets
|
|
12
|
+
import signal
|
|
13
|
+
import socket
|
|
14
|
+
import sys
|
|
15
|
+
import time
|
|
16
|
+
import webbrowser
|
|
17
|
+
from contextlib import ExitStack
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from tempfile import TemporaryDirectory
|
|
20
|
+
|
|
21
|
+
from ._report_validation import _constant, _object, validate_report
|
|
22
|
+
from .adapters import _TRL_FILE, _VERL_FILE, _files, load_trl, load_verl
|
|
23
|
+
from .benchmark_reference import load_benchmark_reference
|
|
24
|
+
from .comparison import compare_rollouts, resolve_comparison_citations
|
|
25
|
+
from .comparison_report import (
|
|
26
|
+
comparison_json,
|
|
27
|
+
render_comparison_report,
|
|
28
|
+
validate_comparison_report,
|
|
29
|
+
)
|
|
30
|
+
from .html_report import render_report
|
|
31
|
+
from .inspection import resolve_exemplars
|
|
32
|
+
from .investigation import (
|
|
33
|
+
approve_aria_intervention,
|
|
34
|
+
compare_aria_intervention,
|
|
35
|
+
execute_aria_intervention,
|
|
36
|
+
prepare_aria_intervention,
|
|
37
|
+
render_aria_comparison,
|
|
38
|
+
)
|
|
39
|
+
from .pipeline import analyze_rollouts
|
|
40
|
+
from .replay import SandboxConfig, _json_output, replay_aria, serialize_replay
|
|
41
|
+
from .report import _check_output_path, _write_text, build_report, write_report
|
|
42
|
+
from .watch import (
|
|
43
|
+
WatchSession,
|
|
44
|
+
load_watch_source,
|
|
45
|
+
render_watch_report,
|
|
46
|
+
resolve_watch_exemplars,
|
|
47
|
+
seal_watch_step,
|
|
48
|
+
validate_watch_report,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
_COMPARISON_FORMATS = ["trl", "verl", "openhands", "swe-smith", "inspect"]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _terminal(value: object) -> str:
|
|
55
|
+
if value is None:
|
|
56
|
+
return "Unavailable"
|
|
57
|
+
return "".join(
|
|
58
|
+
char if char.isprintable() else ascii(char)[1:-1] for char in str(value)
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class _Parser(argparse.ArgumentParser):
|
|
63
|
+
def error(self, message):
|
|
64
|
+
super().error(_terminal(message))
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class _Once(argparse.Action):
|
|
68
|
+
def __call__(self, parser, namespace, values, option_string=None):
|
|
69
|
+
if getattr(namespace, self.dest, None) is not None:
|
|
70
|
+
parser.error(f"{option_string} may be supplied only once")
|
|
71
|
+
setattr(namespace, self.dest, values)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _nonempty(value: str) -> str:
|
|
75
|
+
if not value.strip():
|
|
76
|
+
raise argparse.ArgumentTypeError("must be nonempty")
|
|
77
|
+
return value
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _positive_integer(value: str) -> int:
|
|
81
|
+
try:
|
|
82
|
+
result = int(value)
|
|
83
|
+
except ValueError:
|
|
84
|
+
raise argparse.ArgumentTypeError("must be a positive integer") from None
|
|
85
|
+
if result <= 0:
|
|
86
|
+
raise argparse.ArgumentTypeError("must be a positive integer")
|
|
87
|
+
return result
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _step_range(value: str) -> tuple[int, int]:
|
|
91
|
+
if not re.fullmatch(r"[0-9]+:[0-9]+", value):
|
|
92
|
+
raise argparse.ArgumentTypeError("use an inclusive START:END step range")
|
|
93
|
+
start, end = map(int, value.split(":"))
|
|
94
|
+
if start > end:
|
|
95
|
+
raise argparse.ArgumentTypeError("range start must not exceed its end")
|
|
96
|
+
return start, end
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _port(value: str) -> int:
|
|
100
|
+
try:
|
|
101
|
+
number = int(value)
|
|
102
|
+
except ValueError:
|
|
103
|
+
raise argparse.ArgumentTypeError(
|
|
104
|
+
"port must be an integer from 0 to 65535"
|
|
105
|
+
) from None
|
|
106
|
+
if not 0 <= number <= 65535:
|
|
107
|
+
raise argparse.ArgumentTypeError("port must be an integer from 0 to 65535")
|
|
108
|
+
return number
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _parser() -> argparse.ArgumentParser:
|
|
112
|
+
parser = _Parser(prog="rollout", allow_abbrev=False)
|
|
113
|
+
commands = parser.add_subparsers(dest="command", required=True)
|
|
114
|
+
analyze = commands.add_parser(
|
|
115
|
+
"analyze", help="analyze a TRL or verl rollout dump", allow_abbrev=False
|
|
116
|
+
)
|
|
117
|
+
analyze.add_argument("input", type=_nonempty, help="rollout file or directory")
|
|
118
|
+
analyze.add_argument(
|
|
119
|
+
"--format", required=True, choices=["trl", "verl"], action=_Once
|
|
120
|
+
)
|
|
121
|
+
analyze.add_argument("--run", required=True, type=_nonempty, action=_Once)
|
|
122
|
+
analyze.add_argument("--output", required=True, type=_nonempty, action=_Once)
|
|
123
|
+
analyze.add_argument(
|
|
124
|
+
"--retrospective",
|
|
125
|
+
action="store_true",
|
|
126
|
+
help="also run the experimental retrospective mean-shift detector and report its boundaries (disabled by default)",
|
|
127
|
+
)
|
|
128
|
+
analyze.add_argument(
|
|
129
|
+
"--sensitivity",
|
|
130
|
+
choices=("default", "lenient"),
|
|
131
|
+
action=_Once,
|
|
132
|
+
help="lenient opts into the experimental structural-binary reconstruction: baseline 15, z>4, sustain 3, minimum increase 0; current uncertainty guards, vocabulary and ranking remain. Exports analysis report version 3.",
|
|
133
|
+
)
|
|
134
|
+
analyze.add_argument(
|
|
135
|
+
"--reward-column", type=_nonempty, action=_Once, help="TRL reward column"
|
|
136
|
+
)
|
|
137
|
+
references = analyze.add_mutually_exclusive_group()
|
|
138
|
+
references.add_argument(
|
|
139
|
+
"--benchmark-results",
|
|
140
|
+
type=_nonempty,
|
|
141
|
+
action=_Once,
|
|
142
|
+
help="use supplied benchmark-reference JSON instead of the bundled historical reference",
|
|
143
|
+
)
|
|
144
|
+
references.add_argument(
|
|
145
|
+
"--no-benchmark-reference",
|
|
146
|
+
action="store_true",
|
|
147
|
+
help="omit historical benchmark figures from this analysis report",
|
|
148
|
+
)
|
|
149
|
+
_diff_parser(commands)
|
|
150
|
+
_watch_parser(commands)
|
|
151
|
+
investigate = commands.add_parser(
|
|
152
|
+
"investigate",
|
|
153
|
+
help="open the local investigation application",
|
|
154
|
+
allow_abbrev=False,
|
|
155
|
+
description=(
|
|
156
|
+
"Browse and analyze TRL/verl dumps inside an explicitly selected data root. "
|
|
157
|
+
"The application binds to loopback and uses a private session URL. "
|
|
158
|
+
"Saved replay and intervention evidence is read-only. Responses are never executed."
|
|
159
|
+
),
|
|
160
|
+
)
|
|
161
|
+
investigate.add_argument(
|
|
162
|
+
"--root",
|
|
163
|
+
required=True,
|
|
164
|
+
type=_nonempty,
|
|
165
|
+
action=_Once,
|
|
166
|
+
help="directory the application may read",
|
|
167
|
+
)
|
|
168
|
+
investigate.add_argument(
|
|
169
|
+
"--port",
|
|
170
|
+
type=_port,
|
|
171
|
+
action=_Once,
|
|
172
|
+
help="loopback port (default: 8765; 0 chooses a free port)",
|
|
173
|
+
)
|
|
174
|
+
investigate.add_argument(
|
|
175
|
+
"--no-browser",
|
|
176
|
+
action="store_true",
|
|
177
|
+
help="print the session URL without opening a browser",
|
|
178
|
+
)
|
|
179
|
+
render = commands.add_parser(
|
|
180
|
+
"render", help="render analysis or comparison JSON as HTML", allow_abbrev=False
|
|
181
|
+
)
|
|
182
|
+
render.add_argument(
|
|
183
|
+
"input", type=_nonempty, help="analysis or comparison JSON report"
|
|
184
|
+
)
|
|
185
|
+
render.add_argument("--output", required=True, type=_nonempty, action=_Once)
|
|
186
|
+
inspect = commands.add_parser(
|
|
187
|
+
"inspect", help="inspect selected exemplar responses", allow_abbrev=False
|
|
188
|
+
)
|
|
189
|
+
inspect.add_argument("input", type=_nonempty, help="rollout file or directory")
|
|
190
|
+
inspect.add_argument(
|
|
191
|
+
"--format", required=True, choices=_COMPARISON_FORMATS, action=_Once
|
|
192
|
+
)
|
|
193
|
+
inspect.add_argument("--run", required=True, type=_nonempty, action=_Once)
|
|
194
|
+
inspect.add_argument("--report", required=True, type=_nonempty, action=_Once)
|
|
195
|
+
inspect.add_argument(
|
|
196
|
+
"--event-rank",
|
|
197
|
+
type=_positive_integer,
|
|
198
|
+
action=_Once,
|
|
199
|
+
help="within-run event rank from the saved report",
|
|
200
|
+
)
|
|
201
|
+
inspect.add_argument(
|
|
202
|
+
"--side",
|
|
203
|
+
choices=["a", "b"],
|
|
204
|
+
action=_Once,
|
|
205
|
+
help="comparison-report side to resolve",
|
|
206
|
+
)
|
|
207
|
+
inspect.add_argument(
|
|
208
|
+
"--feature",
|
|
209
|
+
type=_nonempty,
|
|
210
|
+
action=_Once,
|
|
211
|
+
help="comparison-report feature to inspect",
|
|
212
|
+
)
|
|
213
|
+
inspect.add_argument(
|
|
214
|
+
"--reward-column",
|
|
215
|
+
type=_nonempty,
|
|
216
|
+
action=_Once,
|
|
217
|
+
help="TRL/trajectory reward column or Inspect scorer name",
|
|
218
|
+
)
|
|
219
|
+
inspect.add_argument(
|
|
220
|
+
"--model", type=_nonempty, action=_Once, help="trajectory source model filter"
|
|
221
|
+
)
|
|
222
|
+
inspect.add_argument(
|
|
223
|
+
"--with-absent",
|
|
224
|
+
action="store_true",
|
|
225
|
+
help="compare each selected exemplar with a feature-absent sibling",
|
|
226
|
+
)
|
|
227
|
+
replay = commands.add_parser(
|
|
228
|
+
"replay-aria",
|
|
229
|
+
help="explicitly replay one pinned Aria completion",
|
|
230
|
+
allow_abbrev=False,
|
|
231
|
+
)
|
|
232
|
+
replay.add_argument("input", type=_nonempty, help="checksum-pinned Aria parquet")
|
|
233
|
+
for name in (
|
|
234
|
+
"run",
|
|
235
|
+
"sample-id",
|
|
236
|
+
"response-sha256",
|
|
237
|
+
"artifacts",
|
|
238
|
+
"docker-host",
|
|
239
|
+
"output",
|
|
240
|
+
):
|
|
241
|
+
replay.add_argument("--" + name, required=True, type=_nonempty, action=_Once)
|
|
242
|
+
replay.add_argument(
|
|
243
|
+
"--image",
|
|
244
|
+
type=_nonempty,
|
|
245
|
+
action=_Once,
|
|
246
|
+
help="immutable local Python image SHA-256",
|
|
247
|
+
)
|
|
248
|
+
replay.add_argument(
|
|
249
|
+
"--allow-execution",
|
|
250
|
+
action="store_true",
|
|
251
|
+
help="authorize disposable container execution for this completion",
|
|
252
|
+
)
|
|
253
|
+
_investigation_parser(commands)
|
|
254
|
+
return parser
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _watch_parser(commands):
|
|
258
|
+
watch = commands.add_parser(
|
|
259
|
+
"watch",
|
|
260
|
+
help="watch explicitly completed rollout steps",
|
|
261
|
+
allow_abbrev=False,
|
|
262
|
+
description="Passive prefix analysis with durable local step commits. Responses are parsed as data and never executed.",
|
|
263
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
264
|
+
epilog="""Producer interfaces (Python; trainer packages remain optional):
|
|
265
|
+
with AsyncRolloutProducer(output, spool, format="trl", run="training",
|
|
266
|
+
start_step=10, step_stride=10) as producer:
|
|
267
|
+
trainer.add_callback(make_trl_callback(
|
|
268
|
+
trainer, producer, expected_groups=groups_by_step,
|
|
269
|
+
reward_column="quality"))
|
|
270
|
+
# The caller starts training; close drains publication and reports errors.
|
|
271
|
+
|
|
272
|
+
TRL requires trl==1.18.0 and transformers==4.56.2. Support is limited to
|
|
273
|
+
GRPOTrainer, num_iterations=1, steps_per_generation=gradient_accumulation_steps,
|
|
274
|
+
full drop-last batches, integral step logging, and disabled first-step logging,
|
|
275
|
+
evaluation, custom rollouts, tools, environments and multimodal inputs. Only
|
|
276
|
+
world rank zero owns a producer; other ranks pass None. Independent expected
|
|
277
|
+
groups must describe each planned logged generation before observing outputs.
|
|
278
|
+
The callback witnesses optimizer-step boundaries and publishes the latest
|
|
279
|
+
complete gathered generation at selected logging steps. Those completions
|
|
280
|
+
precede the update; unlogged generations and checkpoint-weight parity are unknown.
|
|
281
|
+
Use a producer output separate from TRL's native output directory.
|
|
282
|
+
|
|
283
|
+
For verl==0.9.1, create VerlCompletionLogger(producer,
|
|
284
|
+
expected_groups=groups_by_step) and call logger.bind(trainer) before fit(). This
|
|
285
|
+
replaces the training _dump_generations boundary and bypasses its unbounded
|
|
286
|
+
executor. Validation-data dumping must be disabled. Keep the producer context
|
|
287
|
+
open through training and close it afterward to surface final worker failures.
|
|
288
|
+
|
|
289
|
+
submit(step, canonical_rows, expected_groups=groups) returns a pending receipt.
|
|
290
|
+
receipt.acknowledge() waits for durable spool storage; receipt.result() also
|
|
291
|
+
waits for published data and marker. Default buffering is 8 steps / 64 MiB of
|
|
292
|
+
canonical payloads. Overflow rejects before acceptance. Acknowledged records
|
|
293
|
+
recover on restart; unacknowledged memory submissions can be lost. Spool disk
|
|
294
|
+
use grows with the run. Resume requires matching source, code and configuration.
|
|
295
|
+
|
|
296
|
+
FixedProbeHook(producer, prompts=fixed_prompts, repeats=2, sample=your_sampler,
|
|
297
|
+
training_run="training", training_source=training_output) uses a separate run
|
|
298
|
+
and output. Call hook(step, snapshot_id=...) explicitly. Cadence comes from the
|
|
299
|
+
probe producer. The caller freezes weights and supplies complete scored rows;
|
|
300
|
+
sampling is synchronous and its inference cost is outside publication timings.
|
|
301
|
+
Interrupted attempts remain spent and are never sampled again automatically.
|
|
302
|
+
|
|
303
|
+
All integrations import from rolloutdiff. Use watch run to consume their output.
|
|
304
|
+
watch notify explicitly sends a bounded opaque summary of one durable snapshot;
|
|
305
|
+
it does not retry or promise exactly-once remote delivery.""",
|
|
306
|
+
)
|
|
307
|
+
operations = watch.add_subparsers(dest="watch_operation", required=True)
|
|
308
|
+
seal = operations.add_parser(
|
|
309
|
+
"seal",
|
|
310
|
+
allow_abbrev=False,
|
|
311
|
+
help="publish a completion marker for a closed step",
|
|
312
|
+
description="Seal one closed TRL/verl step against independently supplied complete prompt-group counts. The groups JSON maps canonical prompt IDs to positive expected completion counts. Publish data atomically before sealing; never append or replace committed inputs. File-size stability does not prove completeness.",
|
|
313
|
+
)
|
|
314
|
+
seal.add_argument("input", type=_nonempty, help="one complete supported step file")
|
|
315
|
+
seal.add_argument("--format", choices=["trl", "verl"], required=True, action=_Once)
|
|
316
|
+
seal.add_argument("--run", type=_nonempty, required=True, action=_Once)
|
|
317
|
+
seal.add_argument(
|
|
318
|
+
"--groups",
|
|
319
|
+
type=_nonempty,
|
|
320
|
+
required=True,
|
|
321
|
+
action=_Once,
|
|
322
|
+
help='JSON object such as {"prompt-1":16,"prompt-2":16}; do not derive expectations from a partial file',
|
|
323
|
+
)
|
|
324
|
+
seal.add_argument("--reward-column", type=_nonempty, action=_Once)
|
|
325
|
+
run = operations.add_parser(
|
|
326
|
+
"run",
|
|
327
|
+
allow_abbrev=False,
|
|
328
|
+
help="watch or resume a completed-step directory",
|
|
329
|
+
description="Finalize only start-step + k * step-stride, in order. Missing markers wait and later steps remain pending. There is no gap-skipping mode. Each file needs its .complete.json marker from watch seal. Empty steps, aliases, partial groups and changed committed inputs fail. Source and state must use real directory paths without symbolic-link components. State contains private source copies and must be separate from the input directory. Resume verifies the original source, settings and runtime. A reviewed code upgrade can continue a legacy session with both detectors. Use the newer writer for subsequent steps, or start a new state to choose a different policy. Extraction is cached; new steps still recompute full-prefix analysis. First-emission steps identify the prefix where the watch issued a record. They are separate from inferred onset and detector alert steps and do not give wall-clock training or delivery time. Journals in numbered commits are authoritative; stdout delivery is best-effort across crashes. Render commits/<sequence>/report.json with rollout render. Inspect using that commit's source directory and watch report, not the growing input directory.",
|
|
330
|
+
)
|
|
331
|
+
run.add_argument(
|
|
332
|
+
"input", type=_nonempty, help="directory of step files and completion markers"
|
|
333
|
+
)
|
|
334
|
+
run.add_argument("--format", choices=["trl", "verl"], required=True, action=_Once)
|
|
335
|
+
run.add_argument("--run", type=_nonempty, required=True, action=_Once)
|
|
336
|
+
run.add_argument(
|
|
337
|
+
"--state",
|
|
338
|
+
type=_nonempty,
|
|
339
|
+
required=True,
|
|
340
|
+
action=_Once,
|
|
341
|
+
help="new or matching private watch-state directory",
|
|
342
|
+
)
|
|
343
|
+
policy = run.add_mutually_exclusive_group()
|
|
344
|
+
policy.add_argument(
|
|
345
|
+
"--retrospective",
|
|
346
|
+
action="store_true",
|
|
347
|
+
default=None,
|
|
348
|
+
help="enable experimental boundaries in each prefix; on resume, omission uses the recorded policy",
|
|
349
|
+
)
|
|
350
|
+
policy.add_argument(
|
|
351
|
+
"--no-retrospective",
|
|
352
|
+
dest="retrospective",
|
|
353
|
+
action="store_false",
|
|
354
|
+
default=None,
|
|
355
|
+
help="require sequential-only watch state (default for new sessions)",
|
|
356
|
+
)
|
|
357
|
+
run.add_argument("--start-step", type=int, required=True, action=_Once)
|
|
358
|
+
run.add_argument(
|
|
359
|
+
"--step-stride",
|
|
360
|
+
type=_positive_integer,
|
|
361
|
+
action=_Once,
|
|
362
|
+
help="declared logging cadence (default:1)",
|
|
363
|
+
)
|
|
364
|
+
run.add_argument(
|
|
365
|
+
"--reward-column",
|
|
366
|
+
type=_nonempty,
|
|
367
|
+
action=_Once,
|
|
368
|
+
help="TRL reward column (default:reward)",
|
|
369
|
+
)
|
|
370
|
+
run.add_argument(
|
|
371
|
+
"--sampling-units",
|
|
372
|
+
type=_nonempty,
|
|
373
|
+
action=_Once,
|
|
374
|
+
help="fixed JSON records with run, prompt_id and sampling_unit_id; future prompts must have a declared unit",
|
|
375
|
+
)
|
|
376
|
+
run.add_argument(
|
|
377
|
+
"--max-sparse-per-tier",
|
|
378
|
+
type=int,
|
|
379
|
+
action=_Once,
|
|
380
|
+
help="prefix vocabulary cap (default:2000); frozen on resume",
|
|
381
|
+
)
|
|
382
|
+
run.add_argument(
|
|
383
|
+
"--poll-interval",
|
|
384
|
+
type=float,
|
|
385
|
+
action=_Once,
|
|
386
|
+
help="idle seconds between polls (default:1)",
|
|
387
|
+
)
|
|
388
|
+
run.add_argument(
|
|
389
|
+
"--max-steps",
|
|
390
|
+
type=_positive_integer,
|
|
391
|
+
action=_Once,
|
|
392
|
+
help="maximum new commits per poll",
|
|
393
|
+
)
|
|
394
|
+
run.add_argument(
|
|
395
|
+
"--once",
|
|
396
|
+
action="store_true",
|
|
397
|
+
help="process one finite marker inventory, then exit",
|
|
398
|
+
)
|
|
399
|
+
notify = operations.add_parser(
|
|
400
|
+
"notify",
|
|
401
|
+
allow_abbrev=False,
|
|
402
|
+
help="send one explicit notification for a durable watch snapshot",
|
|
403
|
+
description="Send at most 256 KiB of opaque identifiers, timing and counts to an explicit HTTPS or loopback HTTP endpoint. Responses are limited to 8 KiB. Delivery state records the attempt before transport. Duplicate completed requests reuse the receipt; uncertain or failed attempts are never retried automatically. No response, prompt or feature literal is sent.",
|
|
404
|
+
)
|
|
405
|
+
notify.add_argument("input", type=_nonempty, help="commits/<sequence>/report.json")
|
|
406
|
+
notify.add_argument("--url", required=True, type=_nonempty, action=_Once)
|
|
407
|
+
notify.add_argument(
|
|
408
|
+
"--state",
|
|
409
|
+
required=True,
|
|
410
|
+
type=_nonempty,
|
|
411
|
+
action=_Once,
|
|
412
|
+
help="private delivery state, separate from watch state",
|
|
413
|
+
)
|
|
414
|
+
notify.add_argument(
|
|
415
|
+
"--timeout",
|
|
416
|
+
type=float,
|
|
417
|
+
action=_Once,
|
|
418
|
+
help="transport seconds, greater than 0 and at most 30 (default:5)",
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _watch(args):
|
|
423
|
+
if args.watch_operation == "notify":
|
|
424
|
+
from .notifications import deliver_watch_notification
|
|
425
|
+
|
|
426
|
+
receipt = deliver_watch_notification(
|
|
427
|
+
args.input,
|
|
428
|
+
url=args.url,
|
|
429
|
+
state=args.state,
|
|
430
|
+
timeout=5.0 if args.timeout is None else args.timeout,
|
|
431
|
+
)
|
|
432
|
+
print(json.dumps(receipt, sort_keys=True, allow_nan=False))
|
|
433
|
+
return 0 if receipt["status"] == "delivered" else 1
|
|
434
|
+
if args.watch_operation == "seal":
|
|
435
|
+
marker = seal_watch_step(
|
|
436
|
+
args.input,
|
|
437
|
+
format=args.format,
|
|
438
|
+
run=args.run,
|
|
439
|
+
expected_groups=_json_object(args.groups),
|
|
440
|
+
reward_column=args.reward_column,
|
|
441
|
+
)
|
|
442
|
+
print(f"Completion marker: {_terminal(marker)}")
|
|
443
|
+
return 0
|
|
444
|
+
interval = 1.0 if args.poll_interval is None else args.poll_interval
|
|
445
|
+
if not math.isfinite(interval) or interval <= 0:
|
|
446
|
+
raise ValueError("poll interval must be finite and positive")
|
|
447
|
+
units = None
|
|
448
|
+
if args.sampling_units is not None:
|
|
449
|
+
import pandas as pd
|
|
450
|
+
|
|
451
|
+
try:
|
|
452
|
+
with Path(args.sampling_units).open(encoding="utf-8") as stream:
|
|
453
|
+
value = json.load(
|
|
454
|
+
stream, object_pairs_hook=_object, parse_constant=_constant
|
|
455
|
+
)
|
|
456
|
+
except (ValueError, RecursionError):
|
|
457
|
+
raise ValueError("invalid watch sampling-unit JSON") from None
|
|
458
|
+
names = {"run", "prompt_id", "sampling_unit_id"}
|
|
459
|
+
if not isinstance(value, list) or any(
|
|
460
|
+
type(row) is not dict or set(row) != names for row in value
|
|
461
|
+
):
|
|
462
|
+
raise ValueError(
|
|
463
|
+
"sampling units must be JSON records with run, prompt_id and sampling_unit_id"
|
|
464
|
+
)
|
|
465
|
+
units = pd.DataFrame(value, columns=sorted(names))
|
|
466
|
+
previous = None
|
|
467
|
+
|
|
468
|
+
def stop(signum, frame):
|
|
469
|
+
raise KeyboardInterrupt
|
|
470
|
+
|
|
471
|
+
if not args.once:
|
|
472
|
+
previous = signal.signal(signal.SIGTERM, stop)
|
|
473
|
+
try:
|
|
474
|
+
with WatchSession(
|
|
475
|
+
args.input,
|
|
476
|
+
args.state,
|
|
477
|
+
format=args.format,
|
|
478
|
+
run=args.run,
|
|
479
|
+
start_step=args.start_step,
|
|
480
|
+
step_stride=args.step_stride or 1,
|
|
481
|
+
reward_column=args.reward_column,
|
|
482
|
+
sampling_units=units,
|
|
483
|
+
retrospective=args.retrospective,
|
|
484
|
+
max_sparse_per_tier=2000
|
|
485
|
+
if args.max_sparse_per_tier is None
|
|
486
|
+
else args.max_sparse_per_tier,
|
|
487
|
+
) as watcher:
|
|
488
|
+
waiting = None
|
|
489
|
+
while True:
|
|
490
|
+
for receipt in watcher.poll_once(max_steps=args.max_steps):
|
|
491
|
+
print(
|
|
492
|
+
json.dumps(receipt, sort_keys=True, allow_nan=False), flush=True
|
|
493
|
+
)
|
|
494
|
+
if waiting != watcher.next_step:
|
|
495
|
+
waiting = watcher.next_step
|
|
496
|
+
print(
|
|
497
|
+
f"Waiting for completed step {waiting}.",
|
|
498
|
+
file=sys.stderr,
|
|
499
|
+
flush=True,
|
|
500
|
+
)
|
|
501
|
+
if args.once:
|
|
502
|
+
return 0
|
|
503
|
+
time.sleep(interval)
|
|
504
|
+
except KeyboardInterrupt:
|
|
505
|
+
print("Watch stopped; committed snapshots are preserved.", file=sys.stderr)
|
|
506
|
+
return 0
|
|
507
|
+
finally:
|
|
508
|
+
if previous is not None:
|
|
509
|
+
signal.signal(signal.SIGTERM, previous)
|
|
510
|
+
|
|
511
|
+
|
|
512
|
+
def _diff_parser(commands):
|
|
513
|
+
diff = commands.add_parser(
|
|
514
|
+
"diff",
|
|
515
|
+
help="compare selected windows or two rollout dumps",
|
|
516
|
+
allow_abbrev=False,
|
|
517
|
+
description=(
|
|
518
|
+
"Compare inclusive step ranges A and B in one run, or two rollout dumps. "
|
|
519
|
+
"One dump requires --a and --b; ranges may overlap or appear in either order. "
|
|
520
|
+
"Two dumps default to each full observed range. Completions are pooled "
|
|
521
|
+
"within each exact prompt across each selected range. Exact prompts "
|
|
522
|
+
"receive equal weight, and both ranges share one feature vocabulary. "
|
|
523
|
+
"Pairing requires exact prompt content, including chat metadata. "
|
|
524
|
+
"Approximate 95% normal intervals use measured standard errors. "
|
|
525
|
+
"Partial prompt overlap leaves the overall interval unavailable. "
|
|
526
|
+
"Available difference estimates and shared-prompt comparisons remain "
|
|
527
|
+
"reported. Differences may reflect prompt composition."
|
|
528
|
+
),
|
|
529
|
+
)
|
|
530
|
+
diff.add_argument("input", type=_nonempty, help="side A dump (or shared dump)")
|
|
531
|
+
diff.add_argument("other", nargs="?", type=_nonempty, help="optional side B dump")
|
|
532
|
+
diff.add_argument(
|
|
533
|
+
"--format", choices=_COMPARISON_FORMATS, required=True, action=_Once
|
|
534
|
+
)
|
|
535
|
+
diff.add_argument("--run", type=_nonempty, required=True, action=_Once)
|
|
536
|
+
diff.add_argument(
|
|
537
|
+
"--other-format",
|
|
538
|
+
choices=_COMPARISON_FORMATS,
|
|
539
|
+
action=_Once,
|
|
540
|
+
help="side B format; defaults to --format",
|
|
541
|
+
)
|
|
542
|
+
diff.add_argument(
|
|
543
|
+
"--other-run",
|
|
544
|
+
type=_nonempty,
|
|
545
|
+
action=_Once,
|
|
546
|
+
help="side B run name; defaults to --run",
|
|
547
|
+
)
|
|
548
|
+
diff.add_argument("--a", type=_step_range, action=_Once, help="side A START:END")
|
|
549
|
+
diff.add_argument("--b", type=_step_range, action=_Once, help="side B START:END")
|
|
550
|
+
diff.add_argument(
|
|
551
|
+
"--reward-column",
|
|
552
|
+
type=_nonempty,
|
|
553
|
+
action=_Once,
|
|
554
|
+
help="side A TRL/trajectory reward column or Inspect scorer name",
|
|
555
|
+
)
|
|
556
|
+
diff.add_argument(
|
|
557
|
+
"--other-reward-column",
|
|
558
|
+
type=_nonempty,
|
|
559
|
+
action=_Once,
|
|
560
|
+
help="side B TRL/trajectory reward column or Inspect scorer name; TRL defaults to reward",
|
|
561
|
+
)
|
|
562
|
+
diff.add_argument(
|
|
563
|
+
"--model",
|
|
564
|
+
type=_nonempty,
|
|
565
|
+
action=_Once,
|
|
566
|
+
help="side A trajectory source model filter",
|
|
567
|
+
)
|
|
568
|
+
diff.add_argument(
|
|
569
|
+
"--other-model",
|
|
570
|
+
type=_nonempty,
|
|
571
|
+
action=_Once,
|
|
572
|
+
help="side B trajectory source model filter",
|
|
573
|
+
)
|
|
574
|
+
diff.add_argument(
|
|
575
|
+
"--tool-calls",
|
|
576
|
+
action="store_true",
|
|
577
|
+
help="compare recorded assistant tool calls; exports comparison version 2",
|
|
578
|
+
)
|
|
579
|
+
diff.add_argument(
|
|
580
|
+
"--keyword",
|
|
581
|
+
type=_nonempty,
|
|
582
|
+
action="append",
|
|
583
|
+
default=[],
|
|
584
|
+
help="literal prompt keyword for mix evidence; repeatable",
|
|
585
|
+
)
|
|
586
|
+
diff.add_argument(
|
|
587
|
+
"--output",
|
|
588
|
+
type=_nonempty,
|
|
589
|
+
required=True,
|
|
590
|
+
action=_Once,
|
|
591
|
+
help="new comparison JSON path (never overwritten)",
|
|
592
|
+
)
|
|
593
|
+
diff.add_argument(
|
|
594
|
+
"--html",
|
|
595
|
+
type=_nonempty,
|
|
596
|
+
action=_Once,
|
|
597
|
+
help="optional new self-contained HTML path",
|
|
598
|
+
)
|
|
599
|
+
|
|
600
|
+
|
|
601
|
+
def _investigation_parser(commands):
|
|
602
|
+
investigation = commands.add_parser(
|
|
603
|
+
"investigate-aria",
|
|
604
|
+
help="prepare and compare one Aria intervention",
|
|
605
|
+
allow_abbrev=False,
|
|
606
|
+
)
|
|
607
|
+
actions = investigation.add_subparsers(dest="investigation_action", required=True)
|
|
608
|
+
prepare = actions.add_parser(
|
|
609
|
+
"prepare", help="freeze a control and one exact edit", allow_abbrev=False
|
|
610
|
+
)
|
|
611
|
+
prepare.add_argument("control", type=_nonempty)
|
|
612
|
+
for name in (
|
|
613
|
+
"source",
|
|
614
|
+
"artifacts",
|
|
615
|
+
"edit",
|
|
616
|
+
"hypothesis",
|
|
617
|
+
"predictions",
|
|
618
|
+
"docker-host",
|
|
619
|
+
"output",
|
|
620
|
+
):
|
|
621
|
+
prepare.add_argument("--" + name, required=True, type=_nonempty, action=_Once)
|
|
622
|
+
approve = actions.add_parser(
|
|
623
|
+
"approve", help="record case-specific execution approval", allow_abbrev=False
|
|
624
|
+
)
|
|
625
|
+
approve.add_argument("case", type=_nonempty)
|
|
626
|
+
for name in ("case-sha256", "variant-sha256"):
|
|
627
|
+
approve.add_argument("--" + name, required=True, type=_nonempty, action=_Once)
|
|
628
|
+
approve.add_argument(
|
|
629
|
+
"--max-containers", required=True, type=_positive_integer, action=_Once
|
|
630
|
+
)
|
|
631
|
+
approve.add_argument("--allow-execution", action="store_true")
|
|
632
|
+
run = actions.add_parser(
|
|
633
|
+
"run", help="consume approval and execute once", allow_abbrev=False
|
|
634
|
+
)
|
|
635
|
+
run.add_argument("case", type=_nonempty)
|
|
636
|
+
for name in ("source", "artifacts"):
|
|
637
|
+
run.add_argument("--" + name, required=True, type=_nonempty, action=_Once)
|
|
638
|
+
run.add_argument("--allow-execution", action="store_true")
|
|
639
|
+
compare = actions.add_parser(
|
|
640
|
+
"compare", help="compare saved evidence without execution", allow_abbrev=False
|
|
641
|
+
)
|
|
642
|
+
compare.add_argument("case", type=_nonempty)
|
|
643
|
+
compare.add_argument("result", type=_nonempty)
|
|
644
|
+
compare.add_argument("--provenance", type=_nonempty, action=_Once)
|
|
645
|
+
for name in ("output", "html"):
|
|
646
|
+
compare.add_argument("--" + name, required=True, type=_nonempty, action=_Once)
|
|
647
|
+
|
|
648
|
+
|
|
649
|
+
def _json_object(path):
|
|
650
|
+
value = _json_output(Path(path).read_bytes())
|
|
651
|
+
if type(value) is not dict:
|
|
652
|
+
raise ValueError("input must be a valid JSON object without duplicate members")
|
|
653
|
+
return value
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _investigate(args):
|
|
657
|
+
import uvicorn
|
|
658
|
+
|
|
659
|
+
from .app import _require_local_platform, create_app
|
|
660
|
+
|
|
661
|
+
_require_local_platform()
|
|
662
|
+
root = Path(args.root).resolve(strict=True)
|
|
663
|
+
if not root.is_dir():
|
|
664
|
+
raise ValueError("investigation root must be a directory")
|
|
665
|
+
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener:
|
|
666
|
+
listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
|
667
|
+
listener.bind(("127.0.0.1", 8765 if args.port is None else args.port))
|
|
668
|
+
listener.listen(128)
|
|
669
|
+
port = listener.getsockname()[1]
|
|
670
|
+
origin = f"http://127.0.0.1:{port}"
|
|
671
|
+
token = secrets.token_urlsafe(32)
|
|
672
|
+
app = create_app(root, token=token, origin=origin)
|
|
673
|
+
server = uvicorn.Server(
|
|
674
|
+
uvicorn.Config(
|
|
675
|
+
app,
|
|
676
|
+
host="127.0.0.1",
|
|
677
|
+
port=port,
|
|
678
|
+
proxy_headers=False,
|
|
679
|
+
access_log=False,
|
|
680
|
+
log_level="warning",
|
|
681
|
+
)
|
|
682
|
+
)
|
|
683
|
+
url = f"{origin}/#token={token}"
|
|
684
|
+
print("Local investigation. Keep the session URL private.", flush=True)
|
|
685
|
+
print(url, flush=True)
|
|
686
|
+
if not args.no_browser:
|
|
687
|
+
webbrowser.open(url)
|
|
688
|
+
try:
|
|
689
|
+
server.run(sockets=[listener])
|
|
690
|
+
except KeyboardInterrupt:
|
|
691
|
+
return 130
|
|
692
|
+
return 0
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def _investigation(args):
|
|
696
|
+
if args.investigation_action == "prepare":
|
|
697
|
+
output = Path(args.output)
|
|
698
|
+
_check_output_path(output)
|
|
699
|
+
case = prepare_aria_intervention(
|
|
700
|
+
args.control,
|
|
701
|
+
args.source,
|
|
702
|
+
artifacts=args.artifacts,
|
|
703
|
+
edit=_json_object(args.edit),
|
|
704
|
+
hypothesis=args.hypothesis,
|
|
705
|
+
predictions=_json_object(args.predictions),
|
|
706
|
+
directory=output,
|
|
707
|
+
config=SandboxConfig(docker_host=args.docker_host),
|
|
708
|
+
)
|
|
709
|
+
print(f"Case SHA-256: {_terminal(case['case_sha256'])}")
|
|
710
|
+
print(f"Variant SHA-256: {_terminal(case['variant_response_sha256'])}")
|
|
711
|
+
print(
|
|
712
|
+
"Prepared without execution. Inspect the case and source diff before approval."
|
|
713
|
+
)
|
|
714
|
+
return 0
|
|
715
|
+
if args.investigation_action == "approve":
|
|
716
|
+
approve_aria_intervention(
|
|
717
|
+
args.case,
|
|
718
|
+
case_sha256=args.case_sha256,
|
|
719
|
+
variant_sha256=args.variant_sha256,
|
|
720
|
+
max_containers=args.max_containers,
|
|
721
|
+
execution_authorized=args.allow_execution,
|
|
722
|
+
)
|
|
723
|
+
print("Case-specific approval recorded. Execution has not started.")
|
|
724
|
+
return 0
|
|
725
|
+
if args.investigation_action == "run":
|
|
726
|
+
comparison = execute_aria_intervention(
|
|
727
|
+
args.case,
|
|
728
|
+
args.source,
|
|
729
|
+
artifacts=args.artifacts,
|
|
730
|
+
execution_authorized=args.allow_execution,
|
|
731
|
+
)
|
|
732
|
+
print(
|
|
733
|
+
f"Investigation: {_terminal(comparison['status'])}. Historical claims remain untrusted."
|
|
734
|
+
)
|
|
735
|
+
return 1 if comparison["status"] == "stopped" else 0
|
|
736
|
+
output, html = Path(args.output), Path(args.html)
|
|
737
|
+
if output.resolve() == html.resolve():
|
|
738
|
+
raise ValueError("JSON and HTML output paths must be distinct")
|
|
739
|
+
_check_output_path(output)
|
|
740
|
+
_check_output_path(html)
|
|
741
|
+
comparison = compare_aria_intervention(
|
|
742
|
+
args.case, args.result, provenance=args.provenance
|
|
743
|
+
)
|
|
744
|
+
rendered = render_aria_comparison(comparison)
|
|
745
|
+
serialized = (
|
|
746
|
+
json.dumps(comparison, sort_keys=True, indent=2, allow_nan=False) + "\n"
|
|
747
|
+
)
|
|
748
|
+
_write_text(serialized, output)
|
|
749
|
+
_write_text(rendered, html)
|
|
750
|
+
print("Comparison saved. Historical claims remain untrusted.")
|
|
751
|
+
return 0
|
|
752
|
+
|
|
753
|
+
|
|
754
|
+
def _comparison(comparison: dict) -> list[str]:
|
|
755
|
+
lines = [
|
|
756
|
+
f" Matched comparison: {_terminal(comparison['status'])}",
|
|
757
|
+
" Descriptive same-prompt, same-step comparison.",
|
|
758
|
+
]
|
|
759
|
+
if comparison["reason"] is not None:
|
|
760
|
+
lines.append(f" Reason: {_terminal(comparison['reason'])}")
|
|
761
|
+
for field, label in (
|
|
762
|
+
("n_rollouts", "Group completions"),
|
|
763
|
+
("n_present", "Feature-present completions"),
|
|
764
|
+
("n_absent", "Feature-absent completions"),
|
|
765
|
+
("reward_present", "Present-group mean reward"),
|
|
766
|
+
("reward_absent", "Absent-group mean reward"),
|
|
767
|
+
("difference", "Present-minus-absent mean reward"),
|
|
768
|
+
):
|
|
769
|
+
lines.append(f" {label}: {_terminal(comparison[field])}")
|
|
770
|
+
counterpart = comparison["counterpart"]
|
|
771
|
+
if counterpart is None:
|
|
772
|
+
lines.append(" Matched response: Unavailable.")
|
|
773
|
+
else:
|
|
774
|
+
for field, label in (
|
|
775
|
+
("run", "Matched run"),
|
|
776
|
+
("sample_id", "Matched sample ID"),
|
|
777
|
+
("step", "Matched step"),
|
|
778
|
+
("prompt_id", "Matched prompt ID"),
|
|
779
|
+
("reward", "Matched reward"),
|
|
780
|
+
("feature_value", "Matched feature value"),
|
|
781
|
+
):
|
|
782
|
+
lines.append(f" {label}: {_terminal(counterpart[field])}")
|
|
783
|
+
lines.append(" Matched response (terminal controls escaped):")
|
|
784
|
+
lines.extend(
|
|
785
|
+
f" {_terminal(line)}" for line in counterpart["response"].split("\n")
|
|
786
|
+
)
|
|
787
|
+
return lines
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _inspection(event: dict) -> str:
|
|
791
|
+
lines = [
|
|
792
|
+
f"Run: {_terminal(event['run'])}",
|
|
793
|
+
f"Event: {_terminal(event['event_id'])} (rank {event['rank']})",
|
|
794
|
+
f"Representative: {_terminal(event['representative_feature'])}",
|
|
795
|
+
f"Ranking status: {_terminal(event['ranking_status'])}",
|
|
796
|
+
f"Onset: {event['onset_step']} | Alert: {event['alert_step']}",
|
|
797
|
+
]
|
|
798
|
+
for member in event["members"]:
|
|
799
|
+
lines.extend(
|
|
800
|
+
[
|
|
801
|
+
"",
|
|
802
|
+
f"Feature: {_terminal(member['feature'])}",
|
|
803
|
+
f"Membership status: {_terminal(member['status'])}",
|
|
804
|
+
f"Exemplar records: {_terminal(member['exemplar_status'])}",
|
|
805
|
+
]
|
|
806
|
+
)
|
|
807
|
+
if not member["exemplars"]:
|
|
808
|
+
lines.append(" No exemplar records in the report.")
|
|
809
|
+
for example in member["exemplars"]:
|
|
810
|
+
lines.append("")
|
|
811
|
+
for field, label in (
|
|
812
|
+
("status", "Status"),
|
|
813
|
+
("selection_order", "Selection order"),
|
|
814
|
+
("run", "Run"),
|
|
815
|
+
("sample_id", "Sample ID"),
|
|
816
|
+
("step", "Step"),
|
|
817
|
+
("prompt_id", "Prompt ID"),
|
|
818
|
+
("reward", "Reward"),
|
|
819
|
+
("feature_value", "Feature value"),
|
|
820
|
+
("selection_reason", "Selection reason"),
|
|
821
|
+
("reward_deviation", "Reward deviation"),
|
|
822
|
+
("reward_deviation_status", "Reward deviation status"),
|
|
823
|
+
):
|
|
824
|
+
if field == "reward" and "comparison" in example:
|
|
825
|
+
label = "Exemplar reward"
|
|
826
|
+
lines.append(f" {label}: {_terminal(example[field])}")
|
|
827
|
+
if example["response"] is None:
|
|
828
|
+
lines.append(" Response: Unavailable (no selected reference).")
|
|
829
|
+
else:
|
|
830
|
+
lines.append(" Response (terminal controls escaped):")
|
|
831
|
+
# Only literal LF creates lines; other controls remain visible escapes.
|
|
832
|
+
lines.extend(
|
|
833
|
+
f" {_terminal(line)}" for line in example["response"].split("\n")
|
|
834
|
+
)
|
|
835
|
+
if "comparison" in example:
|
|
836
|
+
lines.extend(_comparison(example["comparison"]))
|
|
837
|
+
return "\n".join(lines)
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _summary(report: dict) -> None:
|
|
841
|
+
for run in report["runs"]:
|
|
842
|
+
counts = run["counts"]
|
|
843
|
+
print(
|
|
844
|
+
f"Run {run['run']}: {counts['screened_series']} screened, "
|
|
845
|
+
f"{counts['sequential_alerts']} alerted; "
|
|
846
|
+
f"{counts['behavioral_events']} behavioral and "
|
|
847
|
+
f"{counts['contextual_events']} contextual events."
|
|
848
|
+
)
|
|
849
|
+
leading = [
|
|
850
|
+
event for event in run["events"] if event["ranking_status"] == "behavioral"
|
|
851
|
+
][:3]
|
|
852
|
+
if leading:
|
|
853
|
+
print(
|
|
854
|
+
"Leading representatives: "
|
|
855
|
+
+ ", ".join(
|
|
856
|
+
f"{event['rank']}. {event['representative_feature']}"
|
|
857
|
+
for event in leading
|
|
858
|
+
)
|
|
859
|
+
)
|
|
860
|
+
policy = report["retrospective"]
|
|
861
|
+
if policy["status"] == "disabled":
|
|
862
|
+
print("Retrospective detection: disabled.")
|
|
863
|
+
else:
|
|
864
|
+
print(f"Experimental retrospective detection: {policy['algorithm']}.")
|
|
865
|
+
for run in report["runs"]:
|
|
866
|
+
counts = run["retrospective"]["screening"]
|
|
867
|
+
print(
|
|
868
|
+
f" {_terminal(run['run'])}: {counts['n_changes']} changes, "
|
|
869
|
+
f"{counts['n_screened']} screened, {counts['n_unscorable']} unscorable."
|
|
870
|
+
)
|
|
871
|
+
print(policy["warning"])
|
|
872
|
+
profile = report.get("sensitivity")
|
|
873
|
+
if profile is not None:
|
|
874
|
+
print(f"Experimental sensitivity: {profile['id']}")
|
|
875
|
+
print(
|
|
876
|
+
"Effective settings: "
|
|
877
|
+
+ json.dumps(profile["effective_settings"], sort_keys=True)
|
|
878
|
+
)
|
|
879
|
+
for group, label in (
|
|
880
|
+
("default_full", "Default, full"),
|
|
881
|
+
("default_eligible", "Default, same cohort"),
|
|
882
|
+
("experimental_eligible", "Experimental, same cohort"),
|
|
883
|
+
):
|
|
884
|
+
for row in profile["counts"][group]:
|
|
885
|
+
print(
|
|
886
|
+
f" {_terminal(row['run'])}: {label}: {row['n_alerted']} alerts, {row['n_screened']}/{row['n_series']} screened, {row['n_unscorable']} unscorable."
|
|
887
|
+
)
|
|
888
|
+
for run in report["runs"]:
|
|
889
|
+
print(
|
|
890
|
+
f" {_terminal(run['run'])}: {run['counts']['excluded_series']} excluded binary series."
|
|
891
|
+
)
|
|
892
|
+
print(
|
|
893
|
+
"Offline reconstruction; bundled benchmark rates describe the default profile."
|
|
894
|
+
)
|
|
895
|
+
reference = report.get("benchmark")
|
|
896
|
+
if reference is not None:
|
|
897
|
+
results = reference.get("results", reference)
|
|
898
|
+
status = reference.get("applicability", {}).get("status", "unverified")
|
|
899
|
+
print(f"Historical benchmark reference; configuration check: {status}.")
|
|
900
|
+
for row in results["synthetic_null"]:
|
|
901
|
+
rate = (
|
|
902
|
+
f"{100 * row['n_alerts'] / row['n_scorable']:.4g}%"
|
|
903
|
+
if row["n_scorable"]
|
|
904
|
+
else "unavailable"
|
|
905
|
+
)
|
|
906
|
+
print(
|
|
907
|
+
f" {row['detector']}: {row['n_alerts']}/{row['n_scorable']} scorable null trials ({rate})."
|
|
908
|
+
)
|
|
909
|
+
if reference["schema_version"] == 2:
|
|
910
|
+
ceiling = reference["provenance"]["original_acceptance"][
|
|
911
|
+
"max_null_false_alert_rate"
|
|
912
|
+
]
|
|
913
|
+
for row in results["synthetic_null"]:
|
|
914
|
+
if row["n_scorable"] and row["n_alerts"] / row["n_scorable"] > ceiling:
|
|
915
|
+
print(
|
|
916
|
+
f" {row['detector']} exceeds the original {100 * ceiling:g}% null ceiling."
|
|
917
|
+
)
|
|
918
|
+
print("Historical benchmark rates do not calibrate this run.")
|
|
919
|
+
|
|
920
|
+
|
|
921
|
+
def _source_files(path, format):
|
|
922
|
+
root = Path(path)
|
|
923
|
+
if format == "trl":
|
|
924
|
+
if root.is_dir() and (root / "completions").is_dir():
|
|
925
|
+
root /= "completions"
|
|
926
|
+
files = _files(root, _TRL_FILE, "completions_*.parquet")
|
|
927
|
+
elif format == "verl":
|
|
928
|
+
files = _files(root, _VERL_FILE, "*.jsonl*")
|
|
929
|
+
elif format == "inspect":
|
|
930
|
+
if not root.is_file():
|
|
931
|
+
raise ValueError("Inspect comparison requires one local log file")
|
|
932
|
+
files = [root]
|
|
933
|
+
elif format in {"openhands", "swe-smith"}:
|
|
934
|
+
suffixes = {".json", ".jsonl", ".parquet"}
|
|
935
|
+
files = (
|
|
936
|
+
sorted(
|
|
937
|
+
file
|
|
938
|
+
for file in root.iterdir()
|
|
939
|
+
if file.is_file() and file.suffix in suffixes
|
|
940
|
+
)
|
|
941
|
+
if root.is_dir()
|
|
942
|
+
else [root]
|
|
943
|
+
)
|
|
944
|
+
if not files:
|
|
945
|
+
raise ValueError("no trajectory files found")
|
|
946
|
+
else:
|
|
947
|
+
raise ValueError("unsupported comparison input format")
|
|
948
|
+
records = []
|
|
949
|
+
for file in files:
|
|
950
|
+
with file.open("rb") as stream:
|
|
951
|
+
digest = hashlib.file_digest(stream, "sha256").hexdigest()
|
|
952
|
+
records.append({"sha256": digest, "size_bytes": file.stat().st_size})
|
|
953
|
+
return records
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def _comparison_input(path, format, run, reward_column, model=None):
|
|
957
|
+
if model is not None and format not in {"openhands", "swe-smith"}:
|
|
958
|
+
raise ValueError("--model requires a trajectory input format")
|
|
959
|
+
before = _source_files(path, format)
|
|
960
|
+
if format == "trl":
|
|
961
|
+
rows = load_trl(path, run=run, reward_column=reward_column)
|
|
962
|
+
elif format == "verl":
|
|
963
|
+
rows = load_verl(path, run=run)
|
|
964
|
+
elif format == "inspect":
|
|
965
|
+
from .inspect_logs import load_inspect
|
|
966
|
+
|
|
967
|
+
rows = load_inspect(path, run=run, score=reward_column)
|
|
968
|
+
else:
|
|
969
|
+
from .trajectories import load_trajectories
|
|
970
|
+
|
|
971
|
+
rows = load_trajectories(
|
|
972
|
+
path, run=run, format=format, model=model, reward_column=reward_column
|
|
973
|
+
)
|
|
974
|
+
if before != _source_files(path, format):
|
|
975
|
+
raise ValueError("comparison source changed while loading")
|
|
976
|
+
source = {
|
|
977
|
+
"format": format,
|
|
978
|
+
"files": before,
|
|
979
|
+
"reward_column": (reward_column or "reward")
|
|
980
|
+
if format == "trl"
|
|
981
|
+
else reward_column,
|
|
982
|
+
}
|
|
983
|
+
if model is not None:
|
|
984
|
+
source["model"] = model
|
|
985
|
+
return rows, source
|
|
986
|
+
|
|
987
|
+
|
|
988
|
+
def _saved_report(path):
|
|
989
|
+
value = _json_object(path)
|
|
990
|
+
if value.get("report_type") == "watch":
|
|
991
|
+
return validate_watch_report(value)
|
|
992
|
+
return (
|
|
993
|
+
validate_comparison_report(value)
|
|
994
|
+
if value.get("report_type") == "comparison"
|
|
995
|
+
else validate_report(value)
|
|
996
|
+
)
|
|
997
|
+
|
|
998
|
+
|
|
999
|
+
def _comparison_outputs(outputs):
|
|
1000
|
+
# Prepare complete files before publication; remove only our own links on failure.
|
|
1001
|
+
with ExitStack() as cleanup:
|
|
1002
|
+
staged = []
|
|
1003
|
+
for text, destination in outputs:
|
|
1004
|
+
directory = cleanup.enter_context(
|
|
1005
|
+
TemporaryDirectory(prefix=".rolloutdiff-", dir=destination.parent)
|
|
1006
|
+
)
|
|
1007
|
+
temporary = Path(directory) / "content"
|
|
1008
|
+
_write_text(text, temporary)
|
|
1009
|
+
staged.append((temporary, destination))
|
|
1010
|
+
published = []
|
|
1011
|
+
try:
|
|
1012
|
+
for temporary, destination in staged:
|
|
1013
|
+
published.append((destination, temporary.stat()))
|
|
1014
|
+
os.link(temporary, destination)
|
|
1015
|
+
except BaseException:
|
|
1016
|
+
for destination, owned in reversed(published):
|
|
1017
|
+
try:
|
|
1018
|
+
current = destination.lstat()
|
|
1019
|
+
if (current.st_dev, current.st_ino) == (owned.st_dev, owned.st_ino):
|
|
1020
|
+
destination.unlink()
|
|
1021
|
+
except FileNotFoundError:
|
|
1022
|
+
pass
|
|
1023
|
+
raise
|
|
1024
|
+
|
|
1025
|
+
|
|
1026
|
+
def _diff(args):
|
|
1027
|
+
if args.other is None and any(
|
|
1028
|
+
value is not None
|
|
1029
|
+
for value in (
|
|
1030
|
+
args.other_format,
|
|
1031
|
+
args.other_run,
|
|
1032
|
+
args.other_reward_column,
|
|
1033
|
+
args.other_model,
|
|
1034
|
+
)
|
|
1035
|
+
):
|
|
1036
|
+
raise ValueError("other-side options require a second dump")
|
|
1037
|
+
if args.other is None and (args.a is None or args.b is None):
|
|
1038
|
+
raise ValueError("one-dump comparisons require both --a and --b")
|
|
1039
|
+
other_format = args.other_format or args.format
|
|
1040
|
+
if args.other_reward_column is not None and other_format == "verl":
|
|
1041
|
+
raise ValueError("--other-reward-column requires a TRL second dump")
|
|
1042
|
+
output = Path(args.output)
|
|
1043
|
+
_check_output_path(output)
|
|
1044
|
+
html = Path(args.html) if args.html else None
|
|
1045
|
+
if html is not None:
|
|
1046
|
+
if output.resolve() == html.resolve():
|
|
1047
|
+
raise ValueError("JSON and HTML output paths must differ")
|
|
1048
|
+
_check_output_path(html)
|
|
1049
|
+
a, a_source = _comparison_input(
|
|
1050
|
+
args.input, args.format, args.run, args.reward_column, args.model
|
|
1051
|
+
)
|
|
1052
|
+
b, b_source = None, a_source
|
|
1053
|
+
if args.other is not None:
|
|
1054
|
+
b, b_source = _comparison_input(
|
|
1055
|
+
args.other,
|
|
1056
|
+
other_format,
|
|
1057
|
+
args.other_run or args.run,
|
|
1058
|
+
args.other_reward_column,
|
|
1059
|
+
args.other_model,
|
|
1060
|
+
)
|
|
1061
|
+
options = {"include_tool_calls": True} if args.tool_calls else {}
|
|
1062
|
+
report = compare_rollouts(
|
|
1063
|
+
a, b, a=args.a, b=args.b, keywords=args.keyword, **options
|
|
1064
|
+
)
|
|
1065
|
+
for side, source in zip(report["sides"], (a_source, b_source), strict=True):
|
|
1066
|
+
side["source"].update(source)
|
|
1067
|
+
encoded = comparison_json(report)
|
|
1068
|
+
rendered = render_comparison_report(report) if html else None
|
|
1069
|
+
outputs = [(encoded, output)]
|
|
1070
|
+
if html is not None:
|
|
1071
|
+
outputs.append((rendered, html))
|
|
1072
|
+
_comparison_outputs(outputs)
|
|
1073
|
+
print(
|
|
1074
|
+
f"Compared {len(report['features'])} features; "
|
|
1075
|
+
f"{report['prompt_mix']['shared_prompts']} shared exact prompts."
|
|
1076
|
+
)
|
|
1077
|
+
leading = [f for f in report["features"] if f["kind"] == "binary"][:3]
|
|
1078
|
+
if leading:
|
|
1079
|
+
print(
|
|
1080
|
+
"Largest prevalence differences (B - A, proportions): "
|
|
1081
|
+
+ ", ".join(
|
|
1082
|
+
f"{_terminal(f['feature'])} ({_terminal(f['difference']['estimate'])})"
|
|
1083
|
+
for f in leading
|
|
1084
|
+
)
|
|
1085
|
+
)
|
|
1086
|
+
return 0
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def _inspect_comparison(args, report):
|
|
1090
|
+
if args.side is None or args.feature is None:
|
|
1091
|
+
raise ValueError("comparison inspection requires --side and --feature")
|
|
1092
|
+
if args.event_rank is not None or args.with_absent:
|
|
1093
|
+
raise ValueError("--event-rank and --with-absent require an analysis report")
|
|
1094
|
+
rows, source = _comparison_input(
|
|
1095
|
+
args.input, args.format, args.run, args.reward_column, args.model
|
|
1096
|
+
)
|
|
1097
|
+
expected = next(item for item in report["sides"] if item["side"] == args.side)
|
|
1098
|
+
if expected["source"]["files"] and any(
|
|
1099
|
+
source.get(key) != expected["source"].get(key)
|
|
1100
|
+
for key in ("format", "files", "reward_column", "model")
|
|
1101
|
+
):
|
|
1102
|
+
raise ValueError("comparison source bytes or loader settings do not match")
|
|
1103
|
+
resolved = resolve_comparison_citations(
|
|
1104
|
+
report, rows, side=args.side, feature=args.feature
|
|
1105
|
+
)
|
|
1106
|
+
lines = [
|
|
1107
|
+
f"Side: {args.side}",
|
|
1108
|
+
f"Feature: {_terminal(args.feature)}",
|
|
1109
|
+
f"Citations: {_terminal(resolved['status'])}",
|
|
1110
|
+
]
|
|
1111
|
+
for citation in resolved["items"]:
|
|
1112
|
+
lines.append("")
|
|
1113
|
+
for key in (
|
|
1114
|
+
"run",
|
|
1115
|
+
"sample_id",
|
|
1116
|
+
"step",
|
|
1117
|
+
"prompt_id",
|
|
1118
|
+
"feature_value",
|
|
1119
|
+
"reward",
|
|
1120
|
+
"selection_reason",
|
|
1121
|
+
):
|
|
1122
|
+
lines.append(f"{key}: {_terminal(citation[key])}")
|
|
1123
|
+
lines.append("Response (terminal controls escaped):")
|
|
1124
|
+
lines.extend(
|
|
1125
|
+
f" {_terminal(line)}" for line in citation["response"].split("\n")
|
|
1126
|
+
)
|
|
1127
|
+
if citation.get("tool_evidence"):
|
|
1128
|
+
lines.append("Recorded tool calls (terminal controls escaped):")
|
|
1129
|
+
lines.extend(
|
|
1130
|
+
" " + _terminal(json.dumps(item, ensure_ascii=False, sort_keys=True))
|
|
1131
|
+
for item in citation["tool_evidence"]
|
|
1132
|
+
)
|
|
1133
|
+
print("\n".join(lines))
|
|
1134
|
+
return 0
|
|
1135
|
+
|
|
1136
|
+
|
|
1137
|
+
def main(argv: list[str] | None = None) -> int:
|
|
1138
|
+
parser = _parser()
|
|
1139
|
+
args = parser.parse_args(argv)
|
|
1140
|
+
if (
|
|
1141
|
+
args.command in {"analyze", "inspect", "diff"}
|
|
1142
|
+
and args.reward_column is not None
|
|
1143
|
+
and (
|
|
1144
|
+
args.format == "verl"
|
|
1145
|
+
or (args.command == "analyze" and args.format != "trl")
|
|
1146
|
+
)
|
|
1147
|
+
):
|
|
1148
|
+
parser.error("--reward-column is supported only with --format trl")
|
|
1149
|
+
try:
|
|
1150
|
+
if args.command == "watch":
|
|
1151
|
+
return _watch(args)
|
|
1152
|
+
if args.command == "investigate":
|
|
1153
|
+
return _investigate(args)
|
|
1154
|
+
if args.command == "diff":
|
|
1155
|
+
return _diff(args)
|
|
1156
|
+
if args.command == "investigate-aria":
|
|
1157
|
+
return _investigation(args)
|
|
1158
|
+
if args.command == "replay-aria":
|
|
1159
|
+
output = Path(args.output)
|
|
1160
|
+
_check_output_path(output)
|
|
1161
|
+
options = {"docker_host": args.docker_host}
|
|
1162
|
+
if args.image is not None:
|
|
1163
|
+
options["image"] = args.image
|
|
1164
|
+
record = replay_aria(
|
|
1165
|
+
args.input,
|
|
1166
|
+
run=args.run,
|
|
1167
|
+
sample_id=args.sample_id,
|
|
1168
|
+
response_sha256=args.response_sha256,
|
|
1169
|
+
artifacts=args.artifacts,
|
|
1170
|
+
config=SandboxConfig(**options),
|
|
1171
|
+
execution_authorized=args.allow_execution,
|
|
1172
|
+
)
|
|
1173
|
+
_write_text(serialize_replay(record), output)
|
|
1174
|
+
print(
|
|
1175
|
+
f"Replay: historical={record['historical']['status']}; task={record['task']['status']}; mechanism={record['mechanism_status']}."
|
|
1176
|
+
)
|
|
1177
|
+
return (
|
|
1178
|
+
0
|
|
1179
|
+
if record["historical"]["status"] == "reported"
|
|
1180
|
+
and record["task"]["determinate"]
|
|
1181
|
+
else 1
|
|
1182
|
+
)
|
|
1183
|
+
if args.command == "inspect":
|
|
1184
|
+
report = _saved_report(args.report)
|
|
1185
|
+
if report.get("report_type") == "comparison":
|
|
1186
|
+
return _inspect_comparison(args, report)
|
|
1187
|
+
if args.format not in {"trl", "verl"} or args.model is not None:
|
|
1188
|
+
raise ValueError(
|
|
1189
|
+
"trajectory and Inspect inputs require a comparison report"
|
|
1190
|
+
)
|
|
1191
|
+
if report.get("report_type") == "watch":
|
|
1192
|
+
if (
|
|
1193
|
+
args.event_rank is None
|
|
1194
|
+
or args.side is not None
|
|
1195
|
+
or args.feature is not None
|
|
1196
|
+
):
|
|
1197
|
+
raise ValueError(
|
|
1198
|
+
"watch inspection requires --event-rank and no comparison side/feature"
|
|
1199
|
+
)
|
|
1200
|
+
rows = load_watch_source(
|
|
1201
|
+
report,
|
|
1202
|
+
args.input,
|
|
1203
|
+
format=args.format,
|
|
1204
|
+
run=args.run,
|
|
1205
|
+
reward_column=args.reward_column,
|
|
1206
|
+
)
|
|
1207
|
+
event = resolve_watch_exemplars(
|
|
1208
|
+
report,
|
|
1209
|
+
rows,
|
|
1210
|
+
event_rank=args.event_rank,
|
|
1211
|
+
with_absent=args.with_absent,
|
|
1212
|
+
)
|
|
1213
|
+
print(
|
|
1214
|
+
f"Watch snapshot: step {report['as_of_step']}, revision {report['sequence']}."
|
|
1215
|
+
)
|
|
1216
|
+
print(_inspection(event))
|
|
1217
|
+
return 0
|
|
1218
|
+
if args.event_rank is None:
|
|
1219
|
+
parser.error("analysis inspection requires --event-rank")
|
|
1220
|
+
if args.side is not None or args.feature is not None:
|
|
1221
|
+
raise ValueError("--side and --feature require a comparison report")
|
|
1222
|
+
rollouts = (
|
|
1223
|
+
load_trl(args.input, run=args.run, reward_column=args.reward_column)
|
|
1224
|
+
if args.format == "trl"
|
|
1225
|
+
else load_verl(args.input, run=args.run)
|
|
1226
|
+
)
|
|
1227
|
+
event = resolve_exemplars(
|
|
1228
|
+
report,
|
|
1229
|
+
rollouts,
|
|
1230
|
+
run=args.run,
|
|
1231
|
+
event_rank=args.event_rank,
|
|
1232
|
+
with_absent=args.with_absent,
|
|
1233
|
+
)
|
|
1234
|
+
print(_inspection(event))
|
|
1235
|
+
return 0
|
|
1236
|
+
output = Path(args.output)
|
|
1237
|
+
_check_output_path(output)
|
|
1238
|
+
if args.command == "render":
|
|
1239
|
+
report = _saved_report(args.input)
|
|
1240
|
+
if report.get("report_type") == "comparison":
|
|
1241
|
+
_write_text(render_comparison_report(report), output)
|
|
1242
|
+
print(f"Rendered {len(report['features'])} feature comparisons.")
|
|
1243
|
+
return 0
|
|
1244
|
+
if report.get("report_type") == "watch":
|
|
1245
|
+
_write_text(render_watch_report(report), output)
|
|
1246
|
+
print(f"Rendered watch snapshot through step {report['as_of_step']}.")
|
|
1247
|
+
return 0
|
|
1248
|
+
_write_text(render_report(report), output)
|
|
1249
|
+
else:
|
|
1250
|
+
benchmark = (
|
|
1251
|
+
load_benchmark_reference(args.benchmark_results)
|
|
1252
|
+
if args.benchmark_results is not None
|
|
1253
|
+
else None
|
|
1254
|
+
)
|
|
1255
|
+
rollouts = (
|
|
1256
|
+
load_trl(args.input, run=args.run, reward_column=args.reward_column)
|
|
1257
|
+
if args.format == "trl"
|
|
1258
|
+
else load_verl(args.input, run=args.run)
|
|
1259
|
+
)
|
|
1260
|
+
analysis = analyze_rollouts(
|
|
1261
|
+
rollouts,
|
|
1262
|
+
sensitivity=args.sensitivity or "default",
|
|
1263
|
+
retrospective=args.retrospective,
|
|
1264
|
+
)
|
|
1265
|
+
if args.benchmark_results is not None or args.no_benchmark_reference:
|
|
1266
|
+
report = build_report(analysis, run=args.run, benchmark=benchmark)
|
|
1267
|
+
else:
|
|
1268
|
+
report = build_report(analysis, run=args.run)
|
|
1269
|
+
write_report(report, output)
|
|
1270
|
+
except (OSError, ValueError) as exc:
|
|
1271
|
+
message = (
|
|
1272
|
+
_terminal(exc)
|
|
1273
|
+
if args.command
|
|
1274
|
+
in {"inspect", "diff", "watch", "replay-aria", "investigate-aria"}
|
|
1275
|
+
else str(exc)
|
|
1276
|
+
)
|
|
1277
|
+
print(f"rollout: error: {message}", file=sys.stderr)
|
|
1278
|
+
return 1
|
|
1279
|
+
if args.command == "analyze":
|
|
1280
|
+
_summary(report)
|
|
1281
|
+
else:
|
|
1282
|
+
print(
|
|
1283
|
+
f"Rendered {sum(len(run['events']) for run in report['runs'])} events "
|
|
1284
|
+
f"across {len(report['runs'])} runs."
|
|
1285
|
+
)
|
|
1286
|
+
for run in report["runs"]:
|
|
1287
|
+
print(f"Run {run['run']!r}: {len(run['events'])} events.")
|
|
1288
|
+
return 0
|
|
1289
|
+
|
|
1290
|
+
|
|
1291
|
+
if __name__ == "__main__":
|
|
1292
|
+
raise SystemExit(main())
|