reeval 0.10.0 → 0.12.0

package/dist/{cli-D303Vj5N.js → cli-B5zQCR2r.js}

@@ -1,30 +1,30 @@
 import { n as isOneOf } from "./object-CLf2-tNq.js";
 import { n as UsageError } from "./errors-B6BSMs03.js";
-import { t as parseDuration } from "./duration-CEOQww97.js";
-import { f as sandboxId, s as harnessId } from "./options-Bh2W63dK.js";
-import { A as workspaceLabel, p as environmentSandbox } from "./environment-DNjRk0nU.js";
-import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
-import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-DEgnUiU-.js";
+import { p as parseDuration } from "./shared-CA61BuPJ.js";
+import { f as sandboxId, s as harnessId } from "./options-CPUW97_7.js";
+import { A as workspaceLabel, p as environmentSandbox } from "./environment-BBz1Yes9.js";
+import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
+import { _ as selectProfile, a as configLabel, d as hostCapacity, f as loadConfig, g as resolveConcurrency, h as requireConfig, l as findConfig, m as missingCredentials, o as configPathFor, p as malformedCredentials, u as hasAnalysisModel } from "./config-Dv5MG7Rp.js";
 import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
-import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
+import { U as LABEL, it as imageInfo } from "./src-LekQrr_S.js";
 import { o as PACKAGE_VERSION } from "./root-DICDHCtp.js";
-import { I as LABEL, X as imageInfo } from "./src-B7treWa8.js";
+import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
+import { C as pluginBuildsLabel } from "./plugins-CUWMnSNb.js";
 import { s as debugTime } from "./debug-D24NOQH1.js";
-import { i as sandboxWarningLine, n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
+import { i as formatDuration, n as clip, o as sandboxWarningLine } from "./text-CrVy5ioL.js";
 import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { B as judgeRatesLabel, H as summaryJudgeRates, I as collapsedJudgeLabel, P as cellUsage, T as runOutcome, Y as cellContrast, g as formatUsage, l as evalMarkOutcome, tt as effortLabel, v as hasUngatedCells, x as isUngatedMiss } from "./model-oqcDplJG.js";
-import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "./family-D2b7WD10.js";
-import { n as headToHeadLines, s as sampleReasons } from "./pairs-CnInRu-G.js";
-import { a as contrastLines, f as undiscriminating, i as checksLabel, n as answersLabel, o as gateLabel, p as undiscriminatingLabel, s as liftLabel } from "./summary-BLAVLySU.js";
-import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-BRfa8E_s.js";
-import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BPZbQG2p.js";
-import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-DmE9uPVt.js";
-import { t as budgetMarker } from "./budget-DBUrH4jX.js";
-import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-D7iFPKeK.js";
-import { n as imageHarness } from "./prepare-doGxaAGR.js";
-import { r as createProviders, t as attemptImage } from "./providers-tAOD1NQN.js";
-import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DbGfBAA_.js";
+import { T as runOutcome, g as formatUsage } from "./model-CWlAqmbU.js";
+import { l as suiteResult } from "./suite-C8sw2pox.js";
+import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BXIBUlaK.js";
+import { p as render } from "./summary-0oQNGskv.js";
 import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
+import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-B3Y17oCC.js";
+import { t as budgetMarker } from "./budget-DBUrH4jX.js";
+import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-BEFazUfy.js";
+import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-Bjm7_ylE.js";
+import { n as imageHarness } from "./prepare-CQmK3SBn.js";
+import { i as createProviders, n as attemptImage } from "./providers-eoniqK6w.js";
+import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CQTbP092.js";
 import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { inspect } from "node:util";
@@ -58,6 +58,7 @@ function parseCli(argv) {
 	});
 	for (const [name, spec] of Object.entries(COMMANDS)) {
 		const command = configure(program.command(name), `reeval ${name}`, spec.flags);
+		if (name === "query") takeStatements(command);
 		command.argument("[args...]").action(() => {
 			const max = spec.sub && !spec.help ? Number.POSITIVE_INFINITY : spec.positionals;
 			parsed = done(program, command, [name], spec.flags, max);
@@ -86,6 +87,28 @@ function configure(command, name, flags) {
 	for (const flag of flags) command.addOption(OPTIONS[flag]());
 	return command;
 }
+/**
+* Makes an argument that starts with `-` but has whitespace before any `=` (SQL opening with a
+* `-- comment`) a positional: no flag looks like that, and commander would refuse it as one.
+*/
+function takeStatements(command) {
+	const parseOptions = command.parseOptions.bind(command);
+	command.parseOptions = (args) => {
+		const held = /* @__PURE__ */ new Map();
+		const masked = args.map((arg, i) => {
+			if (!arg.startsWith("-") || !/\s/.test(arg.split("=")[0] ?? "")) return arg;
+			const token = `\0statement-${i}`;
+			held.set(token, arg);
+			return token;
+		});
+		const restore = (list) => list.map((arg) => held.get(arg) ?? arg);
+		const { operands, unknown } = parseOptions(masked);
+		return {
+			operands: restore(operands),
+			unknown: restore(unknown)
+		};
+	};
+}
 function done(program, command, path, flags, max) {
 	const values = {
 		...valuesOf(program, GLOBAL),
@@ -119,16 +142,17 @@ function valuesOf(command, flags) {
 }
 /** Commander's error, in reeval's words, naming the command whose flag it is. */
 function usageError(error, command) {
-	const [first = "", ...rest] = error.message.replace(/^error: /, "").split("\n");
+	const text = error.message.replace(/^error: /, "");
+	const [first = ""] = text.split("\n");
 	if (error.code === "commander.unknownOption") {
-		const arg = /'([^']+)'/.exec(first)?.[1] ?? "";
+		const [, arg = "", suggestion] = /^unknown option '([\s\S]*)'(?:\n\((.*)\))?$/.exec(text) ?? [];
 		const [flag = "", value] = arg.replace(/^--?/, "").split("=");
 		const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
 		const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
 		if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
 		if (known) return new UsageError(`--${long} isn't an option of ${command}. Run reeval --help.`);
-		const hint = rest.join(" ").replace(/^\(|\)$/g, "");
-		return new UsageError(`Unknown option '${arg}'. ${hint ? `${hint} ` : ""}Run reeval --help.`);
+		const shown = arg.replace(/\r/g, "\\r").replace(/\n/g, "\\n");
+		return new UsageError(`Unknown option '${shown}'. ${suggestion ? `${suggestion} ` : ""}Run reeval --help.`);
 	}
 	if (error.code === "commander.optionMissingArgument") {
 		const usage = /'([^']+)'/.exec(first)?.[1] ?? "";
@@ -139,12 +163,15 @@ function usageError(error, command) {
 }
 //#endregion
 //#region src/preflight.ts
-/** How the terminal names a sandbox to prepare: its cell id, `+ install`, and a harness other than Cursor. */
-function prepareLabel({ sandbox, harness }) {
+/**
+* How the terminal names a sandbox to prepare: its cell id, `+ install`, `+ plugin builds`, and a
+* harness other than Cursor.
+*/
+function prepareLabel({ sandbox, harness, plugins }) {
 	if (sandbox.kind !== "docker") return sandboxId(sandbox);
 	const baked = imageHarness(harness);
 	const install = sandbox.install?.length ? " + install" : "";
-	return `${sandboxId(sandbox)}${install}${baked === "cursor" ? "" : ` · ${baked}`}`;
+	return `${sandboxId(sandbox)}${install}${pluginBuildsLabel(plugins)}${baked === "cursor" ? "" : ` · ${baked}`}`;
 }
 /** A run that stopped in preflight on an infra failure: exit 3. */
 var PreflightInfraError = class extends Error {
@@ -221,24 +248,10 @@ function outcomeOf(e) {
 	if (e.meta?.errorKind === "aborted") return "cancelled";
 	return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
 }
-function attemptOutcome(a) {
-	return a.error?.kind === "aborted" ? "cancelled" : a.outcome;
-}
 /** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
 function attemptAxes(a, fallback) {
 	return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
 }
-function mark(c, outcome) {
-	return {
-		pass: c.green("✓"),
-		fail: c.red("✗"),
-		infra_error: c.yellow("!"),
-		skipped: c.dim("↓"),
-		cancelled: c.yellow("✗")
-	}[outcome];
-}
-/** An expected miss's mark: neutral, since it doesn't fail the run. */
-const MISS_MARK = (c) => c.dim("○");
 const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
 /** A command on one line: the first line, cut to 80 characters at a word boundary. */
 function oneLine(text, max = 80) {
@@ -363,330 +376,25 @@ function prepareLines(c, e, max) {
 	return [`${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `  ${c.yellow(l)}`)];
 }
 //#endregion
-//#region src/ui/summary.ts
-const FAILURE_LINES = 30;
-const KINDS = [
-	{
-		title: "Judge failures",
-		test: (a) => a.error?.kind === "judge"
-	},
-	{
-		title: "Assertion failures",
-		test: (a) => a.error?.kind === "assertion"
-	},
-	{
-		title: "Timeouts",
-		test: (a) => a.error?.kind === "timeout"
-	},
-	{
-		title: "Infra errors",
-		test: (_, o) => o === "infra_error"
-	},
-	{
-		title: "Cancelled",
-		test: (_, o) => o === "cancelled"
-	}
-];
-/** The engine's result for an attempt, or one rebuilt from the record (e.g. cancelled ones). */
-function resultFor(entry, attempt, results) {
-	const key = attemptKey(entry.id, attempt.cell, attempt.n);
-	const found = results.find((r) => r.key === key);
-	const errorKind = attempt.error?.kind ?? null;
-	const meta = {
-		outcome: attempt.outcome === "skipped" ? "pass" : attempt.outcome,
-		errorKind,
-		judgements: attempt.judgements.map((j) => ({ ...j })),
-		turnErrors: attempt.turns.flatMap((t, i) => t.error ? [`turn ${i + 1} error: ${t.error}`] : []),
-		recordPath: ""
-	};
-	if (found) return {
-		...found,
-		meta: found.meta ? {
-			...found.meta,
-			errorKind
-		} : meta
-	};
-	return {
-		id: entry.id,
-		key,
-		file: entry.file,
-		group: entry.group,
-		name: entry.name,
-		attempt: void 0,
-		state: attempt.outcome === "pass" ? "passed" : attempt.outcome === "skipped" ? "skipped" : "failed",
-		durationMs: attempt.durationMs,
-		errors: attempt.error ? [attempt.error.message] : [],
-		formattedErrors: [],
-		meta
-	};
-}
-function entryOutcome(e) {
-	const outcomes = e.attempts.map(attemptOutcome);
-	if (outcomes.length && outcomes.every((o) => o === "cancelled")) return "cancelled";
-	return evalMarkOutcome(e);
-}
-/** Every cell is `gate: 'none'`: the eval reports its attempts but nothing gates it. */
-const allUngated = (e) => e.cells.length > 0 && e.cells.every((c) => c.gate === "none");
-const pct = (rate) => rate === null ? "  – " : `${Math.round(rate * 100)}%`.padStart(4);
-function evalColumns(c, e) {
-	const [attempt, ...more] = e.attempts;
-	if (e.todo) return c.dim("todo");
-	if (!attempt || e.outcome === "skipped") return c.dim("skipped");
-	if (more.length) {
-		const pass = e.attempts.filter((a) => a.outcome === "pass").length;
-		const counted = e.attempts.filter((a) => a.outcome === "pass" || a.outcome === "fail").length;
-		const rate = allUngated(e) ? " · not gated" : hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : `  pass@1 ${pct(e.passRate)}`;
-		return c.dim(`${`${pass}/${counted}`.padStart(7)}${rate}  ${e.attempts.length} attempts`);
-	}
-	if (attemptOutcome(attempt) === "cancelled") return c.yellow("cancelled");
-	const cols = [formatDuration(attempt.durationMs).padStart(7)];
-	const judges = collapsedJudgeLabel(attempt.judgements);
-	if (judges) cols.push(judges);
-	const turns = attempt.turns.length;
-	cols.push(`${turns} turn${turns === 1 ? "" : "s"}`);
-	const tokens = formatUsage(attempt.usage);
-	if (tokens) cols.push(tokens);
-	return c.dim(cols.join("  "));
-}
-/** pass@1 per cell for evals with several attempts; with compare, each cell's role and lift. */
-function cellRows(c, e, indent) {
-	if (e.attempts.length < 2) return [];
-	const compare = e.cells.some((cell) => cell.role === "variant");
-	const width = Math.max(...e.cells.map((cell) => cell.key.length));
-	return e.cells.flatMap((cell) => {
-		const role = compare ? `  ${cell.role.padEnd(7)}` : "";
-		const counts = `${cell.pass}/${cell.pass + cell.fail}`.padStart(5);
-		const infra = cell.infraError ? c.yellow(`   ${cell.infraError} infra`) : "";
-		let note = "";
-		if (cell.lift) {
-			const { minLift } = cell.lift;
-			const below = minLift !== void 0 && cell.lift.interval.lo < minLift;
-			const text = `${liftLabel(cell.lift)}${minLift !== void 0 ? ` (minLift ${Math.round(minLift * 100)})` : ""}`;
-			note = `   ${below ? c.red(`${text} ✗`) : text}`;
-		} else if (compare && cell.gate === "none" && cell.fail > 0) note = `   ${c.dim("not gated")}`;
-		if (cell.gate !== "all" && cell.gate !== "none") note += `   ${c.dim(`gate ${gateLabel(cell.gate)}`)}`;
-		const tokens = formatUsage(cellUsage(e, cell.key));
-		const usage = tokens ? `   ${c.dim(tokens)}` : "";
-		const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role}  ${counts}  pass@1 ${pct(cell.passRate)}${note}${infra}${usage}`];
-		const rates = judgeRatesLabel(summaryJudgeRates(e, cell));
-		if (rates) rows.push(`${indent}  ${c.dim(rates)}`);
-		const effort = cellContrast(e, cell.key);
-		if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effortLabel(effort)}`)}`);
-		const checks = checksLabel(cell);
-		if (checks) rows.push(`${indent}  ${c.dim(`checks: ${checks}`)}`);
-		if (cell.overrides.length) rows.push(`${indent}  ${c.dim(`overrides: ${cell.overrides.join(", ")}`)}`);
-		return rows;
-	});
-}
-/** With two variants or more, each pair of arms on one harness: lift, checks and effort. */
-function contrastRows(c, e, indent) {
-	const lines = contrastLines(e);
-	if (!lines.length) return [];
-	return [`${indent}${c.dim("contrasts (each pair of arms, same harness)")}`, ...lines.map((line) => `${indent}  ${c.dim(line)}`)];
-}
-/** One row per compare question: its win rates, or why it has none. They never gate. */
-function questionRows(c, e, indent) {
-	return (e.answers ?? []).flatMap((a) => {
-		const label = answersLabel(a);
-		return [
-			`${indent}${c.dim("question")}  ${a.error !== void 0 ? c.yellow(label) : label}`,
-			...headToHeadLines(a.votes).map((line) => `${indent}  ${line}`),
-			...sampleReasons(a.votes).map((reason) => `${indent}  ${c.dim(reason)}`)
-		];
-	});
-}
-/** A family's mark: its pooled gate's and its cases' worst. */
-function familyOutcome(family, cases) {
-	const outcomes = [family.outcome, ...cases.map(entryOutcome)];
-	for (const o of [
-		"fail",
-		"infra_error",
-		"cancelled"
-	]) if (outcomes.includes(o)) return o;
-	return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
-}
-/** The width case marks wrap at. */
-const CASE_WIDTH = 96;
-/** A pooled family's rows under its header: its cases' marks, then each pooled rate or lift. */
-function familyRows(c, family, cases, indent) {
-	const marks = cases.map((e) => `${mark(c, entryOutcome(e))} ${caseLabel(family.name, e.name)}`);
-	const plain = cases.map((e) => `x ${caseLabel(family.name, e.name)}`);
-	const rows = [];
-	let line = [];
-	let width = 0;
-	marks.forEach((m, i) => {
-		const w = plain[i].length + 2;
-		if (line.length && width + w > CASE_WIDTH) {
-			rows.push(`${indent}${line.join("  ")}`);
-			line = [];
-			width = 0;
-		}
-		line.push(m);
-		width += w;
-	});
-	if (line.length) rows.push(`${indent}${line.join("  ")}`);
-	if (family.lifts.length) {
-		const keyWidth = Math.max(...family.lifts.map((l) => l.variant.length));
-		for (const lift of family.lifts) {
-			const below = lift.outcome === "fail";
-			const gate = lift.minLift !== void 0 ? ` (minLift ${Math.round(lift.minLift * 100)}${below ? " ✗" : ""})` : "";
-			const text = `${familyLiftLabel(lift)}${gate}`;
-			rows.push(`${indent}${c.dim(lift.variant.padEnd(keyWidth))}  ${below ? c.red(text) : text}`);
-			const effort = familyEffortLabel(lift);
-			if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effort}`)}`);
-		}
-	} else if (family.rates.length === 1) rows.push(`${indent}${familyRateLabel(family.rates[0])}`);
-	else {
-		const keyWidth = Math.max(...family.rates.map((r) => r.cell.length));
-		for (const rate of family.rates) rows.push(`${indent}${c.dim(rate.cell.padEnd(keyWidth))}  ${familyRateLabel(rate)}`);
-	}
-	return rows;
-}
-/**
-* The run's evals as a file › group tree, one row per eval and per cell of a matrix. A pooled
-* family is one row, its template, with its cases' marks and the pooled results under it.
-*/
-function tree(c, evals, families = []) {
-	const rows = [];
-	const shown = /* @__PURE__ */ new Set();
-	let path = [];
-	for (const e of evals) {
-		const family = e.family && families.find((f) => f.id === e.family?.id);
-		if (family && shown.has(family.id)) continue;
-		const next = [e.file, ...e.group];
-		let same = 0;
-		while (same < path.length && same < next.length && path[same] === next[same]) same++;
-		for (let i = same; i < next.length; i++) rows.push({
-			indent: i,
-			text: i === 0 ? c.bold(next[i] ?? "") : next[i] ?? ""
-		});
-		path = next;
-		if (family) {
-			shown.add(family.id);
-			rows.push({
-				indent: next.length,
-				text: family.name,
-				family
-			});
-		} else rows.push({
-			indent: next.length,
-			text: e.name,
-			entry: e
-		});
-	}
-	const width = Math.max(...rows.filter((r) => r.entry || r.family).map((r) => r.indent * 2 + 2 + r.text.length));
-	return rows.flatMap(({ indent, text, entry, family }) => {
-		const pad = "  ".repeat(indent);
-		const fill = " ".repeat(Math.max(0, width - indent * 2 - 2 - text.length));
-		if (family) {
-			const cases = familyCases(family, evals);
-			return [` ${pad}${mark(c, familyOutcome(family, cases))} ${text}${fill}  ${c.dim(familyHeadLabel(family))}`, ...familyRows(c, family, cases, ` ${pad}    `)];
-		}
-		if (!entry) return [` ${pad}${text}`];
-		return [
-			` ${pad}${`${entryOutcome(entry) === "pass" && allUngated(entry) && entry.attempts.some((a) => isUngatedMiss(entry, a)) ? MISS_MARK(c) : mark(c, entryOutcome(entry))} ${text}`}${fill}  ${evalColumns(c, entry)}`,
-			...cellRows(c, entry, ` ${pad}    `),
-			...contrastRows(c, entry, ` ${pad}    `),
-			...questionRows(c, entry, ` ${pad}    `)
-		];
-	});
-}
-function totals(c, s, cancelled = 0) {
-	const fail = s.fail - s.notGated - cancelled;
-	const parts = [
-		s.pass && c.green(`${s.pass} passed`),
-		fail > 0 && c.red(`${fail} failed`),
-		s.notGated > 0 && c.dim(`${s.notGated} missed, not gated`),
-		s.infraError && c.yellow(`${s.infraError} infra error`),
-		cancelled && c.yellow(`${cancelled} cancelled`),
-		s.skipped && c.dim(`${s.skipped} skipped`)
-	].filter(Boolean);
-	const scope = s.attempts === s.evals ? `${s.evals}` : `${s.attempts} attempts, ${s.evals} evals`;
-	return parts.length ? `${parts.join(c.dim(" · "))} ${c.dim(`(${scope})`)}` : "no evals ran";
-}
-/** `1 passed · 1 failed (2)`: the families' pooled gates. */
-function familyTotals(c, families) {
-	const fail = families.filter((f) => f.outcome === "fail").length;
-	const pass = families.length - fail;
-	return `${[pass && c.green(`${pass} passed`), fail && c.red(`${fail} failed`)].filter(Boolean).join(c.dim(" · "))} ${c.dim(`(${families.length})`)}`;
-}
-/** Failing attempts, in run order, including expected misses. */
-function failingAttempts(evals) {
-	return evals.flatMap((entry) => entry.attempts.map((attempt) => ({
-		entry,
-		attempt,
-		outcome: attemptOutcome(attempt)
-	})).filter(({ outcome }) => outcome !== "pass" && outcome !== "skipped"));
-}
-function renderSummary(c, e, attemptDir) {
+//#region src/ui/result.ts
+function resultLines(c, e, verbose) {
 	const { record } = e;
-	const lines = [];
-	if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals, record.families));
-	const all = failingAttempts(record.evals);
-	const missed = all.filter(({ entry, attempt }) => isUngatedMiss(entry, attempt));
-	const failing = all.filter((f) => !missed.includes(f));
-	if (failing.length) {
-		lines.push("", c.bold(c.red(`Failures (${failing.length})`)));
-		for (const kind of KINDS) {
-			const members = failing.filter(({ attempt, outcome }) => kind.test(attempt, outcome));
-			if (!members.length) continue;
-			lines.push("", c.bold(`${kind.title} (${members.length})`));
-			for (const { entry, attempt, outcome } of members) {
-				const result = resultFor(entry, attempt, e.results);
-				const which = entry.attempts.length > 1 ? `  ${c.dim(attempt.id)}` : "";
-				const gated = entry.cells.find((x) => x.key === attempt.cell)?.gate !== "none";
-				lines.push(` ${mark(c, outcome)} ${entry.id}${which}${gated ? "" : `  ${c.dim("(not gated)")}`}`);
-				if (outcome !== "cancelled") for (const line of failureLines(c, result, { max: FAILURE_LINES })) lines.push(`   ${line}`);
-				if (!attempt.error?.prepare) lines.push(`   ${c.dim("record")}  ${attemptDir(entry.id, attempt.cell, attempt.n)}`);
-			}
-		}
-	}
-	if (missed.length) {
-		lines.push("", c.bold(`Expected misses, not gated (${missed.length})`));
-		const pooled = /* @__PURE__ */ new Map();
-		for (const { entry, attempt } of missed) {
-			if (!entry.family) continue;
-			const key = `${entry.family.id}\0${attempt.cell}`;
-			pooled.set(key, (pooled.get(key) ?? 0) + 1);
-		}
-		for (const [key, n] of pooled) {
-			const [id, cell] = key.split("\0");
-			lines.push(` ${MISS_MARK(c)} ${id}  ${c.dim(`${n} ${n === 1 ? "attempt" : "attempts"} on ${cell}`)}`);
-		}
-		for (const { entry, attempt } of missed) {
-			if (entry.family) continue;
-			const which = entry.attempts.length > 1 ? `  ${c.dim(attempt.id)}` : "";
-			const [reason] = failureLines(colors(false), resultFor(entry, attempt, e.results), { max: 1 }).filter((l) => l.trim());
-			lines.push(` ${MISS_MARK(c)} ${entry.id}${which}`);
-			if (reason) lines.push(`   ${c.dim(reason.trim())}`);
-			lines.push(`   ${c.dim("record")}  ${attemptDir(entry.id, attempt.cell, attempt.n)}`);
-		}
-	}
-	for (const error of e.unhandledErrors) lines.push("", c.red(`error: ${error}`));
-	const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
-	const label = record.summary.attempts === record.summary.evals ? "   Evals" : "Attempts";
-	const lifted = record.evals.flatMap((entry) => entry.cells.flatMap((cell) => cell.lift?.minLift !== void 0 && cell.lift.interval.lo < cell.lift.minLift ? [`${entry.id}  ${cell.key} ${liftLabel(cell.lift)}: its lower bound is below minLift ${Math.round(cell.lift.minLift * 100)}`] : []));
-	for (const family of record.families ?? []) for (const lift of family.lifts) {
-		if (lift.outcome !== "fail" || lift.minLift === void 0) continue;
-		lifted.push(`${family.id}  ${lift.variant} pooled ${familyLiftLabel(lift)}: its lower bound is below minLift ${Math.round(lift.minLift * 100)}`);
-	}
-	const idle = undiscriminating(record.evals);
-	if (idle.length) {
-		lines.push("", c.bold(`Checks that don't discriminate (${idle.length})`));
-		for (const x of idle) lines.push(` ${c.dim("·")} ${undiscriminatingLabel(x)}`);
-	}
-	if (lifted.length) {
-		lines.push("", c.bold(c.red(`Lift below minLift (${lifted.length})`)));
-		for (const line of lifted) lines.push(` ${c.red("✗")} ${line}`);
-	}
-	if (record.status === "budget") lines.push("", c.yellow("Budget reached: attempts that hadn’t started were skipped"));
-	lines.push("");
-	if (record.families?.length) lines.push(`${c.dim("Families")}  ${familyTotals(c, record.families)}`);
-	lines.push(`${c.dim(label)}  ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")}  ${formatDuration(record.durationMs)}`);
 	const tokens = formatUsage(record.cost.usage);
-	if (tokens) lines.push(`  ${c.dim("Tokens")}  ${tokens}`);
-	lines.push(` ${c.dim("Results")}  ${e.summaryPath}`);
-	return lines;
+	const notes = [
+		...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
+		...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were cancelled")] : [],
+		c.dim([
+			formatDuration(record.durationMs),
+			tokens,
+			`results ${e.summaryPath}`
+		].filter(Boolean).join(" · "))
+	];
+	return render(suiteResult(record), e.insights ?? null, {
+		color: c.enabled,
+		verbose,
+		notes: [...(e.insightLines ?? []).map((l) => c.yellow(l)), ...notes],
+		hint: e.hint
+	});
 }
 //#endregion
 //#region src/ui/tags.ts
@@ -749,16 +457,9 @@ var Tagger = class {
 //#endregion
 //#region src/ui/agent.ts
 const MAX_LINES = 12;
-const WORD = {
-	fail: "FAIL",
-	infra_error: "INFRA",
-	cancelled: "CANCELLED"
-};
 var AgentRenderer = class {
 	#o;
 	#c = colors(false);
-	#logs = /* @__PURE__ */ new Map();
-	#printed = /* @__PURE__ */ new Set();
 	/** `[key]` prefixes for `-v` lines, from each attempt's start. */
 	#labels = /* @__PURE__ */ new Map();
 	#tags = new Tagger();
@@ -768,30 +469,6 @@ var AgentRenderer = class {
 	#write(text) {
 		this.#o.out.write(`${text.trimEnd()}\n`);
 	}
-	#failure(result, cell, n) {
-		const outcome = outcomeOf(result);
-		if (outcome === "pass" || outcome === "skipped") return;
-		this.#printed.add(result.key);
-		const kind = outcome === "fail" ? result.meta?.errorKind ?? "assertion" : null;
-		const detail = [
-			formatDuration(result.durationMs),
-			kind,
-			result.attempt?.gate === "none" ? "not gated" : ""
-		].filter(Boolean).join(", ");
-		const which = result.attempt?.label ? ` [${result.attempt.label}]` : "";
-		const missed = outcome === "fail" && result.attempt?.gate === "none";
-		this.#write(`${missed ? "MISS" : WORD[outcome]} ${result.id}${which} (${detail})`);
-		if (outcome !== "cancelled") {
-			const lines = failureLines(this.#c, result, { max: MAX_LINES });
-			for (const line of missed ? lines.filter((l) => l.trim()).slice(0, 1) : lines) this.#write(`  ${line}`);
-		}
-		const logs = missed ? void 0 : this.#logs.get(result.key);
-		if (logs?.length) {
-			this.#write("  console:");
-			for (const line of capLines(logs.join("\n"), MAX_LINES)) this.#write(`    ${line}`);
-		}
-		if (!result.meta?.prepare) this.#write(`  record: ${this.#o.attemptDir(result.id, cell, n)}`);
-	}
 	onEvent(e) {
 		switch (e.type) {
 			case "plan":
@@ -814,65 +491,22 @@ var AgentRenderer = class {
 				for (const line of debugLines(this.#c, e.line, this.#o.verbosity === 2)) this.#write(`${label} ${line}`);
 				return;
 			}
-			case "log": {
-				if (!e.key) return;
-				const logs = this.#logs.get(e.key) ?? [];
-				logs.push(...e.text.replace(/\n$/, "").split("\n"));
-				this.#logs.set(e.key, logs);
-				return;
-			}
 			case "eval-progress": {
 				if (e.progress.kind !== "browser-download") return;
 				const label = this.#labels.get(e.key) ?? `[${e.key}]`;
 				this.#write(`${label} ${progressLine(this.#c, e.progress)}`);
 				return;
 			}
-			case "eval-end": {
-				const { attempt } = e.result;
-				this.#failure(e.result, attempt?.cell.key ?? "", attempt?.n ?? 1);
-				return;
-			}
 			case "prepare":
 				for (const line of prepareLines(this.#c, e, MAX_LINES)) this.#write(line);
 				return;
 			case "notice":
 				if (e.level !== "info") this.#write(e.text);
 				return;
-			case "run-end": {
-				const failing = failingAttempts(e.record.evals);
-				for (const { entry, attempt, outcome } of failing) {
-					const result = resultFor(entry, attempt, e.results);
-					if (outcome === "cancelled" && !this.#printed.has(result.key)) this.#failure(result, attempt.cell, attempt.n);
-				}
-				for (const error of e.unhandledErrors) this.#write(`ERROR ${error}`);
-				for (const entry of e.record.evals) for (const a of entry.answers ?? []) {
-					this.#write(`question ${entry.id}: ${answersLabel(a)}`);
-					for (const line of headToHeadLines(a.votes)) this.#write(`  ${line}`);
-					for (const reason of sampleReasons(a.votes)) this.#write(`  ${reason}`);
-				}
-				for (const x of undiscriminating(e.record.evals)) this.#write(`check doesn't discriminate: ${undiscriminatingLabel(x)}`);
-				for (const family of e.record.families ?? []) {
-					const pooled = family.lifts.length ? family.lifts.map((l) => {
-						const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)}${l.outcome === "fail" ? ", FAIL" : ""})`;
-						return `${l.variant} ${familyLiftLabel(l)}${gate}`;
-					}) : family.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
-					this.#write(`family ${family.id}: ${familyHeadLabel(family)} · ${pooled.join(" · ")}`);
-				}
-				const s = e.record.summary;
-				const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
-				const parts = [
-					s.pass && `${s.pass} passed`,
-					s.fail - s.notGated - cancelled > 0 && `${s.fail - s.notGated - cancelled} failed`,
-					s.notGated > 0 && `${s.notGated} missed, not gated`,
-					s.infraError && `${s.infraError} infra error`,
-					cancelled && `${cancelled} cancelled`,
-					s.skipped && `${s.skipped} skipped`
-				].filter(Boolean);
-				const scope = s.attempts === s.evals ? "" : ` (${s.attempts} attempts, ${s.evals} evals)`;
-				this.#write(parts.length ? `${parts.join(", ")}${scope}` : "no evals ran");
-				this.#write(`summary: ${e.summaryPath}`);
+			case "run-end":
+				this.#write("");
+				for (const line of resultLines(this.#c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
 				return;
-			}
 			default: return;
 		}
 	}
@@ -889,6 +523,8 @@ const STARTS = {
 	"browser-download": "browser-download"
 };
 const ERROR_LINES = 12;
+/** Back to the start of the line and clear it, for the live status line. */
+const CLEAR = "\r\x1B[2K";
 /** GitHub Actions workflow-command escaping. */
 const ghEscape = (s, property = false) => {
 	const out = s.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
@@ -914,10 +550,20 @@ var PlainRenderer = class {
 	#runStartedAt = 0;
 	#lastWrite = 0;
 	#timer;
+	/** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
+	#live;
+	#status = "";
+	/** Evals and tries collected (not skipped), and the tries that ended. */
+	#evals = /* @__PURE__ */ new Set();
+	#tries = /* @__PURE__ */ new Set();
+	#ended = /* @__PURE__ */ new Set();
+	/** Insights after the last attempt, once they've started: they take the status line. */
+	#insights = null;
 	constructor(options) {
 		this.#o = options;
 		this.#c = options.colors;
 		this.#now = options.now ?? Date.now;
+		this.#live = options.output.requested === "live";
 		const { heartbeatMs } = options.output;
 		if (options.timers !== false && heartbeatMs > 0) {
 			this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
@@ -925,9 +571,29 @@ var PlainRenderer = class {
 		}
 	}
 	#write(text) {
-		this.#o.out.write(`${text.trimEnd()}\n`);
+		const line = `${text.trimEnd()}\n`;
+		this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
 		this.#lastWrite = this.#now();
 	}
+	/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended; then `insights 3/7…`. */
+	#drawStatus() {
+		if (!this.#live) return;
+		let status;
+		if (this.#insights) status = this.#c.dim(`insights ${this.#insights.done}/${this.#insights.total}…`);
+		else {
+			if (!this.#tries.size) return;
+			const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
+			status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
+		}
+		if (status === this.#status) return;
+		this.#o.out.write(`${CLEAR}${status}`);
+		this.#status = status;
+	}
+	#clearStatus() {
+		if (!this.#status) return;
+		this.#o.out.write(CLEAR);
+		this.#status = "";
+	}
 	#attempt(ref, startedAt) {
 		const n = ref.attempt?.n ?? 1;
 		const label = `[${this.#tags.label(ref.id, n)} ${attemptAxes(ref.attempt, this.#o.harness)}]`;
@@ -973,14 +639,20 @@ var PlainRenderer = class {
 				return;
 			case "collected":
 				this.#tags.addFile(e.evals);
-				for (const x of e.evals) if (!x.skipped) this.#queued.add(x.key);
+				for (const x of e.evals) {
+					if (x.skipped) continue;
+					this.#queued.add(x.key);
+					this.#evals.add(x.id);
+					this.#tries.add(x.key);
+				}
+				this.#drawStatus();
 				return;
 			case "eval-start": {
 				const a = this.#attempt(e, e.at);
 				a.last = e;
 				this.#queued.delete(e.key);
 				this.#running.set(e.key, a);
-				this.#write(`${this.#prefix(a)} ${c.dim("▶")} ${e.id}`);
+				if (this.#streaming) this.#write(`${this.#prefix(a)} ${c.dim("▶")} ${e.id}`);
 				return;
 			}
 			case "eval-progress": {
@@ -1028,6 +700,8 @@ var PlainRenderer = class {
 			}
 			case "eval-end":
 				this.#end(e.result, e.at);
+				if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
+				this.#drawStatus();
 				return;
 			case "prepare":
 				for (const line of prepareLines(c, e, ERROR_LINES)) this.#write(line);
@@ -1038,6 +712,14 @@ var PlainRenderer = class {
 					this.#write(`::error title=${title}::${ghEscape(failure.message)}`);
 				}
 				return;
+			case "insights-progress":
+				this.#insights = {
+					done: e.done,
+					total: e.total,
+					startedAt: this.#insights?.startedAt ?? e.at
+				};
+				this.#drawStatus();
+				return;
 			case "notice": {
 				const paint = {
 					info: c.dim,
@@ -1048,8 +730,9 @@ var PlainRenderer = class {
 				return;
 			}
 			case "run-end":
+				this.#clearStatus();
 				this.#write("");
-				for (const line of renderSummary(c, e, this.#o.attemptDir)) this.#write(line);
+				for (const line of resultLines(c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
 				return;
 		}
 	}
@@ -1062,6 +745,11 @@ var PlainRenderer = class {
 		const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
 		this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
 	}
+	/**
+	* An ended attempt's logs, as `--output-logs` picks: grouped, its block under a `▶` header;
+	* streamed, only its error lines, since the rest already printed. A sandbox that failed to
+	* prepare was annotated once, for all its attempts.
+	*/
 	#end(result, at) {
 		const c = this.#c;
 		const a = this.#running.get(result.key) ?? this.#attempt(result, at - result.durationMs);
@@ -1069,18 +757,17 @@ var PlainRenderer = class {
 		this.#queued.delete(result.key);
 		const outcome = outcomeOf(result);
 		const failed = outcome !== "pass" && outcome !== "skipped";
-		const time = outcome === "skipped" ? "" : `  ${c.dim(formatDuration(result.durationMs))}`;
-		const ungated = failed && result.attempt?.gate === "none" ? `  ${c.dim("(not gated)")}` : "";
-		const missed = ungated !== "" && outcome === "fail";
-		const head = `${this.#prefix(a)} ${missed ? MISS_MARK(c) : mark(c, outcome)} ${result.id}${time}${ungated}`;
+		const ungated = failed && result.attempt?.gate === "none";
+		const missed = ungated && outcome === "fail";
 		const { outputLogs, githubActions, logPrefix } = this.#o.output;
 		const showDetail = !this.#streaming && a.lines.length > 0 && (outputLogs === "full" || outputLogs === "errors-only" && failed);
 		const judged = result.meta?.errorKind === "judge" && (showDetail || this.#streaming);
 		const errors = failed && outputLogs !== "none" && !judged ? missed ? failureLines(colors(false), result, { max: 1 }).filter((l) => l.trim()).slice(0, 1).map((l) => c.dim(l)) : failureLines(c, result, { max: ERROR_LINES }) : [];
 		const lines = a.lines.filter((l) => l.kind !== "cleanup").sort((x, y) => x.at - y.at);
 		const prefixed = logPrefix === "task";
+		const head = `${this.#prefix(a)} ${c.dim("▶")} ${result.id}`;
 		const group = githubActions && showDetail && (!failed || missed);
-		this.#write(group ? `::group::${stripAnsi(head)}` : head);
+		if (showDetail || !this.#streaming && errors.length) this.#write(group ? `::group::${stripAnsi(head)}` : head);
 		if (showDetail) for (const line of lines) this.#write(this.#detail(a, line, prefixed));
 		for (const line of errors) this.#write(prefixed ? `${this.#prefix(a)}  ${line}` : ` ${line}`);
 		if (group) this.#write("::endgroup::");
@@ -1093,12 +780,19 @@ var PlainRenderer = class {
 			this.#write(`::error file=${ghEscape(result.file, true)}${where},title=${title}::${message}`);
 		}
 	}
-	/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
+	/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals or insights are running. */
 	tick(now = this.#now()) {
 		const { heartbeatMs } = this.#o.output;
-		if (heartbeatMs <= 0 || this.#running.size === 0) return;
+		if (this.#live || heartbeatMs <= 0) return;
 		if (now - this.#lastWrite < heartbeatMs) return;
 		const c = this.#c;
+		const insights = this.#insights;
+		if (insights) {
+			const elapsed = formatDuration(now - insights.startedAt);
+			if (insights.done < insights.total) this.#write(c.dim(`… insights ${insights.done}/${insights.total} after ${elapsed}`));
+			return;
+		}
+		if (this.#running.size === 0) return;
 		const running = [...this.#running.values()].map((a) => {
 			const progress = a.last?.type === "eval-progress" ? a.last.progress : void 0;
 			return `${this.#prefix(a)} ${phaseOf(progress)} ${c.dim(formatDuration(now - a.startedAt))}`;
@@ -1109,6 +803,7 @@ var PlainRenderer = class {
 	}
 	close() {
 		clearInterval(this.#timer);
+		this.#clearStatus();
 	}
 };
 //#endregion
@@ -1172,7 +867,7 @@ function resolveOutput(input) {
 		mode,
 		logOrder,
 		logPrefix,
-		outputLogs: logs && logs !== "auto" ? logs : "full",
+		outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
 		color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
 		githubActions,
 		heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
@@ -1231,6 +926,11 @@ const invocation = {
 let interrupted = false;
 /** The last signal that cancelled the run: 130 after SIGINT, 143 after SIGTERM. */
 let stopCode = EXIT.interrupted;
+/**
+* Set while insights run, after the attempts and the exit code: Ctrl-C or SIGTERM then stops the
+* insights instead of cancelling the run, which has nothing left to cancel.
+*/
+let onInsightsSignal = null;
 async function run(argv) {
 	const { values, positionals } = parseCli(argv);
 	invocation.flags = values;
@@ -1248,7 +948,7 @@ async function run(argv) {
 		return values.help ? EXIT.ok : EXIT.usage;
 	}
 	if (command === "init") {
-		const { initCommand } = await import("./init-BrOWin3W.js");
+		const { initCommand } = await import("./init-D7Y1ny1z.js");
 		return initCommand(values, filters, {
 			out: (s) => console.log(s),
 			input: process.stdin,
@@ -1257,7 +957,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "doctor") {
-		const { doctorCommand } = await import("./doctor-BYNok4UO.js");
+		const { doctorCommand } = await import("./doctor-CqttrZcs.js");
 		return doctorCommand(values, {
 			out: (s) => console.log(s),
 			color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1265,7 +965,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "report") {
-		const { reportCommand } = await import("./command-BLfT7vnQ.js");
+		const { reportCommand } = await import("./command-CaZgCTIw.js");
 		return reportCommand(values, filters, {
 			out: (s) => console.log(s),
 			err: (s) => console.error(dim(s)),
@@ -1280,7 +980,7 @@ async function run(argv) {
 	}
 	if (command === "studio") {
 		const config = await loadConfig(configPath, "studio");
-		const { studioCommand } = await import("./command-CMJtHKRa.js");
+		const { studioCommand } = await import("./command-DTgBZR0k.js");
 		return studioCommand({
 			config,
 			flags: values,
@@ -1291,7 +991,7 @@ async function run(argv) {
 	if (command === "mcp") {
 		if (!values.http) console.log = console.info = console.debug = console.error;
 		const config = await loadConfig(configPath, "mcp");
-		const { mcpCommand } = await import("./command-BT9a37ZR.js");
+		const { mcpCommand } = await import("./command-BMHXHqMT.js");
 		return mcpCommand({
 			config,
 			flags: values,
@@ -1300,7 +1000,7 @@ async function run(argv) {
 	}
 	if (command in HISTORY_COMMANDS) {
 		const config = await loadConfig(configPath, command);
-		const commands = await import("./commands-1p-bIWhN.js");
+		const commands = await import("./commands-BXzIDB3X.js");
 		const ctx = {
 			config,
 			flags: values,
@@ -1313,7 +1013,7 @@ async function run(argv) {
 			case "compare": return commands.compareCommand(ctx, filters);
 			case "checkout": return commands.checkoutCommand(ctx, filters);
 			case "query": {
-				const { queryCommand } = await import("./query-C_G_svLU.js");
+				const { queryCommand } = await import("./query-CmBDYt0_.js");
 				return queryCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1321,7 +1021,7 @@ async function run(argv) {
 			}
 			case "analyze": {
 				loadEnvFiles(configPath);
-				const { analyzeCommand } = await import("./command-BkFjijN2.js");
+				const { analyzeCommand } = await import("./command-CVtgyqlM.js");
 				return analyzeCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1360,8 +1060,8 @@ async function cleanCommand(flags, configPath) {
 	const where = fromCwd(debugRoot(config.root)) || ".";
 	console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
 	if (flags.images) {
-		const { dockerAvailable, pruneImages } = await import("./src-pJZOrYR9.js");
-		const { requireDocker } = await import("./providers-gDMko8fl.js");
+		const { dockerAvailable, pruneImages } = await import("./src-B9L9yGKI.js");
+		const { requireDocker } = await import("./providers-CXN2RFz-.js");
 		const engines = [];
 		for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
 		if (!engines.length) await requireDocker();
@@ -1382,7 +1082,7 @@ function verbosityOf(flags) {
 * does. Needs a config, as `reeval run` does: there is no default image.
 */
 async function imageBuildCommand(flags, configPath, filters) {
-	const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-0On0cztu.js");
+	const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-BmJRuTpm.js");
 	const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
 		...flags,
 		"dry-run": true
@@ -1442,13 +1142,12 @@ async function runCommand(flags, configPath, filters) {
 	if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
 	const stepStart = process.env.GITHUB_STEP_SUMMARY;
 	if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
-	const { engineVersions } = await import("./node-B3ydHo6t.js");
+	const { engineVersions } = await import("./node-D0mtjRAO.js");
 	const renderer = createRenderer({
-		out: setup.json ? { write: () => true } : process.stdout,
+		out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
 		output: setup.output,
 		verbosity: ctx.verbosity ?? 0,
-		harness: setup.config.harness[0]?.kind ?? "agent",
-		attemptDir: (id, cell, n) => fromCwd(join(ctx.scratch, ctx.runId, evalSlug(id), evalSlug(cell), String(n)))
+		harness: setup.config.harness[0]?.kind ?? "agent"
 	});
 	let executed;
 	try {
@@ -1461,27 +1160,13 @@ async function runCommand(flags, configPath, filters) {
 		});
 		throw error;
 	}
-	const { engineRun, plan, sandboxVersions, budgetReached, live, notes, questions } = executed;
+	const { engineRun, plan, sandboxVersions, budgetReached, live, questions } = executed;
 	invocation.unhandledErrors = engineRun.unhandledErrors;
 	const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
 	if ((interrupted || engineRun.cancelled) && engines.length) {
-		const { removeRunContainers } = await import("./docker-yauK-9X8.js");
+		const { removeRunContainers } = await import("./docker-DC69dKxB.js");
 		await removeRunContainers(ctx.runId, engines);
 	}
-	if (notes) {
-		const cancelled = interrupted || engineRun.cancelled;
-		const written = await notes.finish(cancelled);
-		if (!cancelled && written.length) {
-			const { notesLine } = await import("./agent-CURAwx_q.js");
-			const text = notesLine(ctx.config.root, ctx.runId, written);
-			renderer.onEvent({
-				type: "notice",
-				at: Date.now(),
-				level: "info",
-				text
-			});
-		}
-	}
 	if (engineRun.stopped || !plan) {
 		live?.remove();
 		renderer.close();
@@ -1489,8 +1174,8 @@ async function runCommand(flags, configPath, filters) {
 	}
 	const stopped = interrupted || engineRun.cancelled;
 	const answers = questions && !stopped ? await questions.settle() : void 0;
-	const { lockfiles } = await import("./state-CgjQo29K.js");
-	const { writeScratchRun } = await import("./results-DuauQI3Z.js");
+	const { lockfiles } = await import("./state-BQR_BCm4.js");
+	const { writeScratchRun } = await import("./results-Drtt0uFl.js");
 	const run = await writeScratchRun({
 		ctx,
 		author: ctx.author,
@@ -1514,17 +1199,56 @@ async function runCommand(flags, configPath, filters) {
 		...answers?.size ? { answers } : {}
 	}, () => live?.stop());
 	const { record } = run;
-	if (notes && !stopped) {
-		const { summarizeRun } = await import("./agent-CURAwx_q.js");
-		const { resolveAnalysisModel } = await import("./model-NM3ID0uR.js");
-		const model = resolveAnalysisModel(ctx.config, "agent");
-		const { level, text } = await summarizeRun(ctx.config, record.runId, model);
-		renderer.onEvent({
-			type: "notice",
-			at: Date.now(),
-			level,
-			text
-		});
+	const outcome = runOutcome(record);
+	const code = stopped ? stopCode : outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
+	let insights = run.insights;
+	let insightLines = [];
+	const { runInsights, wantsInsights } = await import("./run-BeMy92Us.js");
+	const wanted = !stopped && wantsInsights(ctx.config, !flags["no-insights"]);
+	if (wanted && interrupted) insightLines = [`insights skipped (${stopCode === EXIT.terminated ? "SIGTERM" : "Ctrl-C"}): reeval analyze ${record.runId} --insights to write them`];
+	else if (wanted) {
+		const { scratchDir } = await import("./files-DqCBW7i8.js");
+		const stop = new AbortController();
+		let stopSignal = "SIGINT";
+		onInsightsSignal = (signal) => {
+			if (stop.signal.aborted) {
+				renderer.close();
+				process.exit(code);
+			}
+			stopSignal = signal;
+			stop.abort(/* @__PURE__ */ new Error(signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"));
+		};
+		try {
+			const done = await runInsights({
+				config: ctx.config,
+				record,
+				runDir: run.artifactDir ?? join(scratchDir(ctx.config.root), record.runId),
+				remarks: run.insights,
+				verbose: (ctx.verbosity ?? 0) > 0,
+				signal: stop.signal,
+				stopSignal: () => stopSignal,
+				onStart: (text) => {
+					renderer.onEvent({
+						type: "notice",
+						at: Date.now(),
+						level: "info",
+						text
+					});
+				},
+				onProgress: (n, total) => {
+					renderer.onEvent({
+						type: "insights-progress",
+						at: Date.now(),
+						done: n,
+						total
+					});
+				}
+			});
+			insights = done.insights ?? insights;
+			insightLines = done.lines;
+		} finally {
+			onInsightsSignal = null;
+		}
 	}
 	const { pruneDebug } = await import("./files-BAt0i00X.js");
 	await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
@@ -1535,7 +1259,9 @@ async function runCommand(flags, configPath, filters) {
 		record,
 		summaryPath: fromCwd(summary),
 		results: engineRun.evals,
-		unhandledErrors: engineRun.unhandledErrors
+		hint: !hasAnalysisModel(ctx.config),
+		insights,
+		insightLines
 	});
 	renderer.close();
 	const stepSummary = process.env.GITHUB_STEP_SUMMARY;
@@ -1543,17 +1269,20 @@ async function runCommand(flags, configPath, filters) {
 	const { config } = ctx;
 	const mode = setup.record;
 	if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
-		const { record: recordCmd } = await import("./commands-1p-bIWhN.js");
+		const { record: recordCmd } = await import("./commands-BXzIDB3X.js");
 		const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
-		console.log(await recordCmd(config, record.runId, {
+		const line = await recordCmd(config, record.runId, {
 			...flags,
 			...allow
-		}));
+		});
+		if (setup.json) console.error(line);
+		else console.log(line);
 	}
-	if (interrupted || engineRun.cancelled) return stopCode;
-	if (engineRun.unhandledErrors.length > 0) return EXIT.failed;
-	const outcome = runOutcome(record);
-	return outcome === "fail" ? EXIT.failed : outcome === "infra_error" ? EXIT.infra : EXIT.ok;
+	if (setup.json) {
+		const { suiteResult } = await import("./suite-BaHPmAGy.js");
+		process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
+	}
+	return code;
 }
 /** Everything that can fail before the evals are collected: output mode, config, profile, flags. */
 async function prepareRun(flags, configPath) {
@@ -1562,13 +1291,13 @@ async function prepareRun(flags, configPath) {
 			ui: flags.ui,
 			logOrder: flags["log-order"],
 			logPrefix: flags["log-prefix"],
-			outputLogs: flags["output-logs"]
+			outputLogs: flags["output-logs"],
+			verbose: verbosityOf(flags) > 0
 		},
 		env: process.env,
-		isTTY: !!process.stdout.isTTY,
+		isTTY: !!(flags.json && !flags["dry-run"] ? process.stderr : process.stdout).isTTY,
 		agent: detectAgent().name
 	});
-	if (flags.json && !flags["dry-run"]) throw new UsageError("reeval run --json only prints the plan: add --dry-run. A run writes its record to .reeval/runs; read it with reeval show --json.");
 	const path = requireConfig(configPath);
 	loadEnvFiles(path);
 	const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
@@ -1583,7 +1312,7 @@ async function prepareRun(flags, configPath) {
 	const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
 		const container = sandboxes.find((s) => s.kind === "docker");
 		if (!container) return hostCapacity();
-		const { dockerCapacity } = await import("./src-pJZOrYR9.js");
+		const { dockerCapacity } = await import("./src-B9L9yGKI.js");
 		return await dockerCapacity(container.engine) ?? hostCapacity();
 	});
 	return {
@@ -1598,6 +1327,8 @@ async function prepareRun(flags, configPath) {
 		dryRun: !!flags["dry-run"],
 		json: !!flags.json,
 		force: !!flags.force,
+		/** `--no-insights` turns them off for this run. */
+		insights: !flags["no-insights"],
 		verbosity: verbosityOf(flags),
 		configFile: configLabel(path),
 		/** Named in the plan line when the project root isn't the current directory. */
@@ -1610,11 +1341,11 @@ async function prepareRun(flags, configPath) {
 async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
 	const startedAt = /* @__PURE__ */ new Date();
 	const git = await gitInfo(config.root, [config.history.dir]);
-	const { worktreePatch } = await import("./state-CgjQo29K.js");
+	const { worktreePatch } = await import("./state-BQR_BCm4.js");
 	const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
 	const who = await author(config.root);
-	const { stagingDir, sweepStaging } = await import("./results-DuauQI3Z.js");
-	const { scratchDir } = await import("./files-ZZPqfAo6.js");
+	const { stagingDir, sweepStaging } = await import("./results-Drtt0uFl.js");
+	const { scratchDir } = await import("./files-DqCBW7i8.js");
 	const scratch = scratchDir(config.root);
 	if (resume !== void 0) {
 		const target = await resolveResume(config.root, resume || void 0);
@@ -1625,7 +1356,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
 			artifacts: target.previousArtifacts
 		} : await loadPrevious(stagingDir(config.root, target.runId));
 		if (!dryRun) try {
-			const { removeRunContainers } = await import("./docker-yauK-9X8.js");
+			const { removeRunContainers } = await import("./docker-DC69dKxB.js");
 			await removeRunContainers(target.runId, ["docker", "podman"]);
 		} catch {}
 		return {
@@ -1696,8 +1427,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 		configFile,
 		completed: []
 	}));
-	const { runEngine } = await import("./node-B3ydHo6t.js");
-	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-C9Cy27Ay.js");
+	const { runEngine } = await import("./node-D0mtjRAO.js");
+	const { buildPlan, checkAttemptBudget, planDocument, planInsights, planLine, planTable } = await import("./plan-OkwUcJQ9.js");
 	let plan;
 	let sandboxVersions = { images: [] };
 	let preflightStop;
@@ -1705,7 +1436,6 @@ async function executeRun(ctx, setup, filters, renderer) {
 	let budgetTimer;
 	let toRun = [];
 	let live;
-	let notes;
 	let questions;
 	const onPlan = async ({ planned, excluded, matched, unmatched }) => {
 		if (unmatched.length) throw unmatchedFilter(config, unmatched);
@@ -1724,7 +1454,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		} : p);
 		toRun = skipped;
 		for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
-		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-NM3ID0uR.js") : void 0;
+		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-BGKYVpoj.js") : void 0;
 		plan = await buildPlan({
 			planned: skipped,
 			profile,
@@ -1732,7 +1462,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 			concurrency,
 			root: config.root,
 			historyDir: config.history.dir,
-			questionModel: models?.questionModelLabel(config) ?? null
+			questionModel: models?.questionModelLabel(config) ?? null,
+			insights: await planInsights(config, setup.insights)
 		});
 		plan.runId = runId;
 		if (completed.size) plan.resume = {
@@ -1740,13 +1471,14 @@ async function executeRun(ctx, setup, filters, renderer) {
 			completed: completed.size
 		};
 		run.completed = [...completed];
-		const { imageStages, imageTargets } = await import("./images-0On0cztu.js");
+		const { imageStages, imageTargets } = await import("./images-BmJRuTpm.js");
 		const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
 			id: p.id,
 			harness: p.attempt.harness,
-			sandbox: p.attempt.sandbox
+			sandbox: p.attempt.sandbox,
+			plugins: p.attempt.pluginBuilds ?? []
 		})));
-		if (setup.json) {
+		if (setup.json && setup.dryRun) {
 			const images = await Promise.all(targets.map(async (t) => ({
 				id: t.id,
 				harness: t.harness,
@@ -1761,8 +1493,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 		if (setup.dryRun) {
 			const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
 			lines.push(...planTable(plan, todos));
-			const { imageStatusLines } = await import("./images-0On0cztu.js");
-			const { dockerAvailable, engineSpec } = await import("./src-pJZOrYR9.js");
+			const { imageStatusLines } = await import("./images-BmJRuTpm.js");
+			const { dockerAvailable, engineSpec } = await import("./src-B9L9yGKI.js");
 			const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
 			const down = [];
 			for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
@@ -1772,6 +1504,10 @@ async function executeRun(ctx, setup, filters, renderer) {
 				lines.push(`images  not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
 			}
 		}
+		if (setup.dryRun && plan.insights) {
+			const { formatTokens } = await import("./usage-BDo2q_yu.js");
+			lines.push(`insights  ${plan.insights.model} · up to ${formatTokens(plan.insights.maxTokens)} tokens`);
+		}
 		lines.push(...plan.warnings.map((w) => `warning: ${w}`));
 		lines.push(...plan.hints.map((h) => `hint: ${h}`));
 		renderer.onEvent({
@@ -1782,8 +1518,6 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (setup.dryRun) return false;
 		checkAttemptBudget(plan, setup.force, config.file);
-		const analysis = config.analysis.agent ? await import("./model-NM3ID0uR.js") : null;
-		analysis?.analysisModelName(config, "agent");
 		const questionModels = /* @__PURE__ */ new Set();
 		const asked = [];
 		for (const row of plan.rows) {
@@ -1813,24 +1547,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 			questionModels: [...questionModels]
 		});
 		if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
-		if (analysis) {
-			const { startNotes } = await import("./agent-CURAwx_q.js");
-			notes = startNotes({
-				config,
-				model: analysis.resolveAnalysisModel(config, "agent"),
-				runId,
-				stagingDir,
-				planned: skipped,
-				warn: (text) => renderer.onEvent({
-					type: "notice",
-					at: Date.now(),
-					level: "warn",
-					text
-				})
-			});
-		}
 		if (asked.length) {
-			const { questionRunner } = await import("./questions-DjAwMxZ3.js");
+			const { questionRunner } = await import("./questions-1XOulevV.js");
 			questions = questionRunner({
 				evals: asked,
 				stagingDir,
@@ -1847,7 +1565,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 				}
 			});
 		}
-		const { prepareWorkspaces, workspaceKey } = await import("./workspace-D8sSgpiy.js");
+		const { prepareWorkspaces, workspaceKey } = await import("./workspace-B90YDU_a.js");
 		const live = skipped.filter((p) => !p.attempt.plan.skip);
 		const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
 			workspace: a.workspace,
@@ -1877,6 +1595,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 				label: `workspace ${o.label}`,
 				sandbox: user.attempt.sandbox,
 				harness: user.attempt.harness,
+				plugins: [],
 				setBy: o.setBy,
 				runWide: false
 			};
@@ -1896,6 +1615,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			sandboxVersions = await preflight(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
 				harness: p.attempt.harness,
 				sandbox: p.attempt.sandbox,
+				plugins: p.attempt.pluginBuilds ?? [],
 				setBy: p.attempt.setBy
 			})), {
 				root: config.root,
@@ -1961,7 +1681,6 @@ async function executeRun(ctx, setup, filters, renderer) {
 				}
 				if (e.type === "eval-end") {
 					live?.end(e.result);
-					notes?.onResult(e.result);
 					questions?.onEvalEnd(e.result);
 				}
 				if (interrupted && e.type === "eval-end" && e.result.state === "skipped") return;
@@ -1969,6 +1688,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			},
 			onCancellable: (cancel) => {
 				const onSignal = (signal) => {
+					if (onInsightsSignal) return onInsightsSignal(signal === "SIGTERM" ? "SIGTERM" : "SIGINT");
 					stopCode = signal === "SIGTERM" ? EXIT.terminated : EXIT.interrupted;
 					if (interrupted) {
 						renderer.close();
@@ -1995,9 +1715,9 @@ async function executeRun(ctx, setup, filters, renderer) {
 		throw error;
 	} finally {
 		clearTimeout(budgetTimer);
-		const { removePrepared } = await import("./preflight-CPWpQeBI.js");
+		const { removePrepared } = await import("./preflight-B6pY89bF.js");
 		removePrepared(config.root, runId);
-		const { removePreparedWorkspaces } = await import("./workspace-D8sSgpiy.js");
+		const { removePreparedWorkspaces } = await import("./workspace-B90YDU_a.js");
 		removePreparedWorkspaces(config.root, runId);
 	}
 	if (preflightStop) throw preflightStop;
@@ -2010,7 +1730,6 @@ async function executeRun(ctx, setup, filters, renderer) {
 		sandboxVersions,
 		budgetReached: () => reached,
 		live,
-		notes,
 		questions
 	};
 }
@@ -2080,4 +1799,4 @@ run(process.argv.slice(2)).finally(() => {
 //#endregion
 export {};
 
-//# sourceMappingURL=cli-D303Vj5N.js.map
+//# sourceMappingURL=cli-B5zQCR2r.js.map