reeval 0.9.0 → 0.11.0

package/dist/{cli-CVrMh_qz.js → cli-ClpPT8kn.js}

@@ -1,33 +1,32 @@
-import { n as UsageError } from "./errors-yDAEwmZo.js";
-import { t as parseDuration } from "./duration-JsJrIWXq.js";
-import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
+import { n as isOneOf } from "./object-CLf2-tNq.js";
+import { n as UsageError } from "./errors-B6BSMs03.js";
+import { p as parseDuration } from "./shared-CA61BuPJ.js";
+import { f as sandboxId, s as harnessId } from "./options-C0d0AzeR.js";
+import { A as workspaceLabel, p as environmentSandbox } from "./environment-Cca6a4wB.js";
+import { a as fromCwd, o as newRunId } from "./ids-Bp6ut7ZD.js";
+import { _ as selectProfile, a as configLabel, d as hostCapacity, f as loadConfig, g as resolveConcurrency, h as requireConfig, l as findConfig, m as missingCredentials, o as configPathFor, p as malformedCredentials, u as hasAnalysisModel, v as inCI, y as loadEnvFiles } from "./config-DrVSEVN0.js";
 import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
-import { o as PACKAGE_VERSION } from "./root-CvdRlRZX.js";
-import { T as workspaceLabel, u as environmentSandbox } from "./environment-1jE-HhO-.js";
-import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
-import { I as LABEL, X as imageInfo } from "./src-8rRXy4u9.js";
-import { d as sandboxId, s as harnessId } from "./options-CKqyPGxX.js";
-import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-CGvBeewH.js";
+import { o as PACKAGE_VERSION } from "./root-DICDHCtp.js";
+import { I as LABEL, X as imageInfo } from "./src-ftu5wPSt.js";
 import { s as debugTime } from "./debug-D24NOQH1.js";
 import { i as sandboxWarningLine, n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
 import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { H as cellContrast, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, P as collapsedJudgeLabel, _ as hasUngatedCells, b as isUngatedMiss, h as formatUsage, q as effortLabel, w as runOutcome } from "./model-BOnlAPzs.js";
-import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "./family-D2b7WD10.js";
-import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
-import { a as gateLabel, d as undiscriminating, f as undiscriminatingLabel, i as checksLabel, n as answersLabel, o as liftLabel } from "./summary-BTksrIxn.js";
-import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-SUz1GskV.js";
-import { n as FLAGS, o as helpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-CKEYa8V2.js";
-import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-D7HK3vzt.js";
-import { t as budgetMarker } from "./budget-DBUrH4jX.js";
-import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-BkoSxf4C.js";
-import { r as imageHarness } from "./prepare-CUAWHebF.js";
-import { n as createProviders } from "./providers-B1DzMvmH.js";
-import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CLaL8AuW.js";
+import { T as runOutcome, g as formatUsage } from "./model-DaX4bNLZ.js";
+import { s as suiteResult } from "./suite-Cw_7G2RM.js";
 import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
+import { a as render } from "./summary-BUKfU-9l.js";
+import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-Ca1c8vix.js";
+import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-qmEnQ25_.js";
+import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-DQ0AXn6c.js";
+import { t as budgetMarker } from "./budget-DBUrH4jX.js";
+import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-BRzD30L5.js";
+import { n as imageHarness } from "./prepare-B_y-AFMv.js";
+import { r as createProviders, t as attemptImage } from "./providers-DoTq1jHW.js";
+import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-BYDPe3fV.js";
 import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { appendFile, readFile, rm } from "node:fs/promises";
 import { inspect } from "node:util";
+import { appendFile, readFile, rm } from "node:fs/promises";
 import { detectAgent } from "std-env";
 import { Command, Option } from "commander";
 //#region src/cli-parse.ts
@@ -57,6 +56,7 @@ function parseCli(argv) {
 	});
 	for (const [name, spec] of Object.entries(COMMANDS)) {
 		const command = configure(program.command(name), `reeval ${name}`, spec.flags);
+		if (name === "query") takeStatements(command);
 		command.argument("[args...]").action(() => {
 			const max = spec.sub && !spec.help ? Number.POSITIVE_INFINITY : spec.positionals;
 			parsed = done(program, command, [name], spec.flags, max);
@@ -65,13 +65,13 @@ function parseCli(argv) {
 			const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
 			subcommand.argument("[args...]").action(() => {
 				const [stray] = Object.keys(valuesOf(command, spec.flags));
-				if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}`);
+				if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
 				parsed = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
 			});
 		}
 	}
 	program.parse(argv, { from: "user" });
-	if (!parsed) throw new UsageError("Run reeval --help.");
+	if (!parsed) throw new UsageError("reeval couldn't read that command line. Run reeval --help.");
 	return parsed;
 }
 function configure(command, name, flags) {
@@ -85,6 +85,28 @@ function configure(command, name, flags) {
 	for (const flag of flags) command.addOption(OPTIONS[flag]());
 	return command;
 }
+/**
+* Makes an argument that starts with `-` but has whitespace before any `=` (SQL opening with a
+* `-- comment`) a positional: no flag looks like that, and commander would refuse it as one.
+*/
+function takeStatements(command) {
+	const parseOptions = command.parseOptions.bind(command);
+	command.parseOptions = (args) => {
+		const held = /* @__PURE__ */ new Map();
+		const masked = args.map((arg, i) => {
+			if (!arg.startsWith("-") || !/\s/.test(arg.split("=")[0] ?? "")) return arg;
+			const token = `\0statement-${i}`;
+			held.set(token, arg);
+			return token;
+		});
+		const restore = (list) => list.map((arg) => held.get(arg) ?? arg);
+		const { operands, unknown } = parseOptions(masked);
+		return {
+			operands: restore(operands),
+			unknown: restore(unknown)
+		};
+	};
+}
 function done(program, command, path, flags, max) {
 	const values = {
 		...valuesOf(program, GLOBAL),
@@ -118,16 +140,22 @@ function valuesOf(command, flags) {
 }
 /** Commander's error, in reeval's words, naming the command whose flag it is. */
 function usageError(error, command) {
-	const [first = "", ...rest] = error.message.replace(/^error: /, "").split("\n");
+	const text = error.message.replace(/^error: /, "");
+	const [first = ""] = text.split("\n");
 	if (error.code === "commander.unknownOption") {
-		const arg = /'([^']+)'/.exec(first)?.[1] ?? "";
+		const [, arg = "", suggestion] = /^unknown option '([\s\S]*)'(?:\n\((.*)\))?$/.exec(text) ?? [];
 		const [flag = "", value] = arg.replace(/^--?/, "").split("=");
 		const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
 		const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
-		if (known && value !== void 0 && known.type === "boolean") return new UsageError(`Option '--${flag}' does not take an argument`);
+		if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
 		if (known) return new UsageError(`--${long} isn't an option of ${command}. Run reeval --help.`);
-		const hint = rest.join(" ").replace(/^\(|\)$/g, "");
-		return new UsageError(`Unknown option '${arg}'. ${hint ? `${hint} ` : ""}Run reeval --help.`);
+		const shown = arg.replace(/\r/g, "\\r").replace(/\n/g, "\\n");
+		return new UsageError(`Unknown option '${shown}'. ${suggestion ? `${suggestion} ` : ""}Run reeval --help.`);
+	}
+	if (error.code === "commander.optionMissingArgument") {
+		const usage = /'([^']+)'/.exec(first)?.[1] ?? "";
+		const flag = usage.split(/[ ,]/).find((part) => part.startsWith("--")) ?? usage;
+		return new UsageError(`${flag} needs a value, e.g. ${usage}. Run reeval --help.`);
 	}
 	return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
 }
@@ -172,10 +200,10 @@ async function preflight(uses, options) {
 	writePrepared(options.root, options.runId, outcomes);
 	const images = [];
 	for (const o of outcomes) {
-		if (!o.ok || o.target.kind !== "docker") continue;
-		const { ref, id, agentCli } = o.handle;
+		if (!o.ok || o.target.sandbox.kind !== "docker") continue;
+		const { ref, id, agentCli } = attemptImage(o.handle);
 		if (images.some((i) => i.id === id)) continue;
-		const info = await imageInfo(ref);
+		const info = await imageInfo(ref, o.target.sandbox.engine);
 		images.push({
 			ref,
 			id,
@@ -215,24 +243,10 @@ function outcomeOf(e) {
 	if (e.meta?.errorKind === "aborted") return "cancelled";
 	return e.meta?.outcome ?? (e.state === "passed" ? "pass" : "fail");
 }
-function attemptOutcome(a) {
-	return a.error?.kind === "aborted" ? "cancelled" : a.outcome;
-}
 /** What tells an eval's attempts apart: the axes that vary (`cursor:composer-2.5 · with-skill`), else the harness. */
 function attemptAxes(a, fallback) {
 	return a?.label.replace(/\s*#\d+$/, "") || a?.harness.kind || fallback;
 }
-function mark(c, outcome) {
-	return {
-		pass: c.green("✓"),
-		fail: c.red("✗"),
-		infra_error: c.yellow("!"),
-		skipped: c.dim("↓"),
-		cancelled: c.yellow("✗")
-	}[outcome];
-}
-/** An expected miss's mark: neutral, since it doesn't fail the run. */
-const MISS_MARK = (c) => c.dim("○");
 const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
 /** A command on one line: the first line, cut to 80 characters at a word boundary. */
 function oneLine(text, max = 80) {
@@ -357,317 +371,25 @@ function prepareLines(c, e, max) {
 	return [`${c.red("✗")} ${outcome.target.label} failed to prepare ${c.dim(`(set by ${who}; only its attempts fail)`)}`, ...tailLines(outcome.failure.message, max).map((l) => `  ${c.yellow(l)}`)];
 }
 //#endregion
-//#region src/ui/summary.ts
-const FAILURE_LINES = 30;
-const KINDS = [
-	{
-		title: "Judge failures",
-		test: (a) => a.error?.kind === "judge"
-	},
-	{
-		title: "Assertion failures",
-		test: (a) => a.error?.kind === "assertion"
-	},
-	{
-		title: "Timeouts",
-		test: (a) => a.error?.kind === "timeout"
-	},
-	{
-		title: "Infra errors",
-		test: (_, o) => o === "infra_error"
-	},
-	{
-		title: "Cancelled",
-		test: (_, o) => o === "cancelled"
-	}
-];
-/** The engine's result for an attempt, or one rebuilt from the record (e.g. cancelled ones). */
-function resultFor(entry, attempt, results) {
-	const key = attemptKey(entry.id, attempt.cell, attempt.n);
-	const found = results.find((r) => r.key === key);
-	const errorKind = attempt.error?.kind ?? null;
-	const meta = {
-		outcome: attempt.outcome === "skipped" ? "pass" : attempt.outcome,
-		errorKind,
-		judgements: attempt.judgements.map((j) => ({ ...j })),
-		turnErrors: attempt.turns.flatMap((t, i) => t.error ? [`turn ${i + 1} error: ${t.error}`] : []),
-		recordPath: ""
-	};
-	if (found) return {
-		...found,
-		meta: found.meta ? {
-			...found.meta,
-			errorKind
-		} : meta
-	};
-	return {
-		id: entry.id,
-		key,
-		file: entry.file,
-		group: entry.group,
-		name: entry.name,
-		attempt: void 0,
-		state: attempt.outcome === "pass" ? "passed" : attempt.outcome === "skipped" ? "skipped" : "failed",
-		durationMs: attempt.durationMs,
-		errors: attempt.error ? [attempt.error.message] : [],
-		formattedErrors: [],
-		meta
-	};
-}
-function entryOutcome(e) {
-	const outcomes = e.attempts.map(attemptOutcome);
-	if (outcomes.length && outcomes.every((o) => o === "cancelled")) return "cancelled";
-	return e.outcome;
-}
-const pct = (rate) => rate === null ? "  – " : `${Math.round(rate * 100)}%`.padStart(4);
-function evalColumns(c, e) {
-	const [attempt, ...more] = e.attempts;
-	if (e.todo) return c.dim("todo");
-	if (!attempt || e.outcome === "skipped") return c.dim("skipped");
-	if (more.length) {
-		const pass = e.attempts.filter((a) => a.outcome === "pass").length;
-		const counted = e.attempts.filter((a) => a.outcome === "pass" || a.outcome === "fail").length;
-		const rate = hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : `  pass@1 ${pct(e.passRate)}`;
-		return c.dim(`${`${pass}/${counted}`.padStart(7)}${rate}  ${e.attempts.length} attempts`);
-	}
-	if (attemptOutcome(attempt) === "cancelled") return c.yellow("cancelled");
-	const cols = [formatDuration(attempt.durationMs).padStart(7)];
-	const judges = collapsedJudgeLabel(attempt.judgements);
-	if (judges) cols.push(judges);
-	const turns = attempt.turns.length;
-	cols.push(`${turns} turn${turns === 1 ? "" : "s"}`);
-	const tokens = formatUsage(attempt.usage);
-	if (tokens) cols.push(tokens);
-	return c.dim(cols.join("  "));
-}
-/** pass@1 per cell for evals with several attempts; with compare, each cell's role and lift. */
-function cellRows(c, e, indent) {
-	if (e.attempts.length < 2) return [];
-	const compare = e.cells.some((cell) => cell.role === "variant");
-	const width = Math.max(...e.cells.map((cell) => cell.key.length));
-	return e.cells.flatMap((cell) => {
-		const role = compare ? `  ${cell.role.padEnd(7)}` : "";
-		const counts = `${cell.pass}/${cell.pass + cell.fail}`.padStart(5);
-		const infra = cell.infraError ? c.yellow(`   ${cell.infraError} infra`) : "";
-		let note = "";
-		if (cell.lift) {
-			const { minLift } = cell.lift;
-			const below = minLift !== void 0 && cell.lift.interval.lo < minLift;
-			const text = `${liftLabel(cell.lift)}${minLift !== void 0 ? ` (minLift ${Math.round(minLift * 100)})` : ""}`;
-			note = `   ${below ? c.red(`${text} ✗`) : text}`;
-		} else if (compare && cell.gate === "none" && cell.fail > 0) note = `   ${c.dim("not gated")}`;
-		if (cell.gate !== "all" && cell.gate !== "none") note += `   ${c.dim(`gate ${gateLabel(cell.gate)}`)}`;
-		const tokens = formatUsage(cellUsage(e, cell.key));
-		const usage = tokens ? `   ${c.dim(tokens)}` : "";
-		const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role}  ${counts}  pass@1 ${pct(cell.passRate)}${note}${infra}${usage}`];
-		const rates = judgeRatesLabel(cellJudgeRates(e, cell.key));
-		if (rates) rows.push(`${indent}  ${c.dim(rates)}`);
-		const effort = cellContrast(e, cell.key);
-		if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effortLabel(effort)}`)}`);
-		const checks = checksLabel(cell);
-		if (checks) rows.push(`${indent}  ${c.dim(`checks: ${checks}`)}`);
-		if (cell.overrides.length) rows.push(`${indent}  ${c.dim(`overrides: ${cell.overrides.join(", ")}`)}`);
-		return rows;
-	});
-}
-/** One row per compare question: its win rates, or why it has none. They never gate. */
-function questionRows(c, e, indent) {
-	return (e.answers ?? []).flatMap((a) => {
-		const label = answersLabel(a);
-		return [`${indent}${c.dim("question")}  ${a.error !== void 0 ? c.yellow(label) : label}`, ...sampleReasons(a.votes).map((reason) => `${indent}  ${c.dim(reason)}`)];
-	});
-}
-/** A family's mark: its pooled gate's and its cases' worst. */
-function familyOutcome(family, cases) {
-	const outcomes = [family.outcome, ...cases.map(entryOutcome)];
-	for (const o of [
-		"fail",
-		"infra_error",
-		"cancelled"
-	]) if (outcomes.includes(o)) return o;
-	return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
-}
-/** The width case marks wrap at. */
-const CASE_WIDTH = 96;
-/** A pooled family's rows under its header: its cases' marks, then each pooled rate or lift. */
-function familyRows(c, family, cases, indent) {
-	const marks = cases.map((e) => `${mark(c, entryOutcome(e))} ${caseLabel(family.name, e.name)}`);
-	const plain = cases.map((e) => `x ${caseLabel(family.name, e.name)}`);
-	const rows = [];
-	let line = [];
-	let width = 0;
-	marks.forEach((m, i) => {
-		const w = plain[i].length + 2;
-		if (line.length && width + w > CASE_WIDTH) {
-			rows.push(`${indent}${line.join("  ")}`);
-			line = [];
-			width = 0;
-		}
-		line.push(m);
-		width += w;
-	});
-	if (line.length) rows.push(`${indent}${line.join("  ")}`);
-	if (family.lifts.length) {
-		const keyWidth = Math.max(...family.lifts.map((l) => l.variant.length));
-		for (const lift of family.lifts) {
-			const below = lift.outcome === "fail";
-			const gate = lift.minLift !== void 0 ? ` (minLift ${Math.round(lift.minLift * 100)}${below ? " ✗" : ""})` : "";
-			const text = `${familyLiftLabel(lift)}${gate}`;
-			rows.push(`${indent}${c.dim(lift.variant.padEnd(keyWidth))}  ${below ? c.red(text) : text}`);
-			const effort = familyEffortLabel(lift);
-			if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effort}`)}`);
-		}
-	} else if (family.rates.length === 1) rows.push(`${indent}${familyRateLabel(family.rates[0])}`);
-	else {
-		const keyWidth = Math.max(...family.rates.map((r) => r.cell.length));
-		for (const rate of family.rates) rows.push(`${indent}${c.dim(rate.cell.padEnd(keyWidth))}  ${familyRateLabel(rate)}`);
-	}
-	return rows;
-}
-/**
-* The run's evals as a file › group tree, one row per eval and per cell of a matrix. A pooled
-* family is one row, its template, with its cases' marks and the pooled results under it.
-*/
-function tree(c, evals, families = []) {
-	const rows = [];
-	const shown = /* @__PURE__ */ new Set();
-	let path = [];
-	for (const e of evals) {
-		const family = e.family && families.find((f) => f.id === e.family?.id);
-		if (family && shown.has(family.id)) continue;
-		const next = [e.file, ...e.group];
-		let same = 0;
-		while (same < path.length && same < next.length && path[same] === next[same]) same++;
-		for (let i = same; i < next.length; i++) rows.push({
-			indent: i,
-			text: i === 0 ? c.bold(next[i] ?? "") : next[i] ?? ""
-		});
-		path = next;
-		if (family) {
-			shown.add(family.id);
-			rows.push({
-				indent: next.length,
-				text: family.name,
-				family
-			});
-		} else rows.push({
-			indent: next.length,
-			text: e.name,
-			entry: e
-		});
-	}
-	const width = Math.max(...rows.filter((r) => r.entry || r.family).map((r) => r.indent * 2 + 2 + r.text.length));
-	return rows.flatMap(({ indent, text, entry, family }) => {
-		const pad = "  ".repeat(indent);
-		const fill = " ".repeat(Math.max(0, width - indent * 2 - 2 - text.length));
-		if (family) {
-			const cases = familyCases(family, evals);
-			return [` ${pad}${mark(c, familyOutcome(family, cases))} ${text}${fill}  ${c.dim(familyHeadLabel(family))}`, ...familyRows(c, family, cases, ` ${pad}    `)];
-		}
-		if (!entry) return [` ${pad}${text}`];
-		return [
-			` ${pad}${`${mark(c, entryOutcome(entry))} ${text}`}${fill}  ${evalColumns(c, entry)}`,
-			...cellRows(c, entry, ` ${pad}    `),
-			...questionRows(c, entry, ` ${pad}    `)
-		];
-	});
-}
-function totals(c, s, cancelled = 0) {
-	const fail = s.fail - s.notGated - cancelled;
-	const parts = [
-		s.pass && c.green(`${s.pass} passed`),
-		fail > 0 && c.red(`${fail} failed`),
-		s.notGated > 0 && c.dim(`${s.notGated} missed, not gated`),
-		s.infraError && c.yellow(`${s.infraError} infra error`),
-		cancelled && c.yellow(`${cancelled} cancelled`),
-		s.skipped && c.dim(`${s.skipped} skipped`)
-	].filter(Boolean);
-	const scope = s.attempts === s.evals ? `${s.evals}` : `${s.attempts} attempts, ${s.evals} evals`;
-	return parts.length ? `${parts.join(c.dim(" · "))} ${c.dim(`(${scope})`)}` : "no evals ran";
-}
-/** `1 passed · 1 failed (2)`: the families' pooled gates. */
-function familyTotals(c, families) {
-	const fail = families.filter((f) => f.outcome === "fail").length;
-	const pass = families.length - fail;
-	return `${[pass && c.green(`${pass} passed`), fail && c.red(`${fail} failed`)].filter(Boolean).join(c.dim(" · "))} ${c.dim(`(${families.length})`)}`;
-}
-/** Failing attempts, in run order, including expected misses. */
-function failingAttempts(evals) {
-	return evals.flatMap((entry) => entry.attempts.map((attempt) => ({
-		entry,
-		attempt,
-		outcome: attemptOutcome(attempt)
-	})).filter(({ outcome }) => outcome !== "pass" && outcome !== "skipped"));
-}
-function renderSummary(c, e, attemptDir) {
+//#region src/ui/result.ts
+function resultLines(c, e, verbose) {
 	const { record } = e;
-	const lines = [];
-	if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals, record.families));
-	const all = failingAttempts(record.evals);
-	const missed = all.filter(({ entry, attempt }) => isUngatedMiss(entry, attempt));
-	const failing = all.filter((f) => !missed.includes(f));
-	if (failing.length) {
-		lines.push("", c.bold(c.red(`Failures (${failing.length})`)));
-		for (const kind of KINDS) {
-			const members = failing.filter(({ attempt, outcome }) => kind.test(attempt, outcome));
-			if (!members.length) continue;
-			lines.push("", c.bold(`${kind.title} (${members.length})`));
-			for (const { entry, attempt, outcome } of members) {
-				const result = resultFor(entry, attempt, e.results);
-				const which = entry.attempts.length > 1 ? `  ${c.dim(attempt.id)}` : "";
-				const gated = entry.cells.find((x) => x.key === attempt.cell)?.gate !== "none";
-				lines.push(` ${mark(c, outcome)} ${entry.id}${which}${gated ? "" : `  ${c.dim("(not gated)")}`}`);
-				if (outcome !== "cancelled") for (const line of failureLines(c, result, { max: FAILURE_LINES })) lines.push(`   ${line}`);
-				if (!attempt.error?.prepare) lines.push(`   ${c.dim("record")}  ${attemptDir(entry.id, attempt.cell, attempt.n)}`);
-			}
-		}
-	}
-	if (missed.length) {
-		lines.push("", c.bold(`Expected misses, not gated (${missed.length})`));
-		const pooled = /* @__PURE__ */ new Map();
-		for (const { entry, attempt } of missed) {
-			if (!entry.family) continue;
-			const key = `${entry.family.id}\0${attempt.cell}`;
-			pooled.set(key, (pooled.get(key) ?? 0) + 1);
-		}
-		for (const [key, n] of pooled) {
-			const [id, cell] = key.split("\0");
-			lines.push(` ${MISS_MARK(c)} ${id}  ${c.dim(`${n} ${n === 1 ? "attempt" : "attempts"} on ${cell}`)}`);
-		}
-		for (const { entry, attempt } of missed) {
-			if (entry.family) continue;
-			const which = entry.attempts.length > 1 ? `  ${c.dim(attempt.id)}` : "";
-			const [reason] = failureLines(colors(false), resultFor(entry, attempt, e.results), { max: 1 }).filter((l) => l.trim());
-			lines.push(` ${MISS_MARK(c)} ${entry.id}${which}`);
-			if (reason) lines.push(`   ${c.dim(reason.trim())}`);
-			lines.push(`   ${c.dim("record")}  ${attemptDir(entry.id, attempt.cell, attempt.n)}`);
-		}
-	}
-	for (const error of e.unhandledErrors) lines.push("", c.red(`error: ${error}`));
-	const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
-	const label = record.summary.attempts === record.summary.evals ? "   Evals" : "Attempts";
-	const lifted = record.evals.flatMap((entry) => entry.cells.flatMap((cell) => cell.lift?.minLift !== void 0 && cell.lift.interval.lo < cell.lift.minLift ? [`${entry.id}  ${cell.key} ${liftLabel(cell.lift)}: its lower bound is below minLift ${Math.round(cell.lift.minLift * 100)}`] : []));
-	for (const family of record.families ?? []) for (const lift of family.lifts) {
-		if (lift.outcome !== "fail" || lift.minLift === void 0) continue;
-		lifted.push(`${family.id}  ${lift.variant} pooled ${familyLiftLabel(lift)}: its lower bound is below minLift ${Math.round(lift.minLift * 100)}`);
-	}
-	const idle = undiscriminating(record.evals);
-	if (idle.length) {
-		lines.push("", c.bold(`Checks that don't discriminate (${idle.length})`));
-		for (const x of idle) lines.push(` ${c.dim("·")} ${undiscriminatingLabel(x)}`);
-	}
-	if (lifted.length) {
-		lines.push("", c.bold(c.red(`Lift below minLift (${lifted.length})`)));
-		for (const line of lifted) lines.push(` ${c.red("✗")} ${line}`);
-	}
-	if (record.status === "budget") lines.push("", c.yellow("Budget reached: attempts that hadn’t started were skipped"));
-	lines.push("");
-	if (record.families?.length) lines.push(`${c.dim("Families")}  ${familyTotals(c, record.families)}`);
-	lines.push(`${c.dim(label)}  ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")}  ${formatDuration(record.durationMs)}`);
 	const tokens = formatUsage(record.cost.usage);
-	if (tokens) lines.push(`  ${c.dim("Tokens")}  ${tokens}`);
-	lines.push(` ${c.dim("Results")}  ${e.summaryPath}`);
-	return lines;
+	const notes = [
+		...record.status === "budget" ? [c.yellow("Budget reached: attempts that hadn’t started were skipped")] : [],
+		...record.status === "interrupted" ? [c.yellow("Interrupted: attempts that hadn’t finished were cancelled")] : [],
+		c.dim([
+			formatDuration(record.durationMs),
+			tokens,
+			`results ${e.summaryPath}`
+		].filter(Boolean).join(" · "))
+	];
+	return [...e.unhandledErrors.flatMap((error) => [c.red(`error: ${error}`), ""]), ...render(suiteResult(record), null, {
+		color: c.enabled,
+		verbose,
+		notes,
+		hint: e.hint
+	})];
 }
 //#endregion
 //#region src/ui/tags.ts
@@ -730,16 +452,9 @@ var Tagger = class {
 //#endregion
 //#region src/ui/agent.ts
 const MAX_LINES = 12;
-const WORD = {
-	fail: "FAIL",
-	infra_error: "INFRA",
-	cancelled: "CANCELLED"
-};
 var AgentRenderer = class {
 	#o;
 	#c = colors(false);
-	#logs = /* @__PURE__ */ new Map();
-	#printed = /* @__PURE__ */ new Set();
 	/** `[key]` prefixes for `-v` lines, from each attempt's start. */
 	#labels = /* @__PURE__ */ new Map();
 	#tags = new Tagger();
@@ -749,30 +464,6 @@ var AgentRenderer = class {
 	#write(text) {
 		this.#o.out.write(`${text.trimEnd()}\n`);
 	}
-	#failure(result, cell, n) {
-		const outcome = outcomeOf(result);
-		if (outcome === "pass" || outcome === "skipped") return;
-		this.#printed.add(result.key);
-		const kind = outcome === "fail" ? result.meta?.errorKind ?? "assertion" : null;
-		const detail = [
-			formatDuration(result.durationMs),
-			kind,
-			result.attempt?.gate === "none" ? "not gated" : ""
-		].filter(Boolean).join(", ");
-		const which = result.attempt?.label ? ` [${result.attempt.label}]` : "";
-		const missed = outcome === "fail" && result.attempt?.gate === "none";
-		this.#write(`${missed ? "MISS" : WORD[outcome]} ${result.id}${which} (${detail})`);
-		if (outcome !== "cancelled") {
-			const lines = failureLines(this.#c, result, { max: MAX_LINES });
-			for (const line of missed ? lines.filter((l) => l.trim()).slice(0, 1) : lines) this.#write(`  ${line}`);
-		}
-		const logs = missed ? void 0 : this.#logs.get(result.key);
-		if (logs?.length) {
-			this.#write("  console:");
-			for (const line of capLines(logs.join("\n"), MAX_LINES)) this.#write(`    ${line}`);
-		}
-		if (!result.meta?.prepare) this.#write(`  record: ${this.#o.attemptDir(result.id, cell, n)}`);
-	}
 	onEvent(e) {
 		switch (e.type) {
 			case "plan":
@@ -795,64 +486,22 @@ var AgentRenderer = class {
 				for (const line of debugLines(this.#c, e.line, this.#o.verbosity === 2)) this.#write(`${label} ${line}`);
 				return;
 			}
-			case "log": {
-				if (!e.key) return;
-				const logs = this.#logs.get(e.key) ?? [];
-				logs.push(...e.text.replace(/\n$/, "").split("\n"));
-				this.#logs.set(e.key, logs);
-				return;
-			}
 			case "eval-progress": {
 				if (e.progress.kind !== "browser-download") return;
 				const label = this.#labels.get(e.key) ?? `[${e.key}]`;
 				this.#write(`${label} ${progressLine(this.#c, e.progress)}`);
 				return;
 			}
-			case "eval-end": {
-				const { attempt } = e.result;
-				this.#failure(e.result, attempt?.cell.key ?? "", attempt?.n ?? 1);
-				return;
-			}
 			case "prepare":
 				for (const line of prepareLines(this.#c, e, MAX_LINES)) this.#write(line);
 				return;
 			case "notice":
 				if (e.level !== "info") this.#write(e.text);
 				return;
-			case "run-end": {
-				const failing = failingAttempts(e.record.evals);
-				for (const { entry, attempt, outcome } of failing) {
-					const result = resultFor(entry, attempt, e.results);
-					if (outcome === "cancelled" && !this.#printed.has(result.key)) this.#failure(result, attempt.cell, attempt.n);
-				}
-				for (const error of e.unhandledErrors) this.#write(`ERROR ${error}`);
-				for (const entry of e.record.evals) for (const a of entry.answers ?? []) {
-					this.#write(`question ${entry.id}: ${answersLabel(a)}`);
-					for (const reason of sampleReasons(a.votes)) this.#write(`  ${reason}`);
-				}
-				for (const x of undiscriminating(e.record.evals)) this.#write(`check doesn't discriminate: ${undiscriminatingLabel(x)}`);
-				for (const family of e.record.families ?? []) {
-					const pooled = family.lifts.length ? family.lifts.map((l) => {
-						const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)}${l.outcome === "fail" ? ", FAIL" : ""})`;
-						return `${l.variant} ${familyLiftLabel(l)}${gate}`;
-					}) : family.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
-					this.#write(`family ${family.id}: ${familyHeadLabel(family)} · ${pooled.join(" · ")}`);
-				}
-				const s = e.record.summary;
-				const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
-				const parts = [
-					s.pass && `${s.pass} passed`,
-					s.fail - s.notGated - cancelled > 0 && `${s.fail - s.notGated - cancelled} failed`,
-					s.notGated > 0 && `${s.notGated} missed, not gated`,
-					s.infraError && `${s.infraError} infra error`,
-					cancelled && `${cancelled} cancelled`,
-					s.skipped && `${s.skipped} skipped`
-				].filter(Boolean);
-				const scope = s.attempts === s.evals ? "" : ` (${s.attempts} attempts, ${s.evals} evals)`;
-				this.#write(parts.length ? `${parts.join(", ")}${scope}` : "no evals ran");
-				this.#write(`summary: ${e.summaryPath}`);
+			case "run-end":
+				this.#write("");
+				for (const line of resultLines(this.#c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
 				return;
-			}
 			default: return;
 		}
 	}
@@ -869,6 +518,8 @@ const STARTS = {
 	"browser-download": "browser-download"
 };
 const ERROR_LINES = 12;
+/** Back to the start of the line and clear it, for the live status line. */
+const CLEAR = "\r\x1B[2K";
 /** GitHub Actions workflow-command escaping. */
 const ghEscape = (s, property = false) => {
 	const out = s.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A");
@@ -894,10 +545,18 @@ var PlainRenderer = class {
 	#runStartedAt = 0;
 	#lastWrite = 0;
 	#timer;
+	/** On a TTY: the status line is drawn under everything else, and redrawn after each write. */
+	#live;
+	#status = "";
+	/** Evals and tries collected (not skipped), and the tries that ended. */
+	#evals = /* @__PURE__ */ new Set();
+	#tries = /* @__PURE__ */ new Set();
+	#ended = /* @__PURE__ */ new Set();
 	constructor(options) {
 		this.#o = options;
 		this.#c = options.colors;
 		this.#now = options.now ?? Date.now;
+		this.#live = options.output.requested === "live";
 		const { heartbeatMs } = options.output;
 		if (options.timers !== false && heartbeatMs > 0) {
 			this.#timer = setInterval(() => this.tick(), Math.min(heartbeatMs, 5e3));
@@ -905,9 +564,24 @@ var PlainRenderer = class {
 		}
 	}
 	#write(text) {
-		this.#o.out.write(`${text.trimEnd()}\n`);
+		const line = `${text.trimEnd()}\n`;
+		this.#o.out.write(this.#status ? `${CLEAR}${line}${this.#status}` : line);
 		this.#lastWrite = this.#now();
 	}
+	/** `7 evals · 12/28 tries`: what's collected so far and how many tries ended. */
+	#drawStatus() {
+		if (!this.#live || !this.#tries.size) return;
+		const evals = `${this.#evals.size} eval${this.#evals.size === 1 ? "" : "s"}`;
+		const status = this.#c.dim(`${evals} · ${this.#ended.size}/${this.#tries.size} tries`);
+		if (status === this.#status) return;
+		this.#o.out.write(`${CLEAR}${status}`);
+		this.#status = status;
+	}
+	#clearStatus() {
+		if (!this.#status) return;
+		this.#o.out.write(CLEAR);
+		this.#status = "";
+	}
 	#attempt(ref, startedAt) {
 		const n = ref.attempt?.n ?? 1;
 		const label = `[${this.#tags.label(ref.id, n)} ${attemptAxes(ref.attempt, this.#o.harness)}]`;
@@ -953,14 +627,20 @@ var PlainRenderer = class {
 				return;
 			case "collected":
 				this.#tags.addFile(e.evals);
-				for (const x of e.evals) if (!x.skipped) this.#queued.add(x.key);
+				for (const x of e.evals) {
+					if (x.skipped) continue;
+					this.#queued.add(x.key);
+					this.#evals.add(x.id);
+					this.#tries.add(x.key);
+				}
+				this.#drawStatus();
 				return;
 			case "eval-start": {
 				const a = this.#attempt(e, e.at);
 				a.last = e;
 				this.#queued.delete(e.key);
 				this.#running.set(e.key, a);
-				this.#write(`${this.#prefix(a)} ${c.dim("▶")} ${e.id}`);
+				if (this.#streaming) this.#write(`${this.#prefix(a)} ${c.dim("▶")} ${e.id}`);
 				return;
 			}
 			case "eval-progress": {
@@ -1008,6 +688,8 @@ var PlainRenderer = class {
 			}
 			case "eval-end":
 				this.#end(e.result, e.at);
+				if (this.#tries.has(e.result.key)) this.#ended.add(e.result.key);
+				this.#drawStatus();
 				return;
 			case "prepare":
 				for (const line of prepareLines(c, e, ERROR_LINES)) this.#write(line);
@@ -1028,8 +710,9 @@ var PlainRenderer = class {
 				return;
 			}
 			case "run-end":
+				this.#clearStatus();
 				this.#write("");
-				for (const line of renderSummary(c, e, this.#o.attemptDir)) this.#write(line);
+				for (const line of resultLines(c, e, (this.#o.verbosity ?? 0) > 0)) this.#write(line);
 				return;
 		}
 	}
@@ -1042,6 +725,11 @@ var PlainRenderer = class {
 		const title = w.kind === "root" ? "Sandbox runs as root" : "Sandbox has extra capabilities";
 		this.#write(`::warning ${file}title=${ghEscape(title, true)}::${ghEscape(w.message)}`);
 	}
+	/**
+	* An ended attempt's logs, as `--output-logs` picks: grouped, its block under a `▶` header;
+	* streamed, only its error lines, since the rest already printed. A sandbox that failed to
+	* prepare was annotated once, for all its attempts.
+	*/
 	#end(result, at) {
 		const c = this.#c;
 		const a = this.#running.get(result.key) ?? this.#attempt(result, at - result.durationMs);
@@ -1049,18 +737,17 @@ var PlainRenderer = class {
 		this.#queued.delete(result.key);
 		const outcome = outcomeOf(result);
 		const failed = outcome !== "pass" && outcome !== "skipped";
-		const time = outcome === "skipped" ? "" : `  ${c.dim(formatDuration(result.durationMs))}`;
-		const ungated = failed && result.attempt?.gate === "none" ? `  ${c.dim("(not gated)")}` : "";
-		const missed = ungated !== "" && outcome === "fail";
-		const head = `${this.#prefix(a)} ${missed ? MISS_MARK(c) : mark(c, outcome)} ${result.id}${time}${ungated}`;
+		const ungated = failed && result.attempt?.gate === "none";
+		const missed = ungated && outcome === "fail";
 		const { outputLogs, githubActions, logPrefix } = this.#o.output;
 		const showDetail = !this.#streaming && a.lines.length > 0 && (outputLogs === "full" || outputLogs === "errors-only" && failed);
 		const judged = result.meta?.errorKind === "judge" && (showDetail || this.#streaming);
 		const errors = failed && outputLogs !== "none" && !judged ? missed ? failureLines(colors(false), result, { max: 1 }).filter((l) => l.trim()).slice(0, 1).map((l) => c.dim(l)) : failureLines(c, result, { max: ERROR_LINES }) : [];
 		const lines = a.lines.filter((l) => l.kind !== "cleanup").sort((x, y) => x.at - y.at);
 		const prefixed = logPrefix === "task";
+		const head = `${this.#prefix(a)} ${c.dim("▶")} ${result.id}`;
 		const group = githubActions && showDetail && (!failed || missed);
-		this.#write(group ? `::group::${stripAnsi(head)}` : head);
+		if (showDetail || !this.#streaming && errors.length) this.#write(group ? `::group::${stripAnsi(head)}` : head);
 		if (showDetail) for (const line of lines) this.#write(this.#detail(a, line, prefixed));
 		for (const line of errors) this.#write(prefixed ? `${this.#prefix(a)}  ${line}` : ` ${line}`);
 		if (group) this.#write("::endgroup::");
@@ -1076,7 +763,7 @@ var PlainRenderer = class {
 	/** Prints the heartbeat if nothing was written for `heartbeatMs` and evals are running. */
 	tick(now = this.#now()) {
 		const { heartbeatMs } = this.#o.output;
-		if (heartbeatMs <= 0 || this.#running.size === 0) return;
+		if (this.#live || heartbeatMs <= 0 || this.#running.size === 0) return;
 		if (now - this.#lastWrite < heartbeatMs) return;
 		const c = this.#c;
 		const running = [...this.#running.values()].map((a) => {
@@ -1089,6 +776,7 @@ var PlainRenderer = class {
 	}
 	close() {
 		clearInterval(this.#timer);
+		this.#clearStatus();
 	}
 };
 //#endregion
@@ -1113,13 +801,13 @@ const CI_VARS = [
 const isCI = (env) => CI_VARS.some((name) => set(env, name));
 function pick(name, value, allowed) {
 	if (value === void 0 || value === "") return void 0;
-	if (value === "auto" || allowed.includes(value)) return value;
-	throw new UsageError(`${name} must be one of ${[...allowed, "auto"].join(", ")}`);
+	if (value === "auto" || isOneOf(allowed, value)) return value;
+	throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
 }
 function parseHeartbeat(value) {
 	if (!value) return 6e4;
 	const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
-	if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s or 0`);
+	if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
 	const unit = {
 		ms: 1,
 		s: 1e3,
@@ -1152,7 +840,7 @@ function resolveOutput(input) {
 		mode,
 		logOrder,
 		logPrefix,
-		outputLogs: logs && logs !== "auto" ? logs : "full",
+		outputLogs: logs && logs !== "auto" ? logs : flags.verbose ? "full" : "errors-only",
 		color: env.NO_COLOR ? false : env.FORCE_COLOR !== void 0 && env.FORCE_COLOR !== "" ? set(env, "FORCE_COLOR") : mode === "agent" ? false : input.isTTY || githubActions,
 		githubActions,
 		heartbeatMs: mode === "agent" ? 0 : parseHeartbeat(env.REEVAL_HEARTBEAT)
@@ -1219,12 +907,16 @@ async function run(argv) {
 		return EXIT.ok;
 	}
 	const [command, ...filters] = positionals;
+	if (values.help && command) {
+		console.log(commandHelpText(version, positionals));
+		return EXIT.ok;
+	}
 	if (values.help || !command) {
 		console.log(HELP);
 		return values.help ? EXIT.ok : EXIT.usage;
 	}
 	if (command === "init") {
-		const { initCommand } = await import("./init-B6vegaP_.js");
+		const { initCommand } = await import("./init-fhOXED8F.js");
 		return initCommand(values, filters, {
 			out: (s) => console.log(s),
 			input: process.stdin,
@@ -1233,7 +925,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "doctor") {
-		const { doctorCommand } = await import("./doctor-DlKycx5H.js");
+		const { doctorCommand } = await import("./doctor-D2MbNO8L.js");
 		return doctorCommand(values, {
 			out: (s) => console.log(s),
 			color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1241,7 +933,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "report") {
-		const { reportCommand } = await import("./command-CzIFp1JQ.js");
+		const { reportCommand } = await import("./command-DbAe1Rzb.js");
 		return reportCommand(values, filters, {
 			out: (s) => console.log(s),
 			err: (s) => console.error(dim(s)),
@@ -1256,7 +948,7 @@ async function run(argv) {
 	}
 	if (command === "studio") {
 		const config = await loadConfig(configPath, "studio");
-		const { studioCommand } = await import("./command-BFP0hKEb.js");
+		const { studioCommand } = await import("./command-BlM0y8jN.js");
 		return studioCommand({
 			config,
 			flags: values,
@@ -1267,7 +959,7 @@ async function run(argv) {
 	if (command === "mcp") {
 		if (!values.http) console.log = console.info = console.debug = console.error;
 		const config = await loadConfig(configPath, "mcp");
-		const { mcpCommand } = await import("./command-C5xFPKHM.js");
+		const { mcpCommand } = await import("./command-TgbzPizb.js");
 		return mcpCommand({
 			config,
 			flags: values,
@@ -1276,7 +968,7 @@ async function run(argv) {
 	}
 	if (command in HISTORY_COMMANDS) {
 		const config = await loadConfig(configPath, command);
-		const commands = await import("./commands-fI3G2Wnh.js");
+		const commands = await import("./commands-CthqwkAn.js");
 		const ctx = {
 			config,
 			flags: values,
@@ -1289,7 +981,7 @@ async function run(argv) {
 			case "compare": return commands.compareCommand(ctx, filters);
 			case "checkout": return commands.checkoutCommand(ctx, filters);
 			case "query": {
-				const { queryCommand } = await import("./query-oPO_XrTy.js");
+				const { queryCommand } = await import("./query-CRQfsw0e.js");
 				return queryCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1297,7 +989,7 @@ async function run(argv) {
 			}
 			case "analyze": {
 				loadEnvFiles(configPath);
-				const { analyzeCommand } = await import("./command-BrmXUC-W.js");
+				const { analyzeCommand } = await import("./command-C-cVjlny.js");
 				return analyzeCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1330,16 +1022,19 @@ async function cleanCommand(flags, configPath) {
 		keep = Number(flags.keep);
 		if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
 	}
-	const { debugRoot, pruneDebug } = await import("./files-BM78_nnF.js");
-	const { formatBytes } = await import("./debug-SEmW4w8f.js");
+	const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
+	const { formatBytes } = await import("./debug-D7bEv_UV.js");
 	const { runs, bytes } = await pruneDebug(config.root, keep);
 	const where = fromCwd(debugRoot(config.root)) || ".";
 	console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
 	if (flags.images) {
-		const { pruneImages } = await import("./src-cdIVjC5b.js");
-		const { requireDocker } = await import("./providers-CpEfNU1F.js");
-		await requireDocker();
-		const removed = await pruneImages();
+		const { dockerAvailable, pruneImages } = await import("./src-DYO3DdnQ.js");
+		const { requireDocker } = await import("./providers-B9yTdIqF.js");
+		const engines = [];
+		for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
+		if (!engines.length) await requireDocker();
+		let removed = 0;
+		for (const engine of engines) removed += await pruneImages(engine);
 		console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
 	}
 	return EXIT.ok;
@@ -1355,40 +1050,21 @@ function verbosityOf(flags) {
 * does. Needs a config, as `reeval run` does: there is no default image.
 */
 async function imageBuildCommand(flags, configPath, filters) {
-	const { buildImages, imagePlanLines, imageTargets } = await import("./images-BMgdpMAX.js");
+	const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-Bi3aGxb9.js");
 	const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
 		...flags,
 		"dry-run": true
 	}, configPath));
 	const root = config.root;
-	const { runEngine } = await import("./node-Co7EXge0.js");
-	let planned = [];
-	await runEngine({
-		run: JSON.parse(JSON.stringify({
-			runId,
-			stagingDir,
-			config,
-			runs,
-			tags,
-			concurrency,
-			profile
-		})),
-		filters,
-		onPlan: async (collected) => {
-			if (collected.unmatched.length) throw unmatchedFilter(config, collected.unmatched);
-			planned = collected.planned.filter((p) => !p.attempt.plan.skip);
-			return false;
-		}
-	});
-	const targets = imageTargets(planned.length ? planned.map((p) => ({
-		id: p.id,
-		harness: p.attempt.harness,
-		sandbox: p.attempt.sandbox
-	})) : config.harness.map((harness) => ({
-		id: "",
-		harness,
-		sandbox: config.sandbox
-	})));
+	const targets = imageTargets(await plannedImageUses({
+		runId,
+		stagingDir,
+		config,
+		runs,
+		tags,
+		concurrency,
+		profile
+	}, filters, (parts) => unmatchedFilter(config, parts)));
 	if (targets.length === 0) {
 		console.log("No eval runs in a Docker sandbox, so there are no images to build.");
 		return EXIT.ok;
@@ -1397,7 +1073,7 @@ async function imageBuildCommand(flags, configPath, filters) {
 		for (const line of await imagePlanLines(targets, root)) console.log(line);
 		return EXIT.ok;
 	}
-	const { PrepareError } = await import("./provider-BDbNOQhQ.js");
+	const { PrepareError } = await import("./provider-DehR4CMN.js");
 	let built;
 	try {
 		built = await buildImages(targets, root, {
@@ -1434,13 +1110,12 @@ async function runCommand(flags, configPath, filters) {
 	if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
 	const stepStart = process.env.GITHUB_STEP_SUMMARY;
 	if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
-	const { engineVersions } = await import("./node-Co7EXge0.js");
+	const { engineVersions } = await import("./node-B3ydHo6t.js");
 	const renderer = createRenderer({
-		out: setup.json ? { write: () => true } : process.stdout,
+		out: setup.json ? setup.dryRun ? { write: () => true } : process.stderr : process.stdout,
 		output: setup.output,
 		verbosity: ctx.verbosity ?? 0,
-		harness: setup.config.harness[0]?.kind ?? "agent",
-		attemptDir: (id, cell, n) => fromCwd(join(ctx.scratch, ctx.runId, evalSlug(id), evalSlug(cell), String(n)))
+		harness: setup.config.harness[0]?.kind ?? "agent"
 	});
 	let executed;
 	try {
@@ -1455,15 +1130,16 @@ async function runCommand(flags, configPath, filters) {
 	}
 	const { engineRun, plan, sandboxVersions, budgetReached, live, notes, questions } = executed;
 	invocation.unhandledErrors = engineRun.unhandledErrors;
-	if ((interrupted || engineRun.cancelled) && plan?.sandboxes.some((s) => s.kind === "docker")) {
-		const { removeRunContainers } = await import("./docker-Df_YOxxB.js");
-		await removeRunContainers(ctx.runId);
+	const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
+	if ((interrupted || engineRun.cancelled) && engines.length) {
+		const { removeRunContainers } = await import("./docker-DNmMPOBb.js");
+		await removeRunContainers(ctx.runId, engines);
 	}
 	if (notes) {
 		const cancelled = interrupted || engineRun.cancelled;
 		const written = await notes.finish(cancelled);
 		if (!cancelled && written.length) {
-			const { notesLine } = await import("./agent-6FefIhX7.js");
+			const { notesLine } = await import("./agent-D483ru7D.js");
 			const text = notesLine(ctx.config.root, ctx.runId, written);
 			renderer.onEvent({
 				type: "notice",
@@ -1480,8 +1156,8 @@ async function runCommand(flags, configPath, filters) {
 	}
 	const stopped = interrupted || engineRun.cancelled;
 	const answers = questions && !stopped ? await questions.settle() : void 0;
-	const { lockfiles } = await import("./state-Bo5zg4da.js");
-	const { writeScratchRun } = await import("./results-Di7m8dQ3.js");
+	const { lockfiles } = await import("./state-DpAxYyED.js");
+	const { writeScratchRun } = await import("./results-BE2f-1XS.js");
 	const run = await writeScratchRun({
 		ctx,
 		author: ctx.author,
@@ -1491,7 +1167,7 @@ async function runCommand(flags, configPath, filters) {
 		interrupted: stopped,
 		budgetReached: budgetReached(),
 		plan,
-		es: version,
+		reeval: version,
 		runtime: runtimeInfo(),
 		language: { name: "typescript" },
 		enginePackages: engineVersions,
@@ -1506,8 +1182,8 @@ async function runCommand(flags, configPath, filters) {
 	}, () => live?.stop());
 	const { record } = run;
 	if (notes && !stopped) {
-		const { summarizeRun } = await import("./agent-6FefIhX7.js");
-		const { resolveAnalysisModel } = await import("./model-FXko2kl7.js");
+		const { summarizeRun } = await import("./agent-D483ru7D.js");
+		const { resolveAnalysisModel } = await import("./model-CXhrziBe.js");
 		const model = resolveAnalysisModel(ctx.config, "agent");
 		const { level, text } = await summarizeRun(ctx.config, record.runId, model);
 		renderer.onEvent({
@@ -1517,7 +1193,7 @@ async function runCommand(flags, configPath, filters) {
 			text
 		});
 	}
-	const { pruneDebug } = await import("./files-BM78_nnF.js");
+	const { pruneDebug } = await import("./files-BAt0i00X.js");
 	await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
 	const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
 	renderer.onEvent({
@@ -1526,7 +1202,8 @@ async function runCommand(flags, configPath, filters) {
 		record,
 		summaryPath: fromCwd(summary),
 		results: engineRun.evals,
-		unhandledErrors: engineRun.unhandledErrors
+		unhandledErrors: engineRun.unhandledErrors,
+		hint: !hasAnalysisModel(ctx.config)
 	});
 	renderer.close();
 	const stepSummary = process.env.GITHUB_STEP_SUMMARY;
@@ -1534,12 +1211,18 @@ async function runCommand(flags, configPath, filters) {
 	const { config } = ctx;
 	const mode = setup.record;
 	if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
-		const { record: recordCmd } = await import("./commands-fI3G2Wnh.js");
+		const { record: recordCmd } = await import("./commands-CthqwkAn.js");
 		const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
-		console.log(await recordCmd(config, record.runId, {
+		const line = await recordCmd(config, record.runId, {
 			...flags,
 			...allow
-		}));
+		});
+		if (setup.json) console.error(line);
+		else console.log(line);
+	}
+	if (setup.json) {
+		const { suiteResult } = await import("./suite-DamAEnNF.js");
+		process.stdout.write(`${JSON.stringify(suiteResult(record), null, 2)}\n`);
 	}
 	if (interrupted || engineRun.cancelled) return stopCode;
 	if (engineRun.unhandledErrors.length > 0) return EXIT.failed;
@@ -1553,13 +1236,13 @@ async function prepareRun(flags, configPath) {
 			ui: flags.ui,
 			logOrder: flags["log-order"],
 			logPrefix: flags["log-prefix"],
-			outputLogs: flags["output-logs"]
+			outputLogs: flags["output-logs"],
+			verbose: verbosityOf(flags) > 0
 		},
 		env: process.env,
-		isTTY: !!process.stdout.isTTY,
+		isTTY: !!(flags.json && !flags["dry-run"] ? process.stderr : process.stdout).isTTY,
 		agent: detectAgent().name
 	});
-	if (flags.json && !flags["dry-run"]) throw new UsageError("reeval run --json only prints the plan: add --dry-run. A run writes its record to .reeval/runs; read it with reeval show --json.");
 	const path = requireConfig(configPath);
 	loadEnvFiles(path);
 	const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
@@ -1572,9 +1255,10 @@ async function prepareRun(flags, configPath) {
 	}
 	const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile?.defaults.environment ? [environmentSandbox(profile.defaults.environment, config.sandbox)] : []];
 	const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
-		if (!sandboxes.some((s) => s.kind === "docker")) return hostCapacity();
-		const { dockerCapacity } = await import("./src-cdIVjC5b.js");
-		return await dockerCapacity() ?? hostCapacity();
+		const container = sandboxes.find((s) => s.kind === "docker");
+		if (!container) return hostCapacity();
+		const { dockerCapacity } = await import("./src-DYO3DdnQ.js");
+		return await dockerCapacity(container.engine) ?? hostCapacity();
 	});
 	return {
 		output,
@@ -1600,11 +1284,11 @@ async function prepareRun(flags, configPath) {
 async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
 	const startedAt = /* @__PURE__ */ new Date();
 	const git = await gitInfo(config.root, [config.history.dir]);
-	const { worktreePatch } = await import("./state-Bo5zg4da.js");
+	const { worktreePatch } = await import("./state-DpAxYyED.js");
 	const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
 	const who = await author(config.root);
-	const { stagingDir, sweepStaging } = await import("./results-Di7m8dQ3.js");
-	const { scratchDir } = await import("./files-BMCvfB4X.js");
+	const { stagingDir, sweepStaging } = await import("./results-BE2f-1XS.js");
+	const { scratchDir } = await import("./files-Bw8lWEjg.js");
 	const scratch = scratchDir(config.root);
 	if (resume !== void 0) {
 		const target = await resolveResume(config.root, resume || void 0);
@@ -1615,8 +1299,8 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
 			artifacts: target.previousArtifacts
 		} : await loadPrevious(stagingDir(config.root, target.runId));
 		if (!dryRun) try {
-			const { removeRunContainers } = await import("./docker-Df_YOxxB.js");
-			await removeRunContainers(target.runId);
+			const { removeRunContainers } = await import("./docker-DNmMPOBb.js");
+			await removeRunContainers(target.runId, ["docker", "podman"]);
 		} catch {}
 		return {
 			runId: target.runId,
@@ -1686,8 +1370,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 		configFile,
 		completed: []
 	}));
-	const { runEngine } = await import("./node-Co7EXge0.js");
-	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DUydFc_1.js");
+	const { runEngine } = await import("./node-B3ydHo6t.js");
+	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DgQxp9yy.js");
 	let plan;
 	let sandboxVersions = { images: [] };
 	let preflightStop;
@@ -1714,7 +1398,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		} : p);
 		toRun = skipped;
 		for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
-		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-FXko2kl7.js") : void 0;
+		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-CXhrziBe.js") : void 0;
 		plan = await buildPlan({
 			planned: skipped,
 			profile,
@@ -1730,13 +1414,13 @@ async function executeRun(ctx, setup, filters, renderer) {
 			completed: completed.size
 		};
 		run.completed = [...completed];
-		const { imageStages, imageTargets } = await import("./images-BMgdpMAX.js");
+		const { imageStages, imageTargets } = await import("./images-Bi3aGxb9.js");
 		const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
 			id: p.id,
 			harness: p.attempt.harness,
 			sandbox: p.attempt.sandbox
 		})));
-		if (setup.json) {
+		if (setup.json && setup.dryRun) {
 			const images = await Promise.all(targets.map(async (t) => ({
 				id: t.id,
 				harness: t.harness,
@@ -1751,10 +1435,16 @@ async function executeRun(ctx, setup, filters, renderer) {
 		if (setup.dryRun) {
 			const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
 			lines.push(...planTable(plan, todos));
-			const { imageStatusLines } = await import("./images-BMgdpMAX.js");
-			const { dockerAvailable } = await import("./src-cdIVjC5b.js");
-			if (targets.length && await dockerAvailable()) lines.push(...await imageStatusLines(targets, config.root));
-			else if (targets.length) lines.push("images  not checked: Docker is not running");
+			const { imageStatusLines } = await import("./images-Bi3aGxb9.js");
+			const { dockerAvailable, engineSpec } = await import("./src-DYO3DdnQ.js");
+			const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
+			const down = [];
+			for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
+			if (targets.length && !down.length) lines.push(...await imageStatusLines(targets, config.root));
+			else if (targets.length) {
+				const names = down.map((e) => engineSpec(e).name).join(" and ");
+				lines.push(`images  not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
+			}
 		}
 		lines.push(...plan.warnings.map((w) => `warning: ${w}`));
 		lines.push(...plan.hints.map((h) => `hint: ${h}`));
@@ -1766,7 +1456,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (setup.dryRun) return false;
 		checkAttemptBudget(plan, setup.force, config.file);
-		const analysis = config.analysis.agent ? await import("./model-FXko2kl7.js") : null;
+		const analysis = config.analysis.agent ? await import("./model-CXhrziBe.js") : null;
 		analysis?.analysisModelName(config, "agent");
 		const questionModels = /* @__PURE__ */ new Set();
 		const asked = [];
@@ -1775,7 +1465,9 @@ async function executeRun(ctx, setup, filters, renderer) {
 			const questions = (row.plan.questions ?? []).map((q) => ({
 				question: q.question,
 				model: models.questionModel(config, q, row.id),
-				...q.across ? { across: q.across } : {}
+				...q.across ? { across: q.across } : {},
+				...q.state ? { state: q.state } : {},
+				...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
 			}));
 			for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
 			if (questions.length) asked.push({
@@ -1796,7 +1488,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
 		if (analysis) {
-			const { startNotes } = await import("./agent-6FefIhX7.js");
+			const { startNotes } = await import("./agent-D483ru7D.js");
 			notes = startNotes({
 				config,
 				model: analysis.resolveAnalysisModel(config, "agent"),
@@ -1812,7 +1504,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			});
 		}
 		if (asked.length) {
-			const { questionRunner } = await import("./questions-B8by4Bbt.js");
+			const { questionRunner } = await import("./questions-C2dym2ce.js");
 			questions = questionRunner({
 				evals: asked,
 				stagingDir,
@@ -1829,7 +1521,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 				}
 			});
 		}
-		const { prepareWorkspaces, workspaceKey } = await import("./workspace-Do_SiWBs.js");
+		const { prepareWorkspaces, workspaceKey } = await import("./workspace-C_X0tdv6.js");
 		const live = skipped.filter((p) => !p.attempt.plan.skip);
 		const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
 			workspace: a.workspace,
@@ -1977,14 +1669,14 @@ async function executeRun(ctx, setup, filters, renderer) {
 		throw error;
 	} finally {
 		clearTimeout(budgetTimer);
-		const { removePrepared } = await import("./preflight-D5LW7Oqi.js");
+		const { removePrepared } = await import("./preflight-DlQvbySb.js");
 		removePrepared(config.root, runId);
-		const { removePreparedWorkspaces } = await import("./workspace-Do_SiWBs.js");
+		const { removePreparedWorkspaces } = await import("./workspace-C_X0tdv6.js");
 		removePreparedWorkspaces(config.root, runId);
 	}
 	if (preflightStop) throw preflightStop;
 	if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) live?.remove();
-	if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files match ${config.include.join(", ")} (config.include) in ${config.root}. Name eval files to match it, or set include in ${config.file}.`);
+	if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files in ${config.root} match include: ${config.include.join(", ")}. Name eval files to match it, or set include in ${config.file}.`);
 	if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) throw noMatch(config, filters, tags);
 	return {
 		engineRun,
@@ -2013,7 +1705,7 @@ function finish(code, error, kind) {
 	const command = commandOf(argv);
 	logInvocation(root, {
 		at: new Date(startedAt).toISOString(),
-		es: version,
+		reeval: version,
 		argv: redactArgv(argv),
 		command,
 		exitCode: code,
@@ -2062,4 +1754,4 @@ run(process.argv.slice(2)).finally(() => {
 //#endregion
 export {};
 
-//# sourceMappingURL=cli-CVrMh_qz.js.map
+//# sourceMappingURL=cli-ClpPT8kn.js.map