reeval 0.9.0 → 0.10.0

package/dist/{cli-CVrMh_qz.js → cli-D303Vj5N.js}

@@ -1,33 +1,34 @@
-import { n as UsageError } from "./errors-yDAEwmZo.js";
-import { t as parseDuration } from "./duration-JsJrIWXq.js";
+import { n as isOneOf } from "./object-CLf2-tNq.js";
+import { n as UsageError } from "./errors-B6BSMs03.js";
+import { t as parseDuration } from "./duration-CEOQww97.js";
+import { f as sandboxId, s as harnessId } from "./options-Bh2W63dK.js";
+import { A as workspaceLabel, p as environmentSandbox } from "./environment-DNjRk0nU.js";
+import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
+import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-DEgnUiU-.js";
 import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
 import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
-import { o as PACKAGE_VERSION } from "./root-CvdRlRZX.js";
-import { T as workspaceLabel, u as environmentSandbox } from "./environment-1jE-HhO-.js";
-import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
-import { I as LABEL, X as imageInfo } from "./src-8rRXy4u9.js";
-import { d as sandboxId, s as harnessId } from "./options-CKqyPGxX.js";
-import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-CGvBeewH.js";
+import { o as PACKAGE_VERSION } from "./root-DICDHCtp.js";
+import { I as LABEL, X as imageInfo } from "./src-B7treWa8.js";
 import { s as debugTime } from "./debug-D24NOQH1.js";
 import { i as sandboxWarningLine, n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
 import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { H as cellContrast, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, P as collapsedJudgeLabel, _ as hasUngatedCells, b as isUngatedMiss, h as formatUsage, q as effortLabel, w as runOutcome } from "./model-BOnlAPzs.js";
+import { B as judgeRatesLabel, H as summaryJudgeRates, I as collapsedJudgeLabel, P as cellUsage, T as runOutcome, Y as cellContrast, g as formatUsage, l as evalMarkOutcome, tt as effortLabel, v as hasUngatedCells, x as isUngatedMiss } from "./model-oqcDplJG.js";
 import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "./family-D2b7WD10.js";
-import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
-import { a as gateLabel, d as undiscriminating, f as undiscriminatingLabel, i as checksLabel, n as answersLabel, o as liftLabel } from "./summary-BTksrIxn.js";
-import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-SUz1GskV.js";
-import { n as FLAGS, o as helpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-CKEYa8V2.js";
-import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-D7HK3vzt.js";
+import { n as headToHeadLines, s as sampleReasons } from "./pairs-CnInRu-G.js";
+import { a as contrastLines, f as undiscriminating, i as checksLabel, n as answersLabel, o as gateLabel, p as undiscriminatingLabel, s as liftLabel } from "./summary-BLAVLySU.js";
+import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-BRfa8E_s.js";
+import { a as commandHelpText, n as FLAGS, r as GLOBAL, s as helpText, t as COMMANDS } from "./cli-spec-BPZbQG2p.js";
+import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-DmE9uPVt.js";
 import { t as budgetMarker } from "./budget-DBUrH4jX.js";
-import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-BkoSxf4C.js";
-import { r as imageHarness } from "./prepare-CUAWHebF.js";
-import { n as createProviders } from "./providers-B1DzMvmH.js";
-import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CLaL8AuW.js";
+import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-D7iFPKeK.js";
+import { n as imageHarness } from "./prepare-doGxaAGR.js";
+import { r as createProviders, t as attemptImage } from "./providers-tAOD1NQN.js";
+import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DbGfBAA_.js";
 import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
 import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { appendFile, readFile, rm } from "node:fs/promises";
 import { inspect } from "node:util";
+import { appendFile, readFile, rm } from "node:fs/promises";
 import { detectAgent } from "std-env";
 import { Command, Option } from "commander";
 //#region src/cli-parse.ts
@@ -65,13 +66,13 @@ function parseCli(argv) {
 			const subcommand = configure(command.command(subName), `reeval ${name} ${subName}`, sub.flags);
 			subcommand.argument("[args...]").action(() => {
 				const [stray] = Object.keys(valuesOf(command, spec.flags));
-				if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}`);
+				if (stray) throw new UsageError(`--${stray} isn't an option of reeval ${name} ${subName}. Run reeval --help.`);
 				parsed = done(program, subcommand, [name, subName], sub.flags, sub.positionals);
 			});
 		}
 	}
 	program.parse(argv, { from: "user" });
-	if (!parsed) throw new UsageError("Run reeval --help.");
+	if (!parsed) throw new UsageError("reeval couldn't read that command line. Run reeval --help.");
 	return parsed;
 }
 function configure(command, name, flags) {
@@ -124,11 +125,16 @@ function usageError(error, command) {
 		const [flag = "", value] = arg.replace(/^--?/, "").split("=");
 		const known = arg.startsWith("--") ? FLAGS[flag] : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[1];
 		const long = arg.startsWith("--") ? flag : Object.entries(FLAGS).find(([, s]) => "short" in s && s.short === flag)?.[0];
-		if (known && value !== void 0 && known.type === "boolean") return new UsageError(`Option '--${flag}' does not take an argument`);
+		if (known && value !== void 0 && known.type === "boolean") return new UsageError(`--${flag} doesn't take a value: pass --${flag} on its own.`);
 		if (known) return new UsageError(`--${long} isn't an option of ${command}. Run reeval --help.`);
 		const hint = rest.join(" ").replace(/^\(|\)$/g, "");
 		return new UsageError(`Unknown option '${arg}'. ${hint ? `${hint} ` : ""}Run reeval --help.`);
 	}
+	if (error.code === "commander.optionMissingArgument") {
+		const usage = /'([^']+)'/.exec(first)?.[1] ?? "";
+		const flag = usage.split(/[ ,]/).find((part) => part.startsWith("--")) ?? usage;
+		return new UsageError(`${flag} needs a value, e.g. ${usage}. Run reeval --help.`);
+	}
 	return new UsageError(first.charAt(0).toUpperCase() + first.slice(1));
 }
 //#endregion
@@ -172,10 +178,10 @@ async function preflight(uses, options) {
 	writePrepared(options.root, options.runId, outcomes);
 	const images = [];
 	for (const o of outcomes) {
-		if (!o.ok || o.target.kind !== "docker") continue;
-		const { ref, id, agentCli } = o.handle;
+		if (!o.ok || o.target.sandbox.kind !== "docker") continue;
+		const { ref, id, agentCli } = attemptImage(o.handle);
 		if (images.some((i) => i.id === id)) continue;
-		const info = await imageInfo(ref);
+		const info = await imageInfo(ref, o.target.sandbox.engine);
 		images.push({
 			ref,
 			id,
@@ -417,8 +423,10 @@ function resultFor(entry, attempt, results) {
 function entryOutcome(e) {
 	const outcomes = e.attempts.map(attemptOutcome);
 	if (outcomes.length && outcomes.every((o) => o === "cancelled")) return "cancelled";
-	return e.outcome;
+	return evalMarkOutcome(e);
 }
+/** Every cell is `gate: 'none'`: the eval reports its attempts but nothing gates it. */
+const allUngated = (e) => e.cells.length > 0 && e.cells.every((c) => c.gate === "none");
 const pct = (rate) => rate === null ? "  – " : `${Math.round(rate * 100)}%`.padStart(4);
 function evalColumns(c, e) {
 	const [attempt, ...more] = e.attempts;
@@ -427,7 +435,7 @@ function evalColumns(c, e) {
 	if (more.length) {
 		const pass = e.attempts.filter((a) => a.outcome === "pass").length;
 		const counted = e.attempts.filter((a) => a.outcome === "pass" || a.outcome === "fail").length;
-		const rate = hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : `  pass@1 ${pct(e.passRate)}`;
+		const rate = allUngated(e) ? " · not gated" : hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : `  pass@1 ${pct(e.passRate)}`;
 		return c.dim(`${`${pass}/${counted}`.padStart(7)}${rate}  ${e.attempts.length} attempts`);
 	}
 	if (attemptOutcome(attempt) === "cancelled") return c.yellow("cancelled");
@@ -460,7 +468,7 @@ function cellRows(c, e, indent) {
 		const tokens = formatUsage(cellUsage(e, cell.key));
 		const usage = tokens ? `   ${c.dim(tokens)}` : "";
 		const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role}  ${counts}  pass@1 ${pct(cell.passRate)}${note}${infra}${usage}`];
-		const rates = judgeRatesLabel(cellJudgeRates(e, cell.key));
+		const rates = judgeRatesLabel(summaryJudgeRates(e, cell));
 		if (rates) rows.push(`${indent}  ${c.dim(rates)}`);
 		const effort = cellContrast(e, cell.key);
 		if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effortLabel(effort)}`)}`);
@@ -470,11 +478,21 @@ function cellRows(c, e, indent) {
 		return rows;
 	});
 }
+/** With two variants or more, each pair of arms on one harness: lift, checks and effort. */
+function contrastRows(c, e, indent) {
+	const lines = contrastLines(e);
+	if (!lines.length) return [];
+	return [`${indent}${c.dim("contrasts (each pair of arms, same harness)")}`, ...lines.map((line) => `${indent}  ${c.dim(line)}`)];
+}
 /** One row per compare question: its win rates, or why it has none. They never gate. */
 function questionRows(c, e, indent) {
 	return (e.answers ?? []).flatMap((a) => {
 		const label = answersLabel(a);
-		return [`${indent}${c.dim("question")}  ${a.error !== void 0 ? c.yellow(label) : label}`, ...sampleReasons(a.votes).map((reason) => `${indent}  ${c.dim(reason)}`)];
+		return [
+			`${indent}${c.dim("question")}  ${a.error !== void 0 ? c.yellow(label) : label}`,
+			...headToHeadLines(a.votes).map((line) => `${indent}  ${line}`),
+			...sampleReasons(a.votes).map((reason) => `${indent}  ${c.dim(reason)}`)
+		];
 	});
 }
 /** A family's mark: its pooled gate's and its cases' worst. */
@@ -566,8 +584,9 @@ function tree(c, evals, families = []) {
 		}
 		if (!entry) return [` ${pad}${text}`];
 		return [
-			` ${pad}${`${mark(c, entryOutcome(entry))} ${text}`}${fill}  ${evalColumns(c, entry)}`,
+			` ${pad}${`${entryOutcome(entry) === "pass" && allUngated(entry) && entry.attempts.some((a) => isUngatedMiss(entry, a)) ? MISS_MARK(c) : mark(c, entryOutcome(entry))} ${text}`}${fill}  ${evalColumns(c, entry)}`,
 			...cellRows(c, entry, ` ${pad}    `),
+			...contrastRows(c, entry, ` ${pad}    `),
 			...questionRows(c, entry, ` ${pad}    `)
 		];
 	});
@@ -828,6 +847,7 @@ var AgentRenderer = class {
 				for (const error of e.unhandledErrors) this.#write(`ERROR ${error}`);
 				for (const entry of e.record.evals) for (const a of entry.answers ?? []) {
 					this.#write(`question ${entry.id}: ${answersLabel(a)}`);
+					for (const line of headToHeadLines(a.votes)) this.#write(`  ${line}`);
 					for (const reason of sampleReasons(a.votes)) this.#write(`  ${reason}`);
 				}
 				for (const x of undiscriminating(e.record.evals)) this.#write(`check doesn't discriminate: ${undiscriminatingLabel(x)}`);
@@ -1113,13 +1133,13 @@ const CI_VARS = [
 const isCI = (env) => CI_VARS.some((name) => set(env, name));
 function pick(name, value, allowed) {
 	if (value === void 0 || value === "") return void 0;
-	if (value === "auto" || allowed.includes(value)) return value;
-	throw new UsageError(`${name} must be one of ${[...allowed, "auto"].join(", ")}`);
+	if (value === "auto" || isOneOf(allowed, value)) return value;
+	throw new UsageError(`${name} is one of ${[...allowed, "auto"].join(", ")} (got ${value})`);
 }
 function parseHeartbeat(value) {
 	if (!value) return 6e4;
 	const match = /^(\d+(?:\.\d+)?)(ms|s|m)?$/.exec(value);
-	if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s or 0`);
+	if (!match) throw new UsageError(`REEVAL_HEARTBEAT must be a duration like 30s, or 0 for none (got ${value})`);
 	const unit = {
 		ms: 1,
 		s: 1e3,
@@ -1219,12 +1239,16 @@ async function run(argv) {
 		return EXIT.ok;
 	}
 	const [command, ...filters] = positionals;
+	if (values.help && command) {
+		console.log(commandHelpText(version, positionals));
+		return EXIT.ok;
+	}
 	if (values.help || !command) {
 		console.log(HELP);
 		return values.help ? EXIT.ok : EXIT.usage;
 	}
 	if (command === "init") {
-		const { initCommand } = await import("./init-B6vegaP_.js");
+		const { initCommand } = await import("./init-BrOWin3W.js");
 		return initCommand(values, filters, {
 			out: (s) => console.log(s),
 			input: process.stdin,
@@ -1233,7 +1257,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "doctor") {
-		const { doctorCommand } = await import("./doctor-DlKycx5H.js");
+		const { doctorCommand } = await import("./doctor-BYNok4UO.js");
 		return doctorCommand(values, {
 			out: (s) => console.log(s),
 			color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1241,7 +1265,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "report") {
-		const { reportCommand } = await import("./command-CzIFp1JQ.js");
+		const { reportCommand } = await import("./command-BLfT7vnQ.js");
 		return reportCommand(values, filters, {
 			out: (s) => console.log(s),
 			err: (s) => console.error(dim(s)),
@@ -1256,7 +1280,7 @@ async function run(argv) {
 	}
 	if (command === "studio") {
 		const config = await loadConfig(configPath, "studio");
-		const { studioCommand } = await import("./command-BFP0hKEb.js");
+		const { studioCommand } = await import("./command-CMJtHKRa.js");
 		return studioCommand({
 			config,
 			flags: values,
@@ -1267,7 +1291,7 @@ async function run(argv) {
 	if (command === "mcp") {
 		if (!values.http) console.log = console.info = console.debug = console.error;
 		const config = await loadConfig(configPath, "mcp");
-		const { mcpCommand } = await import("./command-C5xFPKHM.js");
+		const { mcpCommand } = await import("./command-BT9a37ZR.js");
 		return mcpCommand({
 			config,
 			flags: values,
@@ -1276,7 +1300,7 @@ async function run(argv) {
 	}
 	if (command in HISTORY_COMMANDS) {
 		const config = await loadConfig(configPath, command);
-		const commands = await import("./commands-fI3G2Wnh.js");
+		const commands = await import("./commands-1p-bIWhN.js");
 		const ctx = {
 			config,
 			flags: values,
@@ -1289,7 +1313,7 @@ async function run(argv) {
 			case "compare": return commands.compareCommand(ctx, filters);
 			case "checkout": return commands.checkoutCommand(ctx, filters);
 			case "query": {
-				const { queryCommand } = await import("./query-oPO_XrTy.js");
+				const { queryCommand } = await import("./query-C_G_svLU.js");
 				return queryCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1297,7 +1321,7 @@ async function run(argv) {
 			}
 			case "analyze": {
 				loadEnvFiles(configPath);
-				const { analyzeCommand } = await import("./command-BrmXUC-W.js");
+				const { analyzeCommand } = await import("./command-BkFjijN2.js");
 				return analyzeCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1330,16 +1354,19 @@ async function cleanCommand(flags, configPath) {
 		keep = Number(flags.keep);
 		if (!(Number.isInteger(keep) && keep >= 0)) throw new UsageError(`--keep must be a whole number (got ${flags.keep})`);
 	}
-	const { debugRoot, pruneDebug } = await import("./files-BM78_nnF.js");
-	const { formatBytes } = await import("./debug-SEmW4w8f.js");
+	const { debugRoot, pruneDebug } = await import("./files-BAt0i00X.js");
+	const { formatBytes } = await import("./debug-D7bEv_UV.js");
 	const { runs, bytes } = await pruneDebug(config.root, keep);
 	const where = fromCwd(debugRoot(config.root)) || ".";
 	console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
 	if (flags.images) {
-		const { pruneImages } = await import("./src-cdIVjC5b.js");
-		const { requireDocker } = await import("./providers-CpEfNU1F.js");
-		await requireDocker();
-		const removed = await pruneImages();
+		const { dockerAvailable, pruneImages } = await import("./src-pJZOrYR9.js");
+		const { requireDocker } = await import("./providers-gDMko8fl.js");
+		const engines = [];
+		for (const engine of ["docker", "podman"]) if (await dockerAvailable(engine)) engines.push(engine);
+		if (!engines.length) await requireDocker();
+		let removed = 0;
+		for (const engine of engines) removed += await pruneImages(engine);
 		console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
 	}
 	return EXIT.ok;
@@ -1355,40 +1382,21 @@ function verbosityOf(flags) {
 * does. Needs a config, as `reeval run` does: there is no default image.
 */
 async function imageBuildCommand(flags, configPath, filters) {
-	const { buildImages, imagePlanLines, imageTargets } = await import("./images-BMgdpMAX.js");
+	const { buildImages, imagePlanLines, imageTargets, plannedImageUses } = await import("./images-0On0cztu.js");
 	const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
 		...flags,
 		"dry-run": true
 	}, configPath));
 	const root = config.root;
-	const { runEngine } = await import("./node-Co7EXge0.js");
-	let planned = [];
-	await runEngine({
-		run: JSON.parse(JSON.stringify({
-			runId,
-			stagingDir,
-			config,
-			runs,
-			tags,
-			concurrency,
-			profile
-		})),
-		filters,
-		onPlan: async (collected) => {
-			if (collected.unmatched.length) throw unmatchedFilter(config, collected.unmatched);
-			planned = collected.planned.filter((p) => !p.attempt.plan.skip);
-			return false;
-		}
-	});
-	const targets = imageTargets(planned.length ? planned.map((p) => ({
-		id: p.id,
-		harness: p.attempt.harness,
-		sandbox: p.attempt.sandbox
-	})) : config.harness.map((harness) => ({
-		id: "",
-		harness,
-		sandbox: config.sandbox
-	})));
+	const targets = imageTargets(await plannedImageUses({
+		runId,
+		stagingDir,
+		config,
+		runs,
+		tags,
+		concurrency,
+		profile
+	}, filters, (parts) => unmatchedFilter(config, parts)));
 	if (targets.length === 0) {
 		console.log("No eval runs in a Docker sandbox, so there are no images to build.");
 		return EXIT.ok;
@@ -1397,7 +1405,7 @@ async function imageBuildCommand(flags, configPath, filters) {
 		for (const line of await imagePlanLines(targets, root)) console.log(line);
 		return EXIT.ok;
 	}
-	const { PrepareError } = await import("./provider-BDbNOQhQ.js");
+	const { PrepareError } = await import("./provider-DehR4CMN.js");
 	let built;
 	try {
 		built = await buildImages(targets, root, {
@@ -1434,7 +1442,7 @@ async function runCommand(flags, configPath, filters) {
 	if (githubOut) await appendFile(githubOut, `run-id=${ctx.runId}\n`).catch(() => {});
 	const stepStart = process.env.GITHUB_STEP_SUMMARY;
 	if (stepStart) await appendFile(stepStart, `run ${ctx.runId}\n`).catch(() => {});
-	const { engineVersions } = await import("./node-Co7EXge0.js");
+	const { engineVersions } = await import("./node-B3ydHo6t.js");
 	const renderer = createRenderer({
 		out: setup.json ? { write: () => true } : process.stdout,
 		output: setup.output,
@@ -1455,15 +1463,16 @@ async function runCommand(flags, configPath, filters) {
 	}
 	const { engineRun, plan, sandboxVersions, budgetReached, live, notes, questions } = executed;
 	invocation.unhandledErrors = engineRun.unhandledErrors;
-	if ((interrupted || engineRun.cancelled) && plan?.sandboxes.some((s) => s.kind === "docker")) {
-		const { removeRunContainers } = await import("./docker-Df_YOxxB.js");
-		await removeRunContainers(ctx.runId);
+	const engines = [...new Set(plan?.sandboxes.flatMap((s) => s.kind === "docker" ? [s.engine ?? "docker"] : []))];
+	if ((interrupted || engineRun.cancelled) && engines.length) {
+		const { removeRunContainers } = await import("./docker-yauK-9X8.js");
+		await removeRunContainers(ctx.runId, engines);
 	}
 	if (notes) {
 		const cancelled = interrupted || engineRun.cancelled;
 		const written = await notes.finish(cancelled);
 		if (!cancelled && written.length) {
-			const { notesLine } = await import("./agent-6FefIhX7.js");
+			const { notesLine } = await import("./agent-CURAwx_q.js");
 			const text = notesLine(ctx.config.root, ctx.runId, written);
 			renderer.onEvent({
 				type: "notice",
@@ -1480,8 +1489,8 @@ async function runCommand(flags, configPath, filters) {
 	}
 	const stopped = interrupted || engineRun.cancelled;
 	const answers = questions && !stopped ? await questions.settle() : void 0;
-	const { lockfiles } = await import("./state-Bo5zg4da.js");
-	const { writeScratchRun } = await import("./results-Di7m8dQ3.js");
+	const { lockfiles } = await import("./state-CgjQo29K.js");
+	const { writeScratchRun } = await import("./results-DuauQI3Z.js");
 	const run = await writeScratchRun({
 		ctx,
 		author: ctx.author,
@@ -1491,7 +1500,7 @@ async function runCommand(flags, configPath, filters) {
 		interrupted: stopped,
 		budgetReached: budgetReached(),
 		plan,
-		es: version,
+		reeval: version,
 		runtime: runtimeInfo(),
 		language: { name: "typescript" },
 		enginePackages: engineVersions,
@@ -1506,8 +1515,8 @@ async function runCommand(flags, configPath, filters) {
 	}, () => live?.stop());
 	const { record } = run;
 	if (notes && !stopped) {
-		const { summarizeRun } = await import("./agent-6FefIhX7.js");
-		const { resolveAnalysisModel } = await import("./model-FXko2kl7.js");
+		const { summarizeRun } = await import("./agent-CURAwx_q.js");
+		const { resolveAnalysisModel } = await import("./model-NM3ID0uR.js");
 		const model = resolveAnalysisModel(ctx.config, "agent");
 		const { level, text } = await summarizeRun(ctx.config, record.runId, model);
 		renderer.onEvent({
@@ -1517,7 +1526,7 @@ async function runCommand(flags, configPath, filters) {
 			text
 		});
 	}
-	const { pruneDebug } = await import("./files-BM78_nnF.js");
+	const { pruneDebug } = await import("./files-BAt0i00X.js");
 	await pruneDebug(ctx.config.root, ctx.config.debug.keep).catch(() => {});
 	const summary = run.artifactDir ? join(run.artifactDir, "summary.md") : run.recordPath;
 	renderer.onEvent({
@@ -1534,7 +1543,7 @@ async function runCommand(flags, configPath, filters) {
 	const { config } = ctx;
 	const mode = setup.record;
 	if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
-		const { record: recordCmd } = await import("./commands-fI3G2Wnh.js");
+		const { record: recordCmd } = await import("./commands-1p-bIWhN.js");
 		const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
 		console.log(await recordCmd(config, record.runId, {
 			...flags,
@@ -1572,9 +1581,10 @@ async function prepareRun(flags, configPath) {
 	}
 	const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile?.defaults.environment ? [environmentSandbox(profile.defaults.environment, config.sandbox)] : []];
 	const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
-		if (!sandboxes.some((s) => s.kind === "docker")) return hostCapacity();
-		const { dockerCapacity } = await import("./src-cdIVjC5b.js");
-		return await dockerCapacity() ?? hostCapacity();
+		const container = sandboxes.find((s) => s.kind === "docker");
+		if (!container) return hostCapacity();
+		const { dockerCapacity } = await import("./src-pJZOrYR9.js");
+		return await dockerCapacity(container.engine) ?? hostCapacity();
 	});
 	return {
 		output,
@@ -1600,11 +1610,11 @@ async function prepareRun(flags, configPath) {
 async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
 	const startedAt = /* @__PURE__ */ new Date();
 	const git = await gitInfo(config.root, [config.history.dir]);
-	const { worktreePatch } = await import("./state-Bo5zg4da.js");
+	const { worktreePatch } = await import("./state-CgjQo29K.js");
 	const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
 	const who = await author(config.root);
-	const { stagingDir, sweepStaging } = await import("./results-Di7m8dQ3.js");
-	const { scratchDir } = await import("./files-BMCvfB4X.js");
+	const { stagingDir, sweepStaging } = await import("./results-DuauQI3Z.js");
+	const { scratchDir } = await import("./files-ZZPqfAo6.js");
 	const scratch = scratchDir(config.root);
 	if (resume !== void 0) {
 		const target = await resolveResume(config.root, resume || void 0);
@@ -1615,8 +1625,8 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
 			artifacts: target.previousArtifacts
 		} : await loadPrevious(stagingDir(config.root, target.runId));
 		if (!dryRun) try {
-			const { removeRunContainers } = await import("./docker-Df_YOxxB.js");
-			await removeRunContainers(target.runId);
+			const { removeRunContainers } = await import("./docker-yauK-9X8.js");
+			await removeRunContainers(target.runId, ["docker", "podman"]);
 		} catch {}
 		return {
 			runId: target.runId,
@@ -1686,8 +1696,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 		configFile,
 		completed: []
 	}));
-	const { runEngine } = await import("./node-Co7EXge0.js");
-	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DUydFc_1.js");
+	const { runEngine } = await import("./node-B3ydHo6t.js");
+	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-C9Cy27Ay.js");
 	let plan;
 	let sandboxVersions = { images: [] };
 	let preflightStop;
@@ -1714,7 +1724,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		} : p);
 		toRun = skipped;
 		for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
-		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-FXko2kl7.js") : void 0;
+		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-NM3ID0uR.js") : void 0;
 		plan = await buildPlan({
 			planned: skipped,
 			profile,
@@ -1730,7 +1740,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			completed: completed.size
 		};
 		run.completed = [...completed];
-		const { imageStages, imageTargets } = await import("./images-BMgdpMAX.js");
+		const { imageStages, imageTargets } = await import("./images-0On0cztu.js");
 		const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
 			id: p.id,
 			harness: p.attempt.harness,
@@ -1751,10 +1761,16 @@ async function executeRun(ctx, setup, filters, renderer) {
 		if (setup.dryRun) {
 			const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
 			lines.push(...planTable(plan, todos));
-			const { imageStatusLines } = await import("./images-BMgdpMAX.js");
-			const { dockerAvailable } = await import("./src-cdIVjC5b.js");
-			if (targets.length && await dockerAvailable()) lines.push(...await imageStatusLines(targets, config.root));
-			else if (targets.length) lines.push("images  not checked: Docker is not running");
+			const { imageStatusLines } = await import("./images-0On0cztu.js");
+			const { dockerAvailable, engineSpec } = await import("./src-pJZOrYR9.js");
+			const engines = [...new Set(targets.map((t) => t.sandbox.engine ?? "docker"))];
+			const down = [];
+			for (const engine of engines) if (!await dockerAvailable(engine)) down.push(engine);
+			if (targets.length && !down.length) lines.push(...await imageStatusLines(targets, config.root));
+			else if (targets.length) {
+				const names = down.map((e) => engineSpec(e).name).join(" and ");
+				lines.push(`images  not checked: ${names} ${down.length === 1 ? "is" : "are"} not running`);
+			}
 		}
 		lines.push(...plan.warnings.map((w) => `warning: ${w}`));
 		lines.push(...plan.hints.map((h) => `hint: ${h}`));
@@ -1766,7 +1782,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (setup.dryRun) return false;
 		checkAttemptBudget(plan, setup.force, config.file);
-		const analysis = config.analysis.agent ? await import("./model-FXko2kl7.js") : null;
+		const analysis = config.analysis.agent ? await import("./model-NM3ID0uR.js") : null;
 		analysis?.analysisModelName(config, "agent");
 		const questionModels = /* @__PURE__ */ new Set();
 		const asked = [];
@@ -1775,7 +1791,9 @@ async function executeRun(ctx, setup, filters, renderer) {
 			const questions = (row.plan.questions ?? []).map((q) => ({
 				question: q.question,
 				model: models.questionModel(config, q, row.id),
-				...q.across ? { across: q.across } : {}
+				...q.across ? { across: q.across } : {},
+				...q.state ? { state: q.state } : {},
+				...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
 			}));
 			for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
 			if (questions.length) asked.push({
@@ -1796,7 +1814,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
 		if (analysis) {
-			const { startNotes } = await import("./agent-6FefIhX7.js");
+			const { startNotes } = await import("./agent-CURAwx_q.js");
 			notes = startNotes({
 				config,
 				model: analysis.resolveAnalysisModel(config, "agent"),
@@ -1812,7 +1830,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			});
 		}
 		if (asked.length) {
-			const { questionRunner } = await import("./questions-B8by4Bbt.js");
+			const { questionRunner } = await import("./questions-DjAwMxZ3.js");
 			questions = questionRunner({
 				evals: asked,
 				stagingDir,
@@ -1829,7 +1847,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 				}
 			});
 		}
-		const { prepareWorkspaces, workspaceKey } = await import("./workspace-Do_SiWBs.js");
+		const { prepareWorkspaces, workspaceKey } = await import("./workspace-D8sSgpiy.js");
 		const live = skipped.filter((p) => !p.attempt.plan.skip);
 		const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
 			workspace: a.workspace,
@@ -1977,14 +1995,14 @@ async function executeRun(ctx, setup, filters, renderer) {
 		throw error;
 	} finally {
 		clearTimeout(budgetTimer);
-		const { removePrepared } = await import("./preflight-D5LW7Oqi.js");
+		const { removePrepared } = await import("./preflight-CPWpQeBI.js");
 		removePrepared(config.root, runId);
-		const { removePreparedWorkspaces } = await import("./workspace-Do_SiWBs.js");
+		const { removePreparedWorkspaces } = await import("./workspace-D8sSgpiy.js");
 		removePreparedWorkspaces(config.root, runId);
 	}
 	if (preflightStop) throw preflightStop;
 	if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) live?.remove();
-	if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files match ${config.include.join(", ")} (config.include) in ${config.root}. Name eval files to match it, or set include in ${config.file}.`);
+	if (engineRun.noFiles && !filters.length && !tags.length) throw new UsageError(`No eval files in ${config.root} match include: ${config.include.join(", ")}. Name eval files to match it, or set include in ${config.file}.`);
 	if (engineRun.noFiles || !engineRun.stopped && engineRun.evals.length === 0 && !interrupted) throw noMatch(config, filters, tags);
 	return {
 		engineRun,
@@ -2013,7 +2031,7 @@ function finish(code, error, kind) {
 	const command = commandOf(argv);
 	logInvocation(root, {
 		at: new Date(startedAt).toISOString(),
-		es: version,
+		reeval: version,
 		argv: redactArgv(argv),
 		command,
 		exitCode: code,
@@ -2062,4 +2080,4 @@ run(process.argv.slice(2)).finally(() => {
 //#endregion
 export {};
 
-//# sourceMappingURL=cli-CVrMh_qz.js.map
+//# sourceMappingURL=cli-D303Vj5N.js.map