reeval 0.7.0 → 0.9.0

package/dist/{cli-r9CDqoFL.js → cli-CVrMh_qz.js}

@@ -3,24 +3,26 @@ import { t as parseDuration } from "./duration-JsJrIWXq.js";
 import { n as loadEnvFiles, t as inCI } from "./env-BQqj93gu.js";
 import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
 import { o as PACKAGE_VERSION } from "./root-CvdRlRZX.js";
-import { T as workspaceLabel, u as environmentSandbox } from "./environment-DiCljXMD.js";
+import { T as workspaceLabel, u as environmentSandbox } from "./environment-1jE-HhO-.js";
 import { a as fromCwd, i as evalSlug, n as attemptKey, o as newRunId } from "./ids-Bp6ut7ZD.js";
-import { I as LABEL, X as imageInfo } from "./src-79JhW-zG.js";
-import { d as sandboxId, s as harnessId } from "./options-Ge_x32XE.js";
-import { c as hostCapacity, d as missingCredentials, f as resolveConcurrency, i as configPathFor, l as loadConfig, p as selectProfile, r as configLabel, s as findConfig, u as malformedCredentials } from "./config-DuQDCx0B.js";
+import { I as LABEL, X as imageInfo } from "./src-8rRXy4u9.js";
+import { d as sandboxId, s as harnessId } from "./options-CKqyPGxX.js";
+import { a as configPathFor, c as findConfig, d as malformedCredentials, f as missingCredentials, h as selectProfile, i as configLabel, l as hostCapacity, m as resolveConcurrency, p as requireConfig, u as loadConfig } from "./config-CGvBeewH.js";
 import { s as debugTime } from "./debug-D24NOQH1.js";
 import { i as sandboxWarningLine, n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
 import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { B as cellContrast, G as effortLabel, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, P as collapsedJudgeLabel, _ as hasUngatedCells, b as isUngatedMiss, h as formatUsage, w as runOutcome } from "./model-DyQbu7KF.js";
-import { a as liftLabel, i as gateLabel, n as answersLabel } from "./summary--FBuB_sr.js";
-import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-BL4Yw7Ld.js";
-import { n as FLAGS, o as helpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-DOyxAMc2.js";
-import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-Byl2T6LJ.js";
+import { H as cellContrast, L as judgeRatesLabel, M as cellJudgeRates, N as cellUsage, P as collapsedJudgeLabel, _ as hasUngatedCells, b as isUngatedMiss, h as formatUsage, q as effortLabel, w as runOutcome } from "./model-BOnlAPzs.js";
+import { a as familyLiftLabel, i as familyHeadLabel, n as familyCases, o as familyRateLabel, r as familyEffortLabel, t as caseLabel } from "./family-D2b7WD10.js";
+import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
+import { a as gateLabel, d as undiscriminating, f as undiscriminatingLabel, i as checksLabel, n as answersLabel, o as liftLabel } from "./summary-BTksrIxn.js";
+import { c as writeLiveProgress, i as prepareResume, o as resolveResume, r as loadPrevious, s as writeCurrent, t as completedKeysFor } from "./resume-SUz1GskV.js";
+import { n as FLAGS, o as helpText, r as GLOBAL, t as COMMANDS } from "./cli-spec-CKEYa8V2.js";
+import { i as runtimeInfo, n as gitInfo, r as runnerInfo, t as author } from "./provenance-D7HK3vzt.js";
 import { t as budgetMarker } from "./budget-DBUrH4jX.js";
-import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-CyaAZE9r.js";
-import { r as imageHarness } from "./prepare-Zr1xOMUX.js";
-import { n as createProviders } from "./providers-Cflw7Ks-.js";
-import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-DArYWyXN.js";
+import { n as prepareAll, o as writePrepared, r as prepareTargets, t as RunStopError } from "./preflight-BkoSxf4C.js";
+import { r as imageHarness } from "./prepare-CUAWHebF.js";
+import { n as createProviders } from "./providers-B1DzMvmH.js";
+import { a as redactArgv, r as commandOf, t as logInvocation } from "./invocations-CLaL8AuW.js";
 import { n as stripAnsi, t as colors } from "./colors-QV9dFVhN.js";
 import { existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
@@ -425,7 +427,7 @@ function evalColumns(c, e) {
 	if (more.length) {
 		const pass = e.attempts.filter((a) => a.outcome === "pass").length;
 		const counted = e.attempts.filter((a) => a.outcome === "pass" || a.outcome === "fail").length;
-		const rate = hasUngatedCells(e.cells) ? ` · ${pct(e.passRate).trim()} gated` : `  ${pct(e.passRate)}`;
+		const rate = hasUngatedCells(e.cells) ? ` · pass@1 ${pct(e.passRate).trim()} gated` : `  pass@1 ${pct(e.passRate)}`;
 		return c.dim(`${`${pass}/${counted}`.padStart(7)}${rate}  ${e.attempts.length} attempts`);
 	}
 	if (attemptOutcome(attempt) === "cancelled") return c.yellow("cancelled");
@@ -438,7 +440,7 @@ function evalColumns(c, e) {
 	if (tokens) cols.push(tokens);
 	return c.dim(cols.join("  "));
 }
-/** Pass rate per cell for evals with several attempts; with compare, each cell's role and lift. */
+/** pass@1 per cell for evals with several attempts; with compare, each cell's role and lift. */
 function cellRows(c, e, indent) {
 	if (e.attempts.length < 2) return [];
 	const compare = e.cells.some((cell) => cell.role === "variant");
@@ -457,27 +459,82 @@ function cellRows(c, e, indent) {
 		if (cell.gate !== "all" && cell.gate !== "none") note += `   ${c.dim(`gate ${gateLabel(cell.gate)}`)}`;
 		const tokens = formatUsage(cellUsage(e, cell.key));
 		const usage = tokens ? `   ${c.dim(tokens)}` : "";
-		const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role}  ${counts}  ${pct(cell.passRate)}${note}${infra}${usage}`];
+		const rows = [`${indent}${c.dim(cell.key.padEnd(width))}${role}  ${counts}  pass@1 ${pct(cell.passRate)}${note}${infra}${usage}`];
 		const rates = judgeRatesLabel(cellJudgeRates(e, cell.key));
 		if (rates) rows.push(`${indent}  ${c.dim(rates)}`);
 		const effort = cellContrast(e, cell.key);
 		if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effortLabel(effort)}`)}`);
+		const checks = checksLabel(cell);
+		if (checks) rows.push(`${indent}  ${c.dim(`checks: ${checks}`)}`);
 		if (cell.overrides.length) rows.push(`${indent}  ${c.dim(`overrides: ${cell.overrides.join(", ")}`)}`);
 		return rows;
 	});
 }
 /** One row per compare question: its win rates, or why it has none. They never gate. */
 function questionRows(c, e, indent) {
-	return (e.answers ?? []).map((a) => {
+	return (e.answers ?? []).flatMap((a) => {
 		const label = answersLabel(a);
-		return `${indent}${c.dim("question")}  ${a.error !== void 0 ? c.yellow(label) : label}`;
+		return [`${indent}${c.dim("question")}  ${a.error !== void 0 ? c.yellow(label) : label}`, ...sampleReasons(a.votes).map((reason) => `${indent}  ${c.dim(reason)}`)];
 	});
 }
-/** The run's evals as a file › group tree, one row per eval and per cell of a matrix. */
-function tree(c, evals) {
+/** A family's mark: its pooled gate's and its cases' worst. */
+function familyOutcome(family, cases) {
+	const outcomes = [family.outcome, ...cases.map(entryOutcome)];
+	for (const o of [
+		"fail",
+		"infra_error",
+		"cancelled"
+	]) if (outcomes.includes(o)) return o;
+	return outcomes.every((o) => o === "skipped") ? "skipped" : "pass";
+}
+/** The width case marks wrap at. */
+const CASE_WIDTH = 96;
+/** A pooled family's rows under its header: its cases' marks, then each pooled rate or lift. */
+function familyRows(c, family, cases, indent) {
+	const marks = cases.map((e) => `${mark(c, entryOutcome(e))} ${caseLabel(family.name, e.name)}`);
+	const plain = cases.map((e) => `x ${caseLabel(family.name, e.name)}`);
 	const rows = [];
+	let line = [];
+	let width = 0;
+	marks.forEach((m, i) => {
+		const w = plain[i].length + 2;
+		if (line.length && width + w > CASE_WIDTH) {
+			rows.push(`${indent}${line.join("  ")}`);
+			line = [];
+			width = 0;
+		}
+		line.push(m);
+		width += w;
+	});
+	if (line.length) rows.push(`${indent}${line.join("  ")}`);
+	if (family.lifts.length) {
+		const keyWidth = Math.max(...family.lifts.map((l) => l.variant.length));
+		for (const lift of family.lifts) {
+			const below = lift.outcome === "fail";
+			const gate = lift.minLift !== void 0 ? ` (minLift ${Math.round(lift.minLift * 100)}${below ? " ✗" : ""})` : "";
+			const text = `${familyLiftLabel(lift)}${gate}`;
+			rows.push(`${indent}${c.dim(lift.variant.padEnd(keyWidth))}  ${below ? c.red(text) : text}`);
+			const effort = familyEffortLabel(lift);
+			if (effort) rows.push(`${indent}  ${c.dim(`effort: ${effort}`)}`);
+		}
+	} else if (family.rates.length === 1) rows.push(`${indent}${familyRateLabel(family.rates[0])}`);
+	else {
+		const keyWidth = Math.max(...family.rates.map((r) => r.cell.length));
+		for (const rate of family.rates) rows.push(`${indent}${c.dim(rate.cell.padEnd(keyWidth))}  ${familyRateLabel(rate)}`);
+	}
+	return rows;
+}
+/**
+* The run's evals as a file › group tree, one row per eval and per cell of a matrix. A pooled
+* family is one row, its template, with its cases' marks and the pooled results under it.
+*/
+function tree(c, evals, families = []) {
+	const rows = [];
+	const shown = /* @__PURE__ */ new Set();
 	let path = [];
 	for (const e of evals) {
+		const family = e.family && families.find((f) => f.id === e.family?.id);
+		if (family && shown.has(family.id)) continue;
 		const next = [e.file, ...e.group];
 		let same = 0;
 		while (same < path.length && same < next.length && path[same] === next[same]) same++;
@@ -486,18 +543,30 @@ function tree(c, evals) {
 			text: i === 0 ? c.bold(next[i] ?? "") : next[i] ?? ""
 		});
 		path = next;
-		rows.push({
+		if (family) {
+			shown.add(family.id);
+			rows.push({
+				indent: next.length,
+				text: family.name,
+				family
+			});
+		} else rows.push({
 			indent: next.length,
 			text: e.name,
 			entry: e
 		});
 	}
-	const width = Math.max(...rows.filter((r) => r.entry).map((r) => r.indent * 2 + 2 + r.text.length));
-	return rows.flatMap(({ indent, text, entry }) => {
+	const width = Math.max(...rows.filter((r) => r.entry || r.family).map((r) => r.indent * 2 + 2 + r.text.length));
+	return rows.flatMap(({ indent, text, entry, family }) => {
 		const pad = "  ".repeat(indent);
+		const fill = " ".repeat(Math.max(0, width - indent * 2 - 2 - text.length));
+		if (family) {
+			const cases = familyCases(family, evals);
+			return [` ${pad}${mark(c, familyOutcome(family, cases))} ${text}${fill}  ${c.dim(familyHeadLabel(family))}`, ...familyRows(c, family, cases, ` ${pad}    `)];
+		}
 		if (!entry) return [` ${pad}${text}`];
 		return [
-			` ${pad}${`${mark(c, entryOutcome(entry))} ${text}`}${" ".repeat(Math.max(0, width - indent * 2 - 2 - text.length))}  ${evalColumns(c, entry)}`,
+			` ${pad}${`${mark(c, entryOutcome(entry))} ${text}`}${fill}  ${evalColumns(c, entry)}`,
 			...cellRows(c, entry, ` ${pad}    `),
 			...questionRows(c, entry, ` ${pad}    `)
 		];
@@ -516,6 +585,12 @@ function totals(c, s, cancelled = 0) {
 	const scope = s.attempts === s.evals ? `${s.evals}` : `${s.attempts} attempts, ${s.evals} evals`;
 	return parts.length ? `${parts.join(c.dim(" · "))} ${c.dim(`(${scope})`)}` : "no evals ran";
 }
+/** `1 passed · 1 failed (2)`: the families' pooled gates. */
+function familyTotals(c, families) {
+	const fail = families.filter((f) => f.outcome === "fail").length;
+	const pass = families.length - fail;
+	return `${[pass && c.green(`${pass} passed`), fail && c.red(`${fail} failed`)].filter(Boolean).join(c.dim(" · "))} ${c.dim(`(${families.length})`)}`;
+}
 /** Failing attempts, in run order, including expected misses. */
 function failingAttempts(evals) {
 	return evals.flatMap((entry) => entry.attempts.map((attempt) => ({
@@ -527,7 +602,7 @@ function failingAttempts(evals) {
 function renderSummary(c, e, attemptDir) {
 	const { record } = e;
 	const lines = [];
-	if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals));
+	if (record.evals.length) lines.push(c.bold("Summary"), ...tree(c, record.evals, record.families));
 	const all = failingAttempts(record.evals);
 	const missed = all.filter(({ entry, attempt }) => isUngatedMiss(entry, attempt));
 	const failing = all.filter((f) => !missed.includes(f));
@@ -549,7 +624,18 @@ function renderSummary(c, e, attemptDir) {
 	}
 	if (missed.length) {
 		lines.push("", c.bold(`Expected misses, not gated (${missed.length})`));
+		const pooled = /* @__PURE__ */ new Map();
 		for (const { entry, attempt } of missed) {
+			if (!entry.family) continue;
+			const key = `${entry.family.id}\0${attempt.cell}`;
+			pooled.set(key, (pooled.get(key) ?? 0) + 1);
+		}
+		for (const [key, n] of pooled) {
+			const [id, cell] = key.split("\0");
+			lines.push(` ${MISS_MARK(c)} ${id}  ${c.dim(`${n} ${n === 1 ? "attempt" : "attempts"} on ${cell}`)}`);
+		}
+		for (const { entry, attempt } of missed) {
+			if (entry.family) continue;
 			const which = entry.attempts.length > 1 ? `  ${c.dim(attempt.id)}` : "";
 			const [reason] = failureLines(colors(false), resultFor(entry, attempt, e.results), { max: 1 }).filter((l) => l.trim());
 			lines.push(` ${MISS_MARK(c)} ${entry.id}${which}`);
@@ -561,12 +647,23 @@ function renderSummary(c, e, attemptDir) {
 	const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
 	const label = record.summary.attempts === record.summary.evals ? "   Evals" : "Attempts";
 	const lifted = record.evals.flatMap((entry) => entry.cells.flatMap((cell) => cell.lift?.minLift !== void 0 && cell.lift.interval.lo < cell.lift.minLift ? [`${entry.id}  ${cell.key} ${liftLabel(cell.lift)}: its lower bound is below minLift ${Math.round(cell.lift.minLift * 100)}`] : []));
+	for (const family of record.families ?? []) for (const lift of family.lifts) {
+		if (lift.outcome !== "fail" || lift.minLift === void 0) continue;
+		lifted.push(`${family.id}  ${lift.variant} pooled ${familyLiftLabel(lift)}: its lower bound is below minLift ${Math.round(lift.minLift * 100)}`);
+	}
+	const idle = undiscriminating(record.evals);
+	if (idle.length) {
+		lines.push("", c.bold(`Checks that don't discriminate (${idle.length})`));
+		for (const x of idle) lines.push(` ${c.dim("·")} ${undiscriminatingLabel(x)}`);
+	}
 	if (lifted.length) {
 		lines.push("", c.bold(c.red(`Lift below minLift (${lifted.length})`)));
 		for (const line of lifted) lines.push(` ${c.red("✗")} ${line}`);
 	}
 	if (record.status === "budget") lines.push("", c.yellow("Budget reached: attempts that hadn’t started were skipped"));
-	lines.push("", `${c.dim(label)}  ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")}  ${formatDuration(record.durationMs)}`);
+	lines.push("");
+	if (record.families?.length) lines.push(`${c.dim("Families")}  ${familyTotals(c, record.families)}`);
+	lines.push(`${c.dim(label)}  ${totals(c, record.summary, cancelled)}`, `${c.dim("Duration")}  ${formatDuration(record.durationMs)}`);
 	const tokens = formatUsage(record.cost.usage);
 	if (tokens) lines.push(`  ${c.dim("Tokens")}  ${tokens}`);
 	lines.push(` ${c.dim("Results")}  ${e.summaryPath}`);
@@ -729,7 +826,18 @@ var AgentRenderer = class {
 					if (outcome === "cancelled" && !this.#printed.has(result.key)) this.#failure(result, attempt.cell, attempt.n);
 				}
 				for (const error of e.unhandledErrors) this.#write(`ERROR ${error}`);
-				for (const entry of e.record.evals) for (const a of entry.answers ?? []) this.#write(`question ${entry.id}: ${answersLabel(a)}`);
+				for (const entry of e.record.evals) for (const a of entry.answers ?? []) {
+					this.#write(`question ${entry.id}: ${answersLabel(a)}`);
+					for (const reason of sampleReasons(a.votes)) this.#write(`  ${reason}`);
+				}
+				for (const x of undiscriminating(e.record.evals)) this.#write(`check doesn't discriminate: ${undiscriminatingLabel(x)}`);
+				for (const family of e.record.families ?? []) {
+					const pooled = family.lifts.length ? family.lifts.map((l) => {
+						const gate = l.minLift === void 0 ? "" : ` (minLift ${Math.round(l.minLift * 100)}${l.outcome === "fail" ? ", FAIL" : ""})`;
+						return `${l.variant} ${familyLiftLabel(l)}${gate}`;
+					}) : family.rates.map((r) => `${r.cell} ${familyRateLabel(r)}`);
+					this.#write(`family ${family.id}: ${familyHeadLabel(family)} · ${pooled.join(" · ")}`);
+				}
 				const s = e.record.summary;
 				const cancelled = failing.filter((f) => f.outcome === "cancelled").length;
 				const parts = [
@@ -1116,7 +1224,7 @@ async function run(argv) {
 		return values.help ? EXIT.ok : EXIT.usage;
 	}
 	if (command === "init") {
-		const { initCommand } = await import("./init-1THeGfut.js");
+		const { initCommand } = await import("./init-B6vegaP_.js");
 		return initCommand(values, filters, {
 			out: (s) => console.log(s),
 			input: process.stdin,
@@ -1125,7 +1233,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "doctor") {
-		const { doctorCommand } = await import("./doctor-Bk6aJeec.js");
+		const { doctorCommand } = await import("./doctor-DlKycx5H.js");
 		return doctorCommand(values, {
 			out: (s) => console.log(s),
 			color: !!process.stdout.isTTY && !process.env.NO_COLOR,
@@ -1133,7 +1241,7 @@ async function run(argv) {
 		});
 	}
 	if (command === "report") {
-		const { reportCommand } = await import("./command-Cej2t3y-.js");
+		const { reportCommand } = await import("./command-CzIFp1JQ.js");
 		return reportCommand(values, filters, {
 			out: (s) => console.log(s),
 			err: (s) => console.error(dim(s)),
@@ -1148,7 +1256,7 @@ async function run(argv) {
 	}
 	if (command === "studio") {
 		const config = await loadConfig(configPath, "studio");
-		const { studioCommand } = await import("./command-l4ZFCeMh.js");
+		const { studioCommand } = await import("./command-BFP0hKEb.js");
 		return studioCommand({
 			config,
 			flags: values,
@@ -1159,7 +1267,7 @@ async function run(argv) {
 	if (command === "mcp") {
 		if (!values.http) console.log = console.info = console.debug = console.error;
 		const config = await loadConfig(configPath, "mcp");
-		const { mcpCommand } = await import("./command-DMR69l5G.js");
+		const { mcpCommand } = await import("./command-C5xFPKHM.js");
 		return mcpCommand({
 			config,
 			flags: values,
@@ -1168,7 +1276,7 @@ async function run(argv) {
 	}
 	if (command in HISTORY_COMMANDS) {
 		const config = await loadConfig(configPath, command);
-		const commands = await import("./commands-DgOUkcSO.js");
+		const commands = await import("./commands-fI3G2Wnh.js");
 		const ctx = {
 			config,
 			flags: values,
@@ -1181,7 +1289,7 @@ async function run(argv) {
 			case "compare": return commands.compareCommand(ctx, filters);
 			case "checkout": return commands.checkoutCommand(ctx, filters);
 			case "query": {
-				const { queryCommand } = await import("./query-CvutWHaL.js");
+				const { queryCommand } = await import("./query-oPO_XrTy.js");
 				return queryCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1189,7 +1297,7 @@ async function run(argv) {
 			}
 			case "analyze": {
 				loadEnvFiles(configPath);
-				const { analyzeCommand } = await import("./command-Cxfgghqo.js");
+				const { analyzeCommand } = await import("./command-BrmXUC-W.js");
 				return analyzeCommand({
 					...ctx,
 					err: (s) => console.error(dim(s))
@@ -1228,8 +1336,8 @@ async function cleanCommand(flags, configPath) {
 	const where = fromCwd(debugRoot(config.root)) || ".";
 	console.log(runs.length ? `Removed debug data for ${runs.length} run${runs.length === 1 ? "" : "s"} (${formatBytes(bytes)}) from ${where}` : `No debug data to remove in ${where}`);
 	if (flags.images) {
-		const { pruneImages } = await import("./src-DH0jhpoi.js");
-		const { requireDocker } = await import("./providers-SBMExI53.js");
+		const { pruneImages } = await import("./src-cdIVjC5b.js");
+		const { requireDocker } = await import("./providers-CpEfNU1F.js");
 		await requireDocker();
 		const removed = await pruneImages();
 		console.log(removed ? `Removed ${removed} image${removed === 1 ? "" : "s"} reeval built that nothing uses` : "No leftover images to remove");
@@ -1244,55 +1352,43 @@ function verbosityOf(flags) {
 /**
 * `reeval image build`: collects the evals (as `--dry-run` does) and builds every image their Docker
 * sandboxes need; with `--dry-run`, lists them instead. `--profile` plans as `reeval run --profile`
-* does. Without a config, just reeval's image.
+* does. Needs a config, as `reeval run` does: there is no default image.
 */
 async function imageBuildCommand(flags, configPath, filters) {
-	const { buildImages, imagePlanLines, imageTargets } = await import("./images-BpwFDT1E.js");
-	const { docker } = await import("./sandbox.js");
-	let root = process.cwd();
-	const { fakeHarness } = await import("./harness.js");
-	let uses = [{
+	const { buildImages, imagePlanLines, imageTargets } = await import("./images-BMgdpMAX.js");
+	const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
+		...flags,
+		"dry-run": true
+	}, configPath));
+	const root = config.root;
+	const { runEngine } = await import("./node-Co7EXge0.js");
+	let planned = [];
+	await runEngine({
+		run: JSON.parse(JSON.stringify({
+			runId,
+			stagingDir,
+			config,
+			runs,
+			tags,
+			concurrency,
+			profile
+		})),
+		filters,
+		onPlan: async (collected) => {
+			if (collected.unmatched.length) throw unmatchedFilter(config, collected.unmatched);
+			planned = collected.planned.filter((p) => !p.attempt.plan.skip);
+			return false;
+		}
+	});
+	const targets = imageTargets(planned.length ? planned.map((p) => ({
+		id: p.id,
+		harness: p.attempt.harness,
+		sandbox: p.attempt.sandbox
+	})) : config.harness.map((harness) => ({
 		id: "",
-		harness: fakeHarness(),
-		sandbox: docker()
-	}];
-	if (!configPath && flags.profile) selectProfile(await loadConfig(void 0), flags.profile);
-	if (configPath) {
-		const { runId, stagingDir, config, runs, tags, concurrency, profile } = await startRun(await prepareRun({
-			...flags,
-			"dry-run": true
-		}, configPath));
-		root = config.root;
-		const { runEngine } = await import("./node-Co7EXge0.js");
-		let planned = [];
-		await runEngine({
-			run: JSON.parse(JSON.stringify({
-				runId,
-				stagingDir,
-				config,
-				runs,
-				tags,
-				concurrency,
-				profile
-			})),
-			filters,
-			onPlan: async (collected) => {
-				if (collected.unmatched.length) throw unmatchedFilter(config, collected.unmatched);
-				planned = collected.planned.filter((p) => !p.attempt.plan.skip);
-				return false;
-			}
-		});
-		uses = planned.length ? planned.map((p) => ({
-			id: p.id,
-			harness: p.attempt.harness,
-			sandbox: p.attempt.sandbox
-		})) : config.harness.map((harness) => ({
-			id: "",
-			harness,
-			sandbox: config.sandbox
-		}));
-	}
-	const targets = imageTargets(uses);
+		harness,
+		sandbox: config.sandbox
+	})));
 	if (targets.length === 0) {
 		console.log("No eval runs in a Docker sandbox, so there are no images to build.");
 		return EXIT.ok;
@@ -1360,14 +1456,14 @@ async function runCommand(flags, configPath, filters) {
 	const { engineRun, plan, sandboxVersions, budgetReached, live, notes, questions } = executed;
 	invocation.unhandledErrors = engineRun.unhandledErrors;
 	if ((interrupted || engineRun.cancelled) && plan?.sandboxes.some((s) => s.kind === "docker")) {
-		const { removeRunContainers } = await import("./docker-Dnya8eo9.js");
+		const { removeRunContainers } = await import("./docker-Df_YOxxB.js");
 		await removeRunContainers(ctx.runId);
 	}
 	if (notes) {
 		const cancelled = interrupted || engineRun.cancelled;
 		const written = await notes.finish(cancelled);
 		if (!cancelled && written.length) {
-			const { notesLine } = await import("./agent-Dhd5FJCZ.js");
+			const { notesLine } = await import("./agent-6FefIhX7.js");
 			const text = notesLine(ctx.config.root, ctx.runId, written);
 			renderer.onEvent({
 				type: "notice",
@@ -1384,8 +1480,8 @@ async function runCommand(flags, configPath, filters) {
 	}
 	const stopped = interrupted || engineRun.cancelled;
 	const answers = questions && !stopped ? await questions.settle() : void 0;
-	const { lockfiles } = await import("./state-DkqbCVVp.js");
-	const { writeScratchRun } = await import("./results-owJR-NFm.js");
+	const { lockfiles } = await import("./state-Bo5zg4da.js");
+	const { writeScratchRun } = await import("./results-Di7m8dQ3.js");
 	const run = await writeScratchRun({
 		ctx,
 		author: ctx.author,
@@ -1410,8 +1506,8 @@ async function runCommand(flags, configPath, filters) {
 	}, () => live?.stop());
 	const { record } = run;
 	if (notes && !stopped) {
-		const { summarizeRun } = await import("./agent-Dhd5FJCZ.js");
-		const { resolveAnalysisModel } = await import("./model-BeoRxCcJ.js");
+		const { summarizeRun } = await import("./agent-6FefIhX7.js");
+		const { resolveAnalysisModel } = await import("./model-FXko2kl7.js");
 		const model = resolveAnalysisModel(ctx.config, "agent");
 		const { level, text } = await summarizeRun(ctx.config, record.runId, model);
 		renderer.onEvent({
@@ -1438,7 +1534,7 @@ async function runCommand(flags, configPath, filters) {
 	const { config } = ctx;
 	const mode = setup.record;
 	if ((flags.record || mode === "always" || mode === "ci" && inCI()) && record.status !== "interrupted") {
-		const { record: recordCmd } = await import("./commands-DgOUkcSO.js");
+		const { record: recordCmd } = await import("./commands-fI3G2Wnh.js");
 		const allow = record.status === "budget" ? { "allow-incomplete": true } : {};
 		console.log(await recordCmd(config, record.runId, {
 			...flags,
@@ -1464,9 +1560,10 @@ async function prepareRun(flags, configPath) {
 		agent: detectAgent().name
 	});
 	if (flags.json && !flags["dry-run"]) throw new UsageError("reeval run --json only prints the plan: add --dry-run. A run writes its record to .reeval/runs; read it with reeval show --json.");
-	loadEnvFiles(configPath);
+	const path = requireConfig(configPath);
+	loadEnvFiles(path);
 	const named = flags.profile ?? (process.env.REEVAL_PROFILE || void 0);
-	const config = await loadConfig(configPath, "run", named);
+	const config = await loadConfig(path, "run", named);
 	const profile = selectProfile(config, flags.profile);
 	let runs;
 	if (flags.runs !== void 0) {
@@ -1476,7 +1573,7 @@ async function prepareRun(flags, configPath) {
 	const sandboxes = [environmentSandbox(config.environment, config.sandbox), ...profile?.defaults.environment ? [environmentSandbox(profile.defaults.environment, config.sandbox)] : []];
 	const concurrency = await resolveConcurrency(config, flags.concurrency, async () => {
 		if (!sandboxes.some((s) => s.kind === "docker")) return hostCapacity();
-		const { dockerCapacity } = await import("./src-DH0jhpoi.js");
+		const { dockerCapacity } = await import("./src-cdIVjC5b.js");
 		return await dockerCapacity() ?? hostCapacity();
 	});
 	return {
@@ -1492,9 +1589,9 @@ async function prepareRun(flags, configPath) {
 		json: !!flags.json,
 		force: !!flags.force,
 		verbosity: verbosityOf(flags),
-		configFile: configPath && configLabel(configPath),
+		configFile: configLabel(path),
 		/** Named in the plan line when the project root isn't the current directory. */
-		configElsewhere: !!configPath && dirname(configPath) !== process.cwd(),
+		configElsewhere: dirname(path) !== process.cwd(),
 		/** `undefined` is a new run; `''` is newest resumable; a string is a run id or prefix. */
 		resume: flags.resume
 	};
@@ -1503,11 +1600,11 @@ async function prepareRun(flags, configPath) {
 async function startRun({ config, profile, runs, tags, concurrency, verbosity, configFile, resume, dryRun }) {
 	const startedAt = /* @__PURE__ */ new Date();
 	const git = await gitInfo(config.root, [config.history.dir]);
-	const { worktreePatch } = await import("./state-DkqbCVVp.js");
+	const { worktreePatch } = await import("./state-Bo5zg4da.js");
 	const patch = git?.dirty ? await worktreePatch(config.root, [config.history.dir]) : null;
 	const who = await author(config.root);
-	const { stagingDir, sweepStaging } = await import("./results-owJR-NFm.js");
-	const { scratchDir } = await import("./files-DmZGPsWK.js");
+	const { stagingDir, sweepStaging } = await import("./results-Di7m8dQ3.js");
+	const { scratchDir } = await import("./files-BMCvfB4X.js");
 	const scratch = scratchDir(config.root);
 	if (resume !== void 0) {
 		const target = await resolveResume(config.root, resume || void 0);
@@ -1518,7 +1615,7 @@ async function startRun({ config, profile, runs, tags, concurrency, verbosity, c
 			artifacts: target.previousArtifacts
 		} : await loadPrevious(stagingDir(config.root, target.runId));
 		if (!dryRun) try {
-			const { removeRunContainers } = await import("./docker-Dnya8eo9.js");
+			const { removeRunContainers } = await import("./docker-Df_YOxxB.js");
 			await removeRunContainers(target.runId);
 		} catch {}
 		return {
@@ -1590,7 +1687,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		completed: []
 	}));
 	const { runEngine } = await import("./node-Co7EXge0.js");
-	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-BqBadHYj.js");
+	const { buildPlan, checkAttemptBudget, planDocument, planLine, planTable } = await import("./plan-DUydFc_1.js");
 	let plan;
 	let sandboxVersions = { images: [] };
 	let preflightStop;
@@ -1617,7 +1714,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		} : p);
 		toRun = skipped;
 		for (const p of skipped) for (const name of p.attempt.secretVars) registerSecret(process.env[name]);
-		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-BeoRxCcJ.js") : void 0;
+		const models = skipped.some((p) => !p.attempt.plan.skip && p.attempt.plan.questions?.length) ? await import("./model-FXko2kl7.js") : void 0;
 		plan = await buildPlan({
 			planned: skipped,
 			profile,
@@ -1633,7 +1730,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			completed: completed.size
 		};
 		run.completed = [...completed];
-		const { imageStages, imageTargets } = await import("./images-BpwFDT1E.js");
+		const { imageStages, imageTargets } = await import("./images-BMgdpMAX.js");
 		const targets = imageTargets(skipped.filter((p) => !p.attempt.plan.skip).map((p) => ({
 			id: p.id,
 			harness: p.attempt.harness,
@@ -1654,8 +1751,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 		if (setup.dryRun) {
 			const todos = excluded.filter((e) => e.mode === "todo").map((e) => e.id);
 			lines.push(...planTable(plan, todos));
-			const { imageStatusLines } = await import("./images-BpwFDT1E.js");
-			const { dockerAvailable } = await import("./src-DH0jhpoi.js");
+			const { imageStatusLines } = await import("./images-BMgdpMAX.js");
+			const { dockerAvailable } = await import("./src-cdIVjC5b.js");
 			if (targets.length && await dockerAvailable()) lines.push(...await imageStatusLines(targets, config.root));
 			else if (targets.length) lines.push("images  not checked: Docker is not running");
 		}
@@ -1669,7 +1766,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (setup.dryRun) return false;
 		checkAttemptBudget(plan, setup.force, config.file);
-		const analysis = config.analysis.agent ? await import("./model-BeoRxCcJ.js") : null;
+		const analysis = config.analysis.agent ? await import("./model-FXko2kl7.js") : null;
 		analysis?.analysisModelName(config, "agent");
 		const questionModels = /* @__PURE__ */ new Set();
 		const asked = [];
@@ -1677,7 +1774,8 @@ async function executeRun(ctx, setup, filters, renderer) {
 			if (row.plan.skip || !models) continue;
 			const questions = (row.plan.questions ?? []).map((q) => ({
 				question: q.question,
-				model: models.questionModel(config, q, row.id)
+				model: models.questionModel(config, q, row.id),
+				...q.across ? { across: q.across } : {}
 			}));
 			for (const q of questions) if (typeof q.model === "string") questionModels.add(q.model);
 			if (questions.length) asked.push({
@@ -1698,7 +1796,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 		});
 		if (malformed.length) throw new UsageError(`Credentials with whitespace or a line break in them: ${malformed.join(", ")}. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
 		if (analysis) {
-			const { startNotes } = await import("./agent-Dhd5FJCZ.js");
+			const { startNotes } = await import("./agent-6FefIhX7.js");
 			notes = startNotes({
 				config,
 				model: analysis.resolveAnalysisModel(config, "agent"),
@@ -1714,7 +1812,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 			});
 		}
 		if (asked.length) {
-			const { questionRunner } = await import("./questions-ZdT3JDyQ.js");
+			const { questionRunner } = await import("./questions-B8by4Bbt.js");
 			questions = questionRunner({
 				evals: asked,
 				stagingDir,
@@ -1731,7 +1829,7 @@ async function executeRun(ctx, setup, filters, renderer) {
 				}
 			});
 		}
-		const { prepareWorkspaces, workspaceKey } = await import("./workspace-wudSz_W-.js");
+		const { prepareWorkspaces, workspaceKey } = await import("./workspace-Do_SiWBs.js");
 		const live = skipped.filter((p) => !p.attempt.plan.skip);
 		const outcomes = await prepareWorkspaces(live.flatMap(({ attempt: a }) => a.workspace ? [{
 			workspace: a.workspace,
@@ -1879,9 +1977,9 @@ async function executeRun(ctx, setup, filters, renderer) {
 		throw error;
 	} finally {
 		clearTimeout(budgetTimer);
-		const { removePrepared } = await import("./preflight-CFtZNTLt.js");
+		const { removePrepared } = await import("./preflight-D5LW7Oqi.js");
 		removePrepared(config.root, runId);
-		const { removePreparedWorkspaces } = await import("./workspace-wudSz_W-.js");
+		const { removePreparedWorkspaces } = await import("./workspace-Do_SiWBs.js");
 		removePreparedWorkspaces(config.root, runId);
 	}
 	if (preflightStop) throw preflightStop;
@@ -1964,4 +2062,4 @@ run(process.argv.slice(2)).finally(() => {
 //#endregion
 export {};
 
-//# sourceMappingURL=cli-r9CDqoFL.js.map
+//# sourceMappingURL=cli-CVrMh_qz.js.map