reeval 0.7.0 → 0.9.0

package/dist/index.js

@@ -1,24 +1,26 @@
-import { a as getRunContext, d as expect, h as HarnessCapabilityError, i as collectingFile, o as registerGroup, r as withLease, s as registerTest } from "./client-LsfDiD-X.js";
+import { i as HarnessCapabilityError, r as withLease } from "./client-Bbo9SPlK.js";
 import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-yDAEwmZo.js";
 import { t as parseDuration } from "./duration-JsJrIWXq.js";
 import { t as inCI } from "./env-BQqj93gu.js";
 import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
-import { C as stepCommand, E as workspaceOf, T as workspaceLabel, _ as isSecret, b as parseGitRepo, c as defineEnvironment, d as filesOf, h as isHostFile, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES, v as isSecretTemplate, y as mcpHosts } from "./environment-DiCljXMD.js";
+import { C as stepCommand, E as workspaceOf, T as workspaceLabel, _ as isSecret, b as parseGitRepo, c as defineEnvironment, d as filesOf, h as isHostFile, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES, v as isSecretTemplate, y as mcpHosts } from "./environment-1jE-HhO-.js";
 import { t as InfraError } from "./types-C5vI3g1C.js";
-import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-BKbyriEa.js";
+import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-BhzNh7Yi.js";
 import { c as relativeFile, i as evalSlug, l as sha256, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
-import { c as judgeModelName, d as isApiError, i as badJudgeList, u as runJudge } from "./judge-7WjNdU9z.js";
-import { S as placeholder, et as run, tt as shellQuote } from "./src-79JhW-zG.js";
-import { a as evalPlan, d as sandboxId, f as sandboxSetBy, h as unbrokeredError, i as checkOptions, l as matchesTags, m as unbrokered, n as cascade, o as expand, p as secretVarsIn, s as harnessId, t as brokeredIn } from "./options-Ge_x32XE.js";
-import { n as badCharacterAt, o as defineConfig } from "./config-DuQDCx0B.js";
+import { c as judgeModelName, d as isApiError, i as badJudgeList, u as runJudge } from "./judge-CeYPN8P_.js";
+import { S as placeholder } from "./src-8rRXy4u9.js";
+import { n as shellQuote, t as run } from "./process-CSVO_jsc.js";
+import { a as evalPlan, d as sandboxId, f as sandboxSetBy, h as unbrokeredError, i as checkOptions, l as matchesTags, m as unbrokered, n as cascade, o as expand, p as secretVarsIn, s as harnessId, t as brokeredIn } from "./options-CKqyPGxX.js";
+import { n as badCharacterAt, s as defineConfig } from "./config-CGvBeewH.js";
+import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile, u as expect } from "./runtime-DdsfnRz_.js";
 import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CGPV3BZY.js";
 import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
 import { n as clip, r as formatDuration, t as capMiddle } from "./text-gEcKjAyp.js";
-import { a as asCheck, i as writeCaptureSidecars, n as serveCommand, o as createChecks, t as capturePage } from "./capture-CisAmy-G.js";
-import { t as contextTaker } from "./context-CvUv7DUb.js";
+import { a as asCheck, c as snapshot, i as writeCaptureSidecars, n as serveCommand, o as createChecks, s as diffSinceSnapshot, t as capturePage } from "./capture-Bp74Z6Fr.js";
+import { t as contextTaker } from "./context-DbSMEVKs.js";
 import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { i as preparedWorkspace } from "./workspace-C4_J0pqb.js";
-import { n as judgeDiff } from "./patch-CGCVipPu.js";
+import { i as preparedWorkspace } from "./workspace-C-TrNIUc.js";
+import { n as judgeDiff } from "./patch-bDV8YFW0.js";
 import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
 import { basename, isAbsolute, join, relative, sep } from "node:path";
 import { setTimeout as setTimeout$1 } from "node:timers/promises";
@@ -864,7 +866,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
 //#region src/harness/index.ts
 async function createHarness(descriptor) {
 	if (descriptor.kind === "fake") return createFakeHarness(descriptor);
-	const { createAgentHarness } = await import("./agent-gjBiZis8.js");
+	const { createAgentHarness } = await import("./agent-WSmTaesf.js");
 	return createAgentHarness(descriptor);
 }
 /** Where each harness's agent looks for repo skills, relative to the workspace. */
@@ -891,8 +893,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
 	if (debug) traceEnvironment(debug, spec, env);
 	record.image = null;
 	if (spec.sandbox.kind === "docker") {
-		const { preparedFor } = await import("./preflight-CFtZNTLt.js");
-		const { dockerProvider } = await import("./providers-SBMExI53.js");
+		const { preparedFor } = await import("./preflight-D5LW7Oqi.js");
+		const { dockerProvider } = await import("./providers-CpEfNU1F.js");
 		signal.throwIfAborted();
 		const prepared = preparedFor(root, runId, dockerProvider(), spec.sandbox, spec.harness);
 		if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
@@ -903,6 +905,62 @@ async function prepareEnvironment(a, root, runId, where, signal) {
 	} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
 }
 //#endregion
+//#region src/judge/artifacts.ts
+/** A capture handle: a name plus the file the attempt wrote. */
+function isCaptureHandle(value) {
+	if (typeof value !== "object" || value === null) return false;
+	const v = value;
+	return typeof v.name === "string" && typeof v.file?.path === "string";
+}
+/** The first `artifacts` value that isn't a list of capture handles or names. */
+function badJudgeArtifacts(artifacts) {
+	if (artifacts === void 0) return void 0;
+	if (!Array.isArray(artifacts)) return `must be a list of capture handles or names (got ${JSON.stringify(artifacts)})`;
+	for (const item of artifacts) {
+		if (typeof item === "string" && item.trim()) continue;
+		if (isCaptureHandle(item)) continue;
+		return `must be a list of capture handles or names (got ${JSON.stringify(item)})`;
+	}
+}
+const nameOf = (item) => typeof item === "string" ? item.trim() : item.name;
+/**
+* Resolves call `artifacts` (a capture handle or its name) to PNG bytes this attempt captured.
+* Unknown names throw: a mistake in the call, before the provider is asked.
+*/
+function resolveJudgeArtifacts(artifacts, blobs) {
+	if (!artifacts?.length) return [];
+	return artifacts.map((item) => {
+		const name = nameOf(item);
+		const data = blobs.get(name);
+		if (!data) throw new Error(`c.judge() artifacts names a capture this attempt doesn't have: ${JSON.stringify(name)}`);
+		return {
+			name,
+			path: `artifacts/${name}.png`,
+			sha256: sha256(data),
+			bytes: data.byteLength,
+			mediaType: "image/png",
+			data
+		};
+	});
+}
+const filePartsOf = (resolved) => resolved.map((a) => ({
+	type: "file",
+	mediaType: a.mediaType,
+	data: a.data
+}));
+const artifactRefsOf = (resolved) => resolved.map(({ data: _data, ...ref }) => ref);
+/**
+* JSON-only state stays an object so TypeSafe and other JSON judges are unchanged.
+* File parts are an ordered decide-state array: one JSON part, then each capture.
+*/
+function decideState(json, files) {
+	if (files.length === 0) return json;
+	return [{
+		type: "json",
+		value: json
+	}, ...files];
+}
+//#endregion
 //#region src/judge/size.ts
 /** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
 const CHARS_PER_TOKEN = 4;
@@ -989,12 +1047,14 @@ async function judgeState(turn, output, { include, files }, readFile) {
 * checks after it still run.
 */
 function judgeCaller(a, readFile) {
-	return async (question, { on, ...call } = {}) => {
+	const ask = async (question, { on, ...call } = {}) => {
 		if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
 		const { threshold } = call;
 		if (threshold !== void 0 && !(typeof threshold === "number" && threshold >= 0 && threshold <= 1)) throw new Error(`c.judge() threshold must be between 0 and 1 (got ${String(threshold)})`);
 		const bad = badJudgeList(call);
 		if (bad) throw new Error(`c.judge() ${bad.key} must be a list of non-empty strings (got ${bad.got})`);
+		const badArts = badJudgeArtifacts(call.artifacts);
+		if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
 		const subject = isTurn(on) ? on : a.turns.at(-1);
 		if (!subject) throw new Error("c.judge() has no turn to grade: call c.send() first");
 		const options = {
@@ -1011,11 +1071,15 @@ function judgeCaller(a, readFile) {
 			include,
 			files
 		}, readFile);
+		const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
+		const refs = artifactRefsOf(resolved);
+		const decide = decideState(state, filePartsOf(resolved));
 		const name = judgeModelName(model);
 		const input = {
 			question,
 			model: name,
 			state,
+			...refs.length ? { artifacts: refs } : {},
 			...dropped.length ? { dropped } : {}
 		};
 		a.record.judgeInputs?.push(input);
@@ -1053,20 +1117,25 @@ function judgeCaller(a, readFile) {
 				threshold: atLeast,
 				question,
 				model: name,
-				error: message
+				error: message,
+				...refs.length ? { artifacts: refs } : {}
 			};
 		};
 		const provider = name.split("/")[0];
 		try {
-			const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, state, {
+			const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
 				...judgeOptions,
 				model,
 				threshold: atLeast
 			}, a.signal), a.signal));
-			a.record.judgements.push(retries.length ? {
+			const recorded = refs.length ? {
 				...verdict,
+				artifacts: refs
+			} : verdict;
+			a.record.judgements.push(retries.length ? {
+				...recorded,
 				retries
-			} : verdict);
+			} : recorded);
 			const { score } = verdict;
 			debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
 			debugTrace("judge", "response", {
@@ -1081,7 +1150,7 @@ function judgeCaller(a, readFile) {
 				threshold: verdict.threshold,
 				ms: ms()
 			});
-			return verdict;
+			return recorded;
 		} catch (error) {
 			if (a.signal.aborted) {
 				a.judgeErrored = true;
@@ -1092,6 +1161,7 @@ function judgeCaller(a, readFile) {
 			return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
 		}
 	};
+	return async (question, options) => Object.defineProperty(await ask(question, options), SUBJECT, { value: question });
 }
 //#endregion
 //#region src/sandbox/index.ts
@@ -1100,7 +1170,7 @@ async function createSandbox(descriptor, options) {
 		switch (descriptor.kind) {
 			case "fake": return await createFakeSandbox(descriptor, options.env);
 			case "docker": {
-				const { createDockerSandbox } = await import("./docker-Dnya8eo9.js");
+				const { createDockerSandbox } = await import("./docker-Df_YOxxB.js");
 				return await createDockerSandbox(descriptor, options);
 			}
 		}
@@ -1156,6 +1226,27 @@ function recordFailure(a, error, timedOut) {
 	record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
 }
 /**
+* A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
+* whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
+*/
+async function recordOom(a, sandbox) {
+	const oom = await sandbox?.oomKills?.().catch(() => null);
+	const at = oom && `the ${oom.limit} limit${oom.swap ? ` (+${oom.swap} swap)` : ""}`;
+	if (oom) debugLog("teardown", `oom_kill ${oom.kills} at ${at}`);
+	if (!oom?.kills) return void 0;
+	const message = `out of memory: ${oom.kills === 1 ? "1 process" : `${oom.kills} processes`} killed at ${at}; raise docker({ memory })`;
+	a.record.error = {
+		kind: "infra",
+		message,
+		resource: {
+			cause: "oom",
+			kills: oom.kills
+		}
+	};
+	a.record.outcome = "infra_error";
+	return new InfraError(message);
+}
+/**
 * The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
 * Refused hosts the harness always tries are `expectedBlocked`, summarised in one `-v` line.
 */
@@ -1212,36 +1303,6 @@ function turnErrors(transcripts) {
 	});
 }
 //#endregion
-//#region src/diff.ts
-const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null";
-/** Records the workspace state before the agent runs, in a git dir outside the workspace. */
-async function snapshot(sandbox, signal) {
-	const r = await sandbox.exec(`export GIT_DIR="$PWD/../.reeval-git-$(basename "$PWD")" GIT_WORK_TREE="$PWD" && ${GIT} init -q && ${GIT} add -A && ${GIT} commit -q --allow-empty -m base`, { signal });
-	if (r.exitCode !== 0) throw new InfraError(`Failed to snapshot the workspace: ${r.stderr.trim()}`);
-}
-async function diffSinceSnapshot(sandbox, signal) {
-	const env = `export GIT_DIR="$PWD/../.reeval-git-$(basename "$PWD")" GIT_WORK_TREE="$PWD" && ${GIT} add -A && `;
-	const patch = await sandbox.exec(`${env}${GIT} diff --cached --binary HEAD`, { signal });
-	const names = await sandbox.exec(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
-	if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
-	const status = {
-		A: "added",
-		M: "modified",
-		D: "deleted"
-	};
-	const files = names.stdout.split("\n").filter(Boolean).map((line) => {
-		const [code = "M", ...rest] = line.split("	");
-		return {
-			path: rest.join("	"),
-			status: status[code[0]] ?? "modified"
-		};
-	});
-	return {
-		patch: patch.stdout,
-		files
-	};
-}
-//#endregion
 //#region src/sandbox/phases.ts
 const strip = ({ phase: _, ...sub }) => sub;
 /**
@@ -1642,7 +1703,7 @@ async function runAttempt(spec, fn, handle) {
 	});
 	return withDebug(debug, () => attemptBody(spec, fn, handle, id, debug));
 }
-async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited }, id, debug) {
+async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited, assertions }, id, debug) {
 	const setupStart = performance.now();
 	const ctx = getRunContext();
 	const { runId, stagingDir, config } = ctx;
@@ -1665,7 +1726,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		return starting;
 	};
 	const takeContext = contextTaker(a);
-	const { exec, readFile } = createChecks(a);
+	const { exec, readFile, writeFile } = createChecks(a);
 	const c = {
 		get signal() {
 			return a.signal;
@@ -1673,6 +1734,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		turns: a.turns,
 		exec,
 		readFile,
+		writeFile,
 		serve: (command, port) => serveCommand(a, command, port),
 		capture: (target, options) => capturePage(a, target, options),
 		attempt: {
@@ -1694,6 +1756,8 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		judge: judgeCaller(a, readFile)
 	};
 	const timedOut = () => !!timer?.aborted && !testSignal.aborted;
+	let failure;
+	let oom;
 	try {
 		a.report({ kind: "sandbox" });
 		const where = `${id} [${attemptId(spec.cell.key, spec.n)}]`;
@@ -1710,8 +1774,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
 	} catch (error) {
 		recordFailure(a, error, timedOut());
-		if (timedOut()) throw new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`);
-		throw error;
+		failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
 	} finally {
 		if (!a.signal.aborted) await unawaited().catch(() => {});
 		clock?.clear();
@@ -1724,9 +1787,16 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		await a.session?.close().catch(() => {});
 		if (a.session) debug.log("teardown", "harness session closed");
 		recordEgress(a, sandbox);
+		oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
 		await sandbox?.destroy();
 		if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
 		const { record } = a;
+		const checked = assertions();
+		if (checked.length) record.assertions = checked;
+		for (const x of checked) {
+			const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
+			debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
+		}
 		const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
 		debug.log("teardown", `outcome ${record.outcome}${error}`);
 		const rel = (path) => relativeFile(config.root, path) || ".";
@@ -1735,6 +1805,8 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		setMeta(await writeRecord(a, stagingDir, id));
 		await debug.finish(debug.verbosity === 2 || record.outcome !== "pass");
 	}
+	if (oom) throw oom;
+	if (failure) throw failure.error;
 }
 //#endregion
 //#region src/plan-eval.ts
@@ -1793,6 +1865,11 @@ function planEval(declaration, ctx) {
 		if (mcp) throw new UsageError(`${where} on ${harnessId(planned.harness)}: ${mcp}`);
 	}
 	const path = ctx.file ? relativeFile(ctx.config.root, ctx.file) : null;
+	const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
+	const family = declaration.family && {
+		id: evalId(rankFile, [...groups, declaration.family.name]),
+		...declaration.family
+	};
 	const harnessBy = resolved.sources?.harness ?? ctx.config.file;
 	const plan = evalPlan(resolvedEval, plannedAttempts, {
 		configFile: ctx.config.file,
@@ -1819,7 +1896,8 @@ function planEval(declaration, ctx) {
 			brokered: [],
 			secretVars: [],
 			egress: first.egress,
-			plan
+			plan,
+			...family ? { family } : {}
 		};
 		return [{
 			name: `${name} [${attemptId(first.cell.key, 1)}]`,
@@ -1833,7 +1911,6 @@ function planEval(declaration, ctx) {
 		const workspace = workspaceOf(planned.environment.layers);
 		const files = filesOf(planned.environment.layers);
 		const setBy = setByOf(planned);
-		const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
 		const id = evalId(rankFile, [...groups, name]);
 		const key = attemptKey(id, planned.cell.key, planned.n);
 		const skip = ctx.completed?.includes(key) ? "already completed in this run" : void 0;
@@ -1858,7 +1935,8 @@ function planEval(declaration, ctx) {
 				...plan,
 				skip
 			} : plan,
-			...skip ? { skip } : {}
+			...skip ? { skip } : {},
+			...family ? { family } : {}
 		};
 		return {
 			name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
@@ -1981,7 +2059,7 @@ function checkUnique(where, groups, name, hint) {
 	if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
 	declared.add(key);
 }
-function register(mode, name, own, fn, hint = "") {
+function register(mode, name, own, fn, hint = "", family) {
 	const groups = groupStack.map((g) => g.name);
 	checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
 	const body = `b${bodies.size + 1}`;
@@ -1991,7 +2069,8 @@ function register(mode, name, own, fn, hint = "") {
 		name,
 		mode,
 		options: data(own),
-		body
+		body,
+		...family ? { family } : {}
 	});
 	const tests = planEval({
 		name,
@@ -2000,7 +2079,8 @@ function register(mode, name, own, fn, hint = "") {
 			name,
 			options
 		})),
-		options: own
+		options: own,
+		...family ? { family } : {}
 	}, {
 		...getRunContext(),
 		file: collectingFile(),
@@ -2021,19 +2101,37 @@ const defineEval = (mode) => (name, a, b) => {
 function formatName(template, item, index) {
 	return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => typeof item === "object" && item !== null && key in item ? String(item[key]) : whole);
 }
-/** One eval per case: `evaluate.for(cases)('adds $a and $b', async (c, { a, b }) => …)`. */
-const evalFor = (cases) => (name, a, b) => {
-	const [own, fn] = splitArgs("evaluate.for", name, a, b);
-	const hint = ` For evaluate.for, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
+function checkEachOptions(options) {
+	if (options === void 0) return {};
+	const keys = typeof options === "object" && options !== null ? Object.keys(options) : null;
+	const pool = options?.pool;
+	if (keys?.every((k) => k === "pool") && (pool === void 0 || typeof pool === "boolean")) return options;
+	throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
+}
+/**
+* One eval per case, pooled into one suite:
+* `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
+*/
+const evalEach = (cases, options) => (name, a, b) => {
+	const { pool = true } = checkEachOptions(options);
+	if (!Array.isArray(cases)) throw new UsageError(`evaluate.each takes an array of cases (got ${typeof cases}), e.g. evaluate.each([{ n: 1 }, { n: 2 }])("case $n", async (c, item) => { … }).`);
+	const [own, fn] = splitArgs("evaluate.each", name, a, b);
+	const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
+	const pooled = pool && cases.length >= 2;
 	cases.forEach((item, i) => {
-		register("run", formatName(name, item, i), own, (c) => fn(c, item), hint);
+		const family = pooled ? {
+			name,
+			case: i,
+			cases: cases.length
+		} : void 0;
+		register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
 	});
 };
 const evaluate = Object.assign(defineEval("run"), {
 	skip: defineEval("skip"),
 	only: defineEval("only"),
 	todo: registerTodo,
-	for: evalFor
+	each: evalEach
 });
 //#endregion
 export { DEFAULT_DIRECTORY_IGNORES, DEFAULT_EGRESS_ALLOW, DEFAULT_SKILL_EXCLUDE, HarnessCapabilityError, defineConfig, defineEnvironment, evaluate, expect, group };