reeval 0.2.0 → 0.4.0

package/dist/index.js

@@ -1,22 +1,22 @@
 import { n as isTimeout, t as UsageError } from "./errors-CSxfJ4qu.js";
-import { b as parseDuration, d as isSecret, f as isSecretTemplate, g as stepCommand, l as isHostFile, m as parseGitRepo, o as defineEnvironment, p as mcpHosts, t as DEFAULT_SKILL_EXCLUDE, v as workspaceLabel, y as workspaceOf } from "./environment-9WHSWLxH.js";
+import { C as workspaceLabel, T as parseDuration, _ as isSecretTemplate, c as defineEnvironment, g as isSecret, m as isHostFile, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES, u as filesOf, v as mcpHosts, w as workspaceOf, x as stepCommand, y as parseGitRepo } from "./environment-lDFQY2BF.js";
 import { t as inCI } from "./env-BQqj93gu.js";
-import { a as registerSecret, i as redactSecrets } from "./redact-C4fL_bZP.js";
+import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
 import { t as InfraError } from "./types-C5vI3g1C.js";
-import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-QJMzVtbw.js";
+import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-C7Obb71D.js";
 import { c as relativeFile, i as evalSlug, l as sha256, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
-import { l as runJudge, s as judgeModelName } from "./judge-D8Uf4dA9.js";
-import { S as placeholder, et as run, tt as shellQuote } from "./src-DwwiVLqv.js";
-import { a as evalPlan, d as secretVarsIn, f as unbrokered, i as checkOptions, l as matchesTags, n as cascade, o as expand, p as unbrokeredError, s as harnessId, t as brokeredIn, u as sandboxId } from "./options-k25fjaER.js";
-import { n as badCharacterAt, o as defineConfig } from "./config-DqJvSqYo.js";
-import { a as withLease, c as registerGroup, l as registerTest, o as collectingFile, p as expect, s as getRunContext, t as HarnessCapabilityError } from "./matchers-DyLvm7EG.js";
-import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-BdRQ9jKX.js";
+import { c as judgeModelName, d as isApiError, i as badJudgeList, u as runJudge } from "./judge-D6wqgTDp.js";
+import { S as placeholder, et as run, tt as shellQuote } from "./src-V-iU5X9S.js";
+import { a as evalPlan, d as sandboxId, f as secretVarsIn, i as checkOptions, l as matchesTags, m as unbrokeredError, n as cascade, o as expand, p as unbrokered, s as harnessId, t as brokeredIn } from "./options-Ftb2gE01.js";
+import { n as badCharacterAt, o as defineConfig } from "./config-B93Csktg.js";
+import { a as withLease, c as registerGroup, l as registerTest, o as collectingFile, p as expect, s as getRunContext, t as HarnessCapabilityError } from "./matchers-DbFPZQgA.js";
+import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CGPV3BZY.js";
 import { t as attemptDebugDir } from "./files-BLlrhU4-.js";
 import { n as clip, r as formatDuration, t as capMiddle } from "./text-gEcKjAyp.js";
 import { t as contextTaker } from "./context-loi9OCZv.js";
 import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { i as preparedWorkspace } from "./workspace-uPapQj16.js";
-import { r as patchForJudge, t as changedFilesForJudge } from "./patch-BkKje4cp.js";
+import { i as preparedWorkspace } from "./workspace-QtlJtRut.js";
+import { n as judgeDiff } from "./patch-BxqggH-T.js";
 import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
 import { basename, isAbsolute, join, relative, sep } from "node:path";
 import { setTimeout as setTimeout$1 } from "node:timers/promises";
@@ -26,12 +26,12 @@ import { parse } from "yaml";
 /** Runs a check or setup step with the checks' brokered secrets swapped in (see brokerPhases). */
 const asCheck = (a, fn) => a.phases ? a.phases.checks(fn) : fn();
 function requireSandbox(a) {
-	if (!a.sandbox) throw new Error("No sandbox yet: call t.send() or await t.sandbox() before t.exec() or t.readFile()");
+	if (!a.sandbox) throw new Error("No sandbox yet: call c.send() or await c.sandbox() before c.exec() or c.readFile()");
 	return a.sandbox;
 }
 /**
-* Phase 4, checks: `t.exec` runs with the checks' variables and secrets, each run recorded and
-* timed; `t.readFile` reads the sandbox as is.
+* Phase 4, checks: `c.exec` runs with the checks' variables and secrets, each run recorded and
+* timed; `c.readFile` reads the sandbox as is.
 */
 function createChecks(a) {
 	const checkEnv = () => a.env?.checkVars ?? {};
@@ -552,7 +552,8 @@ async function resolveEnvironment(environment, optionsEnv, root, where, rulesFil
 	}
 	const fileList = [...files].map(([path, v]) => ({
 		path,
-		content: isHostFile(v) ? readHostFile(v, envWhere) : v
+		content: isHostFile(v) ? readHostFile(v, envWhere) : v,
+		...isHostFile(v) && statSync(v.path).mode & 64 ? { executable: true } : {}
 	}));
 	const rulesContent = rules === void 0 ? void 0 : isHostFile(rules) ? Buffer.from(readHostFile(rules, envWhere)).toString("utf8") : rules;
 	const skillList = [...skills.values()];
@@ -568,6 +569,7 @@ async function resolveEnvironment(environment, optionsEnv, root, where, rulesFil
 		path: f.path,
 		sha256: sha256(f.content)
 	}));
+	const executable = fileList.flatMap((f) => f.executable ? [f.path] : []);
 	const rulesRecord = rulesContent === void 0 ? void 0 : {
 		path: rulesFile,
 		sha256: sha256(rulesContent)
@@ -614,6 +616,7 @@ async function resolveEnvironment(environment, optionsEnv, root, where, rulesFil
 		]),
 		mcp: mcpShape,
 		files: fileRecords,
+		...executable.length ? { executable } : {},
 		rules: rulesRecord?.sha256,
 		vars: literalVars,
 		secrets: secretNames,
@@ -905,7 +908,7 @@ async function callMcpTool(sandbox, servers, name, tool, signal) {
 //#region src/harness/index.ts
 async function createHarness(descriptor) {
 	if (descriptor.kind === "fake") return createFakeHarness(descriptor);
-	const { createAgentHarness } = await import("./agent-D-6z0Z3F.js");
+	const { createAgentHarness } = await import("./agent-M70OWyfG.js");
 	return createAgentHarness(descriptor);
 }
 /** Where each harness's agent looks for repo skills, relative to the workspace. */
@@ -932,8 +935,8 @@ async function prepareEnvironment(a, root, runId, where, signal) {
 	if (debug) traceEnvironment(debug, spec, env);
 	record.image = null;
 	if (spec.sandbox.kind === "docker") {
-		const { preparedFor } = await import("./preflight-D2Hce5Jp.js");
-		const { dockerProvider } = await import("./providers-BMIGzPPG.js");
+		const { preparedFor } = await import("./preflight-D3cJETui.js");
+		const { dockerProvider } = await import("./providers-Cay8pI0y.js");
 		signal.throwIfAborted();
 		const prepared = preparedFor(root, runId, dockerProvider(), spec.sandbox, spec.harness);
 		if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
@@ -944,29 +947,100 @@ async function prepareEnvironment(a, root, runId, where, signal) {
 	} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
 }
 //#endregion
+//#region src/judge/size.ts
+/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
+const CHARS_PER_TOKEN = 4;
+const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
+const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
+const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
+	name: /^diff --git a\/.+ b\/(.+)$/m.exec(part)?.[1] ?? "",
+	chars: part.length
+})).filter((item) => item.name);
+/** The state's parts by size, each with its biggest files where it has any. */
+function partsOf(state, question) {
+	return [...Object.entries(state).map(([name, value]) => {
+		const text = typeof value === "string" ? value : JSON.stringify(value);
+		const items = name === "diff" && typeof value === "string" ? diffItems(value) : value && typeof value === "object" && !Array.isArray(value) ? Object.entries(value).map(([file, content]) => ({
+			name: file,
+			chars: typeof content === "string" ? content.length : JSON.stringify(content).length
+		})) : [];
+		return {
+			name,
+			chars: text?.length ?? 0,
+			items
+		};
+	}), {
+		name: "question",
+		chars: question.length,
+		items: []
+	}].sort((a, b) => b.chars - a.chars);
+}
+/** The estimated size of what the judge reads, in tokens. */
+const inputTokens = (state, question) => tokensOf(JSON.stringify(state).length + question.length);
+/**
+* `diff 306k, of which dist/assets/index-….js is 296k; output 1k; …`: the state's parts by
+* estimated tokens, naming the biggest files in each.
+*/
+function describeSize(state, question) {
+	const parts = partsOf(state, question);
+	const big = parts.filter((p) => tokensOf(p.chars) >= 1e3);
+	return (big.length ? big : parts.slice(0, 1)).map((p) => {
+		const top = p.items.sort((a, b) => b.chars - a.chars).slice(0, 3).filter((item) => tokensOf(item.chars) >= 1e3).map((item) => `${item.name} is ${k(tokensOf(item.chars))}`);
+		return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
+	}).join("; ");
+}
+const HINT = "Narrow what the judge sees with include, e.g. c.judge(question, { include: ['src/'] }), or set judge: { include } in the config; files shows chosen files in full.";
+/**
+* A provider's error, as the provider worded it, then what the judge's input held: its estimated
+* size and biggest parts and files, and how to narrow it.
+*/
+const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}. ${HINT}`;
+//#endregion
 //#region src/attempt/judge.ts
-const noJudgeModel = (configFile) => `t.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.13.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
+const noJudgeModel = (configFile) => `c.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.13.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
 const isTurn = (value) => typeof value === "object" && value !== null && "transcript" in value && "diff" in value;
-function judgeState(turn, output = turn.text) {
-	return {
+const workspacePath = (path) => path.replace(/^\.\//, "");
+/**
+* The turn's task and output, its changed files and diff as `judgeDiff` shows them, and the whole
+* current content of each of `files` (null when there's no such file).
+*/
+async function judgeState(turn, output, { include, files }, readFile) {
+	const inFull = files?.length ? files : void 0;
+	const shown = judgeDiff(turn.diff, {
+		...include ? { include } : {},
+		...inFull ? { inFull: new Set(inFull.map(workspacePath)) } : {}
+	});
+	const state = {
 		task: turn.prompt,
 		output,
-		files: changedFilesForJudge(turn.diff.files),
-		diff: patchForJudge(turn.diff.patch)
+		files: shown.files,
+		diff: shown.patch
+	};
+	if (inFull) {
+		const contents = {};
+		for (const path of inFull) contents[workspacePath(path)] = await readFile(path);
+		state.contents = contents;
+	}
+	return {
+		state,
+		dropped: shown.dropped
 	};
 }
 /**
 * Phase 4, the judge: grades a turn (the last by default) or a value against a question, under
 * the judge provider's concurrency lease. Every call's input is recorded, including calls that
-* fail, and a failure marks the attempt's assertion errors as the judge's.
+* fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
+* checks after it still run.
 */
-function judgeCaller(a) {
+function judgeCaller(a, readFile) {
 	return async (question, { on, ...call } = {}) => {
-		if (typeof question !== "string" || !question.trim()) throw new Error(`t.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
+		if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
 		const { threshold } = call;
-		if (threshold !== void 0 && !(typeof threshold === "number" && threshold >= 0 && threshold <= 1)) throw new Error(`t.judge() threshold must be between 0 and 1 (got ${String(threshold)})`);
+		if (threshold !== void 0 && !(typeof threshold === "number" && threshold >= 0 && threshold <= 1)) throw new Error(`c.judge() threshold must be between 0 and 1 (got ${String(threshold)})`);
+		const bad = badJudgeList(call);
+		if (bad) throw new Error(`c.judge() ${bad.key} must be a list of non-empty strings (got ${bad.got})`);
 		const subject = isTurn(on) ? on : a.turns.at(-1);
-		if (!subject) throw new Error("t.judge() has no turn to grade: call t.send() first");
+		if (!subject) throw new Error("c.judge() has no turn to grade: call c.send() first");
 		const options = {
 			...a.spec.judge,
 			...call
@@ -976,35 +1050,69 @@ function judgeCaller(a) {
 			a.judgeErrored = true;
 			throw new Error(noJudgeModel(getRunContext().config.file));
 		}
-		const state = judgeState(subject, on === void 0 || isTurn(on) ? void 0 : on);
+		const { include, files, maxConcurrent, ...judgeOptions } = options;
+		const { state, dropped } = await judgeState(subject, on === void 0 || isTurn(on) ? subject.text : on, {
+			include,
+			files
+		}, readFile);
+		const name = judgeModelName(model);
 		const input = {
 			question,
-			model: judgeModelName(model),
-			state
+			model: name,
+			state,
+			...dropped.length ? { dropped } : {}
 		};
 		a.record.judgeInputs?.push(input);
 		const start = performance.now();
 		const ms = () => Math.round(performance.now() - start);
+		const atLeast = options.threshold ?? .8;
 		a.report({
 			kind: "judge-start",
 			question
 		});
-		debugLog("judge", `asking ${input.model}: ${JSON.stringify(clip(question, 160))}`);
+		debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
+		if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
 		debugTrace("judge", "prompt", input, "judge prompt");
-		const { maxConcurrent, ...judgeOptions } = options;
-		const provider = judgeModelName(model).split("/")[0];
+		const failed = (message, retries) => {
+			a.judgeErrored = true;
+			a.judgeErrors.push(`c.judge(${JSON.stringify(clip(question, 80))}) failed: ${message}`);
+			input.error = message;
+			if (retries?.length) input.retries = retries;
+			debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
+			debugTrace("judge", "error", {
+				error: message,
+				retries
+			}, "judge error");
+			a.report({
+				kind: "judge-end",
+				question,
+				verdict: "error",
+				score: null,
+				threshold: null,
+				ms: ms()
+			});
+			return {
+				verdict: "error",
+				score: null,
+				threshold: atLeast,
+				question,
+				model: name,
+				error: message
+			};
+		};
+		const provider = name.split("/")[0];
 		try {
 			const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, state, {
 				...judgeOptions,
 				model,
-				threshold: options.threshold ?? .8
+				threshold: atLeast
 			}, a.signal), a.signal));
 			a.record.judgements.push(retries.length ? {
 				...verdict,
 				retries
 			} : verdict);
-			const { score, threshold } = verdict;
-			debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= threshold ? "≥" : "<"} ${threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= threshold ? "at or above" : "below"} the ${threshold} threshold`);
+			const { score } = verdict;
+			debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
 			debugTrace("judge", "response", {
 				verdict,
 				retries
@@ -1014,29 +1122,18 @@ function judgeCaller(a) {
 				question,
 				verdict: verdict.verdict,
 				score,
-				threshold,
+				threshold: verdict.threshold,
 				ms: ms()
 			});
 			return verdict;
 		} catch (error) {
-			a.judgeErrored = true;
+			if (a.signal.aborted) {
+				a.judgeErrored = true;
+				throw error;
+			}
 			const { retries } = error;
-			input.error = error instanceof Error ? error.message : String(error);
-			if (retries?.length) input.retries = retries;
-			debugLog("judge", `${input.model} · ${formatDuration(ms())} · error: ${input.error}`);
-			debugTrace("judge", "error", {
-				error: input.error,
-				retries
-			}, "judge error");
-			a.report({
-				kind: "judge-end",
-				question,
-				verdict: "error",
-				score: null,
-				threshold: null,
-				ms: ms()
-			});
-			throw error;
+			const message = error instanceof Error ? error.message : String(error);
+			return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
 		}
 	};
 }
@@ -1047,7 +1144,7 @@ async function createSandbox(descriptor, options) {
 		switch (descriptor.kind) {
 			case "fake": return await createFakeSandbox(descriptor, options.env);
 			case "docker": {
-				const { createDockerSandbox } = await import("./docker-Buu3a60z.js");
+				const { createDockerSandbox } = await import("./docker-Bv1ZC6V7.js");
 				return await createDockerSandbox(descriptor, options);
 			}
 		}
@@ -1258,7 +1355,8 @@ function createAttempt(spec, signal, progress) {
 			}
 		},
 		signal,
-		judgeErrored: false
+		judgeErrored: false,
+		judgeErrors: []
 	};
 }
 const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s");
@@ -1423,7 +1521,13 @@ async function startSession(a, runId, setupStart) {
 	for (const f of env.files) {
 		await box.writeFile(f.path, f.content);
 		const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
-		debugLog("env", `file ${f.path} (${size} bytes)`);
+		debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
+	}
+	const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
+	if (executable.length) {
+		const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
+		const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
+		if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
 	}
 	const skillDir = workspaceSkillDir(spec.harness);
 	for (const skill of env.workspaceSkills) {
@@ -1605,7 +1709,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 	};
 	const takeContext = contextTaker(a);
 	const { exec, readFile } = createChecks(a);
-	const t = {
+	const c = {
 		get signal() {
 			return a.signal;
 		},
@@ -1628,7 +1732,7 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 			exec,
 			readFile
 		}),
-		judge: judgeCaller(a)
+		judge: judgeCaller(a, readFile)
 	};
 	const timedOut = () => !!timer?.aborted && !testSignal.aborted;
 	try {
@@ -1639,10 +1743,12 @@ async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, un
 		timer = clock.signal;
 		a.signal = AbortSignal.any([testSignal, timer]);
 		timer.addEventListener("abort", destroy, { once: true });
-		await fn(t);
+		await fn(c);
 		a.signal.throwIfAborted();
 		const missed = await unawaited();
 		if (missed) throw missed;
+		const [only, ...more] = a.judgeErrors;
+		if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
 	} catch (error) {
 		recordFailure(a, error, timedOut());
 		if (timedOut()) throw new Error(`Eval timed out after ${spec.timeoutMs / 1e3}s`);
@@ -1763,6 +1869,7 @@ function planEval(declaration, ctx) {
 	return plannedAttempts.map((planned) => {
 		if (!cells.includes(planned.cell.key)) cells.push(planned.cell.key);
 		const workspace = workspaceOf(planned.environment.layers);
+		const files = filesOf(planned.environment.layers);
 		const attempt = {
 			name,
 			groups,
@@ -1779,6 +1886,7 @@ function planEval(declaration, ctx) {
 			secretVars: secretVarsIn(resolved.env, planned),
 			egress: planned.egress,
 			...workspace ? { workspace } : {},
+			...files.length ? { files } : {},
 			plan
 		};
 		const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
@@ -1839,7 +1947,7 @@ function splitArgs(what, name, a, b) {
 	if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environment: …) or in the config's env.`);
 	if (typeof a === "function") return [b ?? {}, a];
 	if (typeof b === "function") return [a ?? {}, b];
-	throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (t) => { … }).`);
+	throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
 }
 /** `evaluate.only` and `group.only` are refused in CI, where they'd silently skip the rest. */
 function checkOnly(mode, what) {
@@ -1939,12 +2047,12 @@ const defineEval = (mode) => (name, a, b) => {
 function formatName(template, item, index) {
 	return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => typeof item === "object" && item !== null && key in item ? String(item[key]) : whole);
 }
-/** One eval per case: `evaluate.for(cases)('adds $a and $b', async (t, c) => …)`. */
+/** One eval per case: `evaluate.for(cases)('adds $a and $b', async (c, { a, b }) => …)`. */
 const evalFor = (cases) => (name, a, b) => {
 	const [own, fn] = splitArgs("evaluate.for", name, a, b);
 	const hint = ` For evaluate.for, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
 	cases.forEach((item, i) => {
-		register("run", formatName(name, item, i), own, (t) => fn(t, item), hint);
+		register("run", formatName(name, item, i), own, (c) => fn(c, item), hint);
 	});
 };
 const evaluate = Object.assign(defineEval("run"), {
@@ -1954,6 +2062,6 @@ const evaluate = Object.assign(defineEval("run"), {
 	for: evalFor
 });
 //#endregion
-export { DEFAULT_EGRESS_ALLOW, DEFAULT_SKILL_EXCLUDE, HarnessCapabilityError, defineConfig, defineEnvironment, evaluate, expect, group };
+export { DEFAULT_DIRECTORY_IGNORES, DEFAULT_EGRESS_ALLOW, DEFAULT_SKILL_EXCLUDE, HarnessCapabilityError, defineConfig, defineEnvironment, evaluate, expect, group };
 
 //# sourceMappingURL=index.js.map