reeval 0.0.1

package/dist/index.js

@@ -0,0 +1,1959 @@
+import { n as isTimeout, t as UsageError } from "./errors-CSxfJ4qu.js";
+import { _ as stepCommand, b as workspaceOf, d as isSecret, f as isSecretTemplate, h as parseGitRepo, l as isHostFile, m as mcpHosts, o as defineEnvironment, t as DEFAULT_SKILL_EXCLUDE, x as parseDuration, y as workspaceLabel } from "./environment-Mz7wvrtD.js";
+import { t as inCI } from "./env-BQqj93gu.js";
+import { a as registerSecret, i as redactSecrets } from "./redact-C4fL_bZP.js";
+import { t as InfraError } from "./types-C5vI3g1C.js";
+import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-73SQfKdx.js";
+import { c as relativeFile, i as evalSlug, l as sha256, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
+import { l as runJudge, s as judgeModelName } from "./judge-D8Uf4dA9.js";
+import { S as placeholder, et as run, tt as shellQuote } from "./src-DwwiVLqv.js";
+import { a as evalPlan, d as secretVarsIn, f as unbrokered, i as checkOptions, l as matchesTags, n as cascade, o as expand, p as unbrokeredError, s as harnessId, t as brokeredIn, u as sandboxId } from "./options-CAQhu--C.js";
+import { n as badCharacterAt, o as defineConfig } from "./config-Cuj0kVYj.js";
+import { a as withLease, c as registerGroup, l as registerTest, o as collectingFile, p as expect, s as getRunContext, t as HarnessCapabilityError } from "./matchers-DyLvm7EG.js";
+import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-BdRQ9jKX.js";
+import { t as attemptDebugDir } from "./files-BLlrhU4-.js";
+import { n as clip, r as formatDuration, t as capMiddle } from "./text-gEcKjAyp.js";
+import { t as contextTaker } from "./context-loi9OCZv.js";
+import { t as PrepareError } from "./provider-CmAYTQQs.js";
+import { i as preparedWorkspace } from "./workspace-C2S0Z6gl.js";
+import { r as patchForJudge, t as changedFilesForJudge } from "./patch-BkKje4cp.js";
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { basename, isAbsolute, join, relative, sep } from "node:path";
+import { setTimeout as setTimeout$1 } from "node:timers/promises";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { parse } from "yaml";
+//#region src/attempt/checks.ts
+/** Runs a check or setup step with the checks' brokered secrets swapped in (see brokerPhases). */
+const asCheck = (a, fn) => a.phases ? a.phases.checks(fn) : fn();
+function requireSandbox(a) {
+	if (!a.sandbox) throw new Error("No sandbox yet: call t.send() or await t.sandbox() before t.exec() or t.readFile()");
+	return a.sandbox;
+}
+/**
+* Phase 4, checks: `t.exec` runs with the checks' variables and secrets, each run recorded and
+* timed; `t.readFile` reads the sandbox as is.
+*/
+function createChecks(a) {
+	const checkEnv = () => a.env?.checkVars ?? {};
+	const exec = async (command) => {
+		const box = requireSandbox(a);
+		const start = performance.now();
+		a.report({
+			kind: "check-start",
+			command
+		});
+		const r = await a.timed("checksMs", () => asCheck(a, () => box.exec(command, {
+			signal: a.signal,
+			env: checkEnv()
+		})));
+		const durationMs = Math.round(performance.now() - start);
+		a.record.checks.push({
+			command,
+			...r,
+			durationMs
+		});
+		debugLog("check", `$ ${clip(command, 160)} → exit ${r.exitCode} · ${formatDuration(durationMs)}`);
+		debugTrace("check", "output", {
+			command,
+			...r,
+			durationMs
+		}, `check output: ${clip(command, 80)}`);
+		a.report({
+			kind: "check-end",
+			command,
+			exitCode: r.exitCode,
+			ms: durationMs
+		});
+		return r;
+	};
+	const readFile = (path) => requireSandbox(a).readFile(path);
+	return {
+		exec,
+		readFile
+	};
+}
+//#endregion
+//#region src/attempt/debug.ts
+/** Which config and profile the attempt runs under, and where each resolved option came from. */
+function tracePlan(debug, spec, ctx) {
+	const config = ctx.configFile ?? "none (no config file found)";
+	const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
+	const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
+	debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
+	debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harness")}`);
+	debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
+	const env = spec.environment;
+	const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
+	const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
+	debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
+	debug.log("plan", `egress ${egressLabel(spec.egress) || "allow-all"} ${from("egress")}`);
+	const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
+	debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
+	const judge = spec.judge;
+	const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
+	debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
+	for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
+	debug.trace("plan", "config", {
+		config: ctx.config,
+		profile: ctx.profile
+	}, "resolved config");
+	debug.trace("plan", "attempt", {
+		cell: spec.cell,
+		n: spec.n,
+		harness: spec.harness,
+		sandbox: spec.sandbox,
+		egress: spec.egress,
+		gate: spec.gate,
+		timeoutMs: spec.timeoutMs,
+		judge: spec.judge,
+		sources: spec.sources,
+		env: Object.keys(spec.env).sort()
+	}, "resolved attempt");
+}
+/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
+function endpointLabel({ method, host, path }) {
+	const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
+	return `${method?.join("|") ?? "*"} ${where}`;
+}
+/** The resolved environment and what the gateway brokers, by name. */
+function traceEnvironment(debug, spec, env) {
+	const r = env.record;
+	debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
+	const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
+	debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
+	for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
+	const brokering = brokeringFor(spec.harness);
+	if (brokering) {
+		const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
+		debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
+	}
+	debug.trace("env", "environment", {
+		record: r,
+		vars: Object.keys(env.vars).sort(),
+		checkVars: Object.keys(env.checkVars).sort(),
+		files: env.files.map((f) => f.path),
+		setup: env.setup,
+		skills: env.skills.map((s) => s.name),
+		workspaceSkills: env.workspaceSkills.map((s) => s.name),
+		mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
+			transport: m.transport,
+			command: m.command,
+			args: m.args,
+			env: Object.keys(m.env)
+		} : {
+			transport: m.transport,
+			url: m.url,
+			headers: Object.keys(m.headers)
+		}])),
+		brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
+			name,
+			hosts,
+			in: where,
+			phase
+		}))
+	}, "resolved environment");
+}
+//#endregion
+//#region src/environment/resolve.ts
+const SKILL_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+const DEFAULT_SETUP_TIMEOUT = "5m";
+function broker(secrets, name, value, hosts, where = ["header"], phase = "agent") {
+	const at = badCharacterAt(value);
+	if (at && where.includes("header")) throw new InfraError(`${name} has whitespace or a line break in it (character ${at}), so it can't go in a request header. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
+	const sub = {
+		name,
+		placeholder: placeholder(),
+		value,
+		hosts: [...hosts],
+		in: [...where],
+		phase
+	};
+	secrets.brokered.push(sub);
+	return sub.placeholder;
+}
+/** The host value, or its placeholder when it's brokered to `secret.hosts` (else `hosts`). */
+function hostValue(name, secret, where, secrets, hosts = secret.hosts) {
+	const from = secret.from ?? name;
+	const value = process.env[from];
+	if (!value) {
+		if (secret.optional) {
+			secrets.unset.push(name);
+			return;
+		}
+		const source = from === name ? name : `${name} (from ${from})`;
+		throw new InfraError(`Secret ${source} for ${where} isn't set. Set it in the environment or .env.local.`);
+	}
+	registerSecret(value);
+	if (secret.to === "checks") secrets.checks.push(name);
+	else secrets.names.push(name);
+	if (!hosts) return value;
+	return broker(secrets, name, value, hosts, secret.in, secret.to === "checks" ? "checks" : "agent");
+}
+/** Each host variable in the template, or with `hosts` one placeholder per variable. */
+function template(value, where, secrets, hosts) {
+	const placeholders = /* @__PURE__ */ new Map();
+	return value.template.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, name) => {
+		const v = process.env[name];
+		if (!v) throw new InfraError(`secret.template needs ${name} for ${where}, and it isn't set`);
+		registerSecret(v);
+		if (!hosts) return v;
+		const p = placeholders.get(name) ?? broker(secrets, name, v, hosts);
+		placeholders.set(name, p);
+		return p;
+	});
+}
+/** A `SKILL.md` split into its YAML frontmatter's `name` and `description` and its body. */
+function parseSkillMd(text, where) {
+	const match = /^---\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)([\s\S]*)$/.exec(text);
+	if (!match) return { body: text };
+	let front;
+	try {
+		front = parse(match[1] ?? "");
+	} catch (error) {
+		const reason = error.message.split("\n")[0];
+		throw new InfraError(`${where}: the SKILL.md frontmatter isn't valid YAML (${reason}). Quote values that contain ": ".`);
+	}
+	if (front != null && (typeof front !== "object" || Array.isArray(front))) throw new InfraError(`${where}: the SKILL.md frontmatter must be a YAML mapping`);
+	const field = (key) => {
+		const value = front?.[key];
+		if (value == null) return void 0;
+		if (typeof value !== "string") throw new InfraError(`${where}: the SKILL.md ${key} must be a string (got ${typeof value})`);
+		return value.trim() || void 0;
+	};
+	return {
+		name: field("name"),
+		description: field("description"),
+		body: match[2] ?? ""
+	};
+}
+/** A gitignore-style test for `DEFAULT_SKILL_EXCLUDE`-shaped patterns against a skill-relative path. */
+function excludeMatcher(patterns) {
+	const rules = patterns.map((pattern) => {
+		const dirOnly = pattern.endsWith("/");
+		const body = pattern.replace(/\/+$/, "");
+		const anchored = body.includes("/");
+		const glob = body.replace(/^\/+/, "").split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*");
+		return {
+			dirOnly,
+			re: new RegExp(`${anchored ? "^" : "(?:^|/)"}${glob}$`)
+		};
+	});
+	return (path, isDir) => rules.some((r) => (isDir || !r.dirOnly) && r.re.test(path));
+}
+function parseSkill(dir, source, exclude) {
+	const front = parseSkillMd(readFileSync(join(dir, "SKILL.md"), "utf8"), `Skill ${basename(dir)} in ${source}`);
+	const name = front.name || basename(dir);
+	checkSkill(name, front.description, source);
+	const skip = excludeMatcher(exclude);
+	const paths = [];
+	const excluded = [];
+	const walk = (rel) => {
+		for (const entry of readdirSync(join(dir, rel), { withFileTypes: true })) {
+			const path = rel ? `${rel}/${entry.name}` : entry.name;
+			const isDir = entry.isDirectory();
+			if (entry.name === ".git") continue;
+			if (skip(path, isDir)) excluded.push(isDir ? `${path}/` : path);
+			else if (isDir) walk(path);
+			else if (entry.isFile() && path !== "SKILL.md") paths.push(path);
+		}
+	};
+	walk("");
+	const files = paths.sort().map((path) => ({
+		path,
+		content: readFileSync(join(dir, path), "utf8")
+	}));
+	return {
+		skill: {
+			name,
+			description: front.description ?? "",
+			content: front.body,
+			files
+		},
+		excluded: excluded.sort()
+	};
+}
+function checkSkill(name, description, source) {
+	if (!SKILL_NAME.test(name)) throw new InfraError(`Skill "${name}" in ${source}: the name must be kebab-case`);
+	if (!description) throw new InfraError(`Skill "${name}" in ${source} has no description in its SKILL.md frontmatter`);
+}
+function loadSkills(dir, source, only, exclude = DEFAULT_SKILL_EXCLUDE) {
+	if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new InfraError(`Skill source ${source} isn't a directory`);
+	const dirs = existsSync(join(dir, "SKILL.md")) ? [dir] : readdirSync(dir, { withFileTypes: true }).filter((e) => e.isDirectory() && existsSync(join(dir, e.name, "SKILL.md"))).map((e) => join(dir, e.name)).sort();
+	if (dirs.length === 0) throw new InfraError(`Skill source ${source} has no SKILL.md (and no subdirectories with one)`);
+	const skills = dirs.map((d) => parseSkill(d, source, exclude));
+	if (!only) return skills;
+	const missing = only.filter((n) => !skills.some((s) => s.skill.name === n));
+	if (missing.length) throw new InfraError(`only: ${JSON.stringify(missing)} not found in ${source} (found: ${skills.map((s) => s.skill.name).join(", ")})`);
+	return skills.filter((s) => only.includes(s.skill.name));
+}
+async function git(args, cwd, what) {
+	const r = await run("git", args, { cwd });
+	if (r.exitCode !== 0) throw new InfraError(`${what} failed: ${r.stderr.trim()}`);
+	return r.stdout.trim();
+}
+/** One clone per source and ref per worker process; the commit is recorded. */
+const clones = /* @__PURE__ */ new Map();
+function cloneSkills(root, url, ref) {
+	const key = `${url}#${ref ?? ""}`;
+	let clone = clones.get(key);
+	if (!clone) {
+		clone = (async () => {
+			const dir = join(root, ".reeval", "cache", "skills", `${sha256(key).slice(0, 16)}-${process.pid}`);
+			await rm(dir, {
+				recursive: true,
+				force: true
+			});
+			await mkdir(dir, { recursive: true });
+			const what = `Cloning ${url}${ref ? ` at ${ref}` : ""}`;
+			await git(["init", "-q"], dir, what);
+			await git([
+				"fetch",
+				"-q",
+				"--depth",
+				"1",
+				url,
+				ref ?? "HEAD"
+			], dir, what);
+			await git([
+				"-c",
+				"advice.detachedHead=false",
+				"checkout",
+				"-q",
+				"FETCH_HEAD"
+			], dir, what);
+			return {
+				dir,
+				commit: await git(["rev-parse", "HEAD"], dir, what)
+			};
+		})();
+		clones.set(key, clone);
+		clone.catch(() => clones.delete(key));
+	}
+	return clone;
+}
+const packs = /* @__PURE__ */ new Map();
+/** `npm pack` into the cache, unpacked; returns the package directory. */
+function unpackNpm(root, spec) {
+	let pack = packs.get(spec);
+	if (!pack) {
+		pack = (async () => {
+			const dir = join(root, ".reeval", "cache", "skills", `npm-${sha256(spec).slice(0, 16)}-${process.pid}`);
+			await rm(dir, {
+				recursive: true,
+				force: true
+			});
+			await mkdir(dir, { recursive: true });
+			const packed = await run("npm", [
+				"pack",
+				spec,
+				"--silent",
+				"--pack-destination",
+				dir
+			], { cwd: dir });
+			const tarball = packed.stdout.trim().split("\n").at(-1);
+			if (packed.exitCode !== 0 || !tarball) throw new InfraError(`npm pack ${spec} failed: ${packed.stderr.trim()}`);
+			const untar = await run("tar", ["-xzf", tarball], { cwd: dir });
+			if (untar.exitCode !== 0) throw new InfraError(`Unpacking ${spec} failed: ${untar.stderr.trim()}`);
+			return join(dir, "package");
+		})();
+		packs.set(spec, pack);
+		pack.catch(() => packs.delete(spec));
+	}
+	return pack;
+}
+async function resolveSkills(root, s) {
+	switch (s.source) {
+		case "local": {
+			const label = `local:${relative(root, s.path).split(sep).join("/") || "."}`;
+			return {
+				skills: loadSkills(s.path, label, s.only, s.exclude),
+				source: label,
+				commit: null
+			};
+		}
+		case "git": {
+			const { url } = parseGitRepo(s.repo);
+			const { dir, commit } = await cloneSkills(root, url, s.ref);
+			const label = `${s.repo}${s.ref ? `#${s.ref}` : ""}${s.path ? `:${s.path}` : ""}`;
+			return {
+				skills: loadSkills(s.path ? join(dir, s.path) : dir, label, s.only, s.exclude),
+				source: label,
+				commit
+			};
+		}
+		case "registry": {
+			const [repo = "", name = ""] = s.id.split("@");
+			const { dir, commit } = await cloneSkills(root, `https://github.com/${repo}.git`, s.version);
+			const label = `registry:${s.id}${s.version ? `#${s.version}` : ""}`;
+			return {
+				skills: loadSkills(existsSync(join(dir, "skills")) ? join(dir, "skills") : dir, label, [name]),
+				source: label,
+				commit
+			};
+		}
+		case "npm": {
+			const dir = await unpackNpm(root, s.spec);
+			const label = `npm:${isAbsolute(s.spec) ? relative(root, s.spec).split(sep).join("/") : s.spec}:${s.path}`;
+			return {
+				skills: loadSkills(join(dir, s.path), label, s.only, s.exclude),
+				source: label,
+				commit: null
+			};
+		}
+		case "inline": {
+			checkSkill(s.name, s.description, "skill.inline");
+			const files = Object.entries(s.files).sort(([a], [b]) => a < b ? -1 : 1).map(([path, content]) => ({
+				path,
+				content
+			}));
+			return {
+				skills: [{
+					skill: {
+						name: s.name,
+						description: s.description,
+						content: s.content,
+						files
+					},
+					excluded: []
+				}],
+				source: "inline",
+				commit: null
+			};
+		}
+	}
+}
+const skillHash = (s) => sha256(JSON.stringify(s));
+const readHostFile = (f, where) => {
+	if (!existsSync(f.path)) throw new InfraError(`${f.path} for ${where} doesn't exist`);
+	return readFileSync(f.path);
+};
+/**
+* Layers the environment (root first; a later layer wins on shared keys, and each such key is
+* an override), over the options `env` merged down the cascade. `where` names the attempt, and
+* `rulesFile` is the workspace file the harness's agent reads `rules` from.
+*/
+async function resolveEnvironment(environment, optionsEnv, root, where, rulesFile, egress = "allow-all", prepared = () => void 0) {
+	const envWhere = `environment ${environment.name} (${where})`;
+	const overrides = /* @__PURE__ */ new Set();
+	const env = { ...optionsEnv };
+	const files = /* @__PURE__ */ new Map();
+	const mcp = /* @__PURE__ */ new Map();
+	const skills = /* @__PURE__ */ new Map();
+	let rules;
+	const setup = [];
+	const seen = {
+		env: /* @__PURE__ */ new Set(),
+		files: /* @__PURE__ */ new Set(),
+		mcp: /* @__PURE__ */ new Set()
+	};
+	let egressSet = false;
+	let workspace;
+	for (const { spec } of environment.layers) {
+		if (spec.workspace !== void 0) {
+			if (workspace) overrides.add("workspace");
+			workspace = spec.workspace;
+		}
+		if (spec.egress !== void 0) {
+			if (egressSet) overrides.add("egress");
+			egressSet = true;
+		}
+		for (const [k, v] of Object.entries(spec.env ?? {})) {
+			if (seen.env.has(k)) overrides.add(`env.${k}`);
+			seen.env.add(k);
+			env[k] = v;
+		}
+		for (const [path, v] of Object.entries(spec.files ?? {})) {
+			if (seen.files.has(path)) overrides.add(`files.${path}`);
+			seen.files.add(path);
+			files.set(path, v);
+		}
+		if (spec.rules !== void 0) {
+			if (rules !== void 0) overrides.add("rules");
+			rules = spec.rules;
+		}
+		for (const [name, server] of Object.entries(spec.mcp ?? {})) {
+			if (seen.mcp.has(name)) overrides.add(`mcp.${name}`);
+			seen.mcp.add(name);
+			mcp.set(name, server);
+		}
+		for (const step of spec.setup ?? []) {
+			const s = typeof step === "string" ? { run: step } : step;
+			setup.push({
+				run: s.run,
+				timeoutMs: parseDuration(s.timeout ?? DEFAULT_SETUP_TIMEOUT),
+				...s.cwd ? { cwd: s.cwd } : {}
+			});
+		}
+		const layerSkills = /* @__PURE__ */ new Set();
+		for (const source of spec.skills ?? []) {
+			const found = await resolveSkills(root, source);
+			for (const loaded of found.skills) {
+				const name = loaded.skill.name;
+				if (skills.has(name) && !layerSkills.has(name)) overrides.add(`skills.${name}`);
+				layerSkills.add(name);
+				skills.delete(name);
+				skills.set(name, {
+					...loaded,
+					placement: source.placement,
+					source: found.source,
+					commit: found.commit
+				});
+			}
+		}
+	}
+	const secrets = {
+		names: [],
+		checks: [],
+		unset: [],
+		brokered: []
+	};
+	const vars = {};
+	const checkVars = {};
+	for (const [name, value] of Object.entries(env)) {
+		if (!isSecret(value)) {
+			vars[name] = value;
+			continue;
+		}
+		const resolved = hostValue(name, value, envWhere, secrets);
+		if (resolved === void 0) continue;
+		if (value.to === "checks") checkVars[name] = resolved;
+		else vars[name] = resolved;
+	}
+	const secretOf = (value, key, hosts) => {
+		if (typeof value === "string") return value;
+		if (isSecretTemplate(value)) return template(value, envWhere, secrets, hosts);
+		return hostValue(key, value, envWhere, secrets, value.hosts ?? hosts);
+	};
+	const resolvedMcp = {};
+	for (const [name, s] of mcp) if (s.transport === "stdio") {
+		const e = {};
+		for (const [k, v] of Object.entries(s.env)) {
+			const r = secretOf(v, k);
+			if (r !== void 0) e[k] = r;
+		}
+		resolvedMcp[name] = {
+			transport: "stdio",
+			command: s.command,
+			args: s.args,
+			env: e
+		};
+	} else {
+		const h = {};
+		const hosts = mcpHosts(s.url);
+		for (const [k, v] of Object.entries(s.headers)) {
+			const r = secretOf(v, k, hosts);
+			if (r !== void 0) h[k] = r;
+		}
+		resolvedMcp[name] = {
+			transport: s.transport,
+			url: s.url,
+			headers: h
+		};
+	}
+	const fileList = [...files].map(([path, v]) => ({
+		path,
+		content: isHostFile(v) ? readHostFile(v, envWhere) : v
+	}));
+	const rulesContent = rules === void 0 ? void 0 : isHostFile(rules) ? Buffer.from(readHostFile(rules, envWhere)).toString("utf8") : rules;
+	const skillList = [...skills.values()];
+	const skillRecords = skillList.map(({ skill, placement, source, commit, excluded }) => ({
+		name: skill.name,
+		source,
+		commit,
+		sha256: skillHash(skill),
+		placement,
+		excluded
+	}));
+	const fileRecords = fileList.map((f) => ({
+		path: f.path,
+		sha256: sha256(f.content)
+	}));
+	const rulesRecord = rulesContent === void 0 ? void 0 : {
+		path: rulesFile,
+		sha256: sha256(rulesContent)
+	};
+	if (rulesContent !== void 0) fileList.push({
+		path: rulesFile,
+		content: rulesContent
+	});
+	const secretNames = [.../* @__PURE__ */ new Set([...secrets.names, ...secrets.checks])].sort();
+	const brokered = secrets.brokered.map((b) => JSON.stringify([
+		b.name,
+		b.hosts,
+		b.in,
+		b.phase
+	])).sort();
+	const brokeredNames = [...new Set(secrets.brokered.map((b) => b.name))].sort();
+	const varNames = Object.keys(vars).filter((n) => !secrets.names.includes(n)).sort();
+	const literalVars = Object.fromEntries(varNames.map((n) => [n, vars[n]]));
+	const mcpShape = Object.fromEntries(Object.entries(resolvedMcp).map(([name, s]) => s.transport === "stdio" ? [name, {
+		command: s.command,
+		args: s.args,
+		env: Object.keys(s.env).sort()
+	}] : [name, {
+		transport: s.transport,
+		url: s.url,
+		headers: Object.keys(s.headers).sort()
+	}]));
+	const tree = workspace && prepared(workspace);
+	if (workspace && !tree) throw new InfraError(`The workspace git(${JSON.stringify(workspaceLabel(workspace))}) for ${envWhere} wasn't prepared before the run started`);
+	const workspaceRecord = workspace && tree ? {
+		source: isAbsolute(workspace.url) ? relative(root, workspace.url).split(sep).join("/") || "." : workspace.repo,
+		ref: workspace.ref ?? null,
+		path: workspace.path ?? null,
+		commit: tree.commit,
+		tree: tree.tree,
+		...workspace.init ? { init: true } : {}
+	} : void 0;
+	const digest = `sha256:${sha256(JSON.stringify({
+		setup,
+		skills: skillRecords.map((s) => [
+			s.name,
+			s.sha256,
+			s.placement
+		]),
+		mcp: mcpShape,
+		files: fileRecords,
+		rules: rulesRecord?.sha256,
+		vars: literalVars,
+		secrets: secretNames,
+		checks: [...secrets.checks].sort(),
+		unset: [...secrets.unset].sort(),
+		brokered,
+		...egress === "allow-all" ? {} : { egress: typeof egress === "string" ? egress : [...egress.allow].sort() },
+		...workspaceRecord ? { workspace: workspaceRecord.init ? [workspaceRecord.tree, "init"] : workspaceRecord.tree } : {}
+	}))}`;
+	return {
+		record: {
+			name: environment.name,
+			parent: environment.parent,
+			overrides: [...overrides],
+			digest,
+			setup: setup.map(stepCommand),
+			skills: skillRecords,
+			mcp: Object.keys(resolvedMcp).sort(),
+			files: rulesRecord ? [...fileRecords, rulesRecord] : fileRecords,
+			vars: varNames,
+			secrets: secretNames,
+			brokered: brokeredNames,
+			unset: [...secrets.unset].sort(),
+			egress,
+			...workspaceRecord ? { workspace: workspaceRecord } : {}
+		},
+		setup,
+		skills: skillList.filter((s) => s.placement === "harness").map((s) => s.skill),
+		workspaceSkills: skillList.filter((s) => s.placement === "workspace").map((s) => s.skill),
+		mcp: resolvedMcp,
+		files: fileList,
+		vars,
+		checkVars,
+		substitutions: secrets.brokered,
+		...workspace && tree ? { workspace: {
+			tar: tree.tar,
+			label: workspaceLabel(workspace),
+			commit: tree.commit,
+			...workspace.init ? { init: true } : {}
+		} } : {}
+	};
+}
+/** A skill as a `SKILL.md` file plus its other files, for writing into a directory. */
+function skillFiles(skill) {
+	return [{
+		path: "SKILL.md",
+		content: `${`---\nname: ${skill.name}\ndescription: ${JSON.stringify(skill.description)}\n---\n`}${skill.content}`
+	}, ...skill.files];
+}
+//#endregion
+//#region src/harness/fake.ts
+const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
+/**
+* Writes canned files and returns a canned message on every turn, after running its `script`'s
+* real sandbox actions. Harness-placed skills are written under `~/` where its layout's adapter
+* puts them (`~/.agents/skills` by default, as Cursor's does).
+*/
+const createFakeHarness = (descriptor) => {
+	let starts = 0;
+	return { async start(sandbox, _signal, { skills, workspaceSkills, mcp }) {
+		if (descriptor.failWith) throw new InfraError(descriptor.failWith);
+		const hangs = descriptor.startHangs;
+		if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
+		const dir = `~/${layoutOf(descriptor).harnessSkillDir}`;
+		for (const skill of skills) {
+			for (const f of skillFiles(skill)) await sandbox.writeFile(`${dir}/${skill.name}/${f.path}`, f.content);
+			debugLog("env", `skill ${skill.name} placed at ${dir}/${skill.name}/ by the fake agent`);
+		}
+		const names = [...skills.map((s) => s.name), ...workspaceSkills];
+		const loaded = [names.length ? `skills: ${names.join(", ")}` : "", Object.keys(mcp).length ? `mcp: ${Object.keys(mcp).join(", ")}` : ""].filter(Boolean);
+		let calls = 0;
+		let turns = 0;
+		const skilled = names.length > 0 && descriptor.withSkill;
+		const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
+		const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
+		const flaked = Math.random() < (descriptor.flakeRate ?? 0);
+		return {
+			async send(prompt, signal) {
+				const startedAt = Date.now();
+				const script = descriptor.script ?? [];
+				const steps = script[Math.min(turns++, script.length - 1)] ?? [];
+				const at = () => {
+					const ms = Date.now() - startedAt;
+					return {
+						ts: new Date(startedAt + ms).toISOString(),
+						atMs: ms
+					};
+				};
+				const transcript = [{
+					...at(),
+					type: "text",
+					text: `Working on: ${prompt}`,
+					durationMs: 0
+				}, ...loaded.length ? [{
+					...at(),
+					type: "text",
+					text: `Loaded ${loaded.join("; ")}`,
+					durationMs: 0
+				}] : []];
+				const done = (text, status) => ({
+					text,
+					status,
+					startedAt: new Date(startedAt).toISOString(),
+					durationMs: Date.now() - startedAt,
+					transcript,
+					usage: null,
+					model: descriptor.servedModel ?? null,
+					unreported: []
+				});
+				const write = async (path, content) => {
+					const time = at();
+					const before = await sandbox.readFile(path);
+					await sandbox.writeFile(path, content);
+					debugLog("harness", `tool write ${path} (${before === null ? "create" : "modify"})`);
+					transcript.push({
+						...time,
+						type: "file-edit",
+						toolCallId: `call_${++calls}`,
+						path,
+						kind: before === null ? "create" : "modify",
+						added: lineCount(content),
+						removed: before === null ? 0 : lineCount(before),
+						durationMs: Date.now() - startedAt - time.atMs
+					});
+				};
+				if (descriptor.reasoning) transcript.push({
+					...at(),
+					type: "reasoning",
+					text: descriptor.reasoning,
+					durationMs: 0
+				});
+				for (const tool of descriptor.tools ?? []) transcript.push({
+					...at(),
+					type: "tool",
+					toolCallId: `call_${++calls}`,
+					tool: tool.tool,
+					input: tool.input,
+					output: tool.output,
+					ok: tool.ok ?? true,
+					durationMs: 0
+				});
+				if (flaked) {
+					const text = "I ran out of ideas and left the workspace unchanged.";
+					transcript.push({
+						...at(),
+						type: "text",
+						text,
+						durationMs: 0
+					});
+					return done(text, "completed");
+				}
+				let scripted;
+				for (const step of steps) if ("exec" in step) {
+					const time = at();
+					debugLog("harness", `tool shell called: $ ${step.exec}`);
+					const r = await sandbox.exec(step.exec, { signal });
+					debugLog("harness", `tool shell returned exit ${r.exitCode} in ${Date.now() - startedAt - time.atMs}ms`);
+					transcript.push({
+						...time,
+						type: "tool",
+						toolCallId: `call_${++calls}`,
+						tool: "shell",
+						input: { command: step.exec },
+						output: r.stdout,
+						ok: r.exitCode === 0,
+						durationMs: Date.now() - startedAt - time.atMs
+					});
+				} else if ("write" in step) for (const [path, content] of Object.entries(step.write)) await write(path, content);
+				else if ("mcp" in step) {
+					const time = at();
+					const [server = "", tool = ""] = step.mcp.split("/");
+					debugLog("harness", `tool mcp__${server}__${tool} called`);
+					const r = await callMcpTool(sandbox, mcp, server, tool, signal);
+					debugLog("harness", `tool mcp__${server}__${tool} ${r.ok ? "returned" : "failed"}`);
+					transcript.push({
+						...time,
+						type: "tool",
+						toolCallId: `call_${++calls}`,
+						tool: `mcp__${server}__${tool}`,
+						input: {},
+						output: r.output,
+						ok: r.ok,
+						durationMs: Date.now() - startedAt - time.atMs
+					});
+				} else if ("error" in step) {
+					debugLog("harness", `turn failed: ${step.error}`);
+					transcript.push({
+						...at(),
+						type: "error",
+						message: step.error
+					});
+					return done("", "error");
+				} else scripted = step.reply;
+				if (descriptor.delayMs) try {
+					await setTimeout$1(descriptor.delayMs, void 0, descriptor.ignoreAbort ? {} : { signal });
+				} catch {
+					return done("", "aborted");
+				}
+				for (const [path, content] of Object.entries(files)) await write(path, content);
+				const text = scripted ?? reply;
+				transcript.push({
+					...at(),
+					type: "text",
+					text,
+					durationMs: 0
+				});
+				return done(text, "completed");
+			},
+			async close() {}
+		};
+	} };
+};
+/**
+* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
+* arguments: initialize, then the call.
+*/
+async function callMcpTool(sandbox, servers, name, tool, signal) {
+	const server = servers[name];
+	if (!server) {
+		const known = Object.keys(servers);
+		throw new InfraError(`The fake agent's script calls MCP server "${name}", which the environment doesn't have (it has ${known.length ? known.join(", ") : "none"})`);
+	}
+	if (server.transport !== "stdio") throw new InfraError(`The fake agent calls stdio MCP servers only; "${name}" is ${server.transport}`);
+	const messages = [
+		{
+			jsonrpc: "2.0",
+			id: 1,
+			method: "initialize",
+			params: {
+				protocolVersion: "2024-11-05",
+				capabilities: {},
+				clientInfo: {
+					name: "reeval-fake-agent",
+					version: "1"
+				}
+			}
+		},
+		{
+			jsonrpc: "2.0",
+			method: "notifications/initialized"
+		},
+		{
+			jsonrpc: "2.0",
+			id: 2,
+			method: "tools/call",
+			params: {
+				name: tool,
+				arguments: {}
+			}
+		}
+	].map((m) => shellQuote(JSON.stringify(m)));
+	const command = [server.command, ...server.args].map(shellQuote).join(" ");
+	const script = [
+		"d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
+		"{",
+		"  trap '' PIPE",
+		`  printf '%s\\n' ${messages.join(" ")} 2>/dev/null`,
+		"  i=0",
+		`  while [ $i -lt 300 ] && [ ! -f "$d/code" ] && ! grep -Eq '"id": ?2[,}]' "$d/out"; do sleep 0.1; i=$((i + 1)); done`,
+		`} | { ${command} > "$d/out" 2> "$d/err"; echo $? > "$d/code"; }`,
+		"code=$(cat \"$d/code\")",
+		"cat \"$d/out\"; cat \"$d/err\" >&2; rm -rf \"$d\"; exit $code"
+	].join("\n");
+	const r = await sandbox.exec(script, {
+		signal,
+		env: server.env
+	});
+	const reply = r.stdout.split("\n").flatMap((line) => {
+		try {
+			return [JSON.parse(line)];
+		} catch {
+			return [];
+		}
+	}).find((m) => m.id === 2);
+	if (!reply) return {
+		output: r.stderr.trim() || `no reply (exit ${r.exitCode})`,
+		ok: false
+	};
+	if (reply.error) return {
+		output: reply.error.message,
+		ok: false
+	};
+	return {
+		output: (reply.result?.content ?? []).flatMap((c) => c.type === "text" ? [c.text] : []).join("\n"),
+		ok: !reply.result?.isError
+	};
+}
+//#endregion
+//#region src/harness/index.ts
+async function createHarness(descriptor) {
+	if (descriptor.kind === "fake") return createFakeHarness(descriptor);
+	const { createAgentHarness } = await import("./agent-DVHkx8aT.js");
+	return createAgentHarness(descriptor);
+}
+/** Where each harness's agent looks for repo skills, relative to the workspace. */
+const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
+//#endregion
+//#region src/attempt/environment.ts
+/**
+* Phase 1: resolves the attempt's environment (files, skills, secrets and their placeholders)
+* and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
+* signal, before the eval timer starts.
+*/
+async function prepareEnvironment(a, root, runId, where, signal) {
+	const { spec, record } = a;
+	const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w));
+	a.env = env;
+	record.environment = env.record;
+	const { secrets, brokered } = env.record;
+	record.env = {
+		vars: Object.keys(env.vars).filter((n) => !secrets.includes(n)).sort(),
+		secrets,
+		brokered
+	};
+	const debug = attemptDebug();
+	if (debug) traceEnvironment(debug, spec, env);
+	record.image = null;
+	if (spec.sandbox.kind === "docker") {
+		const { preparedFor } = await import("./preflight-BtubXKQ8.js");
+		const { dockerProvider } = await import("./providers-BUOxitcT.js");
+		signal.throwIfAborted();
+		const prepared = preparedFor(root, runId, dockerProvider(), spec.sandbox, spec.harness);
+		if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
+		record.image = prepared;
+		a.image = record.image.ref;
+		const { ref, id, base, install, agentCli } = record.image;
+		debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
+	} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
+}
+//#endregion
+//#region src/attempt/judge.ts
+const noJudgeModel = (configFile) => `t.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.13.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
+const isTurn = (value) => typeof value === "object" && value !== null && "transcript" in value && "diff" in value;
+function judgeState(turn, output = turn.text) {
+	return {
+		task: turn.prompt,
+		output,
+		files: changedFilesForJudge(turn.diff.files),
+		diff: patchForJudge(turn.diff.patch)
+	};
+}
+/**
+* Phase 4, the judge: grades a turn (the last by default) or a value against a question, under
+* the judge provider's concurrency lease. Every call's input is recorded, including calls that
+* fail, and a failure marks the attempt's assertion errors as the judge's.
+*/
+function judgeCaller(a) {
+	return async (question, { on, ...call } = {}) => {
+		if (typeof question !== "string" || !question.trim()) throw new Error(`t.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
+		const { threshold } = call;
+		if (threshold !== void 0 && !(typeof threshold === "number" && threshold >= 0 && threshold <= 1)) throw new Error(`t.judge() threshold must be between 0 and 1 (got ${String(threshold)})`);
+		const subject = isTurn(on) ? on : a.turns.at(-1);
+		if (!subject) throw new Error("t.judge() has no turn to grade: call t.send() first");
+		const options = {
+			...a.spec.judge,
+			...call
+		};
+		const { model } = options;
+		if (model === void 0) {
+			a.judgeErrored = true;
+			throw new Error(noJudgeModel(getRunContext().config.file));
+		}
+		const state = judgeState(subject, on === void 0 || isTurn(on) ? void 0 : on);
+		const input = {
+			question,
+			model: judgeModelName(model),
+			state
+		};
+		a.record.judgeInputs?.push(input);
+		const start = performance.now();
+		const ms = () => Math.round(performance.now() - start);
+		a.report({
+			kind: "judge-start",
+			question
+		});
+		debugLog("judge", `asking ${input.model}: ${JSON.stringify(clip(question, 160))}`);
+		debugTrace("judge", "prompt", input, "judge prompt");
+		const { maxConcurrent, ...judgeOptions } = options;
+		const provider = judgeModelName(model).split("/")[0];
+		try {
+			const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, state, {
+				...judgeOptions,
+				model,
+				threshold: options.threshold ?? .8
+			}, a.signal), a.signal));
+			a.record.judgements.push(retries.length ? {
+				...verdict,
+				retries
+			} : verdict);
+			const { score, threshold } = verdict;
+			debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= threshold ? "≥" : "<"} ${threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= threshold ? "at or above" : "below"} the ${threshold} threshold`);
+			debugTrace("judge", "response", {
+				verdict,
+				retries
+			}, "judge response");
+			a.report({
+				kind: "judge-end",
+				question,
+				verdict: verdict.verdict,
+				score,
+				threshold,
+				ms: ms()
+			});
+			return verdict;
+		} catch (error) {
+			a.judgeErrored = true;
+			const { retries } = error;
+			input.error = error instanceof Error ? error.message : String(error);
+			if (retries?.length) input.retries = retries;
+			debugLog("judge", `${input.model} · ${formatDuration(ms())} · error: ${input.error}`);
+			debugTrace("judge", "error", {
+				error: input.error,
+				retries
+			}, "judge error");
+			a.report({
+				kind: "judge-end",
+				question,
+				verdict: "error",
+				score: null,
+				threshold: null,
+				ms: ms()
+			});
+			throw error;
+		}
+	};
+}
+//#endregion
+//#region src/sandbox/index.ts
+async function createSandbox(descriptor, options) {
+	try {
+		switch (descriptor.kind) {
+			case "fake": return await createFakeSandbox(descriptor, options.env);
+			case "docker": {
+				const { createDockerSandbox } = await import("./docker-DnWAVdEm.js");
+				return await createDockerSandbox(descriptor, options);
+			}
+		}
+	} catch (error) {
+		if (error instanceof InfraError || options.signal.aborted) throw error;
+		throw new InfraError(`Failed to create ${descriptor.kind} sandbox: ${error.message}`);
+	}
+}
+//#endregion
+//#region src/attempt/record.ts
+function classify(error, signal) {
+	const message = error instanceof Error ? error.message : String(error);
+	if (signal.aborted) {
+		const reason = signal.reason instanceof Error ? signal.reason.message : String(signal.reason);
+		return isTimeout(reason) ? {
+			kind: "timeout",
+			message: "The eval timed out"
+		} : {
+			kind: "aborted",
+			message: "The run was cancelled"
+		};
+	}
+	if (error instanceof InfraError) return {
+		kind: "infra",
+		message
+	};
+	return {
+		kind: "assertion",
+		message
+	};
+}
+/**
+* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
+* assertion, which is the judge's when a judge call failed or gave a failing verdict.
+*/
+function recordFailure(a, error, timedOut) {
+	const { record } = a;
+	record.error = timedOut ? {
+		kind: "timeout",
+		message: "The eval timed out"
+	} : classify(error, a.signal);
+	record.error.message = capMiddle(record.error.message);
+	if (error instanceof PrepareError && !timedOut) {
+		const { class: kind, stage, ref, cause, log, setBy } = error.failure;
+		record.error.prepare = {
+			class: kind,
+			stage,
+			ref,
+			cause,
+			log,
+			setBy: setBy ?? []
+		};
+	}
+	const judgeFailed = a.judgeErrored || record.judgements.some((j) => j.verdict !== "pass");
+	if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
+	record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
+}
+/**
+* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
+* Refused hosts the harness always tries are `expectedBlocked`, summarised in one `-v` line.
+*/
+function recordEgress(a, sandbox) {
+	const traffic = sandbox?.traffic?.();
+	const expected = expectedBlockedFor(a.spec.harness);
+	const refused = traffic?.blocked ?? [];
+	const isExpected = (h) => !!expected?.hosts.includes(h.host);
+	const expectedBlocked = refused.filter(isExpected);
+	a.record.egress = {
+		policy: a.spec.egress.policy,
+		allowed: allowedHosts(a.spec.egress),
+		recorded: !!traffic,
+		contacted: traffic?.contacted ?? [],
+		blocked: refused.filter((h) => !isExpected(h)),
+		expectedBlocked
+	};
+	if (expected && expectedBlocked.length) {
+		const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
+		debugLog("gateway", `expected blocks, not logged per request: ${hosts} · ${expected.why}`);
+	}
+	a.record.servedModels = sandbox?.servedModels?.() ?? null;
+}
+/** Phase 6: writes the attempt's record, redacted, and returns what the runner reports. */
+async function writeRecord(a, stagingDir, id) {
+	const { record, spec } = a;
+	const recordPath = join(stagingDir, "evals", recordFileName(attemptKey(id, spec.cell.key, spec.n)));
+	await mkdir(join(stagingDir, "evals"), { recursive: true });
+	const tmp = `${recordPath}.${process.pid}.tmp`;
+	await writeFile(tmp, redactSecrets(JSON.stringify(record, null, 2)));
+	await rename(tmp, recordPath);
+	return {
+		outcome: record.outcome,
+		errorKind: record.error?.kind ?? null,
+		judgements: record.judgements.map((j) => ({
+			verdict: j.verdict,
+			score: j.score,
+			threshold: j.threshold,
+			question: clip(j.question, 200)
+		})),
+		turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
+		...record.error?.prepare ? { prepare: {
+			ref: record.error.prepare.ref,
+			stage: record.error.prepare.stage
+		} } : {}
+	};
+}
+/** `turn N error: <message>` for each turn with an error event. */
+function turnErrors(transcripts) {
+	return transcripts.flatMap((events, i) => {
+		const found = events.find((e) => e.type === "error");
+		return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(clip(found.message.trim(), 500))}`] : [];
+	});
+}
+//#endregion
+//#region src/diff.ts
+const GIT = "git -c user.name=reeval -c user.email=reeval@localhost -c core.hooksPath=/dev/null";
+/** Records the workspace state before the agent runs, in a git dir outside the workspace. */
+async function snapshot(sandbox, signal) {
+	const r = await sandbox.exec(`export GIT_DIR="$PWD/../.reeval-git-$(basename "$PWD")" GIT_WORK_TREE="$PWD" && ${GIT} init -q && ${GIT} add -A && ${GIT} commit -q --allow-empty -m base`, { signal });
+	if (r.exitCode !== 0) throw new InfraError(`Failed to snapshot the workspace: ${r.stderr.trim()}`);
+}
+async function diffSinceSnapshot(sandbox, signal) {
+	const env = `export GIT_DIR="$PWD/../.reeval-git-$(basename "$PWD")" GIT_WORK_TREE="$PWD" && ${GIT} add -A && `;
+	const patch = await sandbox.exec(`${env}${GIT} diff --cached --binary HEAD`, { signal });
+	const names = await sandbox.exec(`${env}${GIT} diff --cached --name-status --no-renames HEAD`, { signal });
+	if (patch.exitCode !== 0 || names.exitCode !== 0) throw new InfraError(`Failed to diff the workspace: ${patch.stderr || names.stderr}`);
+	const status = {
+		A: "added",
+		M: "modified",
+		D: "deleted"
+	};
+	const files = names.stdout.split("\n").filter(Boolean).map((line) => {
+		const [code = "M", ...rest] = line.split("	");
+		return {
+			path: rest.join("	"),
+			status: status[code[0]] ?? "modified"
+		};
+	});
+	return {
+		patch: patch.stdout,
+		files
+	};
+}
+//#endregion
+//#region src/sandbox/phases.ts
+const strip = ({ phase: _, ...sub }) => sub;
+/**
+* Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
+* at least one check or setup step is running, so a request the agent makes between them gets
+* the placeholder.
+*/
+function brokerPhases(box, subs) {
+	const agent = subs.filter((s) => s.phase === "agent").map(strip);
+	const all = subs.map(strip);
+	const perPhase = box.setSubstitutions && all.length > agent.length;
+	let running = 0;
+	return {
+		async start() {
+			if (all.length > 0) await box.setSubstitutions?.(agent);
+		},
+		async checks(fn) {
+			if (!perPhase) return fn();
+			if (running === 0) await box.setSubstitutions?.(all);
+			running++;
+			try {
+				return await fn();
+			} finally {
+				if (--running === 0) await box.setSubstitutions?.(agent);
+			}
+		}
+	};
+}
+//#endregion
+//#region src/attempt/state.ts
+/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
+const SETUP_BUDGET_MS = 12e5;
+function createAttempt(spec, signal, progress) {
+	const record = {
+		startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		outcome: "pass",
+		error: null,
+		turns: [],
+		diff: null,
+		checks: [],
+		judgements: [],
+		timings: {
+			setupMs: 0,
+			agentMs: 0,
+			diffMs: 0,
+			checksMs: 0,
+			judgeMs: 0
+		},
+		judgeInputs: []
+	};
+	return {
+		spec,
+		record,
+		turns: [],
+		report: (update) => progress({
+			at: Date.now(),
+			...update
+		}),
+		async timed(phase, fn) {
+			const start = performance.now();
+			try {
+				return await fn();
+			} finally {
+				record.timings[phase] += Math.round(performance.now() - start);
+			}
+		},
+		signal,
+		judgeErrored: false
+	};
+}
+const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s");
+/**
+* Setup steps and the harness's start (once retried) run outside the eval timeout, so the
+* attempt's own limit makes room for them.
+*/
+function setupAllowance(planned) {
+	const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
+	const { sandbox } = planned;
+	return setupMs + (sandbox.kind === "docker" && !!(sandbox.from || sandbox.dockerfile || sandbox.install?.length) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
+}
+/** The eval's timer, paused while the sandbox and its setup steps start. */
+function evalTimer(ms) {
+	const controller = new AbortController();
+	let left = ms;
+	let since = Date.now();
+	let timer;
+	const fire = () => controller.abort(new DOMException(`Eval timed out after ${ms / 1e3}s`, "TimeoutError"));
+	const start = () => {
+		since = Date.now();
+		timer = setTimeout(fire, Math.max(0, left));
+	};
+	start();
+	return {
+		signal: controller.signal,
+		pause() {
+			clearTimeout(timer);
+			left -= Date.now() - since;
+		},
+		resume: start,
+		clear: () => clearTimeout(timer)
+	};
+}
+//#endregion
+//#region src/attempt/session.ts
+const tailOf = (text) => text.trim().split("\n").slice(-10).join("\n");
+/** Setup steps in order, each with its own timeout; a failure is an infra error. */
+async function runSetup(a, box, env) {
+	const { record, spec } = a;
+	const steps = env.setup;
+	record.setup = [];
+	for (const [i, step] of steps.entries()) {
+		const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
+		const start = performance.now();
+		const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
+		const r = await asCheck(a, () => box.exec(step.run, {
+			signal: stepSignal,
+			env: env.checkVars,
+			...step.cwd ? { cwd: step.cwd } : {}
+		})).catch((error) => {
+			if (a.signal.aborted) throw error;
+			if (!stepSignal.aborted) throw new InfraError(`${what} couldn't run: ${error instanceof Error ? error.message : error}`);
+			return null;
+		});
+		const durationMs = Math.round(performance.now() - start);
+		const output = r ? [r.stdout, r.stderr].filter(Boolean).join("\n") : "";
+		record.setup.push({
+			run: step.run,
+			exitCode: r?.exitCode ?? null,
+			durationMs,
+			output
+		});
+		debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
+		debugTrace("env", "setup", {
+			run: step.run,
+			cwd: step.cwd,
+			exitCode: r?.exitCode ?? null,
+			durationMs,
+			stdout: r?.stdout,
+			stderr: r?.stderr
+		}, `setup ${i + 1} output`);
+		if (r?.exitCode === 0) continue;
+		throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
+	}
+}
+const WORKSPACE_TAR = ".reeval-workspace.tar";
+/** Unpacks the git workspace's tarball into the (empty) workspace. */
+async function copyWorkspace(box, workspace, signal) {
+	const start = performance.now();
+	const tar = await readFile(workspace.tar);
+	await box.writeFile(WORKSPACE_TAR, tar);
+	const r = await box.exec(`tar -xf ${WORKSPACE_TAR} && rm -f ${WORKSPACE_TAR}`, { signal });
+	if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
+	if (workspace.init) {
+		const at = "2000-01-01T00:00:00Z";
+		const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
+			signal,
+			env: {
+				GIT_AUTHOR_NAME: "reeval",
+				GIT_AUTHOR_EMAIL: "reeval@localhost",
+				GIT_AUTHOR_DATE: at,
+				GIT_COMMITTER_NAME: "reeval",
+				GIT_COMMITTER_EMAIL: "reeval@localhost",
+				GIT_COMMITTER_DATE: at
+			}
+		});
+		if (init.exitCode !== 0) throw new InfraError(`Couldn't give the workspace ${workspace.label} a .git (init: true needs git in the image): ${tailOf(init.stderr || init.stdout)}`);
+	}
+	debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
+}
+/**
+* Starts the harness session within its `startTimeout`. A start that doesn't become ready in time
+* is aborted (and closed, should it finish later) and started once more: a stuck bootstrap is
+* usually transient, and nothing has run yet, so the retry can't change what the attempt
+* measures. A second timeout is an infra error; a start that fails outright isn't retried.
+*/
+async function startHarness(a, harness, box, options) {
+	const { harness: descriptor } = a.spec;
+	const ms = startTimeoutMs(descriptor);
+	const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
+	for (let tries = 1;; tries++) {
+		const stop = new AbortController();
+		let timer;
+		const timedOut = new Promise((resolve) => {
+			timer = setTimeout(() => resolve(null), ms);
+		});
+		const starting = harness.start(box, AbortSignal.any([a.signal, stop.signal]), options);
+		const session = await Promise.race([starting, timedOut]).finally(() => clearTimeout(timer));
+		if (session) return session;
+		stop.abort(new InfraError(late));
+		starting.then((s) => s.close().catch(() => {}), () => {});
+		a.signal.throwIfAborted();
+		if (tries === 2) throw new InfraError(`${late} (tried twice)`);
+		a.record.startRetries = [{
+			error: late,
+			waitedMs: ms
+		}];
+		debugLog("harness", `${late}; starting it once more`);
+	}
+}
+/**
+* Phase 2: starts the sandbox (with the brokered secrets its gateway swaps in), writes the
+* environment's files and workspace skills, runs its setup steps, snapshots the workspace for
+* the diff, and starts the harness session with the harness-placed skills and MCP servers.
+*/
+async function startSession(a, runId, setupStart) {
+	const { spec } = a;
+	const env = a.env;
+	const box = await createSandbox(spec.sandbox, {
+		runId,
+		timeoutMs: spec.timeoutMs + setupAllowance(spec) + 6e4,
+		signal: a.signal,
+		env: env.vars,
+		image: a.image,
+		brokering: brokeringFor(spec.harness),
+		expectedBlocked: expectedBlockedFor(spec.harness)?.hosts,
+		substitutions: env.substitutions,
+		egress: spec.egress
+	});
+	a.sandbox = box;
+	if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
+	a.phases = brokerPhases(box, env.substitutions);
+	if (a.signal.aborted) box.destroy();
+	a.signal.throwIfAborted();
+	await a.phases.start();
+	if (env.substitutions.length) {
+		const agent = env.substitutions.filter((s) => s.phase === "agent").map((s) => s.name);
+		debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
+	}
+	if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
+	for (const f of env.files) {
+		await box.writeFile(f.path, f.content);
+		const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
+		debugLog("env", `file ${f.path} (${size} bytes)`);
+	}
+	const skillDir = workspaceSkillDir(spec.harness);
+	for (const skill of env.workspaceSkills) {
+		const files = skillFiles(skill);
+		for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
+		debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
+	}
+	const layout = layoutOf(spec.harness);
+	for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
+	for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
+	await runSetup(a, box, env);
+	await snapshot(box, a.signal);
+	debugLog("env", "workspace snapshot taken: the diff starts here");
+	const harness = await createHarness(spec.harness);
+	const harnessStart = performance.now();
+	debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
+	const session = await startHarness(a, harness, box, {
+		skills: env.skills,
+		workspaceSkills: env.workspaceSkills.map((s) => s.name),
+		mcp: env.mcp
+	});
+	debugLog("harness", `${harnessId(spec.harness)} session ready in ${formatDuration(performance.now() - harnessStart)}`);
+	a.session = session;
+	a.report({
+		kind: "sandbox-ready",
+		ms: Math.round(performance.now() - setupStart)
+	});
+	return [box, session];
+}
+//#endregion
+//#region src/attempt/turns.ts
+/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
+const TURN_STOP_GRACE_MS = 3e3;
+/**
+* Resolves `grace` ms after `signal` aborts, with the turn as cut off; `cancel` drops the timer
+* once the turn settles on its own.
+*/
+function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
+	let timer;
+	let onAbort = () => {};
+	const cutoff = new Promise((resolve) => {
+		onAbort = () => {
+			timer = setTimeout(() => {
+				const message = `The agent didn't stop within ${grace / 1e3}s of being cancelled, so reeval stopped waiting for its turn`;
+				const at = Date.now() - Date.parse(startedAt);
+				resolve({
+					text: "",
+					status: "aborted",
+					transcript: [{
+						type: "error",
+						message,
+						ts: (/* @__PURE__ */ new Date()).toISOString(),
+						atMs: at
+					}],
+					usage: null,
+					model: null,
+					unreported: [],
+					startedAt,
+					durationMs: at
+				});
+			}, grace);
+		};
+		if (signal.aborted) onAbort();
+		else signal.addEventListener("abort", onAbort, { once: true });
+	});
+	const cancel = () => {
+		clearTimeout(timer);
+		signal.removeEventListener("abort", onAbort);
+	};
+	return {
+		cutoff,
+		cancel
+	};
+}
+/**
+* Phase 3: one turn. Sends the prompt to the started session, records the turn as it goes (an
+* errored entry first, so a turn that throws is still in the record), takes the agent's context
+* after the first, and diffs the workspace since the snapshot.
+*/
+function turnSender(a, start, takeContext, checks) {
+	return async (prompt) => {
+		const [box, agent] = await start();
+		const entry = {
+			prompt,
+			text: "",
+			status: "error",
+			transcript: [],
+			usage: null,
+			model: null,
+			unreported: [],
+			startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			durationMs: 0
+		};
+		a.record.turns.push(entry);
+		const turnNumber = a.record.turns.length;
+		a.report({
+			kind: "turn-start",
+			turn: turnNumber
+		});
+		debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
+		debugTrace("harness", "prompt", {
+			turn: turnNumber,
+			prompt
+		}, `turn ${turnNumber} prompt`);
+		const { cutoff, cancel } = cutOff(a.signal, entry.startedAt);
+		const { raw, ...result } = await a.timed("agentMs", () => Promise.race([agent.send(prompt, a.signal), cutoff])).finally(cancel);
+		Object.assign(entry, result, raw ? { raw } : {});
+		for (const event of result.transcript) debugTrace("harness", "transcript", {
+			turn: turnNumber,
+			...event
+		}, `transcript ${event.type}`);
+		await takeContext(box, a.signal);
+		const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
+		const failure = result.transcript.find((e) => e.type === "error");
+		a.report({
+			kind: "turn-end",
+			turn: turnNumber,
+			status: result.status,
+			ms: result.durationMs,
+			tools: count("tool", "web-search", "web-fetch"),
+			fileEdits: count("file-edit"),
+			...failure?.type === "error" ? { error: redactSecrets(clip(failure.message.trim(), 300)) } : {}
+		});
+		const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
+		debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
+		a.signal.throwIfAborted();
+		const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
+		a.record.diff = diff;
+		const data = {
+			prompt,
+			...result,
+			diff
+		};
+		const turn = Object.freeze({
+			...data,
+			...checks
+		});
+		a.turns.push(turn);
+		return turn;
+	};
+}
+//#endregion
+//#region src/attempt/index.ts
+const TRACE_BUFFER_DEFAULT = 8 * 2 ** 20;
+async function runAttempt(spec, fn, handle) {
+	const ctx = getRunContext();
+	const { config, runId } = ctx;
+	const verbosity = ctx.verbosity ?? 0;
+	const id = evalId(relativeFile(config.root, handle.file), [...spec.groups, spec.name]);
+	const debug = new AttemptDebug({
+		dir: attemptDebugDir(config.root, runId, id, spec.cell.key, spec.n),
+		verbosity,
+		traceLimit: config.debug?.traceBufferBytes ?? TRACE_BUFFER_DEFAULT,
+		...verbosity > 0 ? { emit: handle.debug } : {}
+	});
+	return withDebug(debug, () => attemptBody(spec, fn, handle, id, debug));
+}
+async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited }, id, debug) {
+	const setupStart = performance.now();
+	const ctx = getRunContext();
+	const { runId, stagingDir, config } = ctx;
+	tracePlan(debug, spec, ctx);
+	const a = createAttempt(spec, testSignal, progress);
+	let timer;
+	let clock;
+	const destroy = () => void a.sandbox?.destroy();
+	testSignal.addEventListener("abort", destroy, { once: true });
+	let starting;
+	const start = () => {
+		starting ??= (async () => {
+			clock?.pause();
+			try {
+				return await a.timed("setupMs", () => startSession(a, runId, setupStart));
+			} finally {
+				clock?.resume();
+			}
+		})();
+		return starting;
+	};
+	const takeContext = contextTaker(a);
+	const { exec, readFile } = createChecks(a);
+	const t = {
+		get signal() {
+			return a.signal;
+		},
+		turns: a.turns,
+		exec,
+		readFile,
+		attempt: {
+			id: attemptId(spec.cell.key, spec.n),
+			cell: spec.cell,
+			n: spec.n
+		},
+		async sandbox() {
+			await start();
+			return {
+				exec,
+				readFile
+			};
+		},
+		send: turnSender(a, start, takeContext, {
+			exec,
+			readFile
+		}),
+		judge: judgeCaller(a)
+	};
+	const timedOut = () => !!timer?.aborted && !testSignal.aborted;
+	try {
+		a.report({ kind: "sandbox" });
+		const where = `${id} [${attemptId(spec.cell.key, spec.n)}]`;
+		await a.timed("setupMs", () => prepareEnvironment(a, config.root, runId, where, testSignal));
+		clock = evalTimer(spec.timeoutMs);
+		timer = clock.signal;
+		a.signal = AbortSignal.any([testSignal, timer]);
+		timer.addEventListener("abort", destroy, { once: true });
+		await fn(t);
+		a.signal.throwIfAborted();
+		const missed = await unawaited();
+		if (missed) throw missed;
+	} catch (error) {
+		recordFailure(a, error, timedOut());
+		if (timedOut()) throw new Error(`Eval timed out after ${spec.timeoutMs / 1e3}s`);
+		throw error;
+	} finally {
+		if (!a.signal.aborted) await unawaited().catch(() => {});
+		clock?.clear();
+		const { sandbox } = a;
+		if (sandbox && !a.signal.aborted) await takeContext(sandbox, AbortSignal.timeout(3e4));
+		if (sandbox) a.report({ kind: "cleanup" });
+		testSignal.removeEventListener("abort", destroy);
+		const teardown = performance.now();
+		await a.session?.close().catch(() => {});
+		if (a.session) debug.log("teardown", "harness session closed");
+		recordEgress(a, sandbox);
+		await sandbox?.destroy();
+		if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
+		const { record } = a;
+		const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
+		debug.log("teardown", `outcome ${record.outcome}${error}`);
+		const rel = (path) => relativeFile(config.root, path) || ".";
+		const scratch = `.reeval/runs/${runId}`;
+		debug.log("teardown", `record ${scratch}.json · artifacts ${scratch}/${evalSlug(id)}/${evalSlug(spec.cell.key)}/${spec.n} · debug ${rel(debug.dir)}`);
+		setMeta(await writeRecord(a, stagingDir, id));
+		await debug.finish(debug.verbosity === 2 || record.outcome !== "pass");
+	}
+}
+//#endregion
+//#region src/plan-eval.ts
+/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
+const GRACE_MS = 5e3;
+function configLevel(config) {
+	return {
+		harness: config.harness,
+		sandbox: config.sandbox,
+		environment: config.environment,
+		compare: void 0,
+		runs: config.runs,
+		gate: config.gate,
+		env: config.env,
+		egress: config.egress !== void 0 ? { inherited: config.egress } : {},
+		judge: config.judge,
+		timeout: void 0,
+		tags: [],
+		pin: [],
+		sources: {}
+	};
+}
+/**
+* The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
+* `order`). Throws a UsageError for options the plan refuses.
+*/
+function planEval(declaration, ctx) {
+	const { name, mode, groups: enclosing, options: own } = declaration;
+	const groups = enclosing.map((g) => g.name);
+	const where = `evaluate("${[...groups, name].join(" › ")}")`;
+	checkOptions(own, where);
+	const levels = [...enclosing.map((g) => ({
+		options: g.options,
+		where: `group("${g.name}")`
+	})), {
+		options: own,
+		where
+	}];
+	const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
+		profile: ctx.profile,
+		runs: ctx.runs
+	});
+	const { resolved } = resolvedEval;
+	if (!matchesTags(resolved.tags, ctx.tags)) return null;
+	const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
+	const plannedAttempts = resolvedEval.skip ? [] : expand(resolved);
+	if (ctx.config.secrets.unbrokered === "error") {
+		const [first] = unbrokered(resolved.env, plannedAttempts);
+		if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
+	}
+	for (const planned of plannedAttempts) {
+		const why = unenforceable(planned.egress, planned.sandbox, where);
+		if (why) throw new UsageError(why);
+		const mcp = refusedMcp(planned.harness, planned.environment.layers);
+		if (mcp) throw new UsageError(`${where} on ${harnessId(planned.harness)}: ${mcp}`);
+	}
+	const path = ctx.file ? relativeFile(ctx.config.root, ctx.file) : null;
+	const setBy = {
+		sandbox: resolved.sources?.sandbox ?? ctx.config.file,
+		harness: resolved.sources?.harness ?? ctx.config.file
+	};
+	const plan = evalPlan(resolvedEval, plannedAttempts, {
+		configFile: ctx.config.file,
+		file: path
+	});
+	if (resolvedEval.skip) {
+		const [first] = expand(resolved);
+		if (!first) return [];
+		const attempt = {
+			name,
+			groups,
+			tags: resolved.tags,
+			cell: first.cell,
+			n: 1,
+			label: "",
+			harness: first.harness,
+			sandbox: first.sandbox,
+			setBy,
+			gate: "none",
+			brokered: [],
+			secretVars: [],
+			egress: first.egress,
+			plan
+		};
+		return [{
+			name: `${name} [${attemptId(first.cell.key, 1)}]`,
+			mode: "skip",
+			attempt
+		}];
+	}
+	const cells = [];
+	return plannedAttempts.map((planned) => {
+		if (!cells.includes(planned.cell.key)) cells.push(planned.cell.key);
+		const workspace = workspaceOf(planned.environment.layers);
+		const attempt = {
+			name,
+			groups,
+			tags: resolved.tags,
+			cell: planned.cell,
+			n: planned.n,
+			label: planned.label,
+			harness: planned.harness,
+			sandbox: planned.sandbox,
+			setBy,
+			gate: planned.gate,
+			...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
+			brokered: brokeredIn(resolved.env, planned),
+			secretVars: secretVarsIn(resolved.env, planned),
+			egress: planned.egress,
+			...workspace ? { workspace } : {},
+			plan
+		};
+		const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
+		return {
+			name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
+			mode,
+			attempt,
+			timeout: timeoutMs + setupAllowance(planned) + GRACE_MS,
+			gate: {
+				limits: attemptLimits(planned, ctx.concurrency),
+				rank: {
+					eval: evalId(rankFile, [...groups, name]),
+					file: rankFile,
+					order: ctx.order,
+					n: planned.n,
+					cell: cells.indexOf(planned.cell.key)
+				}
+			},
+			spec: {
+				...planned,
+				name,
+				groups,
+				timeoutMs,
+				env: resolved.env,
+				judge: resolved.judge,
+				sources: resolved.sources ?? {},
+				notes: resolvedEval.notes
+			}
+		};
+	});
+}
+/** The run-wide cap, plus the sandbox provider's and harness's own caps when they set one. */
+function attemptLimits(planned, concurrency) {
+	const limits = { attempts: concurrency };
+	const { sandbox, harness } = planned;
+	if (sandbox.maxConcurrent) limits[`sandbox:${sandbox.kind}`] = sandbox.maxConcurrent;
+	if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
+	return limits;
+}
+//#endregion
+//#region src/evaluate.ts
+let groupStack = [];
+/** Each eval file's declarations as `evaluate` and `group` were called, and each eval's body by id. */
+const trees = /* @__PURE__ */ new Map();
+const bodies = /* @__PURE__ */ new Map();
+/** Options as the tree holds them; from TypeScript they may still hold function forms. */
+const data = (options) => options;
+function declare(node) {
+	const parent = groupStack.at(-1)?.node;
+	if (parent) return void parent.children.push(node);
+	const file = collectingFile() ?? "";
+	const tree = trees.get(file) ?? { children: [] };
+	trees.set(file, tree);
+	tree.children.push(node);
+}
+/** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
+function splitArgs(what, name, a, b) {
+	if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environment: …) or in the config's env.`);
+	if (typeof a === "function") return [b ?? {}, a];
+	if (typeof b === "function") return [a ?? {}, b];
+	throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (t) => { … }).`);
+}
+/** `evaluate.only` and `group.only` are refused in CI, where they'd silently skip the rest. */
+function checkOnly(mode, what) {
+	if (mode !== "only" || !inCI()) return;
+	const error = /* @__PURE__ */ new Error(`${what}.only isn't allowed in CI`);
+	error.name = "OnlyInCiError";
+	throw error;
+}
+const defineGroup = (mode) => (name, a, b) => {
+	checkOnly(mode, "group");
+	const [options, fn] = splitArgs("group", name, a, b);
+	checkOptions(options, `group("${name}")`);
+	const node = {
+		type: "group",
+		name,
+		mode,
+		options: data(options),
+		children: []
+	};
+	declare(node);
+	const stack = [...groupStack, {
+		name,
+		options,
+		node
+	}];
+	registerGroup(name, mode, () => {
+		const outer = groupStack;
+		groupStack = stack;
+		try {
+			fn();
+		} finally {
+			groupStack = outer;
+		}
+	});
+};
+const group = Object.assign(defineGroup("run"), {
+	skip: defineGroup("skip"),
+	only: defineGroup("only"),
+	todo: (name) => registerTodo(name)
+});
+/**
+* A placeholder, `evaluate.todo` or `group.todo`: one eval entry that never runs, counted as
+* skipped and shown as todo. The enclosing groups' tags filter it as they do an eval.
+*/
+function registerTodo(name) {
+	declare({
+		type: "todo",
+		name
+	});
+	const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
+	if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
+}
+/** The evals declared so far, by file and name path, so a second one with the same id is refused. */
+const declared = /* @__PURE__ */ new Set();
+function checkUnique(where, groups, name, hint) {
+	const key = `${collectingFile() ?? ""}\0${[...groups, name].join("\0")}`;
+	if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
+	declared.add(key);
+}
+function register(mode, name, own, fn, hint = "") {
+	const groups = groupStack.map((g) => g.name);
+	checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
+	const body = `b${bodies.size + 1}`;
+	bodies.set(body, fn);
+	declare({
+		type: "eval",
+		name,
+		mode,
+		options: data(own),
+		body
+	});
+	const tests = planEval({
+		name,
+		mode,
+		groups: groupStack.map(({ name, options }) => ({
+			name,
+			options
+		})),
+		options: own
+	}, {
+		...getRunContext(),
+		file: collectingFile(),
+		order: registered
+	});
+	if (!tests) return;
+	registered++;
+	for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
+}
+/** Evals registered so far in this worker, for the scheduler's tie-break. */
+let registered = 0;
+const defineEval = (mode) => (name, a, b) => {
+	checkOnly(mode, "evaluate");
+	const [own, fn] = splitArgs("evaluate", name, a, b);
+	register(mode, name, own, fn);
+};
+/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
+function formatName(template, item, index) {
+	return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => typeof item === "object" && item !== null && key in item ? String(item[key]) : whole);
+}
+/** One eval per case: `evaluate.for(cases)('adds $a and $b', async (t, c) => …)`. */
+const evalFor = (cases) => (name, a, b) => {
+	const [own, fn] = splitArgs("evaluate.for", name, a, b);
+	const hint = ` For evaluate.for, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
+	cases.forEach((item, i) => {
+		register("run", formatName(name, item, i), own, (t) => fn(t, item), hint);
+	});
+};
+const evaluate = Object.assign(defineEval("run"), {
+	skip: defineEval("skip"),
+	only: defineEval("only"),
+	todo: registerTodo,
+	for: evalFor
+});
+//#endregion
+export { DEFAULT_EGRESS_ALLOW, DEFAULT_SKILL_EXCLUDE, HarnessCapabilityError, defineConfig, defineEnvironment, evaluate, expect, group };
+
+//# sourceMappingURL=index.js.map