reeval 0.9.0 → 0.11.0

package/dist/evaluate-DKIl54Hb.js

@@ -0,0 +1,2348 @@
+import { t as isObjectLike } from "./object-CLf2-tNq.js";
+import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-B6BSMs03.js";
+import { n as dataProblem, p as parseDuration } from "./shared-CA61BuPJ.js";
+import { D as decideState, E as badJudgeArtifacts, O as filePartsOf, T as artifactRefsOf, _ as unbrokeredError, a as evalPlan, d as refuseIllegalCompare, f as sandboxId, g as unbrokered, h as secretVarsIn, i as checkOptions, k as resolveJudgeArtifacts, l as matchesTags, m as sandboxSetBy, n as cascade, o as expand, p as sandboxProvider, s as harnessId, t as brokeredIn, v as mergeJudge, w as judgeProblem, y as resolveStateOptions } from "./options-C0d0AzeR.js";
+import { A as workspaceLabel, C as parseGitRepo, D as stepCommand, E as sandboxPaths, S as mcpHosts, a as callerDir, b as isSecret, d as environmentFrom, f as environmentPaths, h as globFilter, j as workspaceOf, m as filesOf, n as DEFAULT_SKILL_EXCLUDE, v as isHostFile, x as isSecretTemplate } from "./environment-Cca6a4wB.js";
+import { c as relativeFile, i as evalSlug, l as sha256, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
+import { l as runJudge, s as judgeModelName, u as isApiError } from "./judge-DW_GDOmB.js";
+import { r as badCharacterAt, v as inCI } from "./config-DrVSEVN0.js";
+import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
+import { t as InfraError } from "./types-C5vI3g1C.js";
+import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, v as refusedMcp } from "./egress-2-qzPvGi.js";
+import { S as placeholder, nt as shellQuote, tt as run } from "./src-ftu5wPSt.js";
+import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile } from "./runtime-PX_oXiCS.js";
+import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CGPV3BZY.js";
+import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
+import { n as clip, r as formatDuration, t as capMiddle } from "./text-gEcKjAyp.js";
+import { a as writeCaptureSidecars, c as sandboxFor, d as workspaceFiles, l as diffSinceSnapshot, o as asCheck, r as serveCommand, s as createChecks, t as capturePage, u as snapshot } from "./capture-BQlGHiMC.js";
+import { t as contextTaker } from "./context-C1mxXh80.js";
+import { t as PrepareError } from "./provider-CmAYTQQs.js";
+import { i as preparedWorkspace } from "./workspace-C4RA3srV.js";
+import { n as judgeDiff } from "./patch-B1bnfhN-.js";
+import { t as transcriptForJudge } from "./transcript-BNNANU8A.js";
+import { r as withLease } from "./client-fcwgwOAD.js";
+import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
+import { basename, dirname, isAbsolute, join, relative, sep } from "node:path";
+import { parse } from "yaml";
+import { z } from "zod";
+import { setTimeout as setTimeout$1 } from "node:timers/promises";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+//#region src/attempt/assertions.ts
+const MESSAGE_LIMIT = 1024;
+const LABEL_LIMIT = 60;
+const clipped = (s, limit) => s.length > limit ? `${s.slice(0, limit)}…` : s;
+const text = (error) => error instanceof Error ? error.message : String(error);
+const shown = (value) => {
+	if (typeof value === "string") return value;
+	if (value instanceof RegExp) return String(value);
+	try {
+		return JSON.stringify(value) ?? String(value);
+	} catch {
+		return String(value);
+	}
+};
+const shownSubject = (subject) => typeof subject === "string" && subject ? subject.replace(/\s+/g, " ").trim() : void 0;
+/**
+* The records of an attempt's matcher calls, in call order. Without a label, one is
+* `<matcher> <first argument>` (or what the subject names); repeats get ` #2`, ` #3`.
+*/
+function assertionRecords(raw) {
+	const seen = /* @__PURE__ */ new Map();
+	return raw.map((r) => {
+		const named = r.args.length ? shown(r.args[0]) : shownSubject(r.subject);
+		const base = r.label ?? clipped(named === void 0 ? r.matcher : `${r.matcher} ${named}`, LABEL_LIMIT);
+		const n = (seen.get(base) ?? 0) + 1;
+		seen.set(base, n);
+		return {
+			label: n > 1 ? `${base} #${n}` : base,
+			labelled: r.label !== void 0,
+			matcher: r.matcher,
+			status: r.status,
+			soft: r.soft,
+			...r.status === "pass" ? {} : { message: clipped(text(r.error), MESSAGE_LIMIT) },
+			...r.durationMs === void 0 ? {} : { durationMs: r.durationMs }
+		};
+	});
+}
+/** What fails the attempt at its end for its `expect.soft` calls: the first that didn't pass. */
+function softFailure(raw) {
+	const failed = raw.find((r) => r.soft && r.status !== "pass");
+	if (!failed) return void 0;
+	return failed.error instanceof Error ? failed.error : new Error(text(failed.error));
+}
+//#endregion
+//#region src/attempt/debug.ts
+/** Which config and profile the attempt runs under, and where each resolved option came from. */
+function tracePlan(debug, spec, ctx) {
+	const config = ctx.configFile ?? "none (no config file found)";
+	const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
+	const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
+	debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
+	debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harness")}`);
+	debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
+	const env = spec.environment;
+	const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
+	const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
+	debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
+	debug.log("plan", `egress ${egressLabel(spec.egress) || "allow-all"} ${from("egress")}`);
+	const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
+	debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
+	const judge = spec.judge;
+	const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
+	debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
+	for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
+	debug.trace("plan", "config", {
+		config: ctx.config,
+		profile: ctx.profile
+	}, "resolved config");
+	debug.trace("plan", "attempt", {
+		cell: spec.cell,
+		n: spec.n,
+		harness: spec.harness,
+		sandbox: spec.sandbox,
+		egress: spec.egress,
+		gate: spec.gate,
+		timeoutMs: spec.timeoutMs,
+		judge: spec.judge,
+		sources: spec.sources,
+		env: Object.keys(spec.env).sort()
+	}, "resolved attempt");
+}
+/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
+function endpointLabel({ method, host, path }) {
+	const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
+	return `${method?.join("|") ?? "*"} ${where}`;
+}
+/** The resolved environment and what the gateway brokers, by name. */
+function traceEnvironment(debug, spec, env) {
+	const r = env.record;
+	debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
+	const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
+	debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
+	for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
+	const brokering = brokeringFor(spec.harness);
+	if (brokering) {
+		const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
+		debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
+	}
+	debug.trace("env", "environment", {
+		record: r,
+		vars: Object.keys(env.vars).sort(),
+		checkVars: Object.keys(env.checkVars).sort(),
+		files: env.files.map((f) => f.path),
+		setup: env.setup,
+		skills: env.skills.map((s) => s.name),
+		workspaceSkills: env.workspaceSkills.map((s) => s.name),
+		mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
+			transport: m.transport,
+			command: m.command,
+			args: m.args,
+			env: Object.keys(m.env)
+		} : {
+			transport: m.transport,
+			url: m.url,
+			headers: Object.keys(m.headers)
+		}])),
+		brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
+			name,
+			hosts,
+			in: where,
+			phase
+		}))
+	}, "resolved environment");
+}
+//#endregion
+//#region src/environment/resolve.ts
+const SKILL_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
+const DEFAULT_SETUP_TIMEOUT = "5m";
+function broker(secrets, name, value, hosts, where = ["header"], phase = "agent") {
+	const at = badCharacterAt(value);
+	if (at && where.includes("header")) throw new InfraError(`${name} has whitespace or a line break in it (character ${at}), so it can't go in a request header. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
+	const sub = {
+		name,
+		placeholder: placeholder(),
+		value,
+		hosts: [...hosts],
+		in: [...where],
+		phase
+	};
+	secrets.brokered.push(sub);
+	return sub.placeholder;
+}
+/** The host value, or its placeholder when it's brokered to `secret.hosts` (else `hosts`). */
+function hostValue(name, secret, where, secrets, hosts = secret.hosts) {
+	const from = secret.from ?? name;
+	const value = process.env[from];
+	if (!value) {
+		if (secret.optional) {
+			secrets.unset.push(name);
+			return;
+		}
+		const source = from === name ? name : `${name} (from ${from})`;
+		throw new InfraError(`Secret ${source} for ${where} isn't set. Set it in the environment or .env.local.`);
+	}
+	registerSecret(value);
+	if (secret.to === "checks") secrets.checks.push(name);
+	else secrets.names.push(name);
+	if (!hosts) return value;
+	return broker(secrets, name, value, hosts, secret.in, secret.to === "checks" ? "checks" : "agent");
+}
+/** Each host variable in the template, or with `hosts` one placeholder per variable. */
+function template(value, where, secrets, hosts) {
+	const placeholders = /* @__PURE__ */ new Map();
+	return value.template.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, name) => {
+		const v = process.env[name];
+		if (!v) throw new InfraError(`secret.template for ${where} needs ${name}, which isn't set. Set it in the environment or .env.local.`);
+		registerSecret(v);
+		if (!hosts) return v;
+		const p = placeholders.get(name) ?? broker(secrets, name, v, hosts);
+		placeholders.set(name, p);
+		return p;
+	});
+}
+/** A `SKILL.md` split into its YAML frontmatter's `name` and `description` and its body. */
+function parseSkillMd(text, where) {
+	const match = /^---\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)([\s\S]*)$/.exec(text);
+	if (!match) return { body: text };
+	let front;
+	try {
+		front = parse(match[1] ?? "");
+	} catch (error) {
+		const reason = error.message.split("\n")[0];
+		throw new InfraError(`${where}: the SKILL.md frontmatter isn't valid YAML (${reason}). Quote values that contain ": ".`);
+	}
+	if (front != null && (typeof front !== "object" || Array.isArray(front))) throw new InfraError(`${where}: the SKILL.md frontmatter must be a YAML mapping, e.g. name: my-skill and description: … on their own lines`);
+	const field = (key) => {
+		const value = front?.[key];
+		if (value == null) return void 0;
+		if (typeof value !== "string") throw new InfraError(`${where}: the SKILL.md ${key} must be a string (got ${typeof value})`);
+		return value.trim() || void 0;
+	};
+	return {
+		name: field("name"),
+		description: field("description"),
+		body: match[2] ?? ""
+	};
+}
+/** A gitignore-style test for `DEFAULT_SKILL_EXCLUDE`-shaped patterns against a skill-relative path. */
+function excludeMatcher(patterns) {
+	const rules = patterns.map((pattern) => {
+		const dirOnly = pattern.endsWith("/");
+		const body = pattern.replace(/\/+$/, "");
+		const anchored = body.includes("/");
+		const glob = body.replace(/^\/+/, "").split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*");
+		return {
+			dirOnly,
+			re: new RegExp(`${anchored ? "^" : "(?:^|/)"}${glob}$`)
+		};
+	});
+	return (path, isDir) => rules.some((r) => (isDir || !r.dirOnly) && r.re.test(path));
+}
+function parseSkill(dir, source, exclude) {
+	const front = parseSkillMd(readFileSync(join(dir, "SKILL.md"), "utf8"), `Skill ${basename(dir)} in ${source}`);
+	const name = front.name || basename(dir);
+	checkSkill(name, front.description, source);
+	const skip = excludeMatcher(exclude);
+	const paths = [];
+	const excluded = [];
+	const walk = (rel) => {
+		for (const entry of readdirSync(join(dir, rel), { withFileTypes: true })) {
+			const path = rel ? `${rel}/${entry.name}` : entry.name;
+			const isDir = entry.isDirectory();
+			if (entry.name === ".git") continue;
+			if (skip(path, isDir)) excluded.push(isDir ? `${path}/` : path);
+			else if (isDir) walk(path);
+			else if (entry.isFile() && path !== "SKILL.md") paths.push(path);
+		}
+	};
+	walk("");
+	const files = paths.sort().map((path) => ({
+		path,
+		content: readFileSync(join(dir, path), "utf8")
+	}));
+	return {
+		skill: {
+			name,
+			description: front.description ?? "",
+			content: front.body,
+			files
+		},
+		excluded: excluded.sort()
+	};
+}
+function checkSkill(name, description, source) {
+	if (!SKILL_NAME.test(name)) throw new InfraError(`Skill "${name}" in ${source}: the name must be kebab-case, e.g. my-skill (lower-case letters, digits and -)`);
+	if (!description) throw new InfraError(`Skill "${name}" in ${source} has no description in its SKILL.md frontmatter. Add a description: line saying when to use it.`);
+}
+function loadSkills(dir, source, only, exclude = DEFAULT_SKILL_EXCLUDE) {
+	if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new InfraError(`Skill source ${source} isn't a directory. Point it at a skill's directory, or at a directory of skills.`);
+	const dirs = existsSync(join(dir, "SKILL.md")) ? [dir] : readdirSync(dir, { withFileTypes: true }).filter((e) => e.isDirectory() && existsSync(join(dir, e.name, "SKILL.md"))).map((e) => join(dir, e.name)).sort();
+	if (dirs.length === 0) throw new InfraError(`Skill source ${source} has no SKILL.md (and no subdirectories with one)`);
+	const skills = dirs.map((d) => parseSkill(d, source, exclude));
+	if (!only) return skills;
+	const missing = only.filter((n) => !skills.some((s) => s.skill.name === n));
+	if (missing.length) throw new InfraError(`only: ${JSON.stringify(missing)} not found in ${source} (found: ${skills.map((s) => s.skill.name).join(", ")})`);
+	return skills.filter((s) => only.includes(s.skill.name));
+}
+async function git(args, cwd, what) {
+	const r = await run("git", args, { cwd });
+	if (r.exitCode !== 0) throw new InfraError(`${what} failed: ${r.stderr.trim()}`);
+	return r.stdout.trim();
+}
+/** One clone per source and ref per worker process; the commit is recorded. */
+const clones = /* @__PURE__ */ new Map();
+function cloneSkills(root, url, ref) {
+	const key = `${url}#${ref ?? ""}`;
+	let clone = clones.get(key);
+	if (!clone) {
+		clone = (async () => {
+			const dir = join(root, ".reeval", "cache", "skills", `${sha256(key).slice(0, 16)}-${process.pid}`);
+			await rm(dir, {
+				recursive: true,
+				force: true
+			});
+			await mkdir(dir, { recursive: true });
+			const what = `Cloning ${url}${ref ? ` at ${ref}` : ""}`;
+			await git(["init", "-q"], dir, what);
+			await git([
+				"fetch",
+				"-q",
+				"--depth",
+				"1",
+				url,
+				ref ?? "HEAD"
+			], dir, what);
+			await git([
+				"-c",
+				"advice.detachedHead=false",
+				"checkout",
+				"-q",
+				"FETCH_HEAD"
+			], dir, what);
+			return {
+				dir,
+				commit: await git(["rev-parse", "HEAD"], dir, what)
+			};
+		})();
+		clones.set(key, clone);
+		clone.catch(() => clones.delete(key));
+	}
+	return clone;
+}
+const packs = /* @__PURE__ */ new Map();
+/** `npm pack` into the cache, unpacked; returns the package directory. */
+function unpackNpm(root, spec) {
+	let pack = packs.get(spec);
+	if (!pack) {
+		pack = (async () => {
+			const dir = join(root, ".reeval", "cache", "skills", `npm-${sha256(spec).slice(0, 16)}-${process.pid}`);
+			await rm(dir, {
+				recursive: true,
+				force: true
+			});
+			await mkdir(dir, { recursive: true });
+			const packed = await run("npm", [
+				"pack",
+				spec,
+				"--silent",
+				"--pack-destination",
+				dir
+			], { cwd: dir });
+			const tarball = packed.stdout.trim().split("\n").at(-1);
+			if (packed.exitCode !== 0 || !tarball) throw new InfraError(`npm pack ${spec} failed: ${packed.stderr.trim()}`);
+			const untar = await run("tar", ["-xzf", tarball], { cwd: dir });
+			if (untar.exitCode !== 0) throw new InfraError(`Unpacking ${spec} failed: ${untar.stderr.trim()}`);
+			return join(dir, "package");
+		})();
+		packs.set(spec, pack);
+		pack.catch(() => packs.delete(spec));
+	}
+	return pack;
+}
+async function resolveSkills(root, s) {
+	switch (s.source) {
+		case "local": {
+			const label = `local:${relative(root, s.path).split(sep).join("/") || "."}`;
+			return {
+				skills: loadSkills(s.path, label, s.only, s.exclude),
+				source: label,
+				commit: null
+			};
+		}
+		case "git": {
+			const { url } = parseGitRepo(s.repo);
+			const { dir, commit } = await cloneSkills(root, url, s.ref);
+			const path = s.path ?? (!existsSync(join(dir, "SKILL.md")) && existsSync(join(dir, "skills")) ? "skills" : "");
+			const label = `${s.repo}${s.ref ? `#${s.ref}` : ""}${path ? `:${path}` : ""}`;
+			return {
+				skills: loadSkills(join(dir, path), label, s.only, s.exclude),
+				source: label,
+				commit
+			};
+		}
+		case "registry": {
+			const [repo = "", name = ""] = s.id.split("@");
+			const { dir, commit } = await cloneSkills(root, `https://github.com/${repo}.git`, s.version);
+			const label = `registry:${s.id}${s.version ? `#${s.version}` : ""}`;
+			return {
+				skills: loadSkills(existsSync(join(dir, "skills")) ? join(dir, "skills") : dir, label, [name]),
+				source: label,
+				commit
+			};
+		}
+		case "npm": {
+			const dir = await unpackNpm(root, s.spec);
+			const label = `npm:${isAbsolute(s.spec) ? relative(root, s.spec).split(sep).join("/") : s.spec}:${s.path}`;
+			return {
+				skills: loadSkills(join(dir, s.path), label, s.only, s.exclude),
+				source: label,
+				commit: null
+			};
+		}
+		case "inline": {
+			checkSkill(s.name, s.description, "skill.inline");
+			const files = Object.entries(s.files).sort(([a], [b]) => a < b ? -1 : 1).map(([path, content]) => ({
+				path,
+				content
+			}));
+			return {
+				skills: [{
+					skill: {
+						name: s.name,
+						description: s.description,
+						content: s.content,
+						files
+					},
+					excluded: []
+				}],
+				source: "inline",
+				commit: null
+			};
+		}
+	}
+}
+const skillHash = (s) => sha256(JSON.stringify(s));
+const readHostFile = (f, where) => {
+	if (!existsSync(f.path)) throw new InfraError(`${f.path} for ${where} doesn't exist. file() paths resolve against the file that calls it, or reeval.yaml's directory.`);
+	return readFileSync(f.path);
+};
+/**
+* Layers the environment (root first; a later layer wins on shared keys, and each such key is
+* an override), over the options `env` merged down the cascade. `where` names the attempt, and
+* `rulesFile` is the workspace file the harness's agent reads `rules` from.
+*/
+async function resolveEnvironment(environment, optionsEnv, root, where, rulesFile, egress = "allow-all", prepared = () => void 0) {
+	const envWhere = `environment ${environment.name} (${where})`;
+	const overrides = /* @__PURE__ */ new Set();
+	const env = { ...optionsEnv };
+	const files = /* @__PURE__ */ new Map();
+	const mcp = /* @__PURE__ */ new Map();
+	const skills = /* @__PURE__ */ new Map();
+	let rules;
+	const setup = [];
+	const seen = {
+		env: /* @__PURE__ */ new Set(),
+		files: /* @__PURE__ */ new Set(),
+		mcp: /* @__PURE__ */ new Set()
+	};
+	let egressSet = false;
+	let workspace;
+	let sandboxSet = false;
+	for (const { spec } of environment.layers) {
+		if (spec.sandbox !== void 0) {
+			if (sandboxSet) overrides.add("sandbox");
+			sandboxSet = true;
+		}
+		if (spec.workspace !== void 0) {
+			if (workspace) overrides.add("workspace");
+			workspace = spec.workspace;
+		}
+		if (spec.egress !== void 0) {
+			if (egressSet) overrides.add("egress");
+			egressSet = true;
+		}
+		for (const [k, v] of Object.entries(spec.env ?? {})) {
+			if (seen.env.has(k)) overrides.add(`env.${k}`);
+			seen.env.add(k);
+			env[k] = v;
+		}
+		for (const [path, v] of Object.entries(spec.files ?? {})) {
+			if (seen.files.has(path)) overrides.add(`files.${path}`);
+			seen.files.add(path);
+			files.set(path, v);
+		}
+		if (spec.rules !== void 0) {
+			if (rules !== void 0) overrides.add("rules");
+			rules = spec.rules;
+		}
+		for (const [name, server] of Object.entries(spec.mcp ?? {})) {
+			if (seen.mcp.has(name)) overrides.add(`mcp.${name}`);
+			seen.mcp.add(name);
+			mcp.set(name, server);
+		}
+		for (const step of spec.setup ?? []) {
+			const s = typeof step === "string" ? { run: step } : step;
+			setup.push({
+				run: s.run,
+				timeoutMs: parseDuration(s.timeout ?? DEFAULT_SETUP_TIMEOUT),
+				...s.cwd ? { cwd: s.cwd } : {}
+			});
+		}
+		const layerSkills = /* @__PURE__ */ new Set();
+		for (const source of spec.skills ?? []) {
+			const found = await resolveSkills(root, source);
+			for (const loaded of found.skills) {
+				const name = loaded.skill.name;
+				if (skills.has(name) && !layerSkills.has(name)) overrides.add(`skills.${name}`);
+				layerSkills.add(name);
+				skills.delete(name);
+				skills.set(name, {
+					...loaded,
+					placement: source.placement,
+					source: found.source,
+					commit: found.commit
+				});
+			}
+		}
+	}
+	const secrets = {
+		names: [],
+		checks: [],
+		unset: [],
+		brokered: []
+	};
+	const vars = {};
+	const checkVars = {};
+	for (const [name, value] of Object.entries(env)) {
+		if (!isSecret(value)) {
+			vars[name] = value;
+			continue;
+		}
+		const resolved = hostValue(name, value, envWhere, secrets);
+		if (resolved === void 0) continue;
+		if (value.to === "checks") checkVars[name] = resolved;
+		else vars[name] = resolved;
+	}
+	const secretOf = (value, key, hosts) => {
+		if (typeof value === "string") return value;
+		if (isSecretTemplate(value)) return template(value, envWhere, secrets, hosts);
+		return hostValue(key, value, envWhere, secrets, value.hosts ?? hosts);
+	};
+	const resolvedMcp = {};
+	for (const [name, s] of mcp) if (s.transport === "stdio") {
+		const e = {};
+		for (const [k, v] of Object.entries(s.env)) {
+			const r = secretOf(v, k);
+			if (r !== void 0) e[k] = r;
+		}
+		resolvedMcp[name] = {
+			transport: "stdio",
+			command: s.command,
+			args: s.args,
+			env: e
+		};
+	} else {
+		const h = {};
+		const hosts = mcpHosts(s.url);
+		for (const [k, v] of Object.entries(s.headers)) {
+			const r = secretOf(v, k, hosts);
+			if (r !== void 0) h[k] = r;
+		}
+		resolvedMcp[name] = {
+			transport: s.transport,
+			url: s.url,
+			headers: h
+		};
+	}
+	const fileList = [...files].map(([path, v]) => ({
+		path,
+		content: isHostFile(v) ? readHostFile(v, envWhere) : v,
+		...isHostFile(v) && statSync(v.path).mode & 64 ? { executable: true } : {}
+	}));
+	const rulesContent = rules === void 0 ? void 0 : isHostFile(rules) ? Buffer.from(readHostFile(rules, envWhere)).toString("utf8") : rules;
+	const skillList = [...skills.values()];
+	const skillRecords = skillList.map(({ skill, placement, source, commit, excluded }) => ({
+		name: skill.name,
+		source,
+		commit,
+		sha256: skillHash(skill),
+		placement,
+		excluded
+	}));
+	const fileRecords = fileList.map((f) => ({
+		path: f.path,
+		sha256: sha256(f.content)
+	}));
+	const executable = fileList.flatMap((f) => f.executable ? [f.path] : []);
+	const rulesRecord = rulesContent === void 0 ? void 0 : {
+		path: rulesFile,
+		sha256: sha256(rulesContent)
+	};
+	if (rulesContent !== void 0) fileList.push({
+		path: rulesFile,
+		content: rulesContent
+	});
+	const secretNames = [.../* @__PURE__ */ new Set([...secrets.names, ...secrets.checks])].sort();
+	const brokered = secrets.brokered.map((b) => JSON.stringify([
+		b.name,
+		b.hosts,
+		b.in,
+		b.phase
+	])).sort();
+	const brokeredNames = [...new Set(secrets.brokered.map((b) => b.name))].sort();
+	const varNames = Object.keys(vars).filter((n) => !secrets.names.includes(n)).sort();
+	const literalVars = Object.fromEntries(varNames.map((n) => [n, vars[n]]));
+	const mcpShape = Object.fromEntries(Object.entries(resolvedMcp).map(([name, s]) => s.transport === "stdio" ? [name, {
+		command: s.command,
+		args: s.args,
+		env: Object.keys(s.env).sort()
+	}] : [name, {
+		transport: s.transport,
+		url: s.url,
+		headers: Object.keys(s.headers).sort()
+	}]));
+	const tree = workspace && prepared(workspace);
+	if (workspace && !tree) throw new InfraError(`The workspace git(${JSON.stringify(workspaceLabel(workspace))}) for ${envWhere} wasn't prepared before the run started`);
+	const workspaceRecord = workspace && tree ? {
+		source: isAbsolute(workspace.url) ? relative(root, workspace.url).split(sep).join("/") || "." : workspace.repo,
+		ref: workspace.ref ?? null,
+		path: workspace.path ?? null,
+		commit: tree.commit,
+		tree: tree.tree,
+		...workspace.init ? { init: true } : {}
+	} : void 0;
+	const digest = `sha256:${sha256(JSON.stringify({
+		setup,
+		skills: skillRecords.map((s) => [
+			s.name,
+			s.sha256,
+			s.placement
+		]),
+		mcp: mcpShape,
+		files: fileRecords,
+		...executable.length ? { executable } : {},
+		rules: rulesRecord?.sha256,
+		vars: literalVars,
+		secrets: secretNames,
+		checks: [...secrets.checks].sort(),
+		unset: [...secrets.unset].sort(),
+		brokered,
+		...egress === "allow-all" ? {} : { egress: typeof egress === "string" ? egress : [...egress.allow].sort() },
+		...workspaceRecord ? { workspace: workspaceRecord.init ? [workspaceRecord.tree, "init"] : workspaceRecord.tree } : {}
+	}))}`;
+	return {
+		record: {
+			name: environment.name,
+			parent: environment.parent,
+			overrides: [...overrides],
+			digest,
+			setup: setup.map(stepCommand),
+			skills: skillRecords,
+			mcp: Object.keys(resolvedMcp).sort(),
+			files: rulesRecord ? [...fileRecords, rulesRecord] : fileRecords,
+			vars: varNames,
+			secrets: secretNames,
+			brokered: brokeredNames,
+			unset: [...secrets.unset].sort(),
+			egress,
+			...workspaceRecord ? { workspace: workspaceRecord } : {}
+		},
+		setup,
+		skills: skillList.filter((s) => s.placement === "harness").map((s) => s.skill),
+		workspaceSkills: skillList.filter((s) => s.placement === "workspace").map((s) => s.skill),
+		mcp: resolvedMcp,
+		files: fileList,
+		vars,
+		checkVars,
+		substitutions: secrets.brokered,
+		...workspace && tree ? { workspace: {
+			tar: tree.tar,
+			label: workspaceLabel(workspace),
+			commit: tree.commit,
+			...workspace.init ? { init: true } : {}
+		} } : {}
+	};
+}
+/** A skill as a `SKILL.md` file plus its other files, for writing into a directory. */
+function skillFiles(skill) {
+	return [{
+		path: "SKILL.md",
+		content: `${`---\nname: ${skill.name}\ndescription: ${JSON.stringify(skill.description)}\n---\n`}${skill.content}`
+	}, ...skill.files];
+}
+//#endregion
+//#region src/harness/fake.ts
+const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
+/**
+* Writes canned files and returns a canned message on every turn, after running its `script`'s
+* real sandbox actions. Harness-placed skills are written under `~/` where its layout's adapter
+* puts them (`~/.agents/skills` by default, as Cursor's does).
+*/
+const createFakeHarness = (descriptor) => {
+	let starts = 0;
+	return { async start(sandbox, _signal, { skills, workspaceSkills, mcp }) {
+		if (descriptor.failWith) throw new InfraError(descriptor.failWith);
+		const hangs = descriptor.startHangs;
+		if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
+		const dir = `~/${layoutOf(descriptor).harnessSkillDir}`;
+		for (const skill of skills) {
+			for (const f of skillFiles(skill)) await sandbox.writeFile(`${dir}/${skill.name}/${f.path}`, f.content);
+			debugLog("env", `skill ${skill.name} placed at ${dir}/${skill.name}/ by the fake agent`);
+		}
+		const names = [...skills.map((s) => s.name), ...workspaceSkills];
+		const loaded = [names.length ? `skills: ${names.join(", ")}` : "", Object.keys(mcp).length ? `mcp: ${Object.keys(mcp).join(", ")}` : ""].filter(Boolean);
+		let calls = 0;
+		let turns = 0;
+		const skilled = names.length > 0 && descriptor.withSkill;
+		const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
+		const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
+		const flaked = Math.random() < (descriptor.flakeRate ?? 0);
+		return {
+			async send(prompt, signal) {
+				const startedAt = Date.now();
+				const script = descriptor.script ?? [];
+				const steps = script[Math.min(turns++, script.length - 1)] ?? [];
+				const at = () => {
+					const ms = Date.now() - startedAt;
+					return {
+						ts: new Date(startedAt + ms).toISOString(),
+						atMs: ms
+					};
+				};
+				const transcript = [{
+					...at(),
+					type: "text",
+					text: `Working on: ${prompt}`,
+					durationMs: 0
+				}, ...loaded.length ? [{
+					...at(),
+					type: "text",
+					text: `Loaded ${loaded.join("; ")}`,
+					durationMs: 0
+				}] : []];
+				const done = (text, status) => ({
+					text,
+					status,
+					startedAt: new Date(startedAt).toISOString(),
+					durationMs: Date.now() - startedAt,
+					transcript,
+					usage: null,
+					model: descriptor.servedModel ?? null,
+					unreported: []
+				});
+				const write = async (path, content) => {
+					const time = at();
+					const before = await sandbox.readFile(path);
+					await sandbox.writeFile(path, content);
+					debugLog("harness", `tool write ${path} (${before === null ? "create" : "modify"})`);
+					transcript.push({
+						...time,
+						type: "file-edit",
+						toolCallId: `call_${++calls}`,
+						path,
+						kind: before === null ? "create" : "modify",
+						added: lineCount(content),
+						removed: before === null ? 0 : lineCount(before),
+						durationMs: Date.now() - startedAt - time.atMs
+					});
+				};
+				if (descriptor.reasoning) transcript.push({
+					...at(),
+					type: "reasoning",
+					text: descriptor.reasoning,
+					durationMs: 0
+				});
+				for (const tool of descriptor.tools ?? []) transcript.push({
+					...at(),
+					type: "tool",
+					toolCallId: `call_${++calls}`,
+					tool: tool.tool,
+					input: tool.input,
+					output: tool.output,
+					ok: tool.ok ?? true,
+					durationMs: 0
+				});
+				if (flaked) {
+					const text = "I ran out of ideas and left the workspace unchanged.";
+					transcript.push({
+						...at(),
+						type: "text",
+						text,
+						durationMs: 0
+					});
+					return done(text, "completed");
+				}
+				let scripted;
+				for (const step of steps) if ("exec" in step) {
+					const time = at();
+					debugLog("harness", `tool shell called: $ ${step.exec}`);
+					const r = await sandbox.exec(step.exec, { signal });
+					debugLog("harness", `tool shell returned exit ${r.exitCode} in ${Date.now() - startedAt - time.atMs}ms`);
+					transcript.push({
+						...time,
+						type: "tool",
+						toolCallId: `call_${++calls}`,
+						tool: "shell",
+						input: { command: step.exec },
+						output: r.stdout,
+						ok: r.exitCode === 0,
+						durationMs: Date.now() - startedAt - time.atMs
+					});
+				} else if ("write" in step) for (const [path, content] of Object.entries(step.write)) await write(path, content);
+				else if ("mcp" in step) {
+					const time = at();
+					const [server = "", tool = ""] = step.mcp.split("/");
+					debugLog("harness", `tool mcp__${server}__${tool} called`);
+					const r = await callMcpTool(sandbox, mcp, server, tool, signal);
+					debugLog("harness", `tool mcp__${server}__${tool} ${r.ok ? "returned" : "failed"}`);
+					transcript.push({
+						...time,
+						type: "tool",
+						toolCallId: `call_${++calls}`,
+						tool: `mcp__${server}__${tool}`,
+						input: {},
+						output: r.output,
+						ok: r.ok,
+						durationMs: Date.now() - startedAt - time.atMs
+					});
+				} else if ("error" in step) {
+					debugLog("harness", `turn failed: ${step.error}`);
+					transcript.push({
+						...at(),
+						type: "error",
+						message: step.error
+					});
+					return done("", "error");
+				} else scripted = step.reply;
+				if (descriptor.delayMs) try {
+					await setTimeout$1(descriptor.delayMs, void 0, descriptor.ignoreAbort ? {} : { signal });
+				} catch {
+					return done("", "aborted");
+				}
+				for (const [path, content] of Object.entries(files)) await write(path, content);
+				const text = scripted ?? reply;
+				transcript.push({
+					...at(),
+					type: "text",
+					text,
+					durationMs: 0
+				});
+				return done(text, "completed");
+			},
+			async close() {}
+		};
+	} };
+};
+/**
+* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
+* arguments: initialize, then the call.
+*/
+async function callMcpTool(sandbox, servers, name, tool, signal) {
+	const server = servers[name];
+	if (!server) {
+		const known = Object.keys(servers);
+		throw new InfraError(`The fake agent's script calls MCP server "${name}", which the environment doesn't have (it has ${known.length ? known.join(", ") : "none"})`);
+	}
+	if (server.transport !== "stdio") throw new InfraError(`The fake agent calls stdio MCP servers only; "${name}" is ${server.transport}`);
+	const messages = [
+		{
+			jsonrpc: "2.0",
+			id: 1,
+			method: "initialize",
+			params: {
+				protocolVersion: "2024-11-05",
+				capabilities: {},
+				clientInfo: {
+					name: "reeval-fake-agent",
+					version: "1"
+				}
+			}
+		},
+		{
+			jsonrpc: "2.0",
+			method: "notifications/initialized"
+		},
+		{
+			jsonrpc: "2.0",
+			id: 2,
+			method: "tools/call",
+			params: {
+				name: tool,
+				arguments: {}
+			}
+		}
+	].map((m) => shellQuote(JSON.stringify(m)));
+	const command = [server.command, ...server.args].map(shellQuote).join(" ");
+	const script = [
+		"d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
+		"{",
+		"  trap '' PIPE",
+		`  printf '%s\\n' ${messages.join(" ")} 2>/dev/null`,
+		"  i=0",
+		`  while [ $i -lt 300 ] && [ ! -f "$d/code" ] && ! grep -Eq '"id": ?2[,}]' "$d/out"; do sleep 0.1; i=$((i + 1)); done`,
+		`} | { ${command} > "$d/out" 2> "$d/err"; echo $? > "$d/code"; }`,
+		"code=$(cat \"$d/code\")",
+		"cat \"$d/out\"; cat \"$d/err\" >&2; rm -rf \"$d\"; exit $code"
+	].join("\n");
+	const r = await sandbox.exec(script, {
+		signal,
+		env: server.env
+	});
+	const reply = r.stdout.split("\n").flatMap((line) => {
+		try {
+			return [JSON.parse(line)];
+		} catch {
+			return [];
+		}
+	}).find((m) => m.id === 2);
+	if (!reply) return {
+		output: r.stderr.trim() || `no reply (exit ${r.exitCode})`,
+		ok: false
+	};
+	if (reply.error) return {
+		output: reply.error.message,
+		ok: false
+	};
+	return {
+		output: (reply.result?.content ?? []).flatMap((c) => c.type === "text" ? [c.text] : []).join("\n"),
+		ok: !reply.result?.isError
+	};
+}
+//#endregion
+//#region src/harness/index.ts
+async function createHarness(descriptor) {
+	if (descriptor.kind === "fake") return createFakeHarness(descriptor);
+	const { createAgentHarness } = await import("./agent-DpE80WPz.js");
+	return createAgentHarness(descriptor);
+}
+/** Where each harness's agent looks for repo skills, relative to the workspace. */
+const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
+//#endregion
+//#region src/attempt/environment.ts
+/**
+* Resolves the attempt's environment (files, skills, secrets and their placeholders)
+* and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
+* signal, before the eval timer starts.
+*/
+async function prepareEnvironment(a, root, runId, where, signal) {
+	const { spec, record } = a;
+	const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w));
+	a.env = env;
+	record.environment = env.record;
+	const { secrets, brokered } = env.record;
+	record.env = {
+		vars: Object.keys(env.vars).filter((n) => !secrets.includes(n)).sort(),
+		secrets,
+		brokered
+	};
+	const debug = attemptDebug();
+	if (debug) traceEnvironment(debug, spec, env);
+	record.image = null;
+	if (spec.sandbox.kind === "docker") {
+		const { preparedFor } = await import("./preflight-DlQvbySb.js");
+		const { attemptImage, dockerProvider } = await import("./providers-B9yTdIqF.js");
+		signal.throwIfAborted();
+		const prepared = preparedFor(root, runId, dockerProvider({ engine: spec.sandbox.engine ?? "docker" }), spec.sandbox, spec.harness);
+		if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
+		record.image = attemptImage(prepared);
+		a.image = record.image.ref;
+		const { ref, id, base, install, agentCli } = record.image;
+		debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
+	} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
+}
+//#endregion
+//#region src/judge/size.ts
+/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
+const CHARS_PER_TOKEN = 4;
+const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
+const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
+const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
+	name: /^diff --git a\/.+ b\/(.+)$/m.exec(part)?.[1] ?? "",
+	chars: part.length
+})).filter((item) => item.name);
+/** The state's parts by size, each with its biggest files where it has any. */
+function partsOf(state, question) {
+	return [...Object.entries(state).map(([name, value]) => {
+		const text = typeof value === "string" ? value : JSON.stringify(value);
+		const items = name === "diff" && typeof value === "string" ? diffItems(value) : value && typeof value === "object" && !Array.isArray(value) ? Object.entries(value).map(([file, content]) => ({
+			name: file,
+			chars: typeof content === "string" ? content.length : JSON.stringify(content).length
+		})) : [];
+		return {
+			name,
+			chars: text?.length ?? 0,
+			items
+		};
+	}), {
+		name: "question",
+		chars: question.length,
+		items: []
+	}].sort((a, b) => b.chars - a.chars);
+}
+/** The estimated size of what the judge reads, in tokens. */
+const inputTokens = (state, question) => tokensOf(JSON.stringify(state).length + question.length);
+/**
+* `diff 306k, of which dist/assets/index-….js is 296k; output 1k; …`: the state's parts by
+* estimated tokens, naming the biggest files in each.
+*/
+function describeSize(state, question) {
+	const parts = partsOf(state, question);
+	const big = parts.filter((p) => tokensOf(p.chars) >= 1e3);
+	return (big.length ? big : parts.slice(0, 1)).map((p) => {
+		const top = p.items.sort((a, b) => b.chars - a.chars).slice(0, 3).filter((item) => tokensOf(item.chars) >= 1e3).map((item) => `${item.name} is ${k(tokensOf(item.chars))}`);
+		return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
+	}).join("; ");
+}
+/** `Narrow state.diff, state.transcript or state.output.`: the state's parts, largest first. */
+function stateHint(state, question) {
+	const keys = partsOf(state, question).filter((p) => p.name !== "question").map((p) => `state.${p.name}`);
+	if (!keys.length) return "";
+	const last = keys.pop();
+	return ` Narrow ${keys.length ? `${keys.join(", ")} or ${last}` : last}.`;
+}
+/**
+* A provider's error, as the provider worded it, then what the judge's input held: its estimated
+* size and biggest parts and files, and the `state` keys that were on, largest first. reeval knows
+* no model's limit, so this follows every provider error.
+*/
+const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}.${stateHint(state, question)}`;
+//#endregion
+//#region src/judge/state.ts
+function turnState(t) {
+	return {
+		...t.task !== void 0 ? { task: t.task } : {},
+		...t.output !== void 0 ? { output: t.output } : {},
+		...t.transcript !== void 0 ? { transcript: t.transcript } : {}
+	};
+}
+/** The JSON the judge reads: the graded turn's keys at the top, or every turn under `turns`. */
+function stateOf(d) {
+	const last = d.turns.at(-1) ?? {};
+	const { transcript, ...top } = d.all ? {} : turnState(last);
+	return {
+		...d.all ? { turns: d.turns.map(turnState) } : top,
+		...d.files ? { files: d.files } : {},
+		...d.diff !== void 0 ? { diff: d.diff } : {},
+		...d.contents ? { contents: d.contents } : {},
+		...d.checks ? { checks: d.checks } : {},
+		...transcript !== void 0 ? { transcript } : {}
+	};
+}
+//#endregion
+//#region src/attempt/judge.ts
+const noJudgeModel = (configFile) => `c.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.14.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
+/** A turn this attempt sent becomes its index; any other value is graded as the output. */
+function judgeOn(turns, on) {
+	const turn = turns.indexOf(on);
+	if (turn >= 0) return { turn };
+	if (typeof on === "object" && typeof on?.exec === "function") throw new Error("c.judge() on is a turn this attempt didn't send");
+	return { value: on };
+}
+const workspacePath = (path) => path.replace(/^\.\//, "");
+/** A `contents` entry that names one file rather than matching many. */
+const isExactPath = (entry) => !entry.startsWith("!") && !/[*?[\]{}]/.test(entry);
+/**
+* The judge's state for what the `state` option chose: the graded turn's task and output (or every turn's),
+* its changed files and diff as `judgeDiff` shows them, files whole, `c.exec` results and the
+* transcript.
+*/
+async function draftState(src) {
+	const { subject, wanted } = src;
+	const at = src.turns.indexOf(subject);
+	const turns = (wanted.turns === "all" && at >= 0 ? src.turns.slice(0, at + 1) : [subject]).map((turn) => ({
+		...wanted.task ? { task: turn.prompt } : {},
+		...wanted.output ? { output: turn === subject ? src.output : turn.text } : {},
+		...wanted.transcript ? { transcript: transcriptForJudge(turn.transcript, wanted.transcript).join("\n") } : {}
+	}));
+	const contents = await filesWhole(wanted.contents, src);
+	const shown = judgeDiff(subject.diff, {
+		...Array.isArray(wanted.diff) ? { patterns: wanted.diff } : {},
+		...contents ? { inFull: new Set(Object.keys(contents)) } : {}
+	});
+	const checks = checksShown(wanted.checks, src.checks);
+	return {
+		draft: {
+			all: wanted.turns === "all",
+			turns,
+			...wanted.files ? { files: shown.files } : {},
+			...wanted.diff !== false ? { diff: shown.patch } : {},
+			...contents ? { contents } : {},
+			...checks ? { checks } : {}
+		},
+		dropped: wanted.diff !== false ? shown.dropped : []
+	};
+}
+/** Exact paths (`null` when missing) and every workspace file the patterns match, in path order. */
+async function filesWhole(entries, src) {
+	if (!entries.length) return void 0;
+	const exact = entries.filter(isExactPath).map(workspacePath);
+	const matches = globFilter(entries.map(workspacePath));
+	const matched = entries.some((e) => !isExactPath(e)) ? (await src.listFiles()).filter((path) => matches(path)) : [];
+	const paths = [.../* @__PURE__ */ new Set([...exact, ...matched])].sort();
+	const contents = {};
+	for (const path of paths) {
+		const text = await src.readFile(path);
+		contents[path] = text?.includes("\0") ? "[reeval: dropped binary file]" : text;
+	}
+	return contents;
+}
+function checksShown(which, ran) {
+	if (which === false) return void 0;
+	if (which !== true) {
+		const missing = which.find((command) => !ran.some((r) => r.command === command));
+		if (missing !== void 0) throw new Error(`c.judge() state.checks names a command this attempt hasn't run: ${JSON.stringify(missing)}`);
+	}
+	return ran.filter((r) => which === true || which.includes(r.command)).map(({ command, exitCode, stdout, stderr }) => ({
+		command,
+		exitCode,
+		stdout,
+		stderr
+	}));
+}
+/**
+* Grades a turn (the last by default) or a value against a question, under
+* the judge provider's concurrency lease. Every call's input is recorded, including calls that
+* fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
+* checks after it still run.
+*/
+function judgeCaller(a, readFile) {
+	const ask = async (question, { on, ...call } = {}) => {
+		if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
+		const problem = judgeProblem(call, { at: "call" });
+		if (problem) throw new Error(problem);
+		const badArts = badJudgeArtifacts(call.artifacts);
+		if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
+		const subject = on && "turn" in on ? a.turns[on.turn] : a.turns.at(-1);
+		if (!subject) throw new Error(on && "turn" in on ? `c.judge() on names turn ${on.turn}, but this attempt has ${a.turns.length}` : "c.judge() has no turn to grade: call c.send() first");
+		const options = mergeJudge(a.spec.judge, call);
+		const { model } = options;
+		if (model === void 0) {
+			a.judgeErrored = true;
+			throw new Error(noJudgeModel(getRunContext().config.file));
+		}
+		const wanted = resolveStateOptions(options.state);
+		if (on && "value" in on && !wanted.output) throw new Error("c.judge() on gives the output, but state.output is false");
+		const { state: _state, maxConcurrent, ...judgeOptions } = options;
+		const output = on && "value" in on ? on.value : subject.text;
+		const { draft, dropped } = await draftState({
+			turns: a.turns,
+			subject,
+			output,
+			wanted,
+			checks: a.record.checks,
+			readFile,
+			listFiles: async () => workspaceFiles(await sandboxFor(a, "c.judge()"), a.signal)
+		});
+		const name = judgeModelName(model);
+		const state = stateOf(draft);
+		const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
+		const refs = artifactRefsOf(resolved);
+		const decide = decideState(state, filePartsOf(resolved));
+		const input = {
+			question,
+			model: name,
+			stateOptions: wanted,
+			state,
+			...refs.length ? { artifacts: refs } : {},
+			...dropped.length ? { dropped } : {}
+		};
+		a.record.judgeInputs?.push(input);
+		const start = performance.now();
+		const ms = () => Math.round(performance.now() - start);
+		const atLeast = options.threshold ?? .8;
+		a.report({
+			kind: "judge-start",
+			question
+		});
+		debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
+		if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
+		debugTrace("judge", "prompt", input, "judge prompt");
+		const failed = (message, retries) => {
+			a.judgeErrored = true;
+			a.judgeErrors.push(`c.judge(${JSON.stringify(clip(question, 80))}) failed: ${message}`);
+			input.error = message;
+			if (retries?.length) input.retries = retries;
+			debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
+			debugTrace("judge", "error", {
+				error: message,
+				retries
+			}, "judge error");
+			a.report({
+				kind: "judge-end",
+				question,
+				verdict: "error",
+				score: null,
+				threshold: null,
+				ms: ms()
+			});
+			return {
+				verdict: "error",
+				score: null,
+				threshold: atLeast,
+				question,
+				model: name,
+				error: message,
+				...refs.length ? { artifacts: refs } : {}
+			};
+		};
+		const provider = name.split("/")[0];
+		try {
+			const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
+				...judgeOptions,
+				model,
+				threshold: atLeast
+			}, a.signal), a.signal));
+			const recorded = refs.length ? {
+				...verdict,
+				artifacts: refs
+			} : verdict;
+			a.record.judgements.push(retries.length ? {
+				...recorded,
+				retries
+			} : recorded);
+			const { score } = verdict;
+			debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
+			debugTrace("judge", "response", {
+				verdict,
+				retries
+			}, "judge response");
+			a.report({
+				kind: "judge-end",
+				question,
+				verdict: verdict.verdict,
+				score,
+				threshold: verdict.threshold,
+				ms: ms()
+			});
+			return recorded;
+		} catch (error) {
+			if (a.signal.aborted) {
+				a.judgeErrored = true;
+				throw error;
+			}
+			const { retries } = error;
+			const message = error instanceof Error ? error.message : String(error);
+			return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
+		}
+	};
+	return async (question, { on, ...options } = {}) => {
+		const call = on === void 0 ? options : {
+			...options,
+			on: judgeOn(a.turns, on)
+		};
+		return Object.defineProperty(await ask(question, call), SUBJECT, { value: question });
+	};
+}
+//#endregion
+//#region src/sandbox/index.ts
+async function createSandbox(descriptor, options) {
+	try {
+		switch (descriptor.kind) {
+			case "fake": return await createFakeSandbox(descriptor, options.env);
+			case "docker": {
+				const { createDockerSandbox } = await import("./docker-DNmMPOBb.js");
+				return await createDockerSandbox(descriptor, options);
+			}
+		}
+	} catch (error) {
+		if (error instanceof InfraError || options.signal.aborted) throw error;
+		const kind = descriptor.kind === "docker" ? descriptor.engine ?? "docker" : descriptor.kind;
+		throw new InfraError(`Failed to create ${kind} sandbox: ${error.message}`);
+	}
+}
+//#endregion
+//#region src/attempt/record.ts
+function classify(error, signal) {
+	const message = error instanceof Error ? error.message : String(error);
+	if (signal.aborted) return isTimeout(signal.reason) ? {
+		kind: "timeout",
+		message: "The eval timed out"
+	} : {
+		kind: "aborted",
+		message: "The run was cancelled"
+	};
+	if (error instanceof InfraError) return {
+		kind: "infra",
+		message
+	};
+	return {
+		kind: "assertion",
+		message
+	};
+}
+/**
+* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
+* assertion, which is the judge's when a judge call failed or gave a failing verdict.
+*/
+function recordFailure(a, error, timedOut) {
+	const { record } = a;
+	record.error = timedOut ? {
+		kind: "timeout",
+		message: "The eval timed out"
+	} : classify(error, a.signal);
+	record.error.message = capMiddle(record.error.message);
+	if (error instanceof PrepareError && !timedOut) {
+		const { class: kind, stage, ref, cause, log, setBy } = error.failure;
+		record.error.prepare = {
+			class: kind,
+			stage,
+			ref,
+			cause,
+			log,
+			setBy: setBy ?? []
+		};
+	}
+	const judgeFailed = a.judgeErrored || record.judgements.some((j) => j.verdict !== "pass");
+	if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
+	record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
+}
+/**
+* A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
+* whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
+*/
+async function recordOom(a, sandbox) {
+	const oom = await sandbox?.oomKills?.().catch(() => null);
+	const at = oom && `the ${oom.limit} limit${oom.swap ? ` (+${oom.swap} swap)` : ""}`;
+	if (oom) debugLog("teardown", `oom_kill ${oom.kills} at ${at}`);
+	if (!oom?.kills) return void 0;
+	const message = `out of memory: ${oom.kills === 1 ? "1 process" : `${oom.kills} processes`} killed at ${at}; raise ${sandboxProvider(a.spec.sandbox)}({ memory })`;
+	a.record.error = {
+		kind: "infra",
+		message,
+		resource: {
+			cause: "oom",
+			kills: oom.kills
+		}
+	};
+	a.record.outcome = "infra_error";
+	return new InfraError(message);
+}
+/**
+* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
+* Refused hosts the harness always tries are `expectedBlocked`, summarised in one `-v` line.
+*/
+function recordEgress(a, sandbox) {
+	const traffic = sandbox?.traffic?.();
+	const expected = expectedBlockedFor(a.spec.harness);
+	const refused = traffic?.blocked ?? [];
+	const isExpected = (h) => !!expected?.hosts.includes(h.host);
+	const expectedBlocked = refused.filter(isExpected);
+	a.record.egress = {
+		policy: a.spec.egress.policy,
+		allowed: allowedHosts(a.spec.egress),
+		recorded: !!traffic,
+		contacted: traffic?.contacted ?? [],
+		blocked: refused.filter((h) => !isExpected(h)),
+		expectedBlocked
+	};
+	if (expected && expectedBlocked.length) {
+		const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
+		debugLog("gateway", `expected blocks, not logged per request: ${hosts} · ${expected.why}`);
+	}
+	a.record.servedModels = sandbox?.servedModels?.() ?? null;
+}
+/** Writes the attempt's record, redacted, and returns what the runner reports. */
+async function writeRecord(a, stagingDir, id) {
+	const { record, spec } = a;
+	const recordPath = join(stagingDir, "evals", recordFileName(attemptKey(id, spec.cell.key, spec.n)));
+	await mkdir(join(stagingDir, "evals"), { recursive: true });
+	await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
+	const tmp = `${recordPath}.${process.pid}.tmp`;
+	await writeFile(tmp, redactSecrets(JSON.stringify(record, null, 2)));
+	await rename(tmp, recordPath);
+	return {
+		outcome: record.outcome,
+		errorKind: record.error?.kind ?? null,
+		judgements: record.judgements.map((j) => ({
+			verdict: j.verdict,
+			score: j.score,
+			threshold: j.threshold,
+			question: clip(j.question, 200)
+		})),
+		turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
+		...record.error?.prepare ? { prepare: {
+			ref: record.error.prepare.ref,
+			stage: record.error.prepare.stage
+		} } : {}
+	};
+}
+/** `turn N error: <message>` for each turn with an error event. */
+function turnErrors(transcripts) {
+	return transcripts.flatMap((events, i) => {
+		const found = events.find((e) => e.type === "error");
+		return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(clip(found.message.trim(), 500))}`] : [];
+	});
+}
+//#endregion
+//#region src/sandbox/phases.ts
+const strip = ({ phase: _, ...sub }) => sub;
+/**
+* Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
+* at least one check or setup step is running, so a request the agent makes between them gets
+* the placeholder.
+*/
+function brokerPhases(box, subs) {
+	const agent = subs.filter((s) => s.phase === "agent").map(strip);
+	const all = subs.map(strip);
+	const perPhase = box.setSubstitutions && all.length > agent.length;
+	let running = 0;
+	return {
+		async start() {
+			if (all.length > 0) await box.setSubstitutions?.(agent);
+		},
+		async checks(fn) {
+			if (!perPhase) return fn();
+			if (running === 0) await box.setSubstitutions?.(all);
+			running++;
+			try {
+				return await fn();
+			} finally {
+				if (--running === 0) await box.setSubstitutions?.(agent);
+			}
+		}
+	};
+}
+//#endregion
+//#region src/attempt/state.ts
+/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
+const SETUP_BUDGET_MS = 12e5;
+function createAttempt(spec, signal, progress, start) {
+	const record = {
+		startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+		outcome: "pass",
+		error: null,
+		turns: [],
+		diff: null,
+		checks: [],
+		judgements: [],
+		timings: {
+			setupMs: 0,
+			agentMs: 0,
+			diffMs: 0,
+			checksMs: 0,
+			judgeMs: 0
+		},
+		judgeInputs: []
+	};
+	let starting;
+	let ended = false;
+	const a = {
+		spec,
+		record,
+		turns: [],
+		start(who) {
+			if (!starting && ended) return Promise.reject(/* @__PURE__ */ new Error(`${who} came after the eval's body returned`));
+			starting ??= start(a, who).catch((error) => {
+				a.startFailure = { error };
+				throw error;
+			});
+			return starting;
+		},
+		async endStarts() {
+			ended = true;
+			await starting?.catch(() => {});
+		},
+		report: (update) => progress({
+			at: Date.now(),
+			...update
+		}),
+		async timed(phase, fn) {
+			const start = performance.now();
+			try {
+				return await fn();
+			} finally {
+				record.timings[phase] += Math.round(performance.now() - start);
+			}
+		},
+		signal,
+		judgeErrored: false,
+		judgeErrors: [],
+		captureNames: /* @__PURE__ */ new Set(),
+		backgrounds: [],
+		artifactBlobs: /* @__PURE__ */ new Map()
+	};
+	return a;
+}
+const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s");
+/**
+* Setup steps and the harness's start (once retried) run outside the eval timeout, so the
+* attempt's own limit makes room for them.
+*/
+function setupAllowance(planned) {
+	const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
+	const { sandbox } = planned;
+	return setupMs + (sandbox.kind === "docker" && !!(sandbox.from || sandbox.dockerfile || sandbox.install?.length) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
+}
+/** The eval's timer, paused while the sandbox and its setup steps start. */
+function evalTimer(ms) {
+	const controller = new AbortController();
+	let left = ms;
+	let since = Date.now();
+	let timer;
+	const fire = () => controller.abort(new DOMException(`Eval timed out after ${ms / 1e3}s`, "TimeoutError"));
+	const start = () => {
+		since = Date.now();
+		timer = setTimeout(fire, Math.max(0, left));
+	};
+	start();
+	return {
+		signal: controller.signal,
+		pause() {
+			clearTimeout(timer);
+			left -= Date.now() - since;
+		},
+		resume: start,
+		clear: () => clearTimeout(timer)
+	};
+}
+//#endregion
+//#region src/attempt/session.ts
+const tailOf = (text) => text.trim().split("\n").slice(-10).join("\n");
+/** Setup steps in order, each with its own timeout; a failure is an infra error. */
+async function runSetup(a, box, env) {
+	const { record, spec } = a;
+	const steps = env.setup;
+	record.setup = [];
+	for (const [i, step] of steps.entries()) {
+		const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
+		const start = performance.now();
+		const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
+		const r = await asCheck(a, () => box.exec(step.run, {
+			signal: stepSignal,
+			env: env.checkVars,
+			...step.cwd ? { cwd: step.cwd } : {}
+		})).catch((error) => {
+			if (a.signal.aborted) throw error;
+			if (!stepSignal.aborted) throw new InfraError(`${what} couldn't run: ${error instanceof Error ? error.message : error}`);
+			return null;
+		});
+		const durationMs = Math.round(performance.now() - start);
+		const output = r ? [r.stdout, r.stderr].filter(Boolean).join("\n") : "";
+		record.setup.push({
+			run: step.run,
+			exitCode: r?.exitCode ?? null,
+			durationMs,
+			output
+		});
+		debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
+		debugTrace("env", "setup", {
+			run: step.run,
+			cwd: step.cwd,
+			exitCode: r?.exitCode ?? null,
+			durationMs,
+			stdout: r?.stdout,
+			stderr: r?.stderr
+		}, `setup ${i + 1} output`);
+		if (r?.exitCode === 0) continue;
+		throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
+	}
+}
+const WORKSPACE_TAR = ".reeval-workspace.tar";
+/** Unpacks the git workspace's tarball into the (empty) workspace. */
+async function copyWorkspace(box, workspace, signal) {
+	const start = performance.now();
+	const tar = await readFile(workspace.tar);
+	await box.writeFile(WORKSPACE_TAR, tar);
+	const r = await box.exec(`tar -xf ${WORKSPACE_TAR} && rm -f ${WORKSPACE_TAR}`, { signal });
+	if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
+	if (workspace.init) {
+		const at = "2000-01-01T00:00:00Z";
+		const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
+			signal,
+			env: {
+				GIT_AUTHOR_NAME: "reeval",
+				GIT_AUTHOR_EMAIL: "reeval@localhost",
+				GIT_AUTHOR_DATE: at,
+				GIT_COMMITTER_NAME: "reeval",
+				GIT_COMMITTER_EMAIL: "reeval@localhost",
+				GIT_COMMITTER_DATE: at
+			}
+		});
+		if (init.exitCode !== 0) throw new InfraError(`Couldn't give the workspace ${workspace.label} a .git (init: true needs git in the image): ${tailOf(init.stderr || init.stdout)}`);
+	}
+	debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
+}
+/**
+* Starts the harness session within its `startTimeout`. A start that doesn't become ready in time
+* is aborted (and closed, should it finish later) and started once more: a stuck bootstrap is
+* usually transient, and nothing has run yet, so the retry can't change what the attempt
+* measures. A second timeout is an infra error; a start that fails outright isn't retried.
+*/
+async function startHarness(a, harness, box, options) {
+	const { harness: descriptor } = a.spec;
+	const ms = startTimeoutMs(descriptor);
+	const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
+	for (let tries = 1;; tries++) {
+		const stop = new AbortController();
+		let timer;
+		const timedOut = new Promise((resolve) => {
+			timer = setTimeout(() => resolve(null), ms);
+		});
+		const starting = harness.start(box, AbortSignal.any([a.signal, stop.signal]), options);
+		const session = await Promise.race([starting, timedOut]).finally(() => clearTimeout(timer));
+		if (session) return session;
+		stop.abort(new InfraError(late));
+		starting.then((s) => s.close().catch(() => {}), () => {});
+		a.signal.throwIfAborted();
+		if (tries === 2) throw new InfraError(`${late} (tried twice)`);
+		a.record.startRetries = [{
+			error: late,
+			waitedMs: ms
+		}];
+		debugLog("harness", `${late}; starting it once more`);
+	}
+}
+/**
+* Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
+* environment's files and workspace skills, runs its setup steps, snapshots the workspace for
+* the diff, and starts the harness session with the harness-placed skills and MCP servers.
+*/
+async function startSession(a, runId) {
+	const startedAt = performance.now();
+	const { spec } = a;
+	const env = a.env;
+	const box = await createSandbox(spec.sandbox, {
+		runId,
+		timeoutMs: spec.timeoutMs + setupAllowance(spec) + 6e4,
+		signal: a.signal,
+		env: env.vars,
+		image: a.image,
+		brokering: brokeringFor(spec.harness),
+		expectedBlocked: expectedBlockedFor(spec.harness)?.hosts,
+		substitutions: env.substitutions,
+		egress: spec.egress
+	});
+	a.sandbox = box;
+	if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
+	a.phases = brokerPhases(box, env.substitutions);
+	if (a.signal.aborted) box.destroy();
+	a.signal.throwIfAborted();
+	await a.phases.start();
+	if (env.substitutions.length) {
+		const agent = env.substitutions.filter((s) => s.phase === "agent").map((s) => s.name);
+		debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
+	}
+	if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
+	for (const f of env.files) {
+		await box.writeFile(f.path, f.content);
+		const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
+		debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
+	}
+	const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
+	if (executable.length) {
+		const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
+		const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
+		if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
+	}
+	const skillDir = workspaceSkillDir(spec.harness);
+	for (const skill of env.workspaceSkills) {
+		const files = skillFiles(skill);
+		for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
+		debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
+	}
+	const layout = layoutOf(spec.harness);
+	for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
+	for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
+	await runSetup(a, box, env);
+	await snapshot(box, a.signal);
+	debugLog("env", "workspace snapshot taken: the diff starts here");
+	const harness = await createHarness(spec.harness);
+	const harnessStart = performance.now();
+	debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
+	const session = await startHarness(a, harness, box, {
+		skills: env.skills,
+		workspaceSkills: env.workspaceSkills.map((s) => s.name),
+		mcp: env.mcp
+	});
+	debugLog("harness", `${harnessId(spec.harness)} session ready in ${formatDuration(performance.now() - harnessStart)}`);
+	a.session = session;
+	const startMs = performance.now() - startedAt;
+	debugLog("container", `sandbox and session started in ${formatDuration(startMs)}`);
+	a.report({
+		kind: "sandbox-ready",
+		ms: Math.round(a.record.timings.setupMs + startMs)
+	});
+	return [box, session];
+}
+//#endregion
+//#region src/attempt/turns.ts
+/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
+const TURN_STOP_GRACE_MS = 3e3;
+/**
+* Resolves `grace` ms after `signal` aborts, with the turn as cut off; `cancel` drops the timer
+* once the turn settles on its own.
+*/
+function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
+	let timer;
+	let onAbort = () => {};
+	const cutoff = new Promise((resolve) => {
+		onAbort = () => {
+			timer = setTimeout(() => {
+				const message = `The agent didn't stop within ${grace / 1e3}s of being cancelled, so reeval stopped waiting for its turn`;
+				const at = Date.now() - Date.parse(startedAt);
+				resolve({
+					text: "",
+					status: "aborted",
+					transcript: [{
+						type: "error",
+						message,
+						ts: (/* @__PURE__ */ new Date()).toISOString(),
+						atMs: at
+					}],
+					usage: null,
+					model: null,
+					unreported: [],
+					startedAt,
+					durationMs: at
+				});
+			}, grace);
+		};
+		if (signal.aborted) onAbort();
+		else signal.addEventListener("abort", onAbort, { once: true });
+	});
+	const cancel = () => {
+		clearTimeout(timer);
+		signal.removeEventListener("abort", onAbort);
+	};
+	return {
+		cutoff,
+		cancel
+	};
+}
+/**
+* One turn. Sends the prompt to the started session, records the turn as it goes (an
+* errored entry first, so a turn that throws is still in the record), takes the agent's context
+* after the first, and diffs the workspace since the snapshot.
+*/
+function turnSender(a, takeContext, checks) {
+	return async (prompt) => {
+		const [box, agent] = await a.start("c.send()");
+		const entry = {
+			prompt,
+			text: "",
+			status: "error",
+			transcript: [],
+			usage: null,
+			model: null,
+			unreported: [],
+			startedAt: (/* @__PURE__ */ new Date()).toISOString(),
+			durationMs: 0
+		};
+		a.record.turns.push(entry);
+		const turnNumber = a.record.turns.length;
+		a.report({
+			kind: "turn-start",
+			turn: turnNumber
+		});
+		debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
+		debugTrace("harness", "prompt", {
+			turn: turnNumber,
+			prompt
+		}, `turn ${turnNumber} prompt`);
+		const { cutoff, cancel } = cutOff(a.signal, entry.startedAt);
+		const sendStarted = Date.now();
+		const { raw, ...result } = await a.timed("agentMs", () => Promise.race([agent.send(prompt, a.signal), cutoff])).finally(cancel);
+		const sendWallMs = Date.now() - sendStarted;
+		Object.assign(entry, result, raw ? { raw } : {}, { durationMs: Math.max(result.durationMs, sendWallMs) });
+		for (const event of result.transcript) debugTrace("harness", "transcript", {
+			turn: turnNumber,
+			...event
+		}, `transcript ${event.type}`);
+		await takeContext(box, a.signal);
+		const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
+		const failure = result.transcript.find((e) => e.type === "error");
+		a.report({
+			kind: "turn-end",
+			turn: turnNumber,
+			status: result.status,
+			ms: result.durationMs,
+			tools: count("tool", "web-search", "web-fetch"),
+			fileEdits: count("file-edit"),
+			...failure?.type === "error" ? { error: redactSecrets(clip(failure.message.trim(), 300)) } : {}
+		});
+		const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
+		debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
+		a.signal.throwIfAborted();
+		const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
+		a.record.diff = diff;
+		const data = {
+			prompt,
+			...result,
+			diff
+		};
+		const turn = Object.freeze({
+			...data,
+			...checks
+		});
+		a.turns.push(turn);
+		return turn;
+	};
+}
+//#endregion
+//#region src/attempt/index.ts
+const TRACE_BUFFER_DEFAULT = 8 * 2 ** 20;
+async function runAttempt(spec, fn, handle) {
+	const ctx = getRunContext();
+	const { config, runId } = ctx;
+	const verbosity = ctx.verbosity ?? 0;
+	const id = evalId(relativeFile(config.root, handle.file), [...spec.groups, spec.name]);
+	const debug = new AttemptDebug({
+		dir: attemptDebugDir(config.root, runId, id, spec.cell.key, spec.n),
+		verbosity,
+		traceLimit: config.debug?.traceBufferBytes ?? TRACE_BUFFER_DEFAULT,
+		...verbosity > 0 ? { emit: handle.debug } : {}
+	});
+	return withDebug(debug, () => attemptBody(spec, fn, handle, id, debug));
+}
+async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited, expectError, assertions }, id, debug) {
+	const ctx = getRunContext();
+	const { runId, stagingDir, config } = ctx;
+	tracePlan(debug, spec, ctx);
+	let timer;
+	let clock;
+	const a = createAttempt(spec, testSignal, progress, async (a, who) => {
+		debug.log("container", `starting the sandbox and session for ${who}`);
+		clock?.pause();
+		try {
+			return await a.timed("setupMs", () => startSession(a, runId));
+		} finally {
+			clock?.resume();
+		}
+	});
+	const destroy = () => void a.sandbox?.destroy();
+	testSignal.addEventListener("abort", destroy, { once: true });
+	const takeContext = contextTaker(a);
+	const { exec, readFile, writeFile } = createChecks(a);
+	const c = {
+		get signal() {
+			return a.signal;
+		},
+		turns: a.turns,
+		exec,
+		readFile,
+		writeFile,
+		serve: (command, port) => serveCommand(a, command, port),
+		capture: (target, options) => capturePage(a, target, options),
+		attempt: {
+			id: attemptId(spec.cell.key, spec.n),
+			cell: spec.cell,
+			n: spec.n
+		},
+		send: turnSender(a, takeContext, {
+			exec,
+			readFile
+		}),
+		judge: judgeCaller(a, readFile)
+	};
+	const timedOut = () => !!timer?.aborted && !testSignal.aborted;
+	let failure;
+	let oom;
+	try {
+		a.report({ kind: "sandbox" });
+		const where = `${id} [${attemptId(spec.cell.key, spec.n)}]`;
+		await a.timed("setupMs", () => prepareEnvironment(a, config.root, runId, where, testSignal));
+		clock = evalTimer(spec.timeoutMs);
+		timer = clock.signal;
+		a.signal = AbortSignal.any([testSignal, timer]);
+		timer.addEventListener("abort", destroy, { once: true });
+		await fn(c).catch((error) => {
+			throw a.startFailure ? a.startFailure.error : error;
+		});
+		a.signal.throwIfAborted();
+		const missed = await unawaited() ?? softFailure(assertions()) ?? expectError();
+		if (a.startFailure) throw a.startFailure.error;
+		if (missed) throw missed;
+		const [only, ...more] = a.judgeErrors;
+		if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
+	} catch (error) {
+		recordFailure(a, error, timedOut());
+		failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
+	} finally {
+		if (!a.signal.aborted) await unawaited().catch(() => {});
+		clock?.clear();
+		await a.endStarts();
+		for (const bg of a.backgrounds) await bg.kill().catch(() => {});
+		const { sandbox } = a;
+		if (sandbox && !a.signal.aborted) await takeContext(sandbox, AbortSignal.timeout(3e4));
+		if (sandbox) a.report({ kind: "cleanup" });
+		testSignal.removeEventListener("abort", destroy);
+		const teardown = performance.now();
+		await a.session?.close().catch(() => {});
+		if (a.session) debug.log("teardown", "harness session closed");
+		recordEgress(a, sandbox);
+		oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
+		await sandbox?.destroy();
+		if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
+		const { record } = a;
+		const checked = assertionRecords(assertions());
+		if (checked.length) record.assertions = checked;
+		for (const x of checked) {
+			const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
+			debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
+		}
+		const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
+		debug.log("teardown", `outcome ${record.outcome}${error}`);
+		const rel = (path) => relativeFile(config.root, path) || ".";
+		const scratch = `.reeval/runs/${runId}`;
+		debug.log("teardown", `record ${scratch}.json · artifacts ${scratch}/${evalSlug(id)}/${evalSlug(spec.cell.key)}/${spec.n} · debug ${rel(debug.dir)}`);
+		setMeta(await writeRecord(a, stagingDir, id));
+		await debug.finish(debug.verbosity === 2 || record.outcome !== "pass");
+	}
+	if (oom) throw oom;
+	if (failure) throw failure.error;
+}
+//#endregion
+//#region src/contract/declaration.ts
+/** A node's options: any object, until the cascade checks it. Its JSON Schema is the type's. */
+const evalOptionsDataSchema = z.record(z.string(), z.unknown());
+const declaredModeSchema = z.enum([
+	"run",
+	"skip",
+	"only"
+]).describe("How a group or eval was declared: `group`/`evaluate`, `.skip` or `.only`.");
+const declaredFamilySchema = z.strictObject({
+	name: z.string().describe("The name template passed to `evaluate.each`."),
+	case: z.number().int().min(0).describe("The case's index, from 0."),
+	cases: z.number().int().min(2).describe("How many cases the list has.")
+}).describe("Where a case of a pooled `evaluate.each` sits in its list.");
+const declaredTodoSchema = z.strictObject({
+	type: z.literal("todo"),
+	name: z.string().describe("The name passed to `.todo`.")
+}).describe("`evaluate.todo` or `group.todo`.");
+const declaredEvalSchema = z.strictObject({
+	type: z.literal("eval"),
+	name: z.string().describe("The eval's name, as passed to `evaluate` (a case's, for `evaluate.each`)."),
+	mode: declaredModeSchema,
+	options: evalOptionsDataSchema.describe("The eval's own options, before the cascade."),
+	body: z.string().describe("Names the eval's body to whoever runs it."),
+	family: declaredFamilySchema.optional().describe("Set on each case of a pooled `evaluate.each`: its template, index and the list's length.")
+}).describe("An `evaluate`: one eval, planned into a test per attempt.");
+const declaredGroupSchema = z.strictObject({
+	type: z.literal("group"),
+	name: z.string().describe("The group's name, as passed to `group`."),
+	mode: declaredModeSchema,
+	options: evalOptionsDataSchema.describe("The group's own options, before the cascade."),
+	get children() {
+		return z.array(declaredNodeSchema);
+	}
+}).describe("A `group`: options its evals and groups inherit, and its children in declaration order.");
+const declaredNodeSchema = z.discriminatedUnion("type", [
+	declaredEvalSchema,
+	declaredGroupSchema,
+	declaredTodoSchema
+]).describe("One node of the tree: a group, an eval or a todo.");
+const declarationTreeSchema = z.strictObject({ children: z.array(declaredNodeSchema) }).describe("An eval file's top-level groups, evals and todos, in declaration order.");
+const NODE_SCHEMAS = {
+	eval: declaredEvalSchema,
+	group: declaredGroupSchema,
+	todo: declaredTodoSchema
+};
+/**
+* What's wrong with a declaration tree's shape, or undefined: the field, by its path from the
+* tree's `children`. Options are left to the cascade.
+*/
+const declarationProblem = (children) => dataProblem(declarationTreeSchema, { children }, "The declaration tree", (node) => Object.keys(NODE_SCHEMAS[String(node.type)]?.shape ?? {}));
+//#endregion
+//#region src/plan-eval.ts
+/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
+const GRACE_MS = 5e3;
+function configLevel(config) {
+	return {
+		harness: config.harness,
+		sandbox: config.sandbox,
+		environment: config.environment,
+		compare: void 0,
+		runs: config.runs,
+		gate: config.gate,
+		env: config.env,
+		egress: config.egress !== void 0 ? { inherited: config.egress } : {},
+		judge: config.judge,
+		capture: config.capture,
+		question: config.question,
+		timeout: void 0,
+		tags: [],
+		pin: [],
+		sources: {}
+	};
+}
+/**
+* The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
+* `order`). Throws a UsageError for options the plan refuses.
+*/
+function planEval(declaration, ctx) {
+	const { name, mode, groups: enclosing, options: own } = declaration;
+	const groups = enclosing.map((g) => g.name);
+	const where = `evaluate("${[...groups, name].join(" › ")}")`;
+	checkOptions(own, where);
+	const levels = [...enclosing.map((g) => ({
+		options: g.options,
+		where: `group("${g.name}")`
+	})), {
+		options: own,
+		where
+	}];
+	const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
+		profile: ctx.profile,
+		runs: ctx.runs
+	});
+	const { resolved } = resolvedEval;
+	if (!matchesTags(resolved.tags, ctx.tags)) return null;
+	const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
+	const plannedAttempts = resolvedEval.skip ? [] : expand(resolved);
+	if (ctx.config.secrets.unbrokered === "error") {
+		const [first] = unbrokered(resolved.env, plannedAttempts);
+		if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
+	}
+	for (const planned of plannedAttempts) {
+		const why = unenforceable(planned.egress, planned.sandbox, where);
+		if (why) throw new UsageError(why);
+		const mcp = refusedMcp(planned.harness, planned.environment.layers);
+		if (mcp) throw new UsageError(`${where} on ${harnessId(planned.harness)}: ${mcp}`);
+	}
+	const path = ctx.file ? relativeFile(ctx.config.root, ctx.file) : null;
+	const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
+	const family = declaration.family && {
+		id: evalId(rankFile, [...groups, declaration.family.name]),
+		...declaration.family
+	};
+	const harnessBy = resolved.sources?.harness ?? ctx.config.file;
+	const plan = evalPlan(resolvedEval, plannedAttempts, {
+		configFile: ctx.config.file,
+		file: path
+	});
+	const setByOf = (planned) => ({
+		sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
+		harness: harnessBy
+	});
+	if (resolvedEval.skip) {
+		const [first] = expand(resolved);
+		if (!first) return [];
+		const attempt = {
+			name,
+			groups,
+			tags: resolved.tags,
+			cell: first.cell,
+			n: 1,
+			label: "",
+			harness: first.harness,
+			sandbox: first.sandbox,
+			setBy: setByOf(first),
+			gate: "none",
+			brokered: [],
+			secretVars: [],
+			egress: first.egress,
+			plan,
+			...family ? { family } : {}
+		};
+		return [{
+			name: `${name} [${attemptId(first.cell.key, 1)}]`,
+			mode: "skip",
+			attempt
+		}];
+	}
+	const cells = [];
+	return plannedAttempts.map((planned) => {
+		if (!cells.includes(planned.cell.key)) cells.push(planned.cell.key);
+		const workspace = workspaceOf(planned.environment.layers);
+		const files = filesOf(planned.environment.layers);
+		const setBy = setByOf(planned);
+		const id = evalId(rankFile, [...groups, name]);
+		const key = attemptKey(id, planned.cell.key, planned.n);
+		const skip = ctx.completed?.includes(key) ? "already completed in this run" : void 0;
+		const attempt = {
+			name,
+			groups,
+			tags: resolved.tags,
+			cell: planned.cell,
+			n: planned.n,
+			label: planned.label,
+			harness: planned.harness,
+			sandbox: planned.sandbox,
+			setBy,
+			gate: planned.gate,
+			...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
+			brokered: brokeredIn(resolved.env, planned),
+			secretVars: secretVarsIn(resolved.env, planned),
+			egress: planned.egress,
+			...workspace ? { workspace } : {},
+			...files.length ? { files } : {},
+			plan: skip ? {
+				...plan,
+				skip
+			} : plan,
+			...skip ? { skip } : {},
+			...family ? { family } : {}
+		};
+		return {
+			name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
+			mode: skip ? "skip" : mode,
+			attempt,
+			timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
+			gate: skip ? void 0 : {
+				limits: attemptLimits(planned, ctx.concurrency),
+				rank: {
+					eval: id,
+					file: rankFile,
+					order: ctx.order,
+					n: planned.n,
+					cell: cells.indexOf(planned.cell.key)
+				}
+			},
+			spec: skip ? void 0 : {
+				...planned,
+				name,
+				groups,
+				timeoutMs,
+				env: resolved.env,
+				judge: resolved.judge,
+				capture: resolved.capture,
+				sources: {
+					...resolved.sources,
+					sandbox: setBy.sandbox
+				},
+				notes: resolvedEval.notes
+			}
+		};
+	});
+}
+/** The run-wide cap, plus the sandbox provider's and harness's own caps when they set one. */
+function attemptLimits(planned, concurrency) {
+	const limits = { attempts: concurrency };
+	const { sandbox, harness } = planned;
+	if (sandbox.maxConcurrent) limits[`sandbox:${sandboxProvider(sandbox)}`] = sandbox.maxConcurrent;
+	if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
+	return limits;
+}
+//#endregion
+//#region src/evaluate.ts
+let groupStack = [];
+/** Each eval file's declarations as `evaluate` and `group` were called, and each eval's body by id. */
+const trees = /* @__PURE__ */ new Map();
+const bodies = /* @__PURE__ */ new Map();
+/** Options as the tree holds them; from TypeScript they may still hold function forms. */
+const data = (options) => options;
+function declare(node) {
+	const parent = groupStack.at(-1)?.node;
+	if (parent) return void parent.children.push(node);
+	const file = collectingFile() ?? "";
+	const tree = trees.get(file) ?? { children: [] };
+	trees.set(file, tree);
+	tree.children.push(node);
+}
+const missingIn = (where) => (field, full) => {
+	throw new UsageError(`${field} in ${where}: ${full} doesn't exist. Paths are relative to the file that declares it.`);
+};
+/**
+* The options with a `sandbox` written as data given absolute host paths against `base`, the
+* eval file, as `docker()` and `fakeSandbox()` resolve theirs against the file that calls them.
+*/
+function withPaths(options, where, base = callerDir() ?? process.cwd()) {
+	if (typeof options !== "object" || options === null || !("sandbox" in options)) return options;
+	return {
+		...options,
+		sandbox: sandboxPaths(options.sandbox, base, "sandbox", missingIn(where))
+	};
+}
+/** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
+function splitArgs(what, name, a, b) {
+	if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environment: …) or in the config's env.`);
+	if (typeof a === "function") return [b ?? {}, a];
+	if (typeof b === "function") return [a ?? {}, b];
+	throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
+}
+/** `evaluate.only` and `group.only` are refused in CI, where they'd silently skip the rest. */
+function checkOnly(mode, what) {
+	if (mode !== "only" || !inCI()) return;
+	const error = /* @__PURE__ */ new Error(`${what}.only isn't allowed in CI`);
+	error.name = "OnlyInCiError";
+	throw error;
+}
+const defineGroup = (mode) => (name, a, b) => {
+	checkOnly(mode, "group");
+	const [written, fn] = splitArgs("group", name, a, b);
+	const options = withPaths(written, `group("${name}")`);
+	checkOptions(options, `group("${name}")`);
+	const node = {
+		type: "group",
+		name,
+		mode,
+		options: data(options),
+		children: []
+	};
+	declare(node);
+	const stack = [...groupStack, {
+		name,
+		options,
+		node
+	}];
+	registerGroup(name, mode, () => {
+		const outer = groupStack;
+		groupStack = stack;
+		try {
+			fn();
+		} finally {
+			groupStack = outer;
+		}
+	});
+};
+const group = Object.assign(defineGroup("run"), {
+	skip: defineGroup("skip"),
+	only: defineGroup("only"),
+	todo: (name) => registerTodo(name)
+});
+/**
+* A placeholder, `evaluate.todo` or `group.todo`: one eval entry that never runs, counted as
+* skipped and shown as todo. The enclosing groups' tags filter it as they do an eval.
+*/
+function registerTodo(name) {
+	declare({
+		type: "todo",
+		name
+	});
+	const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
+	if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
+}
+/** The evals declared so far, by file and name path, so a second one with the same id is refused. */
+const declared = /* @__PURE__ */ new Set();
+function checkUnique(where, groups, name, hint) {
+	const key = `${collectingFile() ?? ""}\0${[...groups, name].join("\0")}`;
+	if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
+	declared.add(key);
+}
+function register(mode, name, own, fn, hint = "", family) {
+	const groups = groupStack.map((g) => g.name);
+	checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
+	const body = `b${bodies.size + 1}`;
+	bodies.set(body, fn);
+	declare({
+		type: "eval",
+		name,
+		mode,
+		options: data(own),
+		body,
+		...family ? { family } : {}
+	});
+	const tests = planEval({
+		name,
+		mode,
+		groups: groupStack.map(({ name, options }) => ({
+			name,
+			options
+		})),
+		options: own,
+		...family ? { family } : {}
+	}, {
+		...getRunContext(),
+		file: collectingFile(),
+		order: registered
+	});
+	if (!tests) return;
+	registered++;
+	for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
+}
+/** Evals registered so far in this worker, for the scheduler's tie-break. */
+let registered = 0;
+const defineEval = (mode) => (name, a, b) => {
+	checkOnly(mode, "evaluate");
+	const [own, fn] = splitArgs("evaluate", name, a, b);
+	register(mode, name, withPaths(own, `evaluate("${name}")`), fn);
+};
+/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
+function formatName(template, item, index) {
+	return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => isObjectLike(item) && key in item ? String(item[key]) : whole);
+}
+function checkEachOptions(options) {
+	if (options === void 0) return {};
+	if (isObjectLike(options) && Object.keys(options).every((k) => k === "pool")) {
+		const { pool } = options;
+		if (pool === void 0) return {};
+		if (typeof pool === "boolean") return { pool };
+	}
+	throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
+}
+/**
+* One eval per case, pooled into one suite:
+* `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
+*/
+const evalEach = (cases, options) => (name, a, b) => {
+	const { pool = true } = checkEachOptions(options);
+	if (!Array.isArray(cases)) throw new UsageError(`evaluate.each takes an array of cases (got ${typeof cases}), e.g. evaluate.each([{ n: 1 }, { n: 2 }])("case $n", async (c, item) => { … }).`);
+	const [written, fn] = splitArgs("evaluate.each", name, a, b);
+	const own = withPaths(written, `evaluate.each("${name}")`);
+	const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
+	const pooled = pool && cases.length >= 2;
+	cases.forEach((item, i) => {
+		const family = pooled ? {
+			name,
+			case: i,
+			cases: cases.length
+		} : void 0;
+		register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
+	});
+};
+const evaluate = Object.assign(defineEval("run"), {
+	skip: defineEval("skip"),
+	only: defineEval("only"),
+	todo: registerTodo,
+	each: evalEach
+});
+/**
+* Declares a tree's groups and evals as the calls that made it would, in the file being
+* collected; `bodyFor` gives each eval's body by its id. A tree from another language is
+* checked against the contract first. Its relative host paths resolve against the directory of
+* the file being collected, as a TypeScript eval file's do against its own.
+*/
+function declareTree(nodes, bodyFor) {
+	const problem = declarationProblem(nodes);
+	if (problem) throw new UsageError(problem);
+	const file = collectingFile();
+	declareNodes(nodes, bodyFor, file ? dirname(file) : process.cwd());
+}
+function declareNodes(nodes, bodyFor, base) {
+	for (const node of nodes) if (node.type === "todo") registerTodo(node.name);
+	else if (node.type === "group") {
+		const children = node.children;
+		const where = `group("${node.name}")`;
+		defineGroup(node.mode)(node.name, optionsFrom(node.options, where, base), () => declareNodes(children, bodyFor, base));
+	} else if (node.family) {
+		const options = optionsFrom(node.options, `evaluate("${node.name}")`, base);
+		checkOnly(node.mode, "evaluate");
+		register(node.mode, node.name, options, bodyFor(node.body), "", node.family);
+	} else defineEval(node.mode)(node.name, optionsFrom(node.options, `evaluate("${node.name}")`, base), bodyFor(node.body));
+}
+/**
+* Options from their data forms, with host paths absolute against `base`: environments become
+* `Environment`s again.
+*/
+function optionsFrom(written, where, base) {
+	refuseIllegalCompare(written, where);
+	const { environment, compare, ...rest } = withPaths(written, where, base);
+	const missing = missingIn(where);
+	const asEnv = (env, field) => environmentFrom(environmentPaths(env, base, field, missing));
+	const own = environment !== void 0 ? { environment: asEnv(environment, "environment") } : {};
+	if (compare === void 0) return {
+		...rest,
+		...own
+	};
+	const variants = compare.variants.map((v, i) => asEnv(v, `compare.variants[${i}]`));
+	if (compare.baseline !== void 0) return {
+		...rest,
+		compare: {
+			...compare,
+			baseline: asEnv(compare.baseline, "compare.baseline"),
+			variants
+		}
+	};
+	return {
+		...rest,
+		...own,
+		compare: {
+			...compare,
+			variants
+		}
+	};
+}
+//#endregion
+export { assertionRecords as i, evaluate as n, group as r, declareTree as t };
+
+//# sourceMappingURL=evaluate-DKIl54Hb.js.map