reeval 0.9.0 → 0.10.0

package/dist/index.js

@@ -1,2139 +1,7 @@
-import { i as HarnessCapabilityError, r as withLease } from "./client-Bbo9SPlK.js";
-import { n as UsageError, r as isTimeout, t as TimeoutError } from "./errors-yDAEwmZo.js";
-import { t as parseDuration } from "./duration-JsJrIWXq.js";
-import { t as inCI } from "./env-BQqj93gu.js";
-import { o as redactSecrets, s as registerSecret } from "./redact-DJ9JCShu.js";
-import { C as stepCommand, E as workspaceOf, T as workspaceLabel, _ as isSecret, b as parseGitRepo, c as defineEnvironment, d as filesOf, h as isHostFile, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES, v as isSecretTemplate, y as mcpHosts } from "./environment-1jE-HhO-.js";
-import { t as InfraError } from "./types-C5vI3g1C.js";
-import { _ as layoutOf, a as unenforceable, h as expectedBlockedFor, n as allowedHosts, o as createFakeSandbox, p as brokeringFor, r as egressLabel, t as DEFAULT_EGRESS_ALLOW, v as refusedMcp } from "./egress-BhzNh7Yi.js";
-import { c as relativeFile, i as evalSlug, l as sha256, n as attemptKey, r as evalId, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
-import { c as judgeModelName, d as isApiError, i as badJudgeList, u as runJudge } from "./judge-CeYPN8P_.js";
-import { S as placeholder } from "./src-8rRXy4u9.js";
-import { n as shellQuote, t as run } from "./process-CSVO_jsc.js";
-import { a as evalPlan, d as sandboxId, f as sandboxSetBy, h as unbrokeredError, i as checkOptions, l as matchesTags, m as unbrokered, n as cascade, o as expand, p as secretVarsIn, s as harnessId, t as brokeredIn } from "./options-CKqyPGxX.js";
-import { n as badCharacterAt, s as defineConfig } from "./config-CGvBeewH.js";
-import { i as registerTest, n as getRunContext, o as SUBJECT, r as registerGroup, t as collectingFile, u as expect } from "./runtime-DdsfnRz_.js";
-import { a as withDebug, i as debugTrace, n as attemptDebug, r as debugLog, t as AttemptDebug } from "./attempt-CGPV3BZY.js";
-import { t as attemptDebugDir } from "./files-Ca0d6Uq3.js";
-import { n as clip, r as formatDuration, t as capMiddle } from "./text-gEcKjAyp.js";
-import { a as asCheck, c as snapshot, i as writeCaptureSidecars, n as serveCommand, o as createChecks, s as diffSinceSnapshot, t as capturePage } from "./capture-Bp74Z6Fr.js";
-import { t as contextTaker } from "./context-DbSMEVKs.js";
-import { t as PrepareError } from "./provider-CmAYTQQs.js";
-import { i as preparedWorkspace } from "./workspace-C-TrNIUc.js";
-import { n as judgeDiff } from "./patch-bDV8YFW0.js";
-import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
-import { basename, isAbsolute, join, relative, sep } from "node:path";
-import { setTimeout as setTimeout$1 } from "node:timers/promises";
-import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
-import { parse } from "yaml";
-//#region src/attempt/debug.ts
-/** Which config and profile the attempt runs under, and where each resolved option came from. */
-function tracePlan(debug, spec, ctx) {
-	const config = ctx.configFile ?? "none (no config file found)";
-	const from = (key, fallback = ctx.configFile ? `${config} or its default` : "default") => `(${spec.sources[key] ?? fallback})`;
-	const profile = ctx.profile ? `profile ${ctx.profile.name} (${ctx.profile.reason})` : "no profile";
-	debug.log("plan", `config ${config} · ${profile} · run ${ctx.runId} · attempt ${spec.cell.key}#${spec.n}`);
-	debug.log("plan", `harness ${harnessId(spec.harness)} ${from("harness")}`);
-	debug.log("plan", `sandbox ${sandboxId(spec.sandbox)} ${from("sandbox")}`);
-	const env = spec.environment;
-	const role = spec.cell.role === "variant" ? ` · variant ${spec.cell.variant}` : "";
-	const envFrom = spec.cell.role === "variant" ? from("compare") : from("environment", "none set");
-	debug.log("plan", `environment ${env.name}${role} ${envFrom}`);
-	debug.log("plan", `egress ${egressLabel(spec.egress) || "allow-all"} ${from("egress")}`);
-	const gate = typeof spec.gate === "string" ? spec.gate : `passRate ${spec.gate.passRate}`;
-	debug.log("plan", `run ${spec.n} · runs ${ctx.runs !== void 0 ? "(--runs)" : from("runs")} · gate ${gate} · timeout ${formatDuration(spec.timeoutMs)} ${from("timeout", "config.timeout or 15m")}`);
-	const judge = spec.judge;
-	const model = typeof judge.model === "string" ? judge.model : judge.model && "kind" in judge.model ? "fake judge" : judge.model ? "a model object" : "none";
-	debug.log("plan", `judge ${model}${judge.threshold !== void 0 ? ` · threshold ${judge.threshold}` : ""} ${from("judge")}`);
-	for (const note of spec.notes) debug.log("plan", `profile: ${note}`);
-	debug.trace("plan", "config", {
-		config: ctx.config,
-		profile: ctx.profile
-	}, "resolved config");
-	debug.trace("plan", "attempt", {
-		cell: spec.cell,
-		n: spec.n,
-		harness: spec.harness,
-		sandbox: spec.sandbox,
-		egress: spec.egress,
-		gate: spec.gate,
-		timeoutMs: spec.timeoutMs,
-		judge: spec.judge,
-		sources: spec.sources,
-		env: Object.keys(spec.env).sort()
-	}, "resolved attempt");
-}
-/** `POST api2.cursor.sh/auth/exchange`, `GET host/v1/*` or `POST host (path ~ /re/)`. */
-function endpointLabel({ method, host, path }) {
-	const where = !path ? host : "exact" in path ? `${host}${path.exact}` : "startsWith" in path ? `${host}${path.startsWith}*` : `${host} (path ~ /${path.regex}/)`;
-	return `${method?.join("|") ?? "*"} ${where}`;
-}
-/** The resolved environment and what the gateway brokers, by name. */
-function traceEnvironment(debug, spec, env) {
-	const r = env.record;
-	debug.log("env", `environment ${r.name} (digest ${r.digest.replace(/^sha256:/, "").slice(0, 12)})${r.parent ? ` · extends ${r.parent}` : ""}${r.overrides.length ? ` · overrides ${r.overrides.join(", ")}` : ""}`);
-	const unset = r.unset.length ? ` · unset (optional) ${r.unset.join(", ")}` : "";
-	debug.log("env", `vars ${r.vars.join(", ") || "none"} · secrets ${r.secrets.join(", ") || "none"}${unset}`);
-	for (const s of env.substitutions) debug.log("broker", `${s.name} is a placeholder in the sandbox · swapped in on HTTPS to ${s.hosts.join(", ")} in the ${s.in.join(", ")} · for ${s.phase === "checks" ? "checks and setup only" : "the agent and checks"}`);
-	const brokering = brokeringFor(spec.harness);
-	if (brokering) {
-		const endpoints = brokering.endpoints.map(endpointLabel).join(", ");
-		debug.log("broker", `${brokering.credential} stays on the host · the gateway swaps it in on ${endpoints}`);
-	}
-	debug.trace("env", "environment", {
-		record: r,
-		vars: Object.keys(env.vars).sort(),
-		checkVars: Object.keys(env.checkVars).sort(),
-		files: env.files.map((f) => f.path),
-		setup: env.setup,
-		skills: env.skills.map((s) => s.name),
-		workspaceSkills: env.workspaceSkills.map((s) => s.name),
-		mcp: Object.fromEntries(Object.entries(env.mcp).map(([name, m]) => [name, m.transport === "stdio" ? {
-			transport: m.transport,
-			command: m.command,
-			args: m.args,
-			env: Object.keys(m.env)
-		} : {
-			transport: m.transport,
-			url: m.url,
-			headers: Object.keys(m.headers)
-		}])),
-		brokered: env.substitutions.map(({ name, hosts, in: where, phase }) => ({
-			name,
-			hosts,
-			in: where,
-			phase
-		}))
-	}, "resolved environment");
-}
-//#endregion
-//#region src/environment/resolve.ts
-const SKILL_NAME = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
-const DEFAULT_SETUP_TIMEOUT = "5m";
-function broker(secrets, name, value, hosts, where = ["header"], phase = "agent") {
-	const at = badCharacterAt(value);
-	if (at && where.includes("header")) throw new InfraError(`${name} has whitespace or a line break in it (character ${at}), so it can't go in a request header. A key copied from a wrapped terminal line picks these up; set it again as one line.`);
-	const sub = {
-		name,
-		placeholder: placeholder(),
-		value,
-		hosts: [...hosts],
-		in: [...where],
-		phase
-	};
-	secrets.brokered.push(sub);
-	return sub.placeholder;
-}
-/** The host value, or its placeholder when it's brokered to `secret.hosts` (else `hosts`). */
-function hostValue(name, secret, where, secrets, hosts = secret.hosts) {
-	const from = secret.from ?? name;
-	const value = process.env[from];
-	if (!value) {
-		if (secret.optional) {
-			secrets.unset.push(name);
-			return;
-		}
-		const source = from === name ? name : `${name} (from ${from})`;
-		throw new InfraError(`Secret ${source} for ${where} isn't set. Set it in the environment or .env.local.`);
-	}
-	registerSecret(value);
-	if (secret.to === "checks") secrets.checks.push(name);
-	else secrets.names.push(name);
-	if (!hosts) return value;
-	return broker(secrets, name, value, hosts, secret.in, secret.to === "checks" ? "checks" : "agent");
-}
-/** Each host variable in the template, or with `hosts` one placeholder per variable. */
-function template(value, where, secrets, hosts) {
-	const placeholders = /* @__PURE__ */ new Map();
-	return value.template.replace(/\$\{([A-Za-z_][A-Za-z0-9_]*)\}/g, (_, name) => {
-		const v = process.env[name];
-		if (!v) throw new InfraError(`secret.template needs ${name} for ${where}, and it isn't set`);
-		registerSecret(v);
-		if (!hosts) return v;
-		const p = placeholders.get(name) ?? broker(secrets, name, v, hosts);
-		placeholders.set(name, p);
-		return p;
-	});
-}
-/** A `SKILL.md` split into its YAML frontmatter's `name` and `description` and its body. */
-function parseSkillMd(text, where) {
-	const match = /^---\r?\n(?:([\s\S]*?)\r?\n)?---[ \t]*(?:\r?\n|$)([\s\S]*)$/.exec(text);
-	if (!match) return { body: text };
-	let front;
-	try {
-		front = parse(match[1] ?? "");
-	} catch (error) {
-		const reason = error.message.split("\n")[0];
-		throw new InfraError(`${where}: the SKILL.md frontmatter isn't valid YAML (${reason}). Quote values that contain ": ".`);
-	}
-	if (front != null && (typeof front !== "object" || Array.isArray(front))) throw new InfraError(`${where}: the SKILL.md frontmatter must be a YAML mapping`);
-	const field = (key) => {
-		const value = front?.[key];
-		if (value == null) return void 0;
-		if (typeof value !== "string") throw new InfraError(`${where}: the SKILL.md ${key} must be a string (got ${typeof value})`);
-		return value.trim() || void 0;
-	};
-	return {
-		name: field("name"),
-		description: field("description"),
-		body: match[2] ?? ""
-	};
-}
-/** A gitignore-style test for `DEFAULT_SKILL_EXCLUDE`-shaped patterns against a skill-relative path. */
-function excludeMatcher(patterns) {
-	const rules = patterns.map((pattern) => {
-		const dirOnly = pattern.endsWith("/");
-		const body = pattern.replace(/\/+$/, "");
-		const anchored = body.includes("/");
-		const glob = body.replace(/^\/+/, "").split("*").map((part) => part.replace(/[.+?^${}()|[\]\\]/g, "\\$&")).join("[^/]*");
-		return {
-			dirOnly,
-			re: new RegExp(`${anchored ? "^" : "(?:^|/)"}${glob}$`)
-		};
-	});
-	return (path, isDir) => rules.some((r) => (isDir || !r.dirOnly) && r.re.test(path));
-}
-function parseSkill(dir, source, exclude) {
-	const front = parseSkillMd(readFileSync(join(dir, "SKILL.md"), "utf8"), `Skill ${basename(dir)} in ${source}`);
-	const name = front.name || basename(dir);
-	checkSkill(name, front.description, source);
-	const skip = excludeMatcher(exclude);
-	const paths = [];
-	const excluded = [];
-	const walk = (rel) => {
-		for (const entry of readdirSync(join(dir, rel), { withFileTypes: true })) {
-			const path = rel ? `${rel}/${entry.name}` : entry.name;
-			const isDir = entry.isDirectory();
-			if (entry.name === ".git") continue;
-			if (skip(path, isDir)) excluded.push(isDir ? `${path}/` : path);
-			else if (isDir) walk(path);
-			else if (entry.isFile() && path !== "SKILL.md") paths.push(path);
-		}
-	};
-	walk("");
-	const files = paths.sort().map((path) => ({
-		path,
-		content: readFileSync(join(dir, path), "utf8")
-	}));
-	return {
-		skill: {
-			name,
-			description: front.description ?? "",
-			content: front.body,
-			files
-		},
-		excluded: excluded.sort()
-	};
-}
-function checkSkill(name, description, source) {
-	if (!SKILL_NAME.test(name)) throw new InfraError(`Skill "${name}" in ${source}: the name must be kebab-case`);
-	if (!description) throw new InfraError(`Skill "${name}" in ${source} has no description in its SKILL.md frontmatter`);
-}
-function loadSkills(dir, source, only, exclude = DEFAULT_SKILL_EXCLUDE) {
-	if (!existsSync(dir) || !statSync(dir).isDirectory()) throw new InfraError(`Skill source ${source} isn't a directory`);
-	const dirs = existsSync(join(dir, "SKILL.md")) ? [dir] : readdirSync(dir, { withFileTypes: true }).filter((e) => e.isDirectory() && existsSync(join(dir, e.name, "SKILL.md"))).map((e) => join(dir, e.name)).sort();
-	if (dirs.length === 0) throw new InfraError(`Skill source ${source} has no SKILL.md (and no subdirectories with one)`);
-	const skills = dirs.map((d) => parseSkill(d, source, exclude));
-	if (!only) return skills;
-	const missing = only.filter((n) => !skills.some((s) => s.skill.name === n));
-	if (missing.length) throw new InfraError(`only: ${JSON.stringify(missing)} not found in ${source} (found: ${skills.map((s) => s.skill.name).join(", ")})`);
-	return skills.filter((s) => only.includes(s.skill.name));
-}
-async function git(args, cwd, what) {
-	const r = await run("git", args, { cwd });
-	if (r.exitCode !== 0) throw new InfraError(`${what} failed: ${r.stderr.trim()}`);
-	return r.stdout.trim();
-}
-/** One clone per source and ref per worker process; the commit is recorded. */
-const clones = /* @__PURE__ */ new Map();
-function cloneSkills(root, url, ref) {
-	const key = `${url}#${ref ?? ""}`;
-	let clone = clones.get(key);
-	if (!clone) {
-		clone = (async () => {
-			const dir = join(root, ".reeval", "cache", "skills", `${sha256(key).slice(0, 16)}-${process.pid}`);
-			await rm(dir, {
-				recursive: true,
-				force: true
-			});
-			await mkdir(dir, { recursive: true });
-			const what = `Cloning ${url}${ref ? ` at ${ref}` : ""}`;
-			await git(["init", "-q"], dir, what);
-			await git([
-				"fetch",
-				"-q",
-				"--depth",
-				"1",
-				url,
-				ref ?? "HEAD"
-			], dir, what);
-			await git([
-				"-c",
-				"advice.detachedHead=false",
-				"checkout",
-				"-q",
-				"FETCH_HEAD"
-			], dir, what);
-			return {
-				dir,
-				commit: await git(["rev-parse", "HEAD"], dir, what)
-			};
-		})();
-		clones.set(key, clone);
-		clone.catch(() => clones.delete(key));
-	}
-	return clone;
-}
-const packs = /* @__PURE__ */ new Map();
-/** `npm pack` into the cache, unpacked; returns the package directory. */
-function unpackNpm(root, spec) {
-	let pack = packs.get(spec);
-	if (!pack) {
-		pack = (async () => {
-			const dir = join(root, ".reeval", "cache", "skills", `npm-${sha256(spec).slice(0, 16)}-${process.pid}`);
-			await rm(dir, {
-				recursive: true,
-				force: true
-			});
-			await mkdir(dir, { recursive: true });
-			const packed = await run("npm", [
-				"pack",
-				spec,
-				"--silent",
-				"--pack-destination",
-				dir
-			], { cwd: dir });
-			const tarball = packed.stdout.trim().split("\n").at(-1);
-			if (packed.exitCode !== 0 || !tarball) throw new InfraError(`npm pack ${spec} failed: ${packed.stderr.trim()}`);
-			const untar = await run("tar", ["-xzf", tarball], { cwd: dir });
-			if (untar.exitCode !== 0) throw new InfraError(`Unpacking ${spec} failed: ${untar.stderr.trim()}`);
-			return join(dir, "package");
-		})();
-		packs.set(spec, pack);
-		pack.catch(() => packs.delete(spec));
-	}
-	return pack;
-}
-async function resolveSkills(root, s) {
-	switch (s.source) {
-		case "local": {
-			const label = `local:${relative(root, s.path).split(sep).join("/") || "."}`;
-			return {
-				skills: loadSkills(s.path, label, s.only, s.exclude),
-				source: label,
-				commit: null
-			};
-		}
-		case "git": {
-			const { url } = parseGitRepo(s.repo);
-			const { dir, commit } = await cloneSkills(root, url, s.ref);
-			const label = `${s.repo}${s.ref ? `#${s.ref}` : ""}${s.path ? `:${s.path}` : ""}`;
-			return {
-				skills: loadSkills(s.path ? join(dir, s.path) : dir, label, s.only, s.exclude),
-				source: label,
-				commit
-			};
-		}
-		case "registry": {
-			const [repo = "", name = ""] = s.id.split("@");
-			const { dir, commit } = await cloneSkills(root, `https://github.com/${repo}.git`, s.version);
-			const label = `registry:${s.id}${s.version ? `#${s.version}` : ""}`;
-			return {
-				skills: loadSkills(existsSync(join(dir, "skills")) ? join(dir, "skills") : dir, label, [name]),
-				source: label,
-				commit
-			};
-		}
-		case "npm": {
-			const dir = await unpackNpm(root, s.spec);
-			const label = `npm:${isAbsolute(s.spec) ? relative(root, s.spec).split(sep).join("/") : s.spec}:${s.path}`;
-			return {
-				skills: loadSkills(join(dir, s.path), label, s.only, s.exclude),
-				source: label,
-				commit: null
-			};
-		}
-		case "inline": {
-			checkSkill(s.name, s.description, "skill.inline");
-			const files = Object.entries(s.files).sort(([a], [b]) => a < b ? -1 : 1).map(([path, content]) => ({
-				path,
-				content
-			}));
-			return {
-				skills: [{
-					skill: {
-						name: s.name,
-						description: s.description,
-						content: s.content,
-						files
-					},
-					excluded: []
-				}],
-				source: "inline",
-				commit: null
-			};
-		}
-	}
-}
-const skillHash = (s) => sha256(JSON.stringify(s));
-const readHostFile = (f, where) => {
-	if (!existsSync(f.path)) throw new InfraError(`${f.path} for ${where} doesn't exist`);
-	return readFileSync(f.path);
-};
-/**
-* Layers the environment (root first; a later layer wins on shared keys, and each such key is
-* an override), over the options `env` merged down the cascade. `where` names the attempt, and
-* `rulesFile` is the workspace file the harness's agent reads `rules` from.
-*/
-async function resolveEnvironment(environment, optionsEnv, root, where, rulesFile, egress = "allow-all", prepared = () => void 0) {
-	const envWhere = `environment ${environment.name} (${where})`;
-	const overrides = /* @__PURE__ */ new Set();
-	const env = { ...optionsEnv };
-	const files = /* @__PURE__ */ new Map();
-	const mcp = /* @__PURE__ */ new Map();
-	const skills = /* @__PURE__ */ new Map();
-	let rules;
-	const setup = [];
-	const seen = {
-		env: /* @__PURE__ */ new Set(),
-		files: /* @__PURE__ */ new Set(),
-		mcp: /* @__PURE__ */ new Set()
-	};
-	let egressSet = false;
-	let workspace;
-	let sandboxSet = false;
-	for (const { spec } of environment.layers) {
-		if (spec.sandbox !== void 0) {
-			if (sandboxSet) overrides.add("sandbox");
-			sandboxSet = true;
-		}
-		if (spec.workspace !== void 0) {
-			if (workspace) overrides.add("workspace");
-			workspace = spec.workspace;
-		}
-		if (spec.egress !== void 0) {
-			if (egressSet) overrides.add("egress");
-			egressSet = true;
-		}
-		for (const [k, v] of Object.entries(spec.env ?? {})) {
-			if (seen.env.has(k)) overrides.add(`env.${k}`);
-			seen.env.add(k);
-			env[k] = v;
-		}
-		for (const [path, v] of Object.entries(spec.files ?? {})) {
-			if (seen.files.has(path)) overrides.add(`files.${path}`);
-			seen.files.add(path);
-			files.set(path, v);
-		}
-		if (spec.rules !== void 0) {
-			if (rules !== void 0) overrides.add("rules");
-			rules = spec.rules;
-		}
-		for (const [name, server] of Object.entries(spec.mcp ?? {})) {
-			if (seen.mcp.has(name)) overrides.add(`mcp.${name}`);
-			seen.mcp.add(name);
-			mcp.set(name, server);
-		}
-		for (const step of spec.setup ?? []) {
-			const s = typeof step === "string" ? { run: step } : step;
-			setup.push({
-				run: s.run,
-				timeoutMs: parseDuration(s.timeout ?? DEFAULT_SETUP_TIMEOUT),
-				...s.cwd ? { cwd: s.cwd } : {}
-			});
-		}
-		const layerSkills = /* @__PURE__ */ new Set();
-		for (const source of spec.skills ?? []) {
-			const found = await resolveSkills(root, source);
-			for (const loaded of found.skills) {
-				const name = loaded.skill.name;
-				if (skills.has(name) && !layerSkills.has(name)) overrides.add(`skills.${name}`);
-				layerSkills.add(name);
-				skills.delete(name);
-				skills.set(name, {
-					...loaded,
-					placement: source.placement,
-					source: found.source,
-					commit: found.commit
-				});
-			}
-		}
-	}
-	const secrets = {
-		names: [],
-		checks: [],
-		unset: [],
-		brokered: []
-	};
-	const vars = {};
-	const checkVars = {};
-	for (const [name, value] of Object.entries(env)) {
-		if (!isSecret(value)) {
-			vars[name] = value;
-			continue;
-		}
-		const resolved = hostValue(name, value, envWhere, secrets);
-		if (resolved === void 0) continue;
-		if (value.to === "checks") checkVars[name] = resolved;
-		else vars[name] = resolved;
-	}
-	const secretOf = (value, key, hosts) => {
-		if (typeof value === "string") return value;
-		if (isSecretTemplate(value)) return template(value, envWhere, secrets, hosts);
-		return hostValue(key, value, envWhere, secrets, value.hosts ?? hosts);
-	};
-	const resolvedMcp = {};
-	for (const [name, s] of mcp) if (s.transport === "stdio") {
-		const e = {};
-		for (const [k, v] of Object.entries(s.env)) {
-			const r = secretOf(v, k);
-			if (r !== void 0) e[k] = r;
-		}
-		resolvedMcp[name] = {
-			transport: "stdio",
-			command: s.command,
-			args: s.args,
-			env: e
-		};
-	} else {
-		const h = {};
-		const hosts = mcpHosts(s.url);
-		for (const [k, v] of Object.entries(s.headers)) {
-			const r = secretOf(v, k, hosts);
-			if (r !== void 0) h[k] = r;
-		}
-		resolvedMcp[name] = {
-			transport: s.transport,
-			url: s.url,
-			headers: h
-		};
-	}
-	const fileList = [...files].map(([path, v]) => ({
-		path,
-		content: isHostFile(v) ? readHostFile(v, envWhere) : v,
-		...isHostFile(v) && statSync(v.path).mode & 64 ? { executable: true } : {}
-	}));
-	const rulesContent = rules === void 0 ? void 0 : isHostFile(rules) ? Buffer.from(readHostFile(rules, envWhere)).toString("utf8") : rules;
-	const skillList = [...skills.values()];
-	const skillRecords = skillList.map(({ skill, placement, source, commit, excluded }) => ({
-		name: skill.name,
-		source,
-		commit,
-		sha256: skillHash(skill),
-		placement,
-		excluded
-	}));
-	const fileRecords = fileList.map((f) => ({
-		path: f.path,
-		sha256: sha256(f.content)
-	}));
-	const executable = fileList.flatMap((f) => f.executable ? [f.path] : []);
-	const rulesRecord = rulesContent === void 0 ? void 0 : {
-		path: rulesFile,
-		sha256: sha256(rulesContent)
-	};
-	if (rulesContent !== void 0) fileList.push({
-		path: rulesFile,
-		content: rulesContent
-	});
-	const secretNames = [.../* @__PURE__ */ new Set([...secrets.names, ...secrets.checks])].sort();
-	const brokered = secrets.brokered.map((b) => JSON.stringify([
-		b.name,
-		b.hosts,
-		b.in,
-		b.phase
-	])).sort();
-	const brokeredNames = [...new Set(secrets.brokered.map((b) => b.name))].sort();
-	const varNames = Object.keys(vars).filter((n) => !secrets.names.includes(n)).sort();
-	const literalVars = Object.fromEntries(varNames.map((n) => [n, vars[n]]));
-	const mcpShape = Object.fromEntries(Object.entries(resolvedMcp).map(([name, s]) => s.transport === "stdio" ? [name, {
-		command: s.command,
-		args: s.args,
-		env: Object.keys(s.env).sort()
-	}] : [name, {
-		transport: s.transport,
-		url: s.url,
-		headers: Object.keys(s.headers).sort()
-	}]));
-	const tree = workspace && prepared(workspace);
-	if (workspace && !tree) throw new InfraError(`The workspace git(${JSON.stringify(workspaceLabel(workspace))}) for ${envWhere} wasn't prepared before the run started`);
-	const workspaceRecord = workspace && tree ? {
-		source: isAbsolute(workspace.url) ? relative(root, workspace.url).split(sep).join("/") || "." : workspace.repo,
-		ref: workspace.ref ?? null,
-		path: workspace.path ?? null,
-		commit: tree.commit,
-		tree: tree.tree,
-		...workspace.init ? { init: true } : {}
-	} : void 0;
-	const digest = `sha256:${sha256(JSON.stringify({
-		setup,
-		skills: skillRecords.map((s) => [
-			s.name,
-			s.sha256,
-			s.placement
-		]),
-		mcp: mcpShape,
-		files: fileRecords,
-		...executable.length ? { executable } : {},
-		rules: rulesRecord?.sha256,
-		vars: literalVars,
-		secrets: secretNames,
-		checks: [...secrets.checks].sort(),
-		unset: [...secrets.unset].sort(),
-		brokered,
-		...egress === "allow-all" ? {} : { egress: typeof egress === "string" ? egress : [...egress.allow].sort() },
-		...workspaceRecord ? { workspace: workspaceRecord.init ? [workspaceRecord.tree, "init"] : workspaceRecord.tree } : {}
-	}))}`;
-	return {
-		record: {
-			name: environment.name,
-			parent: environment.parent,
-			overrides: [...overrides],
-			digest,
-			setup: setup.map(stepCommand),
-			skills: skillRecords,
-			mcp: Object.keys(resolvedMcp).sort(),
-			files: rulesRecord ? [...fileRecords, rulesRecord] : fileRecords,
-			vars: varNames,
-			secrets: secretNames,
-			brokered: brokeredNames,
-			unset: [...secrets.unset].sort(),
-			egress,
-			...workspaceRecord ? { workspace: workspaceRecord } : {}
-		},
-		setup,
-		skills: skillList.filter((s) => s.placement === "harness").map((s) => s.skill),
-		workspaceSkills: skillList.filter((s) => s.placement === "workspace").map((s) => s.skill),
-		mcp: resolvedMcp,
-		files: fileList,
-		vars,
-		checkVars,
-		substitutions: secrets.brokered,
-		...workspace && tree ? { workspace: {
-			tar: tree.tar,
-			label: workspaceLabel(workspace),
-			commit: tree.commit,
-			...workspace.init ? { init: true } : {}
-		} } : {}
-	};
-}
-/** A skill as a `SKILL.md` file plus its other files, for writing into a directory. */
-function skillFiles(skill) {
-	return [{
-		path: "SKILL.md",
-		content: `${`---\nname: ${skill.name}\ndescription: ${JSON.stringify(skill.description)}\n---\n`}${skill.content}`
-	}, ...skill.files];
-}
-//#endregion
-//#region src/harness/fake.ts
-const lineCount = (text) => text ? text.split("\n").length - (text.endsWith("\n") ? 1 : 0) : 0;
-/**
-* Writes canned files and returns a canned message on every turn, after running its `script`'s
-* real sandbox actions. Harness-placed skills are written under `~/` where its layout's adapter
-* puts them (`~/.agents/skills` by default, as Cursor's does).
-*/
-const createFakeHarness = (descriptor) => {
-	let starts = 0;
-	return { async start(sandbox, _signal, { skills, workspaceSkills, mcp }) {
-		if (descriptor.failWith) throw new InfraError(descriptor.failWith);
-		const hangs = descriptor.startHangs;
-		if (hangs === true || starts++ < (hangs ?? 0)) return new Promise(() => {});
-		const dir = `~/${layoutOf(descriptor).harnessSkillDir}`;
-		for (const skill of skills) {
-			for (const f of skillFiles(skill)) await sandbox.writeFile(`${dir}/${skill.name}/${f.path}`, f.content);
-			debugLog("env", `skill ${skill.name} placed at ${dir}/${skill.name}/ by the fake agent`);
-		}
-		const names = [...skills.map((s) => s.name), ...workspaceSkills];
-		const loaded = [names.length ? `skills: ${names.join(", ")}` : "", Object.keys(mcp).length ? `mcp: ${Object.keys(mcp).join(", ")}` : ""].filter(Boolean);
-		let calls = 0;
-		let turns = 0;
-		const skilled = names.length > 0 && descriptor.withSkill;
-		const files = (skilled ? descriptor.withSkill?.files : void 0) ?? descriptor.files ?? {};
-		const reply = (skilled ? descriptor.withSkill?.text : void 0) ?? descriptor.text ?? "Done.";
-		const flaked = Math.random() < (descriptor.flakeRate ?? 0);
-		return {
-			async send(prompt, signal) {
-				const startedAt = Date.now();
-				const script = descriptor.script ?? [];
-				const steps = script[Math.min(turns++, script.length - 1)] ?? [];
-				const at = () => {
-					const ms = Date.now() - startedAt;
-					return {
-						ts: new Date(startedAt + ms).toISOString(),
-						atMs: ms
-					};
-				};
-				const transcript = [{
-					...at(),
-					type: "text",
-					text: `Working on: ${prompt}`,
-					durationMs: 0
-				}, ...loaded.length ? [{
-					...at(),
-					type: "text",
-					text: `Loaded ${loaded.join("; ")}`,
-					durationMs: 0
-				}] : []];
-				const done = (text, status) => ({
-					text,
-					status,
-					startedAt: new Date(startedAt).toISOString(),
-					durationMs: Date.now() - startedAt,
-					transcript,
-					usage: null,
-					model: descriptor.servedModel ?? null,
-					unreported: []
-				});
-				const write = async (path, content) => {
-					const time = at();
-					const before = await sandbox.readFile(path);
-					await sandbox.writeFile(path, content);
-					debugLog("harness", `tool write ${path} (${before === null ? "create" : "modify"})`);
-					transcript.push({
-						...time,
-						type: "file-edit",
-						toolCallId: `call_${++calls}`,
-						path,
-						kind: before === null ? "create" : "modify",
-						added: lineCount(content),
-						removed: before === null ? 0 : lineCount(before),
-						durationMs: Date.now() - startedAt - time.atMs
-					});
-				};
-				if (descriptor.reasoning) transcript.push({
-					...at(),
-					type: "reasoning",
-					text: descriptor.reasoning,
-					durationMs: 0
-				});
-				for (const tool of descriptor.tools ?? []) transcript.push({
-					...at(),
-					type: "tool",
-					toolCallId: `call_${++calls}`,
-					tool: tool.tool,
-					input: tool.input,
-					output: tool.output,
-					ok: tool.ok ?? true,
-					durationMs: 0
-				});
-				if (flaked) {
-					const text = "I ran out of ideas and left the workspace unchanged.";
-					transcript.push({
-						...at(),
-						type: "text",
-						text,
-						durationMs: 0
-					});
-					return done(text, "completed");
-				}
-				let scripted;
-				for (const step of steps) if ("exec" in step) {
-					const time = at();
-					debugLog("harness", `tool shell called: $ ${step.exec}`);
-					const r = await sandbox.exec(step.exec, { signal });
-					debugLog("harness", `tool shell returned exit ${r.exitCode} in ${Date.now() - startedAt - time.atMs}ms`);
-					transcript.push({
-						...time,
-						type: "tool",
-						toolCallId: `call_${++calls}`,
-						tool: "shell",
-						input: { command: step.exec },
-						output: r.stdout,
-						ok: r.exitCode === 0,
-						durationMs: Date.now() - startedAt - time.atMs
-					});
-				} else if ("write" in step) for (const [path, content] of Object.entries(step.write)) await write(path, content);
-				else if ("mcp" in step) {
-					const time = at();
-					const [server = "", tool = ""] = step.mcp.split("/");
-					debugLog("harness", `tool mcp__${server}__${tool} called`);
-					const r = await callMcpTool(sandbox, mcp, server, tool, signal);
-					debugLog("harness", `tool mcp__${server}__${tool} ${r.ok ? "returned" : "failed"}`);
-					transcript.push({
-						...time,
-						type: "tool",
-						toolCallId: `call_${++calls}`,
-						tool: `mcp__${server}__${tool}`,
-						input: {},
-						output: r.output,
-						ok: r.ok,
-						durationMs: Date.now() - startedAt - time.atMs
-					});
-				} else if ("error" in step) {
-					debugLog("harness", `turn failed: ${step.error}`);
-					transcript.push({
-						...at(),
-						type: "error",
-						message: step.error
-					});
-					return done("", "error");
-				} else scripted = step.reply;
-				if (descriptor.delayMs) try {
-					await setTimeout$1(descriptor.delayMs, void 0, descriptor.ignoreAbort ? {} : { signal });
-				} catch {
-					return done("", "aborted");
-				}
-				for (const [path, content] of Object.entries(files)) await write(path, content);
-				const text = scripted ?? reply;
-				transcript.push({
-					...at(),
-					type: "text",
-					text,
-					durationMs: 0
-				});
-				return done(text, "completed");
-			},
-			async close() {}
-		};
-	} };
-};
-/**
-* Starts a stdio MCP server in the sandbox, as an agent would, and calls one tool with no
-* arguments: initialize, then the call.
-*/
-async function callMcpTool(sandbox, servers, name, tool, signal) {
-	const server = servers[name];
-	if (!server) {
-		const known = Object.keys(servers);
-		throw new InfraError(`The fake agent's script calls MCP server "${name}", which the environment doesn't have (it has ${known.length ? known.join(", ") : "none"})`);
-	}
-	if (server.transport !== "stdio") throw new InfraError(`The fake agent calls stdio MCP servers only; "${name}" is ${server.transport}`);
-	const messages = [
-		{
-			jsonrpc: "2.0",
-			id: 1,
-			method: "initialize",
-			params: {
-				protocolVersion: "2024-11-05",
-				capabilities: {},
-				clientInfo: {
-					name: "reeval-fake-agent",
-					version: "1"
-				}
-			}
-		},
-		{
-			jsonrpc: "2.0",
-			method: "notifications/initialized"
-		},
-		{
-			jsonrpc: "2.0",
-			id: 2,
-			method: "tools/call",
-			params: {
-				name: tool,
-				arguments: {}
-			}
-		}
-	].map((m) => shellQuote(JSON.stringify(m)));
-	const command = [server.command, ...server.args].map(shellQuote).join(" ");
-	const script = [
-		"d=$(mktemp -d) && : > \"$d/out\" && : > \"$d/err\" || exit 1",
-		"{",
-		"  trap '' PIPE",
-		`  printf '%s\\n' ${messages.join(" ")} 2>/dev/null`,
-		"  i=0",
-		`  while [ $i -lt 300 ] && [ ! -f "$d/code" ] && ! grep -Eq '"id": ?2[,}]' "$d/out"; do sleep 0.1; i=$((i + 1)); done`,
-		`} | { ${command} > "$d/out" 2> "$d/err"; echo $? > "$d/code"; }`,
-		"code=$(cat \"$d/code\")",
-		"cat \"$d/out\"; cat \"$d/err\" >&2; rm -rf \"$d\"; exit $code"
-	].join("\n");
-	const r = await sandbox.exec(script, {
-		signal,
-		env: server.env
-	});
-	const reply = r.stdout.split("\n").flatMap((line) => {
-		try {
-			return [JSON.parse(line)];
-		} catch {
-			return [];
-		}
-	}).find((m) => m.id === 2);
-	if (!reply) return {
-		output: r.stderr.trim() || `no reply (exit ${r.exitCode})`,
-		ok: false
-	};
-	if (reply.error) return {
-		output: reply.error.message,
-		ok: false
-	};
-	return {
-		output: (reply.result?.content ?? []).flatMap((c) => c.type === "text" ? [c.text] : []).join("\n"),
-		ok: !reply.result?.isError
-	};
-}
-//#endregion
-//#region src/harness/index.ts
-async function createHarness(descriptor) {
-	if (descriptor.kind === "fake") return createFakeHarness(descriptor);
-	const { createAgentHarness } = await import("./agent-WSmTaesf.js");
-	return createAgentHarness(descriptor);
-}
-/** Where each harness's agent looks for repo skills, relative to the workspace. */
-const workspaceSkillDir = (descriptor) => layoutOf(descriptor).workspaceSkillDir;
-//#endregion
-//#region src/attempt/environment.ts
-/**
-* Resolves the attempt's environment (files, skills, secrets and their placeholders)
-* and finds the image a Docker sandbox starts from, which preflight prepared. Runs on the test's
-* signal, before the eval timer starts.
-*/
-async function prepareEnvironment(a, root, runId, where, signal) {
-	const { spec, record } = a;
-	const env = await resolveEnvironment(spec.environment, spec.env, root, where, layoutOf(spec.harness).rulesFile, spec.egress.policy, (w) => preparedWorkspace(root, runId, w));
-	a.env = env;
-	record.environment = env.record;
-	const { secrets, brokered } = env.record;
-	record.env = {
-		vars: Object.keys(env.vars).filter((n) => !secrets.includes(n)).sort(),
-		secrets,
-		brokered
-	};
-	const debug = attemptDebug();
-	if (debug) traceEnvironment(debug, spec, env);
-	record.image = null;
-	if (spec.sandbox.kind === "docker") {
-		const { preparedFor } = await import("./preflight-D5LW7Oqi.js");
-		const { dockerProvider } = await import("./providers-CpEfNU1F.js");
-		signal.throwIfAborted();
-		const prepared = preparedFor(root, runId, dockerProvider(), spec.sandbox, spec.harness);
-		if (!prepared) throw new InfraError(`The sandbox image for ${where} wasn't prepared before the run started`);
-		record.image = prepared;
-		a.image = record.image.ref;
-		const { ref, id, base, install, agentCli } = record.image;
-		debugLog("image", `ready ${ref} (${id.replace(/^sha256:/, "").slice(0, 12)}) · prepared before the run${ref !== base ? ` · on ${base}` : ""}${agentCli ? ` · agent CLI ${agentCli}` : ""}${install.length ? ` · install: ${install.join(" && ")}` : ""}`);
-	} else debugLog("image", `none: the ${spec.sandbox.kind} sandbox runs on the host`);
-}
-//#endregion
-//#region src/judge/artifacts.ts
-/** A capture handle: a name plus the file the attempt wrote. */
-function isCaptureHandle(value) {
-	if (typeof value !== "object" || value === null) return false;
-	const v = value;
-	return typeof v.name === "string" && typeof v.file?.path === "string";
-}
-/** The first `artifacts` value that isn't a list of capture handles or names. */
-function badJudgeArtifacts(artifacts) {
-	if (artifacts === void 0) return void 0;
-	if (!Array.isArray(artifacts)) return `must be a list of capture handles or names (got ${JSON.stringify(artifacts)})`;
-	for (const item of artifacts) {
-		if (typeof item === "string" && item.trim()) continue;
-		if (isCaptureHandle(item)) continue;
-		return `must be a list of capture handles or names (got ${JSON.stringify(item)})`;
-	}
-}
-const nameOf = (item) => typeof item === "string" ? item.trim() : item.name;
-/**
-* Resolves call `artifacts` (a capture handle or its name) to PNG bytes this attempt captured.
-* Unknown names throw: a mistake in the call, before the provider is asked.
-*/
-function resolveJudgeArtifacts(artifacts, blobs) {
-	if (!artifacts?.length) return [];
-	return artifacts.map((item) => {
-		const name = nameOf(item);
-		const data = blobs.get(name);
-		if (!data) throw new Error(`c.judge() artifacts names a capture this attempt doesn't have: ${JSON.stringify(name)}`);
-		return {
-			name,
-			path: `artifacts/${name}.png`,
-			sha256: sha256(data),
-			bytes: data.byteLength,
-			mediaType: "image/png",
-			data
-		};
-	});
-}
-const filePartsOf = (resolved) => resolved.map((a) => ({
-	type: "file",
-	mediaType: a.mediaType,
-	data: a.data
-}));
-const artifactRefsOf = (resolved) => resolved.map(({ data: _data, ...ref }) => ref);
-/**
-* JSON-only state stays an object so TypeSafe and other JSON judges are unchanged.
-* File parts are an ordered decide-state array: one JSON part, then each capture.
-*/
-function decideState(json, files) {
-	if (files.length === 0) return json;
-	return [{
-		type: "json",
-		value: json
-	}, ...files];
-}
-//#endregion
-//#region src/judge/size.ts
-/** Roughly four characters a token, as `reeval analyze` counts: an estimate, not the model's. */
-const CHARS_PER_TOKEN = 4;
-const tokensOf = (chars) => Math.ceil(chars / CHARS_PER_TOKEN);
-const k = (tokens) => tokens < 1e3 ? String(tokens) : `${Math.round(tokens / 1e3)}k`;
-const diffItems = (patch) => patch.split(/(?=^diff --git )/m).map((part) => ({
-	name: /^diff --git a\/.+ b\/(.+)$/m.exec(part)?.[1] ?? "",
-	chars: part.length
-})).filter((item) => item.name);
-/** The state's parts by size, each with its biggest files where it has any. */
-function partsOf(state, question) {
-	return [...Object.entries(state).map(([name, value]) => {
-		const text = typeof value === "string" ? value : JSON.stringify(value);
-		const items = name === "diff" && typeof value === "string" ? diffItems(value) : value && typeof value === "object" && !Array.isArray(value) ? Object.entries(value).map(([file, content]) => ({
-			name: file,
-			chars: typeof content === "string" ? content.length : JSON.stringify(content).length
-		})) : [];
-		return {
-			name,
-			chars: text?.length ?? 0,
-			items
-		};
-	}), {
-		name: "question",
-		chars: question.length,
-		items: []
-	}].sort((a, b) => b.chars - a.chars);
-}
-/** The estimated size of what the judge reads, in tokens. */
-const inputTokens = (state, question) => tokensOf(JSON.stringify(state).length + question.length);
-/**
-* `diff 306k, of which dist/assets/index-….js is 296k; output 1k; …`: the state's parts by
-* estimated tokens, naming the biggest files in each.
-*/
-function describeSize(state, question) {
-	const parts = partsOf(state, question);
-	const big = parts.filter((p) => tokensOf(p.chars) >= 1e3);
-	return (big.length ? big : parts.slice(0, 1)).map((p) => {
-		const top = p.items.sort((a, b) => b.chars - a.chars).slice(0, 3).filter((item) => tokensOf(item.chars) >= 1e3).map((item) => `${item.name} is ${k(tokensOf(item.chars))}`);
-		return `${p.name} ${k(tokensOf(p.chars))}${top.length ? `, of which ${top.join(", ")}` : ""}`;
-	}).join("; ");
-}
-const HINT = "Narrow what the judge sees with include, e.g. c.judge(question, { include: ['src/'] }), or set judge: { include } in the config; files shows chosen files in full.";
-/**
-* A provider's error, as the provider worded it, then what the judge's input held: its estimated
-* size and biggest parts and files, and how to narrow it.
-*/
-const withInputSize = (message, state, question) => `${message}\nThe judge's input was ~${k(inputTokens(state, question))} tokens (estimated): ${describeSize(state, question)}. ${HINT}`;
-//#endregion
-//#region src/attempt/judge.ts
-const noJudgeModel = (configFile) => `c.judge() has no model. Set judge: { model: 'typesafe-ai/jev-1.13.0' } in ${configFile}, on the group or eval, or pass { model } to this call.`;
-const isTurn = (value) => typeof value === "object" && value !== null && "transcript" in value && "diff" in value;
-const workspacePath = (path) => path.replace(/^\.\//, "");
-/**
-* The turn's task and output, its changed files and diff as `judgeDiff` shows them, and the whole
-* current content of each of `files` (null when there's no such file).
-*/
-async function judgeState(turn, output, { include, files }, readFile) {
-	const inFull = files?.length ? files : void 0;
-	const shown = judgeDiff(turn.diff, {
-		...include ? { include } : {},
-		...inFull ? { inFull: new Set(inFull.map(workspacePath)) } : {}
-	});
-	const state = {
-		task: turn.prompt,
-		output,
-		files: shown.files,
-		diff: shown.patch
-	};
-	if (inFull) {
-		const contents = {};
-		for (const path of inFull) contents[workspacePath(path)] = await readFile(path);
-		state.contents = contents;
-	}
-	return {
-		state,
-		dropped: shown.dropped
-	};
-}
-/**
-* Grades a turn (the last by default) or a value against a question, under
-* the judge provider's concurrency lease. Every call's input is recorded, including calls that
-* fail. A call that fails returns a `JudgeFailure` and fails the attempt as a judge error after the eval's body, so the judges and
-* checks after it still run.
-*/
-function judgeCaller(a, readFile) {
-	const ask = async (question, { on, ...call } = {}) => {
-		if (typeof question !== "string" || !question.trim()) throw new Error(`c.judge() needs a question: a non-empty string (got ${JSON.stringify(question)})`);
-		const { threshold } = call;
-		if (threshold !== void 0 && !(typeof threshold === "number" && threshold >= 0 && threshold <= 1)) throw new Error(`c.judge() threshold must be between 0 and 1 (got ${String(threshold)})`);
-		const bad = badJudgeList(call);
-		if (bad) throw new Error(`c.judge() ${bad.key} must be a list of non-empty strings (got ${bad.got})`);
-		const badArts = badJudgeArtifacts(call.artifacts);
-		if (badArts) throw new Error(`c.judge() artifacts ${badArts}`);
-		const subject = isTurn(on) ? on : a.turns.at(-1);
-		if (!subject) throw new Error("c.judge() has no turn to grade: call c.send() first");
-		const options = {
-			...a.spec.judge,
-			...call
-		};
-		const { model } = options;
-		if (model === void 0) {
-			a.judgeErrored = true;
-			throw new Error(noJudgeModel(getRunContext().config.file));
-		}
-		const { include, files, maxConcurrent, ...judgeOptions } = options;
-		const { state, dropped } = await judgeState(subject, on === void 0 || isTurn(on) ? subject.text : on, {
-			include,
-			files
-		}, readFile);
-		const resolved = resolveJudgeArtifacts(call.artifacts, a.artifactBlobs);
-		const refs = artifactRefsOf(resolved);
-		const decide = decideState(state, filePartsOf(resolved));
-		const name = judgeModelName(model);
-		const input = {
-			question,
-			model: name,
-			state,
-			...refs.length ? { artifacts: refs } : {},
-			...dropped.length ? { dropped } : {}
-		};
-		a.record.judgeInputs?.push(input);
-		const start = performance.now();
-		const ms = () => Math.round(performance.now() - start);
-		const atLeast = options.threshold ?? .8;
-		a.report({
-			kind: "judge-start",
-			question
-		});
-		debugLog("judge", `asking ${name}: ${JSON.stringify(clip(question, 160))}`);
-		if (dropped.length) debugLog("judge", `left out of the judge's input: ${dropped.join(" · ")}`);
-		debugTrace("judge", "prompt", input, "judge prompt");
-		const failed = (message, retries) => {
-			a.judgeErrored = true;
-			a.judgeErrors.push(`c.judge(${JSON.stringify(clip(question, 80))}) failed: ${message}`);
-			input.error = message;
-			if (retries?.length) input.retries = retries;
-			debugLog("judge", `${name} · ${formatDuration(ms())} · error: ${message}`);
-			debugTrace("judge", "error", {
-				error: message,
-				retries
-			}, "judge error");
-			a.report({
-				kind: "judge-end",
-				question,
-				verdict: "error",
-				score: null,
-				threshold: null,
-				ms: ms()
-			});
-			return {
-				verdict: "error",
-				score: null,
-				threshold: atLeast,
-				question,
-				model: name,
-				error: message,
-				...refs.length ? { artifacts: refs } : {}
-			};
-		};
-		const provider = name.split("/")[0];
-		try {
-			const { verdict, retries } = await a.timed("judgeMs", () => withLease(maxConcurrent ? { [`judge:${provider}`]: maxConcurrent } : {}, () => runJudge(question, decide, {
-				...judgeOptions,
-				model,
-				threshold: atLeast
-			}, a.signal), a.signal));
-			const recorded = refs.length ? {
-				...verdict,
-				artifacts: refs
-			} : verdict;
-			a.record.judgements.push(retries.length ? {
-				...recorded,
-				retries
-			} : recorded);
-			const { score } = verdict;
-			debugLog("judge", `${verdict.resolvedModel ?? verdict.model} · ${formatDuration(ms())} · score ${score.toFixed(2)} ${score >= verdict.threshold ? "≥" : "<"} ${verdict.threshold} · ${verdict.verdict}${retries.length ? ` · ${retries.length} retries` : ""} · reason: P(yes) ${score.toFixed(2)} is ${score >= verdict.threshold ? "at or above" : "below"} the ${verdict.threshold} threshold`);
-			debugTrace("judge", "response", {
-				verdict,
-				retries
-			}, "judge response");
-			a.report({
-				kind: "judge-end",
-				question,
-				verdict: verdict.verdict,
-				score,
-				threshold: verdict.threshold,
-				ms: ms()
-			});
-			return recorded;
-		} catch (error) {
-			if (a.signal.aborted) {
-				a.judgeErrored = true;
-				throw error;
-			}
-			const { retries } = error;
-			const message = error instanceof Error ? error.message : String(error);
-			return failed(isApiError(error) ? withInputSize(message, state, question) : message, retries);
-		}
-	};
-	return async (question, options) => Object.defineProperty(await ask(question, options), SUBJECT, { value: question });
-}
-//#endregion
-//#region src/sandbox/index.ts
-async function createSandbox(descriptor, options) {
-	try {
-		switch (descriptor.kind) {
-			case "fake": return await createFakeSandbox(descriptor, options.env);
-			case "docker": {
-				const { createDockerSandbox } = await import("./docker-Df_YOxxB.js");
-				return await createDockerSandbox(descriptor, options);
-			}
-		}
-	} catch (error) {
-		if (error instanceof InfraError || options.signal.aborted) throw error;
-		throw new InfraError(`Failed to create ${descriptor.kind} sandbox: ${error.message}`);
-	}
-}
-//#endregion
-//#region src/attempt/record.ts
-function classify(error, signal) {
-	const message = error instanceof Error ? error.message : String(error);
-	if (signal.aborted) return isTimeout(signal.reason) ? {
-		kind: "timeout",
-		message: "The eval timed out"
-	} : {
-		kind: "aborted",
-		message: "The run was cancelled"
-	};
-	if (error instanceof InfraError) return {
-		kind: "infra",
-		message
-	};
-	return {
-		kind: "assertion",
-		message
-	};
-}
-/**
-* Why the attempt failed, into the record: the eval's own timeout, a cancel, an infra error, or an
-* assertion, which is the judge's when a judge call failed or gave a failing verdict.
-*/
-function recordFailure(a, error, timedOut) {
-	const { record } = a;
-	record.error = timedOut ? {
-		kind: "timeout",
-		message: "The eval timed out"
-	} : classify(error, a.signal);
-	record.error.message = capMiddle(record.error.message);
-	if (error instanceof PrepareError && !timedOut) {
-		const { class: kind, stage, ref, cause, log, setBy } = error.failure;
-		record.error.prepare = {
-			class: kind,
-			stage,
-			ref,
-			cause,
-			log,
-			setBy: setBy ?? []
-		};
-	}
-	const judgeFailed = a.judgeErrored || record.judgements.some((j) => j.verdict !== "pass");
-	if (record.error.kind === "assertion" && judgeFailed) record.error.kind = "judge";
-	record.outcome = record.error.kind === "infra" ? "infra_error" : "fail";
-}
-/**
-* A process the kernel killed at the sandbox's memory limit makes the attempt an infra error,
-* whatever its checks said: the limit, not the agent, decided it. Returns the error to fail it with.
-*/
-async function recordOom(a, sandbox) {
-	const oom = await sandbox?.oomKills?.().catch(() => null);
-	const at = oom && `the ${oom.limit} limit${oom.swap ? ` (+${oom.swap} swap)` : ""}`;
-	if (oom) debugLog("teardown", `oom_kill ${oom.kills} at ${at}`);
-	if (!oom?.kills) return void 0;
-	const message = `out of memory: ${oom.kills === 1 ? "1 process" : `${oom.kills} processes`} killed at ${at}; raise docker({ memory })`;
-	a.record.error = {
-		kind: "infra",
-		message,
-		resource: {
-			cause: "oom",
-			kills: oom.kills
-		}
-	};
-	a.record.outcome = "infra_error";
-	return new InfraError(message);
-}
-/**
-* The attempt's egress policy and, when its sandbox records it, what it contacted and was refused.
-* Refused hosts the harness always tries are `expectedBlocked`, summarised in one `-v` line.
-*/
-function recordEgress(a, sandbox) {
-	const traffic = sandbox?.traffic?.();
-	const expected = expectedBlockedFor(a.spec.harness);
-	const refused = traffic?.blocked ?? [];
-	const isExpected = (h) => !!expected?.hosts.includes(h.host);
-	const expectedBlocked = refused.filter(isExpected);
-	a.record.egress = {
-		policy: a.spec.egress.policy,
-		allowed: allowedHosts(a.spec.egress),
-		recorded: !!traffic,
-		contacted: traffic?.contacted ?? [],
-		blocked: refused.filter((h) => !isExpected(h)),
-		expectedBlocked
-	};
-	if (expected && expectedBlocked.length) {
-		const hosts = expectedBlocked.map((h) => `${h.host}:${h.port} ×${h.connections}`).join(", ");
-		debugLog("gateway", `expected blocks, not logged per request: ${hosts} · ${expected.why}`);
-	}
-	a.record.servedModels = sandbox?.servedModels?.() ?? null;
-}
-/** Writes the attempt's record, redacted, and returns what the runner reports. */
-async function writeRecord(a, stagingDir, id) {
-	const { record, spec } = a;
-	const recordPath = join(stagingDir, "evals", recordFileName(attemptKey(id, spec.cell.key, spec.n)));
-	await mkdir(join(stagingDir, "evals"), { recursive: true });
-	await writeCaptureSidecars(stagingDir, recordFileName(attemptKey(id, spec.cell.key, spec.n)), a.artifactBlobs);
-	const tmp = `${recordPath}.${process.pid}.tmp`;
-	await writeFile(tmp, redactSecrets(JSON.stringify(record, null, 2)));
-	await rename(tmp, recordPath);
-	return {
-		outcome: record.outcome,
-		errorKind: record.error?.kind ?? null,
-		judgements: record.judgements.map((j) => ({
-			verdict: j.verdict,
-			score: j.score,
-			threshold: j.threshold,
-			question: clip(j.question, 200)
-		})),
-		turnErrors: turnErrors(record.turns.map((t) => t.transcript)),
-		...record.error?.prepare ? { prepare: {
-			ref: record.error.prepare.ref,
-			stage: record.error.prepare.stage
-		} } : {}
-	};
-}
-/** `turn N error: <message>` for each turn with an error event. */
-function turnErrors(transcripts) {
-	return transcripts.flatMap((events, i) => {
-		const found = events.find((e) => e.type === "error");
-		return found?.type === "error" ? [`turn ${i + 1} error: ${redactSecrets(clip(found.message.trim(), 500))}`] : [];
-	});
-}
-//#endregion
-//#region src/sandbox/phases.ts
-const strip = ({ phase: _, ...sub }) => sub;
-/**
-* Which brokered secrets the gateway swaps in when. `to: 'checks'` secrets are active only while
-* at least one check or setup step is running, so a request the agent makes between them gets
-* the placeholder.
-*/
-function brokerPhases(box, subs) {
-	const agent = subs.filter((s) => s.phase === "agent").map(strip);
-	const all = subs.map(strip);
-	const perPhase = box.setSubstitutions && all.length > agent.length;
-	let running = 0;
-	return {
-		async start() {
-			if (all.length > 0) await box.setSubstitutions?.(agent);
-		},
-		async checks(fn) {
-			if (!perPhase) return fn();
-			if (running === 0) await box.setSubstitutions?.(all);
-			running++;
-			try {
-				return await fn();
-			} finally {
-				if (--running === 0) await box.setSubstitutions?.(agent);
-			}
-		}
-	};
-}
-//#endregion
-//#region src/attempt/state.ts
-/** Extra time an attempt gets, outside its eval timeout, when its image may need building. */
-const SETUP_BUDGET_MS = 12e5;
-function createAttempt(spec, signal, progress) {
-	const record = {
-		startedAt: (/* @__PURE__ */ new Date()).toISOString(),
-		outcome: "pass",
-		error: null,
-		turns: [],
-		diff: null,
-		checks: [],
-		judgements: [],
-		timings: {
-			setupMs: 0,
-			agentMs: 0,
-			diffMs: 0,
-			checksMs: 0,
-			judgeMs: 0
-		},
-		judgeInputs: []
-	};
-	return {
-		spec,
-		record,
-		turns: [],
-		report: (update) => progress({
-			at: Date.now(),
-			...update
-		}),
-		async timed(phase, fn) {
-			const start = performance.now();
-			try {
-				return await fn();
-			} finally {
-				record.timings[phase] += Math.round(performance.now() - start);
-			}
-		},
-		signal,
-		judgeErrored: false,
-		judgeErrors: [],
-		captureNames: /* @__PURE__ */ new Set(),
-		backgrounds: [],
-		artifactBlobs: /* @__PURE__ */ new Map()
-	};
-}
-const startTimeoutMs = (harness) => parseDuration(harness.startTimeout ?? "90s");
-/**
-* Setup steps and the harness's start (once retried) run outside the eval timeout, so the
-* attempt's own limit makes room for them.
-*/
-function setupAllowance(planned) {
-	const setupMs = planned.environment.layers.flatMap((l) => l.spec.setup ?? []).reduce((ms, step) => ms + parseDuration((typeof step === "string" ? void 0 : step.timeout) ?? "5m"), 0);
-	const { sandbox } = planned;
-	return setupMs + (sandbox.kind === "docker" && !!(sandbox.from || sandbox.dockerfile || sandbox.install?.length) ? SETUP_BUDGET_MS : 0) + 2 * startTimeoutMs(planned.harness);
-}
-/** The eval's timer, paused while the sandbox and its setup steps start. */
-function evalTimer(ms) {
-	const controller = new AbortController();
-	let left = ms;
-	let since = Date.now();
-	let timer;
-	const fire = () => controller.abort(new DOMException(`Eval timed out after ${ms / 1e3}s`, "TimeoutError"));
-	const start = () => {
-		since = Date.now();
-		timer = setTimeout(fire, Math.max(0, left));
-	};
-	start();
-	return {
-		signal: controller.signal,
-		pause() {
-			clearTimeout(timer);
-			left -= Date.now() - since;
-		},
-		resume: start,
-		clear: () => clearTimeout(timer)
-	};
-}
-//#endregion
-//#region src/attempt/session.ts
-const tailOf = (text) => text.trim().split("\n").slice(-10).join("\n");
-/** Setup steps in order, each with its own timeout; a failure is an infra error. */
-async function runSetup(a, box, env) {
-	const { record, spec } = a;
-	const steps = env.setup;
-	record.setup = [];
-	for (const [i, step] of steps.entries()) {
-		const what = `setup step ${i + 1}/${steps.length} "${clip(step.run, 80)}" in environment ${spec.environment.name}`;
-		const start = performance.now();
-		const stepSignal = AbortSignal.any([a.signal, AbortSignal.timeout(step.timeoutMs)]);
-		const r = await asCheck(a, () => box.exec(step.run, {
-			signal: stepSignal,
-			env: env.checkVars,
-			...step.cwd ? { cwd: step.cwd } : {}
-		})).catch((error) => {
-			if (a.signal.aborted) throw error;
-			if (!stepSignal.aborted) throw new InfraError(`${what} couldn't run: ${error instanceof Error ? error.message : error}`);
-			return null;
-		});
-		const durationMs = Math.round(performance.now() - start);
-		const output = r ? [r.stdout, r.stderr].filter(Boolean).join("\n") : "";
-		record.setup.push({
-			run: step.run,
-			exitCode: r?.exitCode ?? null,
-			durationMs,
-			output
-		});
-		debugLog("env", `setup ${i + 1}/${steps.length} $ ${clip(step.run, 120)} → ${r ? `exit ${r.exitCode}` : `timed out after ${formatDuration(step.timeoutMs)}`} · ${formatDuration(durationMs)}${step.cwd ? ` · in ${step.cwd}` : ""}`);
-		debugTrace("env", "setup", {
-			run: step.run,
-			cwd: step.cwd,
-			exitCode: r?.exitCode ?? null,
-			durationMs,
-			stdout: r?.stdout,
-			stderr: r?.stderr
-		}, `setup ${i + 1} output`);
-		if (r?.exitCode === 0) continue;
-		throw new InfraError(r ? `${what} exited ${r.exitCode} after ${formatDuration(durationMs)} (see setup.log):\n${tailOf(r.stderr || r.stdout)}` : `${what} timed out after ${formatDuration(step.timeoutMs)} (see setup.log)`);
-	}
-}
-const WORKSPACE_TAR = ".reeval-workspace.tar";
-/** Unpacks the git workspace's tarball into the (empty) workspace. */
-async function copyWorkspace(box, workspace, signal) {
-	const start = performance.now();
-	const tar = await readFile(workspace.tar);
-	await box.writeFile(WORKSPACE_TAR, tar);
-	const r = await box.exec(`tar -xf ${WORKSPACE_TAR} && rm -f ${WORKSPACE_TAR}`, { signal });
-	if (r.exitCode !== 0) throw new InfraError(`Couldn't unpack the workspace ${workspace.label} into the sandbox (the image needs tar): ${tailOf(r.stderr || r.stdout)}`);
-	if (workspace.init) {
-		const at = "2000-01-01T00:00:00Z";
-		const init = await box.exec(`git init -q -b main && git add -A && git -c commit.gpgsign=false commit -q --no-verify -m ${shellQuote(`${workspace.label} at ${workspace.commit}`)}`, {
-			signal,
-			env: {
-				GIT_AUTHOR_NAME: "reeval",
-				GIT_AUTHOR_EMAIL: "reeval@localhost",
-				GIT_AUTHOR_DATE: at,
-				GIT_COMMITTER_NAME: "reeval",
-				GIT_COMMITTER_EMAIL: "reeval@localhost",
-				GIT_COMMITTER_DATE: at
-			}
-		});
-		if (init.exitCode !== 0) throw new InfraError(`Couldn't give the workspace ${workspace.label} a .git (init: true needs git in the image): ${tailOf(init.stderr || init.stdout)}`);
-	}
-	debugLog("env", `workspace ${workspace.label} at ${workspace.commit.slice(0, 12)} (${tar.byteLength} bytes of tar${workspace.init ? ", with a .git" : ""}) in ${formatDuration(performance.now() - start)}`);
-}
-/**
-* Starts the harness session within its `startTimeout`. A start that doesn't become ready in time
-* is aborted (and closed, should it finish later) and started once more: a stuck bootstrap is
-* usually transient, and nothing has run yet, so the retry can't change what the attempt
-* measures. A second timeout is an infra error; a start that fails outright isn't retried.
-*/
-async function startHarness(a, harness, box, options) {
-	const { harness: descriptor } = a.spec;
-	const ms = startTimeoutMs(descriptor);
-	const late = `${descriptor.kind}: session did not become ready within ${formatDuration(ms)}`;
-	for (let tries = 1;; tries++) {
-		const stop = new AbortController();
-		let timer;
-		const timedOut = new Promise((resolve) => {
-			timer = setTimeout(() => resolve(null), ms);
-		});
-		const starting = harness.start(box, AbortSignal.any([a.signal, stop.signal]), options);
-		const session = await Promise.race([starting, timedOut]).finally(() => clearTimeout(timer));
-		if (session) return session;
-		stop.abort(new InfraError(late));
-		starting.then((s) => s.close().catch(() => {}), () => {});
-		a.signal.throwIfAborted();
-		if (tries === 2) throw new InfraError(`${late} (tried twice)`);
-		a.record.startRetries = [{
-			error: late,
-			waitedMs: ms
-		}];
-		debugLog("harness", `${late}; starting it once more`);
-	}
-}
-/**
-* Starts the sandbox (with the brokered secrets its gateway swaps in), writes the
-* environment's files and workspace skills, runs its setup steps, snapshots the workspace for
-* the diff, and starts the harness session with the harness-placed skills and MCP servers.
-*/
-async function startSession(a, runId, setupStart) {
-	const { spec } = a;
-	const env = a.env;
-	const box = await createSandbox(spec.sandbox, {
-		runId,
-		timeoutMs: spec.timeoutMs + setupAllowance(spec) + 6e4,
-		signal: a.signal,
-		env: env.vars,
-		image: a.image,
-		brokering: brokeringFor(spec.harness),
-		expectedBlocked: expectedBlockedFor(spec.harness)?.hosts,
-		substitutions: env.substitutions,
-		egress: spec.egress
-	});
-	a.sandbox = box;
-	if (spec.sandbox.kind === "fake") debugLog("container", `fake sandbox at ${box.workdir}`);
-	a.phases = brokerPhases(box, env.substitutions);
-	if (a.signal.aborted) box.destroy();
-	a.signal.throwIfAborted();
-	await a.phases.start();
-	if (env.substitutions.length) {
-		const agent = env.substitutions.filter((s) => s.phase === "agent").map((s) => s.name);
-		debugLog("broker", `gateway swaps in ${agent.join(", ") || "nothing"} for the agent`);
-	}
-	if (env.workspace) await copyWorkspace(box, env.workspace, a.signal);
-	for (const f of env.files) {
-		await box.writeFile(f.path, f.content);
-		const size = typeof f.content === "string" ? Buffer.byteLength(f.content) : f.content.byteLength;
-		debugLog("env", `file ${f.path} (${size} bytes${f.executable ? ", executable" : ""})`);
-	}
-	const executable = env.files.flatMap((f) => f.executable ? [f.path.startsWith("~/") ? `${box.home}/${f.path.slice(2)}` : f.path] : []);
-	if (executable.length) {
-		const paths = executable.map((p) => shellQuote(p.startsWith("-") ? `./${p}` : p));
-		const r = await box.exec(`chmod +x ${paths.join(" ")}`, { signal: a.signal });
-		if (r.exitCode !== 0) throw new InfraError(`Couldn't make ${executable.join(", ")} executable: ${tailOf(r.stderr || r.stdout)}`);
-	}
-	const skillDir = workspaceSkillDir(spec.harness);
-	for (const skill of env.workspaceSkills) {
-		const files = skillFiles(skill);
-		for (const f of files) await box.writeFile(`${skillDir}/${skill.name}/${f.path}`, f.content);
-		debugLog("env", `skill ${skill.name} → ${skillDir}/${skill.name}/ (${files.length} files, workspace)`);
-	}
-	const layout = layoutOf(spec.harness);
-	for (const skill of env.skills) debugLog("env", `skill ${skill.name} → the harness adapter places it (~/${layout.harnessSkillDir}/${skill.name}/ for ${spec.harness.kind === "fake" ? `the fake ${spec.harness.layout ?? "cursor"} layout` : spec.harness.kind})`);
-	for (const [name, server] of Object.entries(env.mcp)) debugLog("env", `mcp ${name} (${server.transport}${server.transport === "stdio" ? `: ${server.command} ${server.args.join(" ")}` : `: ${server.url}`}) → the harness adapter writes its MCP config`);
-	await runSetup(a, box, env);
-	await snapshot(box, a.signal);
-	debugLog("env", "workspace snapshot taken: the diff starts here");
-	const harness = await createHarness(spec.harness);
-	const harnessStart = performance.now();
-	debugLog("harness", `starting ${harnessId(spec.harness)}: bootstrap and session`);
-	const session = await startHarness(a, harness, box, {
-		skills: env.skills,
-		workspaceSkills: env.workspaceSkills.map((s) => s.name),
-		mcp: env.mcp
-	});
-	debugLog("harness", `${harnessId(spec.harness)} session ready in ${formatDuration(performance.now() - harnessStart)}`);
-	a.session = session;
-	a.report({
-		kind: "sandbox-ready",
-		ms: Math.round(performance.now() - setupStart)
-	});
-	return [box, session];
-}
-//#endregion
-//#region src/attempt/turns.ts
-/** How long a turn gets to stop once its attempt is aborted, before reeval stops waiting for it. */
-const TURN_STOP_GRACE_MS = 3e3;
-/**
-* Resolves `grace` ms after `signal` aborts, with the turn as cut off; `cancel` drops the timer
-* once the turn settles on its own.
-*/
-function cutOff(signal, startedAt, grace = TURN_STOP_GRACE_MS) {
-	let timer;
-	let onAbort = () => {};
-	const cutoff = new Promise((resolve) => {
-		onAbort = () => {
-			timer = setTimeout(() => {
-				const message = `The agent didn't stop within ${grace / 1e3}s of being cancelled, so reeval stopped waiting for its turn`;
-				const at = Date.now() - Date.parse(startedAt);
-				resolve({
-					text: "",
-					status: "aborted",
-					transcript: [{
-						type: "error",
-						message,
-						ts: (/* @__PURE__ */ new Date()).toISOString(),
-						atMs: at
-					}],
-					usage: null,
-					model: null,
-					unreported: [],
-					startedAt,
-					durationMs: at
-				});
-			}, grace);
-		};
-		if (signal.aborted) onAbort();
-		else signal.addEventListener("abort", onAbort, { once: true });
-	});
-	const cancel = () => {
-		clearTimeout(timer);
-		signal.removeEventListener("abort", onAbort);
-	};
-	return {
-		cutoff,
-		cancel
-	};
-}
-/**
-* One turn. Sends the prompt to the started session, records the turn as it goes (an
-* errored entry first, so a turn that throws is still in the record), takes the agent's context
-* after the first, and diffs the workspace since the snapshot.
-*/
-function turnSender(a, start, takeContext, checks) {
-	return async (prompt) => {
-		const [box, agent] = await start();
-		const entry = {
-			prompt,
-			text: "",
-			status: "error",
-			transcript: [],
-			usage: null,
-			model: null,
-			unreported: [],
-			startedAt: (/* @__PURE__ */ new Date()).toISOString(),
-			durationMs: 0
-		};
-		a.record.turns.push(entry);
-		const turnNumber = a.record.turns.length;
-		a.report({
-			kind: "turn-start",
-			turn: turnNumber
-		});
-		debugLog("harness", `turn ${turnNumber} started: ${JSON.stringify(clip(prompt, 160))}`);
-		debugTrace("harness", "prompt", {
-			turn: turnNumber,
-			prompt
-		}, `turn ${turnNumber} prompt`);
-		const { cutoff, cancel } = cutOff(a.signal, entry.startedAt);
-		const { raw, ...result } = await a.timed("agentMs", () => Promise.race([agent.send(prompt, a.signal), cutoff])).finally(cancel);
-		Object.assign(entry, result, raw ? { raw } : {});
-		for (const event of result.transcript) debugTrace("harness", "transcript", {
-			turn: turnNumber,
-			...event
-		}, `transcript ${event.type}`);
-		await takeContext(box, a.signal);
-		const count = (...types) => result.transcript.filter((e) => types.includes(e.type)).length;
-		const failure = result.transcript.find((e) => e.type === "error");
-		a.report({
-			kind: "turn-end",
-			turn: turnNumber,
-			status: result.status,
-			ms: result.durationMs,
-			tools: count("tool", "web-search", "web-fetch"),
-			fileEdits: count("file-edit"),
-			...failure?.type === "error" ? { error: redactSecrets(clip(failure.message.trim(), 300)) } : {}
-		});
-		const plural = (n, word) => `${n} ${word}${n === 1 ? "" : "s"}`;
-		debugLog("harness", `turn ${turnNumber} ${result.status} in ${formatDuration(result.durationMs)} · ${plural(count("tool", "web-search", "web-fetch"), "tool call")} · ${plural(count("file-edit"), "file edit")}${failure && "message" in failure ? ` · error: ${clip(String(failure.message), 200)}` : ""}${result.model ? ` · model ${result.model}` : ""}`);
-		a.signal.throwIfAborted();
-		const diff = await a.timed("diffMs", () => diffSinceSnapshot(box, a.signal));
-		a.record.diff = diff;
-		const data = {
-			prompt,
-			...result,
-			diff
-		};
-		const turn = Object.freeze({
-			...data,
-			...checks
-		});
-		a.turns.push(turn);
-		return turn;
-	};
-}
-//#endregion
-//#region src/attempt/index.ts
-const TRACE_BUFFER_DEFAULT = 8 * 2 ** 20;
-async function runAttempt(spec, fn, handle) {
-	const ctx = getRunContext();
-	const { config, runId } = ctx;
-	const verbosity = ctx.verbosity ?? 0;
-	const id = evalId(relativeFile(config.root, handle.file), [...spec.groups, spec.name]);
-	const debug = new AttemptDebug({
-		dir: attemptDebugDir(config.root, runId, id, spec.cell.key, spec.n),
-		verbosity,
-		traceLimit: config.debug?.traceBufferBytes ?? TRACE_BUFFER_DEFAULT,
-		...verbosity > 0 ? { emit: handle.debug } : {}
-	});
-	return withDebug(debug, () => attemptBody(spec, fn, handle, id, debug));
-}
-async function attemptBody(spec, fn, { signal: testSignal, setMeta, progress, unawaited, assertions }, id, debug) {
-	const setupStart = performance.now();
-	const ctx = getRunContext();
-	const { runId, stagingDir, config } = ctx;
-	tracePlan(debug, spec, ctx);
-	const a = createAttempt(spec, testSignal, progress);
-	let timer;
-	let clock;
-	const destroy = () => void a.sandbox?.destroy();
-	testSignal.addEventListener("abort", destroy, { once: true });
-	let starting;
-	const start = () => {
-		starting ??= (async () => {
-			clock?.pause();
-			try {
-				return await a.timed("setupMs", () => startSession(a, runId, setupStart));
-			} finally {
-				clock?.resume();
-			}
-		})();
-		return starting;
-	};
-	const takeContext = contextTaker(a);
-	const { exec, readFile, writeFile } = createChecks(a);
-	const c = {
-		get signal() {
-			return a.signal;
-		},
-		turns: a.turns,
-		exec,
-		readFile,
-		writeFile,
-		serve: (command, port) => serveCommand(a, command, port),
-		capture: (target, options) => capturePage(a, target, options),
-		attempt: {
-			id: attemptId(spec.cell.key, spec.n),
-			cell: spec.cell,
-			n: spec.n
-		},
-		async sandbox() {
-			await start();
-			return {
-				exec,
-				readFile
-			};
-		},
-		send: turnSender(a, start, takeContext, {
-			exec,
-			readFile
-		}),
-		judge: judgeCaller(a, readFile)
-	};
-	const timedOut = () => !!timer?.aborted && !testSignal.aborted;
-	let failure;
-	let oom;
-	try {
-		a.report({ kind: "sandbox" });
-		const where = `${id} [${attemptId(spec.cell.key, spec.n)}]`;
-		await a.timed("setupMs", () => prepareEnvironment(a, config.root, runId, where, testSignal));
-		clock = evalTimer(spec.timeoutMs);
-		timer = clock.signal;
-		a.signal = AbortSignal.any([testSignal, timer]);
-		timer.addEventListener("abort", destroy, { once: true });
-		await fn(c);
-		a.signal.throwIfAborted();
-		const missed = await unawaited();
-		if (missed) throw missed;
-		const [only, ...more] = a.judgeErrors;
-		if (only) throw new Error(more.length ? `${more.length + 1} judge calls failed:\n${a.judgeErrors.join("\n")}` : only);
-	} catch (error) {
-		recordFailure(a, error, timedOut());
-		failure = { error: timedOut() ? new TimeoutError(`Eval timed out after ${spec.timeoutMs / 1e3}s`) : error };
-	} finally {
-		if (!a.signal.aborted) await unawaited().catch(() => {});
-		clock?.clear();
-		for (const bg of a.backgrounds) await bg.kill().catch(() => {});
-		const { sandbox } = a;
-		if (sandbox && !a.signal.aborted) await takeContext(sandbox, AbortSignal.timeout(3e4));
-		if (sandbox) a.report({ kind: "cleanup" });
-		testSignal.removeEventListener("abort", destroy);
-		const teardown = performance.now();
-		await a.session?.close().catch(() => {});
-		if (a.session) debug.log("teardown", "harness session closed");
-		recordEgress(a, sandbox);
-		oom = a.signal.aborted ? void 0 : await recordOom(a, sandbox);
-		await sandbox?.destroy();
-		if (sandbox) debug.log("teardown", `sandbox destroyed (${formatDuration(performance.now() - teardown)} for teardown)`);
-		const { record } = a;
-		const checked = assertions();
-		if (checked.length) record.assertions = checked;
-		for (const x of checked) {
-			const why = x.message ? ` · ${clip(x.message.split("\n")[0] ?? "", 160)}` : "";
-			debug.log("check", `assert ${x.status === "pass" ? "✓" : "✗"} ${x.label} (${x.matcher}, ${x.status})${why}`);
-		}
-		const error = record.error ? ` · ${record.error.kind}: ${record.error.message}` : "";
-		debug.log("teardown", `outcome ${record.outcome}${error}`);
-		const rel = (path) => relativeFile(config.root, path) || ".";
-		const scratch = `.reeval/runs/${runId}`;
-		debug.log("teardown", `record ${scratch}.json · artifacts ${scratch}/${evalSlug(id)}/${evalSlug(spec.cell.key)}/${spec.n} · debug ${rel(debug.dir)}`);
-		setMeta(await writeRecord(a, stagingDir, id));
-		await debug.finish(debug.verbosity === 2 || record.outcome !== "pass");
-	}
-	if (oom) throw oom;
-	if (failure) throw failure.error;
-}
-//#endregion
-//#region src/plan-eval.ts
-/** Lets the attempt's own eval timer fire before the engine's backstop, so the message and cleanup are its own. */
-const GRACE_MS = 5e3;
-function configLevel(config) {
-	return {
-		harness: config.harness,
-		sandbox: config.sandbox,
-		environment: config.environment,
-		compare: void 0,
-		runs: config.runs,
-		gate: config.gate,
-		env: config.env,
-		egress: config.egress !== void 0 ? { inherited: config.egress } : {},
-		judge: config.judge,
-		capture: config.capture,
-		timeout: void 0,
-		tags: [],
-		pin: [],
-		sources: {}
-	};
-}
-/**
-* The tests an eval registers, or `null` when the run's tags leave it out (it isn't counted in
-* `order`). Throws a UsageError for options the plan refuses.
-*/
-function planEval(declaration, ctx) {
-	const { name, mode, groups: enclosing, options: own } = declaration;
-	const groups = enclosing.map((g) => g.name);
-	const where = `evaluate("${[...groups, name].join(" › ")}")`;
-	checkOptions(own, where);
-	const levels = [...enclosing.map((g) => ({
-		options: g.options,
-		where: `group("${g.name}")`
-	})), {
-		options: own,
-		where
-	}];
-	const resolvedEval = cascade(configLevel(ctx.config), levels, where, {
-		profile: ctx.profile,
-		runs: ctx.runs
-	});
-	const { resolved } = resolvedEval;
-	if (!matchesTags(resolved.tags, ctx.tags)) return null;
-	const timeoutMs = resolved.timeout === void 0 ? ctx.config.timeoutMs : parseDuration(resolved.timeout);
-	const plannedAttempts = resolvedEval.skip ? [] : expand(resolved);
-	if (ctx.config.secrets.unbrokered === "error") {
-		const [first] = unbrokered(resolved.env, plannedAttempts);
-		if (first) throw new UsageError(unbrokeredError(first, where, ctx.config.file));
-	}
-	for (const planned of plannedAttempts) {
-		const why = unenforceable(planned.egress, planned.sandbox, where);
-		if (why) throw new UsageError(why);
-		const mcp = refusedMcp(planned.harness, planned.environment.layers);
-		if (mcp) throw new UsageError(`${where} on ${harnessId(planned.harness)}: ${mcp}`);
-	}
-	const path = ctx.file ? relativeFile(ctx.config.root, ctx.file) : null;
-	const rankFile = relativeFile(ctx.config.root, ctx.file ?? "");
-	const family = declaration.family && {
-		id: evalId(rankFile, [...groups, declaration.family.name]),
-		...declaration.family
-	};
-	const harnessBy = resolved.sources?.harness ?? ctx.config.file;
-	const plan = evalPlan(resolvedEval, plannedAttempts, {
-		configFile: ctx.config.file,
-		file: path
-	});
-	const setByOf = (planned) => ({
-		sandbox: sandboxSetBy(planned, resolved, ctx.config.file),
-		harness: harnessBy
-	});
-	if (resolvedEval.skip) {
-		const [first] = expand(resolved);
-		if (!first) return [];
-		const attempt = {
-			name,
-			groups,
-			tags: resolved.tags,
-			cell: first.cell,
-			n: 1,
-			label: "",
-			harness: first.harness,
-			sandbox: first.sandbox,
-			setBy: setByOf(first),
-			gate: "none",
-			brokered: [],
-			secretVars: [],
-			egress: first.egress,
-			plan,
-			...family ? { family } : {}
-		};
-		return [{
-			name: `${name} [${attemptId(first.cell.key, 1)}]`,
-			mode: "skip",
-			attempt
-		}];
-	}
-	const cells = [];
-	return plannedAttempts.map((planned) => {
-		if (!cells.includes(planned.cell.key)) cells.push(planned.cell.key);
-		const workspace = workspaceOf(planned.environment.layers);
-		const files = filesOf(planned.environment.layers);
-		const setBy = setByOf(planned);
-		const id = evalId(rankFile, [...groups, name]);
-		const key = attemptKey(id, planned.cell.key, planned.n);
-		const skip = ctx.completed?.includes(key) ? "already completed in this run" : void 0;
-		const attempt = {
-			name,
-			groups,
-			tags: resolved.tags,
-			cell: planned.cell,
-			n: planned.n,
-			label: planned.label,
-			harness: planned.harness,
-			sandbox: planned.sandbox,
-			setBy,
-			gate: planned.gate,
-			...planned.minLift !== void 0 ? { minLift: planned.minLift } : {},
-			brokered: brokeredIn(resolved.env, planned),
-			secretVars: secretVarsIn(resolved.env, planned),
-			egress: planned.egress,
-			...workspace ? { workspace } : {},
-			...files.length ? { files } : {},
-			plan: skip ? {
-				...plan,
-				skip
-			} : plan,
-			...skip ? { skip } : {},
-			...family ? { family } : {}
-		};
-		return {
-			name: `${name} [${attemptId(planned.cell.key, planned.n)}]`,
-			mode: skip ? "skip" : mode,
-			attempt,
-			timeout: skip ? void 0 : timeoutMs + setupAllowance(planned) + GRACE_MS,
-			gate: skip ? void 0 : {
-				limits: attemptLimits(planned, ctx.concurrency),
-				rank: {
-					eval: id,
-					file: rankFile,
-					order: ctx.order,
-					n: planned.n,
-					cell: cells.indexOf(planned.cell.key)
-				}
-			},
-			spec: skip ? void 0 : {
-				...planned,
-				name,
-				groups,
-				timeoutMs,
-				env: resolved.env,
-				judge: resolved.judge,
-				capture: resolved.capture,
-				sources: {
-					...resolved.sources,
-					sandbox: setBy.sandbox
-				},
-				notes: resolvedEval.notes
-			}
-		};
-	});
-}
-/** The run-wide cap, plus the sandbox provider's and harness's own caps when they set one. */
-function attemptLimits(planned, concurrency) {
-	const limits = { attempts: concurrency };
-	const { sandbox, harness } = planned;
-	if (sandbox.maxConcurrent) limits[`sandbox:${sandbox.kind}`] = sandbox.maxConcurrent;
-	if (harness.maxConcurrent) limits[`harness:${harness.kind}`] = harness.maxConcurrent;
-	return limits;
-}
-//#endregion
-//#region src/evaluate.ts
-let groupStack = [];
-/** Each eval file's declarations as `evaluate` and `group` were called, and each eval's body by id. */
-const trees = /* @__PURE__ */ new Map();
-const bodies = /* @__PURE__ */ new Map();
-/** Options as the tree holds them; from TypeScript they may still hold function forms. */
-const data = (options) => options;
-function declare(node) {
-	const parent = groupStack.at(-1)?.node;
-	if (parent) return void parent.children.push(node);
-	const file = collectingFile() ?? "";
-	const tree = trees.get(file) ?? { children: [] };
-	trees.set(file, tree);
-	tree.children.push(node);
-}
-/** `(name, fn)`, `(name, options, fn)` as in Vitest, or `(name, fn, options)`. */
-function splitArgs(what, name, a, b) {
-	if (typeof a === "function" && typeof b === "function") throw new UsageError(`${what}("${name}") got two functions. Options must be a plain object; helpers such as secret are only available in defineConfig and defineEnvironment callbacks. Put the secret in an environment (environment: …) or in the config's env.`);
-	if (typeof a === "function") return [b ?? {}, a];
-	if (typeof b === "function") return [a ?? {}, b];
-	throw new UsageError(what === "group" ? `group("${name}") has no function. Pass the group's body, e.g. group("${name}", () => { … }).` : `${what}("${name}") has no function. Pass the eval's body, e.g. ${what}("${name}", async (c) => { … }).`);
-}
-/** `evaluate.only` and `group.only` are refused in CI, where they'd silently skip the rest. */
-function checkOnly(mode, what) {
-	if (mode !== "only" || !inCI()) return;
-	const error = /* @__PURE__ */ new Error(`${what}.only isn't allowed in CI`);
-	error.name = "OnlyInCiError";
-	throw error;
-}
-const defineGroup = (mode) => (name, a, b) => {
-	checkOnly(mode, "group");
-	const [options, fn] = splitArgs("group", name, a, b);
-	checkOptions(options, `group("${name}")`);
-	const node = {
-		type: "group",
-		name,
-		mode,
-		options: data(options),
-		children: []
-	};
-	declare(node);
-	const stack = [...groupStack, {
-		name,
-		options,
-		node
-	}];
-	registerGroup(name, mode, () => {
-		const outer = groupStack;
-		groupStack = stack;
-		try {
-			fn();
-		} finally {
-			groupStack = outer;
-		}
-	});
-};
-const group = Object.assign(defineGroup("run"), {
-	skip: defineGroup("skip"),
-	only: defineGroup("only"),
-	todo: (name) => registerTodo(name)
-});
-/**
-* A placeholder, `evaluate.todo` or `group.todo`: one eval entry that never runs, counted as
-* skipped and shown as todo. The enclosing groups' tags filter it as they do an eval.
-*/
-function registerTodo(name) {
-	declare({
-		type: "todo",
-		name
-	});
-	const tags = [...new Set(groupStack.flatMap((g) => g.options.tags ?? []))];
-	if (matchesTags(tags, getRunContext().tags)) registerTest(name, { mode: "todo" }, async () => {});
-}
-/** The evals declared so far, by file and name path, so a second one with the same id is refused. */
-const declared = /* @__PURE__ */ new Set();
-function checkUnique(where, groups, name, hint) {
-	const key = `${collectingFile() ?? ""}\0${[...groups, name].join("\0")}`;
-	if (declared.has(key)) throw new UsageError(`${where} is declared twice in this file. Each eval needs its own name, or its attempts are merged into one eval.${hint}`);
-	declared.add(key);
-}
-function register(mode, name, own, fn, hint = "", family) {
-	const groups = groupStack.map((g) => g.name);
-	checkUnique(`evaluate("${[...groups, name].join(" › ")}")`, groups, name, hint);
-	const body = `b${bodies.size + 1}`;
-	bodies.set(body, fn);
-	declare({
-		type: "eval",
-		name,
-		mode,
-		options: data(own),
-		body,
-		...family ? { family } : {}
-	});
-	const tests = planEval({
-		name,
-		mode,
-		groups: groupStack.map(({ name, options }) => ({
-			name,
-			options
-		})),
-		options: own,
-		...family ? { family } : {}
-	}, {
-		...getRunContext(),
-		file: collectingFile(),
-		order: registered
-	});
-	if (!tests) return;
-	registered++;
-	for (const { name: testName, spec, ...options } of tests) registerTest(testName, options, spec ? (handle) => runAttempt(spec, fn, handle) : async () => {});
-}
-/** Evals registered so far in this worker, for the scheduler's tie-break. */
-let registered = 0;
-const defineEval = (mode) => (name, a, b) => {
-	checkOnly(mode, "evaluate");
-	const [own, fn] = splitArgs("evaluate", name, a, b);
-	register(mode, name, own, fn);
-};
-/** `$key` is a property of the case, `%s` the case itself, `%#` its index. */
-function formatName(template, item, index) {
-	return template.replace(/%#/g, String(index)).replace(/%s/g, typeof item === "string" ? item : JSON.stringify(item)).replace(/\$([A-Za-z_][A-Za-z0-9_]*)/g, (whole, key) => typeof item === "object" && item !== null && key in item ? String(item[key]) : whole);
-}
-function checkEachOptions(options) {
-	if (options === void 0) return {};
-	const keys = typeof options === "object" && options !== null ? Object.keys(options) : null;
-	const pool = options?.pool;
-	if (keys?.every((k) => k === "pool") && (pool === void 0 || typeof pool === "boolean")) return options;
-	throw new UsageError(`evaluate.each(cases, options) takes { pool?: boolean } (got ${JSON.stringify(options)}). Eval options such as compare go after the name: evaluate.each(cases)(name, { compare }, fn).`);
-}
-/**
-* One eval per case, pooled into one suite:
-* `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
-*/
-const evalEach = (cases, options) => (name, a, b) => {
-	const { pool = true } = checkEachOptions(options);
-	if (!Array.isArray(cases)) throw new UsageError(`evaluate.each takes an array of cases (got ${typeof cases}), e.g. evaluate.each([{ n: 1 }, { n: 2 }])("case $n", async (c, item) => { … }).`);
-	const [own, fn] = splitArgs("evaluate.each", name, a, b);
-	const hint = ` For evaluate.each, put $key, %s or %# in the name ("${name}") so each case's name differs.`;
-	const pooled = pool && cases.length >= 2;
-	cases.forEach((item, i) => {
-		const family = pooled ? {
-			name,
-			case: i,
-			cases: cases.length
-		} : void 0;
-		register("run", formatName(name, item, i), own, (c) => fn(c, item), hint, family);
-	});
-};
-const evaluate = Object.assign(defineEval("run"), {
-	skip: defineEval("skip"),
-	only: defineEval("only"),
-	todo: registerTodo,
-	each: evalEach
-});
-//#endregion
+import { t as HarnessCapabilityError } from "./assert-EUIoRdIt.js";
+import { c as defineEnvironment, n as DEFAULT_SKILL_EXCLUDE, t as DEFAULT_DIRECTORY_IGNORES } from "./environment-DNjRk0nU.js";
+import { s as defineConfig } from "./config-DEgnUiU-.js";
+import { t as DEFAULT_EGRESS_ALLOW } from "./egress-C4PZTSSe.js";
+import { l as expect } from "./runtime-PX_oXiCS.js";
+import { n as evaluate, r as group } from "./evaluate-D4zeRGBF.js";
 export { DEFAULT_DIRECTORY_IGNORES, DEFAULT_EGRESS_ALLOW, DEFAULT_SKILL_EXCLUDE, HarnessCapabilityError, defineConfig, defineEnvironment, evaluate, expect, group };
-
-//# sourceMappingURL=index.js.map