zidane 5.9.1 → 5.9.2

package/dist/eval.js

@@ -0,0 +1,1323 @@
+import { n as createProcessContext } from "./contexts-BD2U_xpi.js";
+import { n as runHeadless, t as headlessEventToJsonl } from "./headless-C596K-Lk.js";
+import { i as createMemoryStore, t as createSession } from "./session-7CKYn9qT.js";
+import { join, relative, resolve } from "node:path";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import chalk from "chalk";
+//#region src/eval.ts
+/**
+* Declare an eval's metric set. Returns the same map, typed; pass it to
+* `EvalCaseOptions.metrics`. Declared metrics that are not emitted are recorded
+* as missing with a normalized score of 0; emitting an undeclared metric is an
+* authoring error and throws.
+*/
+function defineMetrics(spec) {
+	return spec;
+}
+/** Always-available efficiency metrics derived from `HeadlessResult`. */
+const EFFICIENCY_METRICS = {
+	"execution-time": {
+		min: 0,
+		max: 3e5,
+		direction: "lower-is-better",
+		tags: ["@efficiency"],
+		description: "Wall-clock run time (ms)."
+	},
+	"provider-tokens": {
+		min: 0,
+		max: 2e5,
+		direction: "lower-is-better",
+		tags: ["@efficiency"],
+		description: "Input + output tokens."
+	},
+	"cache-read-rate": {
+		min: 0,
+		max: 1,
+		direction: "higher-is-better",
+		tags: ["@efficiency"],
+		description: "Cache-read tokens / input tokens."
+	}
+};
+function normalizeMetric(raw, spec) {
+	const span = spec.max - spec.min;
+	if (span === 0) return 1;
+	const normalized = (Math.min(spec.max, Math.max(spec.min, raw)) - spec.min) / span;
+	return spec.direction === "lower-is-better" ? 1 - normalized : normalized;
+}
+/**
+* Thrown by `runEvalCase` when a scorer emits a metric id that was not declared
+* in `defineMetrics`. This is an authoring error (a typo'd or stale metric id),
+* not a low score — it fails the eval test rather than being recorded as a
+* failed scorer. Exported so downstream harnesses can type-narrow on it.
+*/
+var EvalMetricError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "EvalMetricError";
+	}
+};
+/**
+* Wrap an execution context so repeated agent runs share one handle until the
+* caller disposes it. This is useful for eval fixtures that want per-test
+* Docker/process lifetime while still using `runHeadless()` per turn.
+*/
+function createReusableExecutionContext(base) {
+	let handle;
+	let spawnPromise;
+	let disposed = false;
+	let disposePromise;
+	const ensureHandle = (config) => {
+		if (disposed) throw new Error("Reusable execution context has been disposed");
+		if (handle) return Promise.resolve(handle);
+		spawnPromise ??= base.spawn(config).then((next) => {
+			handle = next;
+			return next;
+		}, (err) => {
+			spawnPromise = void 0;
+			throw err;
+		});
+		return spawnPromise;
+	};
+	return {
+		execution: {
+			...base,
+			async spawn(config) {
+				return ensureHandle(config);
+			},
+			async destroy() {}
+		},
+		dispose() {
+			disposePromise ??= (async () => {
+				disposed = true;
+				const active = handle ?? await spawnPromise?.catch(() => void 0);
+				handle = void 0;
+				spawnPromise = void 0;
+				if (active) await base.destroy(active);
+			})();
+			return disposePromise;
+		},
+		handle: () => handle
+	};
+}
+/**
+* Multi-turn eval agent over the low-level headless runner. It keeps session
+* and execution lifetime outside any Playwright/Bolt-specific fixture layer,
+* so downstream platforms can wrap it with their own `agent.run(...)` shape.
+*/
+function createEvalAgent(options) {
+	const { cwd, execution, session: initialSession, store: initialStore, mcpServers: baseMcpServers, onEvent: baseOnEvent, ...headlessDefaults } = options;
+	const executionOwner = createReusableExecutionContext(execution ?? createProcessContext({ cwd }));
+	const store = initialStore ?? createMemoryStore();
+	let sessionPromise = initialSession ? Promise.resolve(initialSession) : void 0;
+	let disposed = false;
+	let runInFlight = false;
+	let disposePromise;
+	const stats = {
+		runs: 0,
+		input: 0,
+		output: 0,
+		cacheRead: 0,
+		cacheCreation: 0,
+		cost: 0,
+		turns: 0,
+		toolCalls: 0,
+		durationMs: 0
+	};
+	async function getSession() {
+		if (disposed) throw new Error("Eval agent has been disposed");
+		sessionPromise ??= createSession({ store });
+		return sessionPromise;
+	}
+	async function resolveMcpServers(runServers) {
+		const merged = [...(typeof baseMcpServers === "function" ? await baseMcpServers() : baseMcpServers) ?? [], ...runServers ?? []];
+		return merged.length > 0 ? [...merged] : void 0;
+	}
+	async function run(runOptions) {
+		if (runInFlight) throw new Error("Eval agent already has a run in progress");
+		runInFlight = true;
+		try {
+			const session = await getSession();
+			const before = session.turns.length;
+			const { mcpServers: runMcpServers, onEvent: runOnEvent, ...restRunOptions } = runOptions;
+			const mcpServers = await resolveMcpServers(runMcpServers);
+			const result = await runHeadless({
+				...headlessDefaults,
+				...restRunOptions,
+				session,
+				execution: executionOwner.execution,
+				...mcpServers ? { mcpServers } : {},
+				onEvent(event) {
+					baseOnEvent?.(event);
+					runOnEvent?.(event);
+				}
+			});
+			const newTranscript = result.transcript.slice(before);
+			const runStats = {
+				input: result.usage.input,
+				output: result.usage.output,
+				cacheRead: result.usage.cacheRead,
+				cacheCreation: result.usage.cacheCreation,
+				cost: result.usage.cost ?? 0,
+				durationMs: result.durationMs,
+				turns: result.turns,
+				toolCalls: countToolCallsInTurns(newTranscript)
+			};
+			accumulateEvalAgentStats(stats, runStats);
+			return {
+				result,
+				stats: runStats,
+				newTranscript
+			};
+		} finally {
+			runInFlight = false;
+		}
+	}
+	function dispose() {
+		disposePromise ??= (async () => {
+			disposed = true;
+			await executionOwner.dispose();
+		})();
+		return disposePromise;
+	}
+	return {
+		run,
+		stats,
+		session: getSession,
+		dispose,
+		[Symbol.asyncDispose]: dispose
+	};
+}
+/**
+* Build an ordered, grouped trajectory from a transcript. Walks assistant
+* turns in order, emitting `think` / `text` / `tool:<name>` steps, grouping
+* only consecutive identical kinds. Per-step duration is approximated from the
+* owning turn's `createdAt` delta to the next turn.
+*/
+function buildTrajectory(transcript) {
+	const steps = [];
+	let total = 0;
+	const push = (kind, name, durationMs) => {
+		total++;
+		const last = steps[steps.length - 1];
+		if (last && last.kind === kind && last.name === name) {
+			last.count++;
+			if (durationMs > 0) last.durationMs = (last.durationMs ?? 0) + durationMs;
+			return;
+		}
+		steps.push({
+			kind,
+			...name ? { name } : {},
+			count: 1,
+			...durationMs > 0 ? { durationMs } : {}
+		});
+	};
+	for (let i = 0; i < transcript.length; i++) {
+		const turn = transcript[i];
+		if (turn.role !== "assistant") continue;
+		const next = transcript[i + 1];
+		const turnDuration = next ? Math.max(0, next.createdAt - turn.createdAt) : 0;
+		const blocks = turn.content.filter((b) => b.type === "thinking" || b.type === "redacted_thinking" || b.type === "text" || b.type === "tool_call");
+		const toolBlocks = blocks.filter((b) => b.type === "tool_call").length;
+		const perTool = toolBlocks > 0 ? Math.round(turnDuration / toolBlocks) : 0;
+		for (const block of blocks) if (block.type === "thinking" || block.type === "redacted_thinking") push("think", void 0, 0);
+		else if (block.type === "text") push("text", void 0, 0);
+		else if (block.type === "tool_call") push("tool", block.name, perTool);
+	}
+	return {
+		steps,
+		totalBlocks: total
+	};
+}
+/** Render a trajectory as a one-line timeline string (no color). */
+function formatTrajectoryLine(trajectory) {
+	return trajectory.steps.map(stepLabel).join(" → ");
+}
+function stepLabel(step) {
+	const base = step.kind === "tool" ? step.name ?? "tool" : step.kind;
+	return step.count > 1 ? `${base} ×${step.count}` : base;
+}
+const evalRegistry = [];
+const STUB_PROVIDER = {
+	name: "stub",
+	meta: { defaultModel: "stub" },
+	formatTools: (tools) => tools,
+	userMessage: () => ({
+		role: "user",
+		content: []
+	}),
+	assistantMessage: () => ({
+		role: "assistant",
+		content: []
+	}),
+	toolResultsMessage: () => ({
+		role: "user",
+		content: []
+	}),
+	stream: async () => {
+		throw new Error("stub provider is not runnable");
+	}
+};
+/**
+* Register an eval definition. Eval files call this at module load; the harness
+* (`buildRegisteredEvals`) materializes them with a provider at run time. The
+* factory keeps definitions lazy so a single registry serves any provider/judge
+* pairing without re-importing eval files.
+*
+* Returns the same factory so a file can also `export default` it for direct,
+* registry-free use (e.g. hermetic unit tests).
+*/
+function defineEval(define) {
+	const id = safeRegistryId(define);
+	if (!evalRegistry.some((entry) => entry.id === id)) evalRegistry.push({
+		id,
+		define
+	});
+	return define;
+}
+/** Snapshot of every registered eval, materialized for one provider/judge pair. */
+function buildRegisteredEvals(ctx) {
+	return evalRegistry.map((entry) => entry.define(ctx));
+}
+/** Drop every registered eval — test isolation helper. */
+function clearRegisteredEvals() {
+	evalRegistry.length = 0;
+}
+function safeRegistryId(define) {
+	try {
+		const stub = define({
+			provider: STUB_PROVIDER,
+			judge: STUB_PROVIDER
+		});
+		return `${stub.suite ?? "eval"}/${stub.id}`;
+	} catch {
+		return `eval/${evalRegistry.length}`;
+	}
+}
+const DEFAULT_JUDGE_SYSTEM = [
+	"You are a strict software eval judge.",
+	"Grade only the supplied output against the rubric.",
+	"Call the `submit_grade` tool with your score and reasoning.",
+	"The score must be between 0 and 1."
+].join("\n");
+/** Forced-output tool the judge must call — the schema-enforcement path. */
+const JUDGE_TOOL = {
+	name: "submit_grade",
+	description: "Submit the grade for the output under evaluation.",
+	inputSchema: {
+		type: "object",
+		properties: {
+			score: {
+				type: "number",
+				minimum: 0,
+				maximum: 1,
+				description: "Quality score from 0 (worst) to 1 (best)."
+			},
+			reasoning: {
+				type: "string",
+				description: "Brief justification for the score."
+			},
+			feedback: {
+				type: "string",
+				description: "Optional actionable feedback."
+			}
+		},
+		required: ["score", "reasoning"],
+		additionalProperties: false
+	}
+};
+async function runEvalCase(options) {
+	const { id, suite, tags = [], artifactDir, workspace, scorers = [], onEvent, ...headless } = options;
+	const { metrics: declaredMetrics, sourceFile, ...headlessRest } = headless;
+	const events = [];
+	const caseArtifactDir = artifactDir ? join(artifactDir, safeSegment(suite ?? "eval"), safeSegment(id)) : void 0;
+	const workspaceState = await prepareWorkspace(headlessRest.execution, workspace);
+	const caseMetricIds = new Set(Object.keys(declaredMetrics ?? {}));
+	const hasCaseMetrics = caseMetricIds.size > 0;
+	const metricSpecs = {
+		...EFFICIENCY_METRICS,
+		...declaredMetrics
+	};
+	const rawMetrics = /* @__PURE__ */ new Map();
+	const emitMetric = (metricId, raw) => {
+		if (!(metricId in metricSpecs)) throw new EvalMetricError(`Eval ${id}: metric "${metricId}" emitted but not declared in defineMetrics`);
+		if (!Number.isFinite(raw)) throw new EvalMetricError(`Eval ${id}: metric "${metricId}" emitted a non-finite value (${raw}); scores require finite numbers`);
+		rawMetrics.set(metricId, raw);
+	};
+	try {
+		const result = await runHeadless({
+			...headlessRest,
+			...workspaceState.execution ? { execution: workspaceState.execution } : {},
+			onEvent(event) {
+				events.push(event);
+				onEvent?.(event);
+			}
+		});
+		emitEfficiencyMetrics(emitMetric, result);
+		const workspaceSnapshot = workspaceState.snapshot();
+		const workspaceError = workspaceState.error();
+		const scores = await runScorers({
+			id,
+			...suite ? { suite } : {},
+			tags,
+			result,
+			events,
+			metric: emitMetric,
+			...workspaceSnapshot ? { workspace: workspaceSnapshot } : {},
+			...caseArtifactDir ? { artifactDir: caseArtifactDir } : {}
+		}, scorers);
+		const metrics = finalizeEvalMetrics(metricSpecs, rawMetrics);
+		const tagScores = computeEvalTagScores(metrics);
+		const caseMetrics = metrics.filter((metric) => caseMetricIds.has(metric.id));
+		const score = hasCaseMetrics && caseMetrics.length > 0 ? meanNormalized(caseMetrics) : aggregateScore(scores);
+		const evalResult = {
+			id,
+			...suite ? { suite } : {},
+			tags,
+			result,
+			score,
+			passed: result.status === "completed" && scores.every((score) => score.passed),
+			scores,
+			metrics,
+			tagScores,
+			trajectory: buildTrajectory(result.transcript),
+			events,
+			...sourceFile ? { sourceFile } : {},
+			...workspaceSnapshot ? { workspace: workspaceSnapshot } : {},
+			...workspaceError ? { workspaceError } : {}
+		};
+		const artifacts = caseArtifactDir ? await writeEvalArtifacts(caseArtifactDir, evalResult) : void 0;
+		return {
+			...evalResult,
+			...artifacts ? { artifacts } : {}
+		};
+	} finally {
+		await workspaceState.cleanup();
+	}
+}
+function statusCompleted(name = "status.completed") {
+	return ({ result }) => ({
+		name,
+		passed: result.status === "completed",
+		score: result.status === "completed" ? 1 : 0,
+		details: result.status === "completed" ? void 0 : {
+			status: result.status,
+			error: result.error
+		}
+	});
+}
+function fileExists(path, name = `file.exists:${path}`) {
+	return ({ workspace }) => {
+		const found = workspace?.files.some((file) => file.path === path) ?? false;
+		return {
+			name,
+			passed: found,
+			score: found ? 1 : 0,
+			details: found ? void 0 : `Missing ${path}`
+		};
+	};
+}
+function fileExistsOneOf(paths, name = `file.existsOneOf:${paths.join("|")}`) {
+	return ({ workspace }) => {
+		const found = paths.find((path) => workspace?.files.some((file) => file.path === path));
+		return {
+			name,
+			passed: Boolean(found),
+			score: found ? 1 : 0,
+			details: found ? void 0 : `Missing one of: ${paths.join(", ")}`
+		};
+	};
+}
+function fileContains(path, expected, name = `file.contains:${path}`) {
+	return ({ workspace }) => {
+		const file = workspace?.files.find((file) => file.path === path);
+		const content = file?.content ?? "";
+		const passed = typeof expected === "string" ? content.includes(expected) : expected.test(content);
+		return {
+			name,
+			passed,
+			score: passed ? 1 : 0,
+			details: passed ? void 0 : file ? `Expected ${path} to contain ${String(expected)}` : `Missing ${path}`
+		};
+	};
+}
+function fileContentQuality(path, expected, name = `file.quality:${path}`) {
+	return ({ workspace }) => {
+		const content = (workspace?.files.find((file) => file.path === path))?.content;
+		if (content === void 0) return {
+			name,
+			passed: false,
+			score: 0,
+			details: `Missing ${path}`
+		};
+		const normalized = content.trim();
+		const expectedNormalized = expected.trim();
+		const score = normalized === expectedNormalized ? 1 : normalized.includes(expectedNormalized) ? .75 : content.includes(expected) ? .5 : 0;
+		return {
+			name,
+			passed: score >= .75,
+			score,
+			details: score >= .75 ? void 0 : `Expected ${path} to closely match ${JSON.stringify(expectedNormalized)}`
+		};
+	};
+}
+function llmJudge(options) {
+	const name = options.name ?? "llm.judge";
+	const model = options.model ?? options.provider.meta.defaultModel;
+	return async (ctx) => {
+		const userMessage = options.provider.userMessage(renderJudgePrompt({
+			rubric: options.rubric,
+			input: options.input?.(ctx) ?? defaultJudgeInput(ctx)
+		}));
+		let text = "";
+		const controller = new AbortController();
+		const timeoutMs = options.timeoutMs ?? 6e4;
+		let timedOut = false;
+		const timer = timeoutMs > 0 ? setTimeout(() => {
+			timedOut = true;
+			controller.abort(/* @__PURE__ */ new Error(`LLM judge timed out after ${timeoutMs}ms`));
+		}, timeoutMs) : void 0;
+		try {
+			const streamPromise = options.provider.stream({
+				model,
+				system: options.system ?? DEFAULT_JUDGE_SYSTEM,
+				tools: options.provider.formatTools([JUDGE_TOOL]),
+				messages: [userMessage],
+				maxTokens: options.maxTokens ?? 600,
+				cache: false,
+				signal: controller.signal,
+				toolChoice: {
+					type: "tool",
+					name: JUDGE_TOOL.name
+				}
+			}, { onText(delta) {
+				text += delta;
+			} });
+			const timeoutPromise = timeoutMs > 0 ? new Promise((_resolve, reject) => {
+				controller.signal.addEventListener("abort", () => {
+					if (timedOut) reject(/* @__PURE__ */ new Error(`LLM judge timed out after ${timeoutMs}ms`));
+				}, { once: true });
+			}) : void 0;
+			const result = timeoutPromise ? await Promise.race([streamPromise, timeoutPromise]) : await streamPromise;
+			const parsed = readJudgeResult(result.toolCalls, text || result.text);
+			if (options.metric) ctx.metric(options.metric, parsed.score);
+			return {
+				name,
+				passed: parsed.score >= .7,
+				score: parsed.score,
+				details: {
+					reasoning: parsed.reasoning,
+					...parsed.feedback ? { feedback: parsed.feedback } : {},
+					...parsed.raw ? { raw: parsed.raw } : {},
+					usage: result.usage
+				}
+			};
+		} finally {
+			if (timer) clearTimeout(timer);
+		}
+	};
+}
+function readJudgeResult(toolCalls, fallbackText) {
+	const call = toolCalls.find((tc) => tc.name === JUDGE_TOOL.name);
+	if (call) {
+		const input = call.input;
+		return {
+			score: clampScore(input.score),
+			reasoning: typeof input.reasoning === "string" ? input.reasoning : "",
+			...typeof input.feedback === "string" ? { feedback: input.feedback } : {}
+		};
+	}
+	return parseJudgeOutput(fallbackText);
+}
+/**
+* Aggregate a group of boolean scorers into a single `0..1` functionality
+* metric (fraction passed) and emit it. Useful for "N of M files present".
+*/
+function functionalityMetric(metricId, scorers, name = metricId) {
+	return async (ctx) => {
+		const results = await runScorers(ctx, scorers);
+		const passed = results.filter((r) => r.passed).length;
+		const fraction = results.length === 0 ? 1 : passed / results.length;
+		ctx.metric(metricId, fraction);
+		return {
+			name,
+			passed: passed === results.length,
+			score: fraction,
+			details: {
+				passed,
+				total: results.length,
+				checks: results.map((r) => ({
+					name: r.name,
+					passed: r.passed
+				}))
+			}
+		};
+	};
+}
+function formatEvalCaseSummary(result) {
+	const lines = [
+		`Eval ${result.suite ? `${result.suite}/` : ""}${result.id}`,
+		`status: ${result.result.status}`,
+		`passed: ${result.passed}`,
+		`score: ${result.score.toFixed(2)}`,
+		`duration: ${result.result.durationMs}ms`,
+		`turns: ${result.result.turns}`,
+		`tool calls: ${result.result.numToolCalls}`,
+		`usage: input=${result.result.usage.input} output=${result.result.usage.output} cacheRead=${result.result.usage.cacheRead} cacheCreation=${result.result.usage.cacheCreation}${result.result.usage.cost !== void 0 ? ` cost=$${result.result.usage.cost.toFixed(6)}` : ""}`
+	];
+	if (result.workspace) lines.push(`workspace files: ${result.workspace.files.length}`);
+	if (result.workspaceError) lines.push(`workspace error: ${result.workspaceError}`);
+	if (result.artifacts) lines.push(`artifacts: ${result.artifacts.dir}`);
+	if (result.scores.length > 0) {
+		lines.push("scores:");
+		for (const score of result.scores) {
+			const value = score.score !== void 0 ? score.score.toFixed(2) : score.passed ? "1.00" : "0.00";
+			lines.push(`  - ${score.name}: ${value} ${score.passed ? "pass" : "fail"}`);
+			if (typeof score.details === "string" && score.details.length > 0) lines.push(`    ${score.details}`);
+			else if (isRecord(score.details) && typeof score.details.reasoning === "string") lines.push(`    ${score.details.reasoning}`);
+		}
+	}
+	return lines.join("\n");
+}
+function formatEvalRunSummary(results) {
+	return formatEvalRunSummaryWithOptions(results);
+}
+/**
+* Register one test per eval case against a test runner, funnel every result
+* into a shared reporter, and print the aggregated summary once after all cases.
+*
+* A failing test means the agent run itself broke (provider/tool/timeout). Low
+* scores are reported, not asserted — see {@link RegisterEvalTestsOptions.failOnIncomplete}.
+*/
+function registerEvalTests(options) {
+	const reporter = options.reporter ?? createEvalRunReporter();
+	const failOnIncomplete = options.failOnIncomplete ?? true;
+	const printSummary = options.printSummary ?? true;
+	const repeat = Math.max(1, options.repeat ?? 1);
+	const concurrency = Math.max(1, options.concurrency ?? 1);
+	options.runner.afterAll(async () => {
+		await reporter.flush();
+		if (printSummary) console.log(`\n${reporter.format()}\n`);
+		if (options.dispose) await options.dispose();
+	});
+	const work = [];
+	for (const evalCase of options.cases) {
+		const baseLabel = `${evalCase.suite ? `${evalCase.suite}/` : ""}${evalCase.id}`;
+		for (let run = 1; run <= repeat; run++) {
+			const label = repeat > 1 ? `${baseLabel} #${run}` : baseLabel;
+			const caseId = repeat > 1 ? `${evalCase.id}-repeat-${run}` : evalCase.id;
+			work.push({
+				label,
+				run: () => runEvalCase({
+					...evalCase,
+					id: caseId,
+					...options.artifactDir ? { artifactDir: options.artifactDir } : {}
+				})
+			});
+		}
+	}
+	const schedule = createLazyPool(concurrency);
+	work.forEach((w) => {
+		options.runner.it(w.label, async () => {
+			const result = await schedule(w.run);
+			await reporter.record(result);
+			if (failOnIncomplete && result.result.status !== "completed") throw new Error(`Eval ${w.label} did not complete: ${JSON.stringify(result.result.error ?? { status: result.result.status }, null, 2)}`);
+		});
+	});
+	return reporter;
+}
+/**
+* Lazily run tasks with a bounded concurrency `limit`. Nothing starts until a
+* runner invokes the corresponding test callback, so filtered/skipped tests do
+* not leak model or Docker work in the background.
+*/
+function createLazyPool(limit) {
+	const queue = [];
+	let active = 0;
+	const drain = () => {
+		const available = Math.min(limit - active, queue.length);
+		for (let i = 0; i < available; i++) queue.shift()?.();
+	};
+	return (task) => new Promise((resolve, reject) => {
+		const run = async () => {
+			active++;
+			try {
+				resolve(await task());
+			} catch (err) {
+				reject(err);
+			} finally {
+				active--;
+				drain();
+			}
+		};
+		queue.push(() => void run());
+		drain();
+	});
+}
+function createEvalRunReporter(options = {}) {
+	const results = [];
+	return {
+		get results() {
+			return results;
+		},
+		async record(result) {
+			results.push(result);
+			if (options.outputDir) {
+				const casePath = join(options.outputDir, "cases", `${safeSegment(result.suite ?? "eval")}--${safeSegment(result.id)}.json`);
+				await mkdir(join(options.outputDir, "cases"), { recursive: true });
+				await writeFile(casePath, `${JSON.stringify(result, null, 2)}\n`);
+			}
+		},
+		async flush() {
+			const summary = buildEvalRunSummary(results);
+			if (options.outputDir) {
+				await mkdir(options.outputDir, { recursive: true });
+				await writeFile(join(options.outputDir, "run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);
+			}
+			return summary;
+		},
+		format() {
+			return formatEvalRunSummaryWithOptions(results, { color: options.color });
+		}
+	};
+}
+/** Stable ordering by `suite/id` so parallel completion order doesn't shuffle output. */
+function sortCases(results) {
+	return [...results].sort((a, b) => {
+		const ka = `${a.suite ?? ""}/${a.id}`;
+		const kb = `${b.suite ?? ""}/${b.id}`;
+		return ka.localeCompare(kb);
+	});
+}
+function buildEvalRunSummary(input) {
+	const results = sortCases(input);
+	const usage = results.reduce((acc, result) => {
+		acc.input += result.result.usage.input;
+		acc.output += result.result.usage.output;
+		acc.cacheRead += result.result.usage.cacheRead;
+		acc.cacheCreation += result.result.usage.cacheCreation;
+		acc.cost += result.result.usage.cost ?? 0;
+		return acc;
+	}, {
+		input: 0,
+		output: 0,
+		cacheRead: 0,
+		cacheCreation: 0,
+		cost: 0
+	});
+	const totalScore = results.reduce((sum, result) => sum + result.score, 0);
+	return {
+		count: results.length,
+		passed: results.filter((result) => result.passed).length,
+		score: results.length > 0 ? totalScore / results.length : 0,
+		durationMs: results.reduce((sum, result) => sum + result.result.durationMs, 0),
+		usage,
+		cases: results.map((result) => ({
+			id: result.id,
+			...result.suite ? { suite: result.suite } : {},
+			passed: result.passed,
+			score: result.score,
+			status: result.result.status,
+			durationMs: result.result.durationMs,
+			scores: result.scores,
+			metrics: result.metrics,
+			tagScores: result.tagScores,
+			trajectory: result.trajectory
+		})),
+		metrics: aggregateMetrics(results),
+		tagScores: aggregateTagScores(results)
+	};
+}
+function aggregateMetrics(results) {
+	const byId = /* @__PURE__ */ new Map();
+	for (const result of results) for (const metric of result.metrics) {
+		const list = byId.get(metric.id) ?? [];
+		list.push(metric);
+		byId.set(metric.id, list);
+	}
+	return [...byId.entries()].map(([id, metrics]) => ({
+		id,
+		direction: metrics[0].direction,
+		tags: metrics[0].tags,
+		raw: metricStats(metrics.map((m) => m.raw)),
+		normalized: metricStats(metrics.map((m) => m.normalized))
+	}));
+}
+function aggregateTagScores(results) {
+	const byTag = /* @__PURE__ */ new Map();
+	for (const result of results) for (const [tag, value] of Object.entries(result.tagScores)) {
+		const list = byTag.get(tag) ?? [];
+		list.push(value);
+		byTag.set(tag, list);
+	}
+	const out = {};
+	for (const [tag, values] of byTag) out[tag] = mean(values);
+	return out;
+}
+function metricStats(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	return {
+		mean: mean(values),
+		min: sorted[0] ?? 0,
+		max: sorted[sorted.length - 1] ?? 0,
+		p50: percentile(sorted, .5),
+		p90: percentile(sorted, .9),
+		zeroCount: values.filter((v) => v === 0).length,
+		values
+	};
+}
+function percentile(sorted, q) {
+	if (sorted.length === 0) return 0;
+	return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1))];
+}
+function formatEvalRunSummaryWithOptions(input, options = {}) {
+	const results = sortCases(input);
+	const color = createEvalColors(options.color);
+	if (results.length === 0) return `${color.heading("Eval run summary")}\n${color.muted("no evals ran")}`;
+	const summary = buildEvalRunSummary(results);
+	const out = [];
+	out.push(color.heading("Eval run summary"));
+	out.push(renderResultsTable(results, summary, color));
+	const tagTable = renderTagTable(summary.tagScores, color);
+	if (tagTable) {
+		out.push("");
+		out.push(tagTable);
+	}
+	out.push("");
+	const hyperlinks = options.color !== false;
+	const linkText = (target) => {
+		const abs = absPath(target);
+		return hyperlinks ? oscLink(abs, abs) : abs;
+	};
+	for (const result of results) {
+		const label = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
+		out.push(`${color.dot(result.passed)} ${color.caseStatus(result.passed, label)}`);
+		out.push(`${color.muted("score")} ${color.score(result.score)}`);
+		const tagEntries = Object.entries(result.tagScores);
+		if (tagEntries.length > 0) out.push(`${color.muted("tags:")} ${tagEntries.map(([tag, value]) => `${color.tag(tag)} ${color.score(value)}`).join("  ")}`);
+		for (const metric of result.metrics) {
+			const note = metric.missing ? color.fail(" (not emitted)") : color.muted(` (${formatMetricRaw(metric)})`);
+			out.push(`${color.muted(padEnd(metric.id, 28))} ${color.score(metric.normalized)}${note}`);
+		}
+		for (const score of result.scores) {
+			const detail = scoreDetailLine(score);
+			if (!score.passed && detail) out.push(`${color.dot(score.passed)} ${color.muted(`${score.name}: ${truncate(detail, 140)}`)}`);
+		}
+		if (result.sourceFile) out.push(`${color.muted("case:")} ${color.muted(linkText(result.sourceFile))}`);
+		const resultTarget = caseLinkTarget(result);
+		if (resultTarget) out.push(`${color.muted("result:")} ${color.muted(linkText(resultTarget))}`);
+		if (result.trajectory.steps.length > 0) out.push(`${color.muted(`turns: ${result.trajectory.totalBlocks}`)} ${formatTrajectoryColored(result.trajectory, color)}`);
+		out.push("");
+	}
+	return out.join("\n").replace(/\n+$/, "");
+}
+function renderTagTable(tagScores, color) {
+	const entries = Object.entries(tagScores);
+	if (entries.length === 0) return void 0;
+	const tagWidth = Math.max(3, ...entries.map(([tag]) => tag.length));
+	const top = `┌${"─".repeat(tagWidth + 2)}┬${"─".repeat(9)}┐`;
+	const mid = `├${"─".repeat(tagWidth + 2)}┼${"─".repeat(9)}┤`;
+	const bot = `└${"─".repeat(tagWidth + 2)}┴${"─".repeat(9)}┘`;
+	return [
+		top,
+		`│ ${color.heading(padEnd("TAG", tagWidth))} │ ${color.heading("SCORE")}   │`,
+		mid,
+		...entries.map(([tag, value]) => `│ ${color.muted(padEnd(tag, tagWidth))} │ ${color.score(value)}    │`),
+		bot
+	].join("\n");
+}
+function formatMetricRaw(metric) {
+	return `raw ${Number.isInteger(metric.raw) ? formatInt(metric.raw) : metric.raw.toFixed(3)} ${metric.direction === "lower-is-better" ? "↓" : "↑"}`;
+}
+/**
+* Resolve a filesystem target to link an eval case to. Prefers the written
+* artifact (`result.json` → its dir), so the link opens the case's output.
+* Returns `undefined` when no artifacts were written.
+*/
+function caseLinkTarget(result) {
+	return result.artifacts?.result ?? result.artifacts?.dir;
+}
+/** Wrap text in an OSC 8 terminal hyperlink to a local path or URL. */
+function oscLink(target, text) {
+	const url = /^(?:file|https?):\/\//.test(target) ? target : `file://${resolve(target)}`;
+	const OSC = "\x1B]8;;";
+	const ST = "\x1B\\";
+	return `${OSC}${url}${ST}${text}${OSC}${ST}`;
+}
+/** Absolute filesystem path (strips any `file://` prefix, resolves relatives). */
+function absPath(target) {
+	return resolve(target.replace(/^file:\/\//, ""));
+}
+function renderResultsTable(results, summary, color) {
+	const header = [
+		"EVAL",
+		"STATUS",
+		"SCORE",
+		"IN",
+		"OUT",
+		"CACHE R",
+		"CACHE W",
+		"COST",
+		"TIME"
+	];
+	const rows = results.map((result) => {
+		const label = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
+		const u = result.result.usage;
+		return {
+			passed: result.passed,
+			cells: [
+				label,
+				result.passed ? "pass" : "fail",
+				result.score.toFixed(2),
+				formatInt(u.input),
+				formatInt(u.output),
+				formatInt(u.cacheRead),
+				formatInt(u.cacheCreation),
+				formatCost(u.cost ?? 0),
+				formatDuration(result.result.durationMs)
+			],
+			score: result.score
+		};
+	});
+	const totals = {
+		cells: [
+			`TOTAL (${summary.passed}/${summary.count})`,
+			summary.passed === summary.count ? "pass" : "fail",
+			summary.score.toFixed(2),
+			formatInt(summary.usage.input),
+			formatInt(summary.usage.output),
+			formatInt(summary.usage.cacheRead),
+			formatInt(summary.usage.cacheCreation),
+			formatCost(summary.usage.cost),
+			formatDuration(summary.durationMs)
+		],
+		passed: summary.passed === summary.count,
+		score: summary.score
+	};
+	const aligns = [
+		"left",
+		"left",
+		"right",
+		"right",
+		"right",
+		"right",
+		"right",
+		"right",
+		"right"
+	];
+	const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r.cells[i].length), totals.cells[i].length));
+	const top = `┌${widths.map((w) => "─".repeat(w + 2)).join("┬")}┐`;
+	const mid = `├${widths.map((w) => "─".repeat(w + 2)).join("┼")}┤`;
+	const bot = `└${widths.map((w) => "─".repeat(w + 2)).join("┴")}┘`;
+	const renderRow = (cells, paint) => {
+		return `│${cells.map((cell, i) => {
+			const padded = aligns[i] === "right" ? padStart(cell, widths[i]) : padEnd(cell, widths[i]);
+			return ` ${paint ? paint(i, cell, padded) : padded} `;
+		}).join("│")}│`;
+	};
+	return [
+		top,
+		renderRow(header, (_i, _raw, padded) => color.heading(padded)),
+		mid,
+		...rows.map((r) => renderRow(r.cells, (i, raw, padded) => paintCell(i, raw, padded, r.passed, r.score, color))),
+		mid,
+		renderRow(totals.cells, (i, raw, padded) => paintCell(i, raw, padded, totals.passed, totals.score, color, true)),
+		bot
+	].join("\n");
+}
+function paintCell(col, raw, padded, passed, score, color, bold = false) {
+	if (col === 0) return bold ? color.heading(padded) : color.caseStatus(passed, padded);
+	if (col === 1) return padded.replace(raw, color.status(passed));
+	if (col === 2) return padded.replace(raw, color.score(score));
+	if (col === 7) return color.cost(padded);
+	return color.muted(padded);
+}
+function scoreDetailLine(score) {
+	if (typeof score.details === "string") return score.details;
+	if (isRecord(score.details) && typeof score.details.reasoning === "string") return score.details.reasoning;
+}
+function formatInt(value) {
+	return value.toLocaleString("en-US");
+}
+function formatCost(value) {
+	return value > 0 ? `$${value.toFixed(4)}` : "-";
+}
+function formatDuration(ms) {
+	const rounded = Math.round(ms);
+	if (rounded < 1e3) return `${rounded}ms`;
+	const s = rounded / 1e3;
+	return `${Number.isInteger(s) ? s : s.toFixed(1)}s`;
+}
+function padEnd(value, width) {
+	return value.length >= width ? value : value + " ".repeat(width - value.length);
+}
+function padStart(value, width) {
+	return value.length >= width ? value : " ".repeat(width - value.length) + value;
+}
+function truncate(value, max) {
+	const oneLine = value.replace(/\s+/g, " ").trim();
+	return oneLine.length > max ? `${oneLine.slice(0, max - 1)}…` : oneLine;
+}
+async function prepareWorkspace(execution, workspace) {
+	if (!workspace) return {
+		snapshot: () => void 0,
+		error: () => void 0,
+		cleanup: async () => {}
+	};
+	let tempDir;
+	let baseExecution = execution;
+	if (!baseExecution) {
+		const cwd = workspace.cwd ?? await mkdtemp(join(tmpdir(), "zidane-eval-"));
+		if (!workspace.cwd) tempDir = cwd;
+		baseExecution = createProcessContext({ cwd });
+	}
+	const captured = {};
+	return {
+		execution: withWorkspaceLifecycle(baseExecution, workspace, captured),
+		snapshot: () => captured.snapshot,
+		error: () => captured.error,
+		cleanup: async () => {
+			if (tempDir && !workspace.retain) await rm(tempDir, {
+				recursive: true,
+				force: true
+			});
+		}
+	};
+}
+function withWorkspaceLifecycle(execution, workspace, captured) {
+	return {
+		...execution,
+		async spawn(config) {
+			const handle = await execution.spawn({
+				...config,
+				...workspace.cwd && !config?.cwd ? { cwd: workspace.cwd } : {}
+			});
+			if (workspace.seedDir) await seedWorkspace(execution, handle, workspace);
+			return handle;
+		},
+		async destroy(handle) {
+			try {
+				captured.snapshot = await captureWorkspace(execution, handle, workspace);
+			} catch (err) {
+				captured.error = err instanceof Error ? err.message : String(err);
+			} finally {
+				await execution.destroy(handle);
+			}
+		}
+	};
+}
+async function seedWorkspace(execution, handle, workspace) {
+	const target = workspace.seedTarget ?? ".";
+	const result = await execution.exec(handle, `mkdir -p ${shellQuote(target)} && cp -R ${shellQuote(`${workspace.seedDir}/.`)} ${shellQuote(target)}`);
+	if (result.exitCode !== 0) throw new Error(`Failed to seed eval workspace: ${result.stderr || result.stdout}`);
+}
+async function captureWorkspace(execution, handle, workspace) {
+	const capturePaths = workspace.capture?.length ? workspace.capture : ["."];
+	const command = [
+		`cd ${shellQuote(handle.cwd)}`,
+		"&&",
+		"for p in",
+		capturePaths.map(shellQuote).join(" "),
+		"; do",
+		"if [ -e \"$p\" ]; then",
+		"find \"$p\" \\( -path \"*/node_modules/*\" -o -path \"*/.git/*\" \\) -prune -o -type f -print0",
+		"; fi",
+		"done"
+	].join(" ");
+	const listed = await execution.exec(handle, command);
+	if (listed.exitCode !== 0) throw new Error(`Failed to list eval workspace: ${listed.stderr || listed.stdout}`);
+	const maxFileChars = workspace.maxFileChars ?? 256 * 1024;
+	const files = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const rawPath of listed.stdout.split("\0").filter(Boolean)) {
+		const path = normalizeSnapshotPath(rawPath);
+		if (seen.has(path)) continue;
+		seen.add(path);
+		const content = await execution.readFile(handle, path);
+		const binary = content.includes("\0");
+		const truncated = content.length > maxFileChars;
+		files.push({
+			path,
+			size: content.length,
+			...binary ? { binary: true } : { content: truncated ? content.slice(0, maxFileChars) : content },
+			...truncated ? { truncated: true } : {}
+		});
+	}
+	return {
+		cwd: handle.cwd,
+		files
+	};
+}
+async function runScorers(ctx, scorers) {
+	const scores = [];
+	for (const scorer of scorers) try {
+		scores.push(await scorer(ctx));
+	} catch (err) {
+		if (err instanceof EvalMetricError) throw err;
+		scores.push({
+			name: scorer.name || "anonymous-scorer",
+			passed: false,
+			score: 0,
+			details: err instanceof Error ? err.message : String(err)
+		});
+	}
+	return scores;
+}
+function aggregateScore(scores) {
+	if (scores.length === 0) return 1;
+	return scores.reduce((sum, score) => sum + (score.score ?? (score.passed ? 1 : 0)), 0) / scores.length;
+}
+function efficiencyMetricValues(result) {
+	const u = result.usage;
+	const readTotal = u.input + u.cacheRead;
+	return {
+		"execution-time": result.durationMs,
+		"provider-tokens": u.input + u.output,
+		"cache-read-rate": readTotal > 0 ? u.cacheRead / readTotal : 0
+	};
+}
+function emitEfficiencyMetrics(emit, result) {
+	const values = efficiencyMetricValues(result);
+	for (const [id, raw] of Object.entries(values)) emit(id, raw);
+}
+function finalizeEvalMetrics(specs, raw) {
+	return Object.entries(specs).map(([metricId, spec]) => {
+		const emitted = raw.has(metricId);
+		const value = emitted ? raw.get(metricId) : spec.min;
+		return {
+			id: metricId,
+			raw: emitted ? value : 0,
+			normalized: emitted ? normalizeMetric(value, spec) : 0,
+			direction: spec.direction,
+			min: spec.min,
+			max: spec.max,
+			tags: spec.tags ?? [],
+			...spec.description ? { description: spec.description } : {},
+			...emitted ? {} : { missing: true }
+		};
+	});
+}
+function computeEvalTagScores(metrics) {
+	const byTag = /* @__PURE__ */ new Map();
+	for (const metric of metrics) for (const tag of metric.tags) {
+		const list = byTag.get(tag) ?? [];
+		list.push(metric.normalized);
+		byTag.set(tag, list);
+	}
+	const out = {};
+	for (const [tag, values] of byTag) out[tag] = mean(values);
+	return out;
+}
+function meanNormalized(metrics) {
+	return metrics.length === 0 ? 0 : mean(metrics.map((m) => m.normalized));
+}
+function accumulateEvalAgentStats(acc, turn) {
+	acc.runs += 1;
+	acc.input += turn.input;
+	acc.output += turn.output;
+	acc.cacheRead += turn.cacheRead;
+	acc.cacheCreation += turn.cacheCreation;
+	acc.cost += turn.cost;
+	acc.turns += turn.turns;
+	acc.toolCalls += turn.toolCalls;
+	acc.durationMs += turn.durationMs;
+}
+function countToolCallsInTurns(turns) {
+	let count = 0;
+	for (const turn of turns) {
+		if (turn.role !== "assistant") continue;
+		count += turn.content.filter((block) => block.type === "tool_call").length;
+	}
+	return count;
+}
+function mean(values) {
+	return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null;
+}
+function createEvalColors(enabled = shouldUseColor()) {
+	if (!enabled) return {
+		heading: (value) => value,
+		pass: (value) => value,
+		fail: (value) => value,
+		muted: (value) => value,
+		cost: (value) => value,
+		status: (passed) => passed ? "pass" : "fail",
+		score: (value) => value.toFixed(2),
+		caseStatus: (_passed, value) => value,
+		dot: (passed) => passed ? "+" : "x",
+		tag: (value) => value,
+		step: (_kind, value) => value
+	};
+	return {
+		heading: (value) => chalk.bold(value),
+		pass: (value) => chalk.green(value),
+		fail: (value) => chalk.red(value),
+		muted: (value) => chalk.gray(value),
+		cost: (value) => chalk.cyan(value),
+		status: (passed) => passed ? chalk.green("pass") : chalk.red("fail"),
+		score: (value) => value >= .9 ? chalk.green(value.toFixed(2)) : value >= .7 ? chalk.yellow(value.toFixed(2)) : chalk.red(value.toFixed(2)),
+		caseStatus: (passed, value) => passed ? chalk.green(value) : chalk.red(value),
+		dot: (passed) => passed ? chalk.green("●") : chalk.red("●"),
+		tag: (value) => chalk.bold(chalk.whiteBright(value)),
+		step: (kind, value) => paintStep(kind, value)
+	};
+}
+/** Per-kind trajectory colors: think=magenta, text=blue, tools by family. */
+function paintStep(kind, value) {
+	if (kind === "think") return chalk.magenta(value);
+	if (kind === "text") return chalk.blue(value);
+	if (/write|edit|create/.test(value)) return chalk.green(value);
+	if (/read|list|glob|grep|search/.test(value)) return chalk.cyan(value);
+	if (/shell|bash|exec|run/.test(value)) return chalk.yellow(value);
+	if (/spawn|task|agent/.test(value)) return chalk.magentaBright(value);
+	return chalk.white(value);
+}
+function formatTrajectoryColored(trajectory, color) {
+	return trajectory.steps.map((step) => {
+		const base = step.kind === "tool" ? step.name ?? "tool" : step.kind;
+		const labelWithCount = step.count > 1 ? `${base} ×${step.count}` : base;
+		const painted = color.step(step.kind, labelWithCount);
+		const timing = step.durationMs && step.durationMs >= 100 ? color.muted(`(${formatDuration(step.durationMs)})`) : "";
+		return timing ? `${painted} ${timing}` : painted;
+	}).join(color.muted(" → "));
+}
+function shouldUseColor() {
+	return Boolean(process.stdout.isTTY && !process.env.NO_COLOR);
+}
+function renderJudgePrompt(input) {
+	return [
+		"Rubric:",
+		input.rubric,
+		"",
+		"Output to grade:",
+		input.input,
+		"",
+		"Return JSON only."
+	].join("\n");
+}
+function defaultJudgeInput(ctx) {
+	const chunks = [`Final answer:\n${ctx.result.finalText}`];
+	if (ctx.workspace) chunks.push(`Workspace files:\n${ctx.workspace.files.map((file) => [`--- ${file.path}${file.truncated ? " (truncated)" : ""} ---`, file.binary ? "[binary omitted]" : file.content ?? ""].join("\n")).join("\n")}`);
+	return chunks.join("\n\n");
+}
+function parseJudgeOutput(raw) {
+	const trimmed = stripCodeFences(raw.trim());
+	const jsonText = extractJsonObject(trimmed);
+	if (jsonText) try {
+		const parsed = JSON.parse(jsonText);
+		if (parsed.score !== void 0 || parsed.reasoning !== void 0) return {
+			score: clampScore(parsed.score),
+			reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
+			...typeof parsed.feedback === "string" ? { feedback: parsed.feedback } : {},
+			...jsonText === trimmed ? {} : { raw: trimmed }
+		};
+	} catch {}
+	const scoreFragment = trimmed.match(/score["']?\s*[:=]\s*(-?\d+(?:\.\d+)?)/i);
+	if (scoreFragment) return {
+		score: clampScore(Number(scoreFragment[1])),
+		reasoning: trimmed.slice(0, 500),
+		raw: trimmed
+	};
+	const bareFraction = trimmed.match(/^(-?\d+(?:\.\d+)?)\s*\/\s*(\d+(?:\.\d+)?)$/);
+	if (bareFraction) {
+		const denom = Number(bareFraction[2]);
+		return {
+			score: clampScore(denom === 0 ? 0 : Number(bareFraction[1]) / denom),
+			reasoning: trimmed,
+			raw: trimmed
+		};
+	}
+	if (trimmed.match(/^-?\d+(?:\.\d+)?$/)) return {
+		score: clampScore(Number(trimmed)),
+		reasoning: "",
+		raw: trimmed
+	};
+	return {
+		score: 0,
+		reasoning: "Judge did not return a parseable score.",
+		raw: trimmed
+	};
+}
+/** Strip a leading/trailing markdown code fence (``` or ```json). */
+function stripCodeFences(text) {
+	const fence = text.match(/^```(?:json)?[ \t]*\n([\s\S]*?)\n```$/i);
+	return fence ? fence[1].trim() : text;
+}
+/** Extract the first balanced `{...}` object, ignoring braces inside strings. */
+function extractJsonObject(text) {
+	const start = text.indexOf("{");
+	if (start === -1) return void 0;
+	let depth = 0;
+	let inString = false;
+	let escaped = false;
+	for (let i = start; i < text.length; i++) {
+		const ch = text[i];
+		if (inString) {
+			if (escaped) escaped = false;
+			else if (ch === "\\") escaped = true;
+			else if (ch === "\"") inString = false;
+			continue;
+		}
+		if (ch === "\"") inString = true;
+		else if (ch === "{") depth++;
+		else if (ch === "}") {
+			depth--;
+			if (depth === 0) return text.slice(start, i + 1);
+		}
+	}
+}
+function clampScore(value) {
+	const n = typeof value === "number" ? value : Number(value);
+	if (!Number.isFinite(n)) return 0;
+	return Math.max(0, Math.min(1, n));
+}
+async function writeEvalArtifacts(dir, result) {
+	await mkdir(dir, { recursive: true });
+	const resultPath = join(dir, "result.json");
+	const eventsPath = join(dir, "events.jsonl");
+	const transcriptPath = join(dir, "transcript.json");
+	const workspacePath = result.workspace ? join(dir, "workspace.json") : void 0;
+	await writeFile(resultPath, `${JSON.stringify(result, null, 2)}\n`);
+	await writeFile(eventsPath, result.events.map(headlessEventToJsonl).join(""));
+	await writeFile(transcriptPath, `${JSON.stringify(result.result.transcript, null, 2)}\n`);
+	if (workspacePath) await writeFile(workspacePath, `${JSON.stringify(result.workspace, null, 2)}\n`);
+	return {
+		dir,
+		result: resultPath,
+		events: eventsPath,
+		transcript: transcriptPath,
+		...workspacePath ? { workspace: workspacePath } : {}
+	};
+}
+function normalizeSnapshotPath(path) {
+	const normalized = path.startsWith("./") ? path.slice(2) : path;
+	return normalized === "" ? "." : normalized;
+}
+function safeSegment(value) {
+	const sanitized = value.replace(/[^\w.-]+/g, "-").replace(/^-+|-+$/g, "");
+	if (sanitized === value && sanitized.length > 0) return sanitized;
+	return `${sanitized || "eval"}-${shortHash(value)}`;
+}
+function shortHash(value) {
+	let hash = 2166136261;
+	for (let i = 0; i < value.length; i++) {
+		hash ^= value.charCodeAt(i);
+		hash = Math.imul(hash, 16777619);
+	}
+	return (hash >>> 0).toString(36).padStart(7, "0").slice(0, 7);
+}
+function shellQuote(value) {
+	return `'${value.replace(/'/g, String.raw`'\''`)}'`;
+}
+function artifactPath(root, result) {
+	return resolve(root, safeSegment(result.suite ?? "eval"), safeSegment(result.id));
+}
+function relativeArtifactPath(root, path) {
+	return relative(root, path);
+}
+//#endregion
+export { EFFICIENCY_METRICS, EvalMetricError, artifactPath, buildEvalRunSummary, buildRegisteredEvals, buildTrajectory, clearRegisteredEvals, computeEvalTagScores, createEvalAgent, createEvalRunReporter, createReusableExecutionContext, defineEval, defineMetrics, efficiencyMetricValues, emitEfficiencyMetrics, fileContains, fileContentQuality, fileExists, fileExistsOneOf, finalizeEvalMetrics, formatEvalCaseSummary, formatEvalRunSummary, formatTrajectoryLine, functionalityMetric, llmJudge, normalizeMetric, registerEvalTests, relativeArtifactPath, runEvalCase, statusCompleted };
+
+//# sourceMappingURL=eval.js.map