zidane 7.2.4 → 8.0.4

package/dist/eval-BNcvle_J.js

@@ -0,0 +1,1595 @@
+import { t as alwaysQuote } from "./shell-quote-BmnhZmdM.js";
+import { d as createMemoryStore, t as createSession } from "./session-xh9JXu6i.js";
+import { t as createProcessContext } from "./contexts-d7UolYot.js";
+import { a as headlessEventToJsonl, c as runHeadless } from "./headless-C_rNUuYC.js";
+import { join, relative, resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import chalk from "chalk";
+//#region src/eval-artifacts.ts
+async function writeEvalArtifacts(dir, result) {
+	await mkdir(dir, { recursive: true });
+	const resultPath = join(dir, "result.json");
+	const eventsPath = join(dir, "events.jsonl");
+	const transcriptPath = join(dir, "transcript.json");
+	const workspacePath = result.workspace ? join(dir, "workspace.json") : void 0;
+	await writeFile(resultPath, `${JSON.stringify(result, null, 2)}\n`);
+	await writeFile(eventsPath, result.events.map(headlessEventToJsonl).join(""));
+	await writeFile(transcriptPath, `${JSON.stringify(result.result.transcript, null, 2)}\n`);
+	if (workspacePath) await writeFile(workspacePath, `${JSON.stringify(result.workspace, null, 2)}\n`);
+	return {
+		dir,
+		result: resultPath,
+		events: eventsPath,
+		transcript: transcriptPath,
+		...workspacePath ? { workspace: workspacePath } : {}
+	};
+}
+function safeSegment(value) {
+	const sanitized = value.replace(/[^\w.-]+/g, "-").replace(/^-+|-+$/g, "");
+	if (sanitized === value && sanitized.length > 0) return sanitized;
+	return `${sanitized || "eval"}-${shortHash(value)}`;
+}
+function shortHash(value) {
+	let hash = 2166136261;
+	for (let i = 0; i < value.length; i++) {
+		hash ^= value.charCodeAt(i);
+		hash = Math.imul(hash, 16777619);
+	}
+	return (hash >>> 0).toString(36).padStart(7, "0").slice(0, 7);
+}
+function artifactPath(root, result) {
+	return resolve(root, safeSegment(result.suite ?? "eval"), safeSegment(result.id));
+}
+function relativeArtifactPath(root, path) {
+	return relative(root, path);
+}
+//#endregion
+//#region src/eval-metrics.ts
+/**
+* Declare an eval's metric set. Returns the same map, typed; pass it to
+* `EvalCaseOptions.metrics`. Declared metrics that are not emitted are recorded
+* as missing with a normalized score of 0; emitting an undeclared metric is an
+* authoring error and throws.
+*/
+function defineMetrics(spec) {
+	return spec;
+}
+/** Always-available efficiency metrics derived from `HeadlessResult`. */
+const EFFICIENCY_METRICS = {
+	"execution-time": {
+		min: 0,
+		max: 3e5,
+		direction: "lower-is-better",
+		tags: ["@efficiency"],
+		description: "Wall-clock run time (ms)."
+	},
+	"provider-tokens": {
+		min: 0,
+		max: 2e5,
+		direction: "lower-is-better",
+		tags: ["@efficiency"],
+		description: "Input + output tokens."
+	},
+	"cache-read-rate": {
+		min: 0,
+		max: 1,
+		direction: "higher-is-better",
+		tags: ["@efficiency"],
+		description: "Cache-read tokens / input tokens."
+	}
+};
+function normalizeMetric(raw, spec) {
+	const span = spec.max - spec.min;
+	if (span === 0) return 1;
+	const normalized = (Math.min(spec.max, Math.max(spec.min, raw)) - spec.min) / span;
+	return spec.direction === "lower-is-better" ? 1 - normalized : normalized;
+}
+/**
+* Thrown by `runEvalCase` when a scorer emits a metric id that was not declared
+* in `defineMetrics`. This is an authoring error (a typo'd or stale metric id),
+* not a low score — it fails the eval test rather than being recorded as a
+* failed scorer. Exported so downstream harnesses can type-narrow on it.
+*/
+var EvalMetricError = class extends Error {
+	constructor(message) {
+		super(message);
+		this.name = "EvalMetricError";
+	}
+};
+function efficiencyMetricValues(result) {
+	const u = result.usage;
+	const readTotal = u.input + u.cacheRead;
+	return {
+		"execution-time": result.durationMs,
+		"provider-tokens": u.input + u.output,
+		"cache-read-rate": readTotal > 0 ? u.cacheRead / readTotal : 0
+	};
+}
+function emitEfficiencyMetrics(emit, result) {
+	const values = efficiencyMetricValues(result);
+	for (const [id, raw] of Object.entries(values)) emit(id, raw);
+}
+function finalizeEvalMetrics(specs, raw) {
+	return Object.entries(specs).map(([metricId, spec]) => {
+		const emitted = raw.has(metricId);
+		const value = emitted ? raw.get(metricId) : spec.min;
+		return {
+			id: metricId,
+			raw: emitted ? value : 0,
+			normalized: emitted ? normalizeMetric(value, spec) : 0,
+			direction: spec.direction,
+			min: spec.min,
+			max: spec.max,
+			tags: spec.tags ?? [],
+			...spec.description ? { description: spec.description } : {},
+			...emitted ? {} : { missing: true }
+		};
+	});
+}
+function computeEvalTagScores(metrics) {
+	const byTag = /* @__PURE__ */ new Map();
+	for (const metric of metrics) for (const tag of metric.tags) {
+		const list = byTag.get(tag) ?? [];
+		list.push(metric.normalized);
+		byTag.set(tag, list);
+	}
+	const out = {};
+	for (const [tag, values] of byTag) out[tag] = mean(values);
+	return out;
+}
+function meanNormalized(metrics) {
+	return metrics.length === 0 ? 0 : mean(metrics.map((m) => m.normalized));
+}
+function mean(values) {
+	return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
+}
+//#endregion
+//#region src/eval-summary.ts
+/** Stable ordering by `variant`, then `suite/id`, so parallel completion order doesn't shuffle output. */
+function sortCases(results) {
+	return [...results].sort((a, b) => {
+		const va = a.variant ?? "";
+		const vb = b.variant ?? "";
+		if (va !== vb) return va.localeCompare(vb);
+		const ka = `${a.suite ?? ""}/${a.id}`;
+		const kb = `${b.suite ?? ""}/${b.id}`;
+		return ka.localeCompare(kb);
+	});
+}
+/** Distinct variant labels across results, in sorted order. */
+function distinctVariants(results) {
+	return [...new Set(results.map((r) => r.variant).filter((v) => Boolean(v)))].sort();
+}
+function buildEvalRunSummary(input) {
+	const results = sortCases(input);
+	const usage = results.reduce((acc, result) => {
+		acc.input += result.result.usage.input;
+		acc.output += result.result.usage.output;
+		acc.cacheRead += result.result.usage.cacheRead;
+		acc.cacheCreation += result.result.usage.cacheCreation;
+		acc.cost += result.result.usage.cost ?? 0;
+		return acc;
+	}, {
+		input: 0,
+		output: 0,
+		cacheRead: 0,
+		cacheCreation: 0,
+		cost: 0
+	});
+	const totalScore = results.reduce((sum, result) => sum + result.score, 0);
+	return {
+		count: results.length,
+		passed: results.filter((result) => result.passed).length,
+		score: results.length > 0 ? totalScore / results.length : 0,
+		durationMs: results.reduce((sum, result) => sum + result.result.durationMs, 0),
+		usage,
+		cases: results.map((result) => ({
+			id: result.id,
+			...result.suite ? { suite: result.suite } : {},
+			...result.variant ? { variant: result.variant } : {},
+			passed: result.passed,
+			score: result.score,
+			status: result.result.status,
+			durationMs: result.result.durationMs,
+			scores: result.scores,
+			metrics: result.metrics,
+			tagScores: result.tagScores,
+			trajectory: result.trajectory
+		})),
+		metrics: aggregateMetrics(results),
+		tagScores: aggregateTagScores(results),
+		...distinctVariants(results).length > 0 ? { variants: aggregateVariants(results) } : {}
+	};
+}
+function aggregateVariants(results) {
+	return distinctVariants(results).map((variant) => {
+		const group = results.filter((r) => r.variant === variant);
+		const usage = group.reduce((acc, result) => {
+			acc.input += result.result.usage.input;
+			acc.output += result.result.usage.output;
+			acc.cacheRead += result.result.usage.cacheRead;
+			acc.cacheCreation += result.result.usage.cacheCreation;
+			acc.cost += result.result.usage.cost ?? 0;
+			return acc;
+		}, {
+			input: 0,
+			output: 0,
+			cacheRead: 0,
+			cacheCreation: 0,
+			cost: 0
+		});
+		return {
+			variant,
+			count: group.length,
+			passed: group.filter((r) => r.passed).length,
+			score: mean(group.map((r) => r.score)),
+			durationMs: group.reduce((sum, r) => sum + r.result.durationMs, 0),
+			usage,
+			turns: group.reduce((sum, r) => sum + r.result.turns, 0),
+			toolCalls: group.reduce((sum, r) => sum + r.result.numToolCalls, 0)
+		};
+	});
+}
+function aggregateMetrics(results) {
+	const byId = /* @__PURE__ */ new Map();
+	for (const result of results) for (const metric of result.metrics) {
+		const list = byId.get(metric.id) ?? [];
+		list.push(metric);
+		byId.set(metric.id, list);
+	}
+	return [...byId.entries()].map(([id, metrics]) => ({
+		id,
+		direction: metrics[0].direction,
+		tags: metrics[0].tags,
+		raw: metricStats(metrics.map((m) => m.raw)),
+		normalized: metricStats(metrics.map((m) => m.normalized))
+	}));
+}
+function aggregateTagScores(results) {
+	const byTag = /* @__PURE__ */ new Map();
+	for (const result of results) for (const [tag, value] of Object.entries(result.tagScores)) {
+		const list = byTag.get(tag) ?? [];
+		list.push(value);
+		byTag.set(tag, list);
+	}
+	const out = {};
+	for (const [tag, values] of byTag) out[tag] = mean(values);
+	return out;
+}
+function metricStats(values) {
+	const sorted = [...values].sort((a, b) => a - b);
+	return {
+		mean: mean(values),
+		min: sorted[0] ?? 0,
+		max: sorted[sorted.length - 1] ?? 0,
+		p50: percentile(sorted, .5),
+		p90: percentile(sorted, .9),
+		zeroCount: values.filter((v) => v === 0).length,
+		values
+	};
+}
+function percentile(sorted, q) {
+	if (sorted.length === 0) return 0;
+	return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(q * sorted.length) - 1))];
+}
+//#endregion
+//#region src/eval-report.ts
+function formatEvalCaseSummary(result) {
+	const lines = [
+		`Eval ${result.suite ? `${result.suite}/` : ""}${result.id}`,
+		...result.variant ? [`variant: ${result.variant}`] : [],
+		`status: ${result.result.status}`,
+		`passed: ${result.passed}`,
+		`score: ${result.score.toFixed(2)}`,
+		`duration: ${result.result.durationMs}ms`,
+		`turns: ${result.result.turns}`,
+		`tool calls: ${result.result.numToolCalls}`,
+		`usage: input=${result.result.usage.input} output=${result.result.usage.output} cacheRead=${result.result.usage.cacheRead} cacheCreation=${result.result.usage.cacheCreation}${result.result.usage.cost !== void 0 ? ` cost=$${result.result.usage.cost.toFixed(6)}` : ""}`
+	];
+	if (result.workspace) lines.push(`workspace files: ${result.workspace.files.length}`);
+	if (result.workspaceError) lines.push(`workspace error: ${result.workspaceError}`);
+	if (result.artifacts) lines.push(`artifacts: ${result.artifacts.dir}`);
+	if (result.scores.length > 0) {
+		lines.push("scores:");
+		for (const score of result.scores) {
+			const value = score.score !== void 0 ? score.score.toFixed(2) : score.passed ? "1.00" : "0.00";
+			lines.push(`  - ${score.name}: ${value} ${score.passed ? "pass" : "fail"}`);
+			if (typeof score.details === "string" && score.details.length > 0) lines.push(`    ${score.details}`);
+			else if (isRecord(score.details) && typeof score.details.reasoning === "string") lines.push(`    ${score.details.reasoning}`);
+		}
+	}
+	return lines.join("\n");
+}
+function formatEvalRunSummary(results) {
+	return formatEvalRunSummaryWithOptions(results);
+}
+function formatEvalRunSummaryWithOptions(input, options = {}) {
+	const results = sortCases(input);
+	const color = createEvalColors(options.color);
+	if (results.length === 0) return `${color.heading("Eval run summary")}\n${color.muted("no evals ran")}`;
+	const summary = buildEvalRunSummary(results);
+	const out = [];
+	const variants = distinctVariants(results);
+	out.push(color.heading("Eval run summary"));
+	if (variants.length > 1) {
+		out.push(renderComparisonTable(results, variants, color));
+		for (const variant of variants) {
+			const group = results.filter((r) => r.variant === variant);
+			out.push("");
+			out.push(color.heading(variant));
+			out.push(renderResultsTable(group, buildEvalRunSummary(group), color));
+		}
+	} else out.push(renderResultsTable(results, summary, color));
+	const tagTable = renderTagTable(summary.tagScores, color);
+	if (tagTable) {
+		out.push("");
+		out.push(tagTable);
+	}
+	out.push("");
+	const hyperlinks = options.color !== false;
+	const linkText = (target) => {
+		const abs = absPath(target);
+		return hyperlinks ? oscLink(abs, abs) : abs;
+	};
+	for (const result of results) {
+		const caseLabel = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
+		const label = result.variant ? `[${result.variant}] ${caseLabel}` : caseLabel;
+		out.push(`${color.dot(result.passed)} ${color.caseStatus(result.passed, label)}`);
+		out.push(`${color.muted("score")} ${color.score(result.score)}`);
+		const tagEntries = Object.entries(result.tagScores);
+		if (tagEntries.length > 0) out.push(`${color.muted("tags:")} ${tagEntries.map(([tag, value]) => `${color.tag(tag)} ${color.score(value)}`).join("  ")}`);
+		for (const metric of result.metrics) {
+			const note = metric.missing ? color.fail(" (not emitted)") : color.muted(` (${formatMetricRaw(metric)})`);
+			out.push(`${color.muted(padEnd(metric.id, 28))} ${color.score(metric.normalized)}${note}`);
+		}
+		for (const score of result.scores) {
+			const detail = scoreDetailLine(score);
+			if (!score.passed && detail) out.push(`${color.dot(score.passed)} ${color.muted(`${score.name}: ${truncate(detail, 140)}`)}`);
+		}
+		if (result.sourceFile) out.push(`${color.muted("case:")} ${color.muted(linkText(result.sourceFile))}`);
+		const resultTarget = caseLinkTarget(result);
+		if (resultTarget) out.push(`${color.muted("result:")} ${color.muted(linkText(resultTarget))}`);
+		if (result.trajectory.steps.length > 0) out.push(`${color.muted(`turns: ${result.trajectory.totalBlocks}`)} ${formatTrajectoryColored(result.trajectory, color)}`);
+		out.push("");
+	}
+	return out.join("\n").replace(/\n+$/, "");
+}
+function renderTagTable(tagScores, color) {
+	const entries = Object.entries(tagScores);
+	if (entries.length === 0) return void 0;
+	const tagWidth = Math.max(3, ...entries.map(([tag]) => tag.length));
+	const top = `┌${"─".repeat(tagWidth + 2)}┬${"─".repeat(9)}┐`;
+	const mid = `├${"─".repeat(tagWidth + 2)}┼${"─".repeat(9)}┤`;
+	const bot = `└${"─".repeat(tagWidth + 2)}┴${"─".repeat(9)}┘`;
+	return [
+		top,
+		`│ ${color.heading(padEnd("TAG", tagWidth))} │ ${color.heading("SCORE")}   │`,
+		mid,
+		...entries.map(([tag, value]) => `│ ${color.muted(padEnd(tag, tagWidth))} │ ${color.score(value)}    │`),
+		bot
+	].join("\n");
+}
+function formatMetricRaw(metric) {
+	return `raw ${Number.isInteger(metric.raw) ? formatInt(metric.raw) : metric.raw.toFixed(3)} ${metric.direction === "lower-is-better" ? "↓" : "↑"}`;
+}
+/**
+* Resolve a filesystem target to link an eval case to. Prefers the written
+* artifact (`result.json` → its dir), so the link opens the case's output.
+* Returns `undefined` when no artifacts were written.
+*/
+function caseLinkTarget(result) {
+	return result.artifacts?.result ?? result.artifacts?.dir;
+}
+/** Wrap text in an OSC 8 terminal hyperlink to a local path or URL. */
+function oscLink(target, text) {
+	const url = /^(?:file|https?):\/\//.test(target) ? target : `file://${resolve(target)}`;
+	const OSC = "\x1B]8;;";
+	const ST = "\x1B\\";
+	return `${OSC}${url}${ST}${text}${OSC}${ST}`;
+}
+/** Absolute filesystem path (strips any `file://` prefix, resolves relatives). */
+function absPath(target) {
+	const clean = target.replace(/^file:\/\//, "");
+	return resolve(clean);
+}
+/**
+* Case key used to line up the same eval across variants in the comparison
+* matrix. Strips the `-repeat-N` suffix `registerEvalTests` appends so a
+* repeated case aggregates into one row (mean score, x/y passed).
+*/
+function comparisonCaseKey(result) {
+	const id = result.id.replace(/-repeat-\d+$/, "");
+	return `${result.suite ? `${result.suite}/` : ""}${id}`;
+}
+/**
+* Cross-variant comparison matrix: one row per eval case, one column per
+* variant (provider/model target), plus rollup rows (score / passed / cost /
+* time) so two models can be read side by side at a glance.
+*/
+function renderComparisonTable(input, variants, color) {
+	const results = sortCases(input);
+	const caseKeys = [...new Set(results.map(comparisonCaseKey))].sort();
+	const byCell = /* @__PURE__ */ new Map();
+	for (const result of results) {
+		const key = `${comparisonCaseKey(result)}\u0000${result.variant ?? ""}`;
+		const list = byCell.get(key) ?? [];
+		list.push(result);
+		byCell.set(key, list);
+	}
+	const cellFor = (caseKey, variant) => {
+		const group = byCell.get(`${caseKey}\u0000${variant}`);
+		if (!group || group.length === 0) return { text: "-" };
+		const score = mean(group.map((r) => r.score));
+		const passedCount = group.filter((r) => r.passed).length;
+		const allPassed = passedCount === group.length;
+		const status = group.length > 1 ? `${passedCount}/${group.length}` : allPassed ? "pass" : "fail";
+		return {
+			text: `${score.toFixed(2)} ${status}`,
+			score,
+			passed: allPassed
+		};
+	};
+	const variantSummaries = aggregateVariants(results);
+	const summaryOf = (variant) => variantSummaries.find((s) => s.variant === variant);
+	const pair = variants.length === 2;
+	const header = [
+		"EVAL",
+		...variants,
+		...pair ? [`Δ ${variants[1]}`] : []
+	];
+	const rows = caseKeys.map((key) => {
+		const cells = variants.map((variant) => cellFor(key, variant));
+		if (pair) cells.push(deltaCell(cells[0]?.score, cells[1]?.score, "higher", "points"));
+		return {
+			label: key,
+			cells
+		};
+	});
+	const rollups = [
+		{
+			label: "SCORE",
+			value: (s) => s.score,
+			format: (v) => v.toFixed(2),
+			better: "higher",
+			delta: "points",
+			paint: (s) => ({ score: s.score })
+		},
+		{
+			label: "PASSED",
+			value: (s) => s.passed,
+			format: (v, s) => `${v}/${s.count}`,
+			better: "higher",
+			delta: "points",
+			paint: (s) => ({ passed: s.passed === s.count })
+		},
+		{
+			label: "COST",
+			value: (s) => s.usage.cost,
+			format: (v) => formatCost(v),
+			better: "lower",
+			delta: "percent"
+		},
+		{
+			label: "TIME",
+			value: (s) => s.durationMs,
+			format: (v) => formatDuration(v),
+			better: "lower",
+			delta: "percent"
+		},
+		{
+			label: "TOKENS",
+			value: (s) => s.usage.input + s.usage.output,
+			format: (v) => formatInt(v),
+			better: "lower",
+			delta: "percent"
+		},
+		{
+			label: "CACHE R",
+			value: (s) => s.usage.cacheRead,
+			format: (v) => formatInt(v),
+			better: "lower",
+			delta: "percent"
+		},
+		{
+			label: "TURNS",
+			value: (s) => s.turns,
+			format: (v) => String(v),
+			better: "lower",
+			delta: "percent"
+		},
+		{
+			label: "TOOL CALLS",
+			value: (s) => s.toolCalls,
+			format: (v) => String(v),
+			better: "lower",
+			delta: "percent"
+		}
+	].map((def) => {
+		const summaries = variants.map(summaryOf);
+		const cells = summaries.map((s) => s ? {
+			text: def.format(def.value(s), s),
+			...def.paint?.(s)
+		} : { text: "-" });
+		if (pair) {
+			const [a, b] = summaries;
+			cells.push(deltaCell(a ? def.value(a) : void 0, b ? def.value(b) : void 0, def.better, def.delta));
+		}
+		return {
+			label: def.label,
+			cells,
+			rollup: true
+		};
+	});
+	const allRows = [...rows, ...rollups];
+	const widths = header.map((h, i) => Math.max(h.length, ...allRows.map((row) => (i === 0 ? row.label : row.cells[i - 1].text).length)));
+	const top = `┌${widths.map((w) => "─".repeat(w + 2)).join("┬")}┐`;
+	const mid = `├${widths.map((w) => "─".repeat(w + 2)).join("┼")}┤`;
+	const bot = `└${widths.map((w) => "─".repeat(w + 2)).join("┴")}┘`;
+	const paintComparisonCell = (cell, padded) => {
+		if (cell.score === void 0 && cell.passed === void 0) return color.muted(padded);
+		if (cell.score !== void 0) {
+			const scoreText = cell.score.toFixed(2);
+			const rest = cell.text.slice(scoreText.length);
+			const coloredRest = cell.passed === void 0 ? color.muted(rest) : cell.passed ? color.pass(rest) : color.fail(rest);
+			return padded.replace(cell.text, `${color.score(cell.score)}${coloredRest}`);
+		}
+		return padded.replace(cell.text, cell.passed ? color.pass(cell.text) : color.fail(cell.text));
+	};
+	const renderRow = (row) => {
+		const labelPadded = padEnd(row.label, widths[0]);
+		return `│ ${row.rollup ? color.heading(labelPadded) : color.muted(labelPadded)} │${row.cells.map((cell, i) => {
+			const padded = padStart(cell.text, widths[i + 1]);
+			return ` ${paintComparisonCell(cell, padded)} `;
+		}).join("│")}│`;
+	};
+	return [
+		top,
+		`│ ${color.heading(padEnd(header[0], widths[0]))} │${header.slice(1).map((v, i) => ` ${color.heading(padStart(v, widths[i + 1]))} `).join("│")}│`,
+		mid,
+		...rows.map(renderRow),
+		mid,
+		...rollups.map(renderRow),
+		bot
+	].join("\n");
+}
+/**
+* The Δ cell of a two-variant comparison: `b` against `a`, as an absolute
+* difference (`points`, for scores and counts of passes) or a relative one
+* (`percent`). Painted pass/fail by whether the change goes the `better` way;
+* unchanged values and missing sides stay muted.
+*/
+function deltaCell(a, b, better, kind) {
+	if (a === void 0 || b === void 0) return { text: "-" };
+	const diff = b - a;
+	if (Math.abs(diff) < 1e-9) return { text: "=" };
+	const sign = diff > 0 ? "+" : "−";
+	const improved = better === "higher" ? diff > 0 : diff < 0;
+	if (kind === "points") {
+		const magnitude = Math.abs(diff);
+		return {
+			text: `${sign}${Number.isInteger(magnitude) ? magnitude : magnitude.toFixed(2)}`,
+			passed: improved
+		};
+	}
+	if (a === 0) return {
+		text: `${sign}∞`,
+		passed: improved
+	};
+	return {
+		text: `${sign}${Math.round(Math.abs(diff / a) * 100)}%`,
+		passed: improved
+	};
+}
+function renderResultsTable(results, summary, color) {
+	const header = [
+		"EVAL",
+		"STATUS",
+		"SCORE",
+		"IN",
+		"OUT",
+		"CACHE R",
+		"CACHE W",
+		"COST",
+		"TIME"
+	];
+	const rows = results.map((result) => {
+		const label = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
+		const u = result.result.usage;
+		return {
+			passed: result.passed,
+			cells: [
+				label,
+				result.passed ? "pass" : "fail",
+				result.score.toFixed(2),
+				formatInt(u.input),
+				formatInt(u.output),
+				formatInt(u.cacheRead),
+				formatInt(u.cacheCreation),
+				formatCost(u.cost ?? 0),
+				formatDuration(result.result.durationMs)
+			],
+			score: result.score
+		};
+	});
+	const totals = {
+		cells: [
+			`TOTAL (${summary.passed}/${summary.count})`,
+			summary.passed === summary.count ? "pass" : "fail",
+			summary.score.toFixed(2),
+			formatInt(summary.usage.input),
+			formatInt(summary.usage.output),
+			formatInt(summary.usage.cacheRead),
+			formatInt(summary.usage.cacheCreation),
+			formatCost(summary.usage.cost),
+			formatDuration(summary.durationMs)
+		],
+		passed: summary.passed === summary.count,
+		score: summary.score
+	};
+	const aligns = [
+		"left",
+		"left",
+		"right",
+		"right",
+		"right",
+		"right",
+		"right",
+		"right",
+		"right"
+	];
+	const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r.cells[i].length), totals.cells[i].length));
+	const top = `┌${widths.map((w) => "─".repeat(w + 2)).join("┬")}┐`;
+	const mid = `├${widths.map((w) => "─".repeat(w + 2)).join("┼")}┤`;
+	const bot = `└${widths.map((w) => "─".repeat(w + 2)).join("┴")}┘`;
+	const renderRow = (cells, paint) => {
+		return `│${cells.map((cell, i) => {
+			const padded = aligns[i] === "right" ? padStart(cell, widths[i]) : padEnd(cell, widths[i]);
+			return ` ${paint ? paint(i, cell, padded) : padded} `;
+		}).join("│")}│`;
+	};
+	return [
+		top,
+		renderRow(header, (_i, _raw, padded) => color.heading(padded)),
+		mid,
+		...rows.map((r) => renderRow(r.cells, (i, raw, padded) => paintCell(i, raw, padded, r.passed, r.score, color))),
+		mid,
+		renderRow(totals.cells, (i, raw, padded) => paintCell(i, raw, padded, totals.passed, totals.score, color, true)),
+		bot
+	].join("\n");
+}
+function paintCell(col, raw, padded, passed, score, color, bold = false) {
+	if (col === 0) return bold ? color.heading(padded) : color.caseStatus(passed, padded);
+	if (col === 1) return padded.replace(raw, color.status(passed));
+	if (col === 2) return padded.replace(raw, color.score(score));
+	if (col === 7) return color.cost(padded);
+	return color.muted(padded);
+}
+function scoreDetailLine(score) {
+	if (typeof score.details === "string") return score.details;
+	if (isRecord(score.details) && typeof score.details.reasoning === "string") return score.details.reasoning;
+}
+function formatInt(value) {
+	return value.toLocaleString("en-US");
+}
+function formatCost(value) {
+	return value > 0 ? `$${value.toFixed(4)}` : "-";
+}
+function formatDuration(ms) {
+	const rounded = Math.round(ms);
+	if (rounded < 1e3) return `${rounded}ms`;
+	const s = rounded / 1e3;
+	return `${Number.isInteger(s) ? s : s.toFixed(1)}s`;
+}
+function padEnd(value, width) {
+	return value.length >= width ? value : value + " ".repeat(width - value.length);
+}
+function padStart(value, width) {
+	return value.length >= width ? value : " ".repeat(width - value.length) + value;
+}
+function truncate(value, max) {
+	const oneLine = value.replace(/\s+/g, " ").trim();
+	return oneLine.length > max ? `${oneLine.slice(0, max - 1)}…` : oneLine;
+}
+function isRecord(value) {
+	return typeof value === "object" && value !== null;
+}
+function createEvalColors(enabled = shouldUseColor()) {
+	if (!enabled) return {
+		heading: (value) => value,
+		pass: (value) => value,
+		fail: (value) => value,
+		muted: (value) => value,
+		cost: (value) => value,
+		status: (passed) => passed ? "pass" : "fail",
+		score: (value) => value.toFixed(2),
+		caseStatus: (_passed, value) => value,
+		dot: (passed) => passed ? "+" : "x",
+		tag: (value) => value,
+		step: (_kind, value) => value
+	};
+	return {
+		heading: (value) => chalk.bold(value),
+		pass: (value) => chalk.green(value),
+		fail: (value) => chalk.red(value),
+		muted: (value) => chalk.gray(value),
+		cost: (value) => chalk.cyan(value),
+		status: (passed) => passed ? chalk.green("pass") : chalk.red("fail"),
+		score: (value) => value >= .9 ? chalk.green(value.toFixed(2)) : value >= .7 ? chalk.yellow(value.toFixed(2)) : chalk.red(value.toFixed(2)),
+		caseStatus: (passed, value) => passed ? chalk.green(value) : chalk.red(value),
+		dot: (passed) => passed ? chalk.green("●") : chalk.red("●"),
+		tag: (value) => chalk.bold(chalk.whiteBright(value)),
+		step: (kind, value) => paintStep(kind, value)
+	};
+}
+/** Per-kind trajectory colors: think=magenta, text=blue, tools by family. */
+function paintStep(kind, value) {
+	if (kind === "think") return chalk.magenta(value);
+	if (kind === "text") return chalk.blue(value);
+	if (/write|edit|create/.test(value)) return chalk.green(value);
+	if (/read|list|glob|grep|search/.test(value)) return chalk.cyan(value);
+	if (/shell|bash|exec|run/.test(value)) return chalk.yellow(value);
+	if (/spawn|task|agent/.test(value)) return chalk.magentaBright(value);
+	return chalk.white(value);
+}
+function formatTrajectoryColored(trajectory, color) {
+	return trajectory.steps.map((step) => {
+		const base = step.kind === "tool" ? step.name ?? "tool" : step.kind;
+		const labelWithCount = step.count > 1 ? `${base} ×${step.count}` : base;
+		const painted = color.step(step.kind, labelWithCount);
+		const timing = step.durationMs && step.durationMs >= 100 ? color.muted(`(${formatDuration(step.durationMs)})`) : "";
+		return timing ? `${painted} ${timing}` : painted;
+	}).join(color.muted(" → "));
+}
+function shouldUseColor() {
+	return Boolean(process.stdout.isTTY && !process.env.NO_COLOR);
+}
+//#endregion
+//#region src/eval-scorers.ts
+function statusCompleted(name = "status.completed") {
+	return ({ result }) => ({
+		name,
+		passed: result.status === "completed",
+		score: result.status === "completed" ? 1 : 0,
+		details: result.status === "completed" ? void 0 : {
+			status: result.status,
+			error: result.error
+		}
+	});
+}
+function fileExists(path, name = `file.exists:${path}`) {
+	return ({ workspace }) => {
+		const found = workspace?.files.some((file) => file.path === path) ?? false;
+		return {
+			name,
+			passed: found,
+			score: found ? 1 : 0,
+			details: found ? void 0 : `Missing ${path}`
+		};
+	};
+}
+function fileExistsOneOf(paths, name = `file.existsOneOf:${paths.join("|")}`) {
+	return ({ workspace }) => {
+		const found = paths.find((path) => workspace?.files.some((file) => file.path === path));
+		return {
+			name,
+			passed: Boolean(found),
+			score: found ? 1 : 0,
+			details: found ? void 0 : `Missing one of: ${paths.join(", ")}`
+		};
+	};
+}
+function fileContains(path, expected, name = `file.contains:${path}`) {
+	return ({ workspace }) => {
+		const file = workspace?.files.find((file) => file.path === path);
+		const content = file?.content ?? "";
+		const passed = typeof expected === "string" ? content.includes(expected) : expected.test(content);
+		return {
+			name,
+			passed,
+			score: passed ? 1 : 0,
+			details: passed ? void 0 : file ? `Expected ${path} to contain ${String(expected)}` : `Missing ${path}`
+		};
+	};
+}
+function fileContentQuality(path, expected, name = `file.quality:${path}`) {
+	return ({ workspace }) => {
+		const content = (workspace?.files.find((file) => file.path === path))?.content;
+		if (content === void 0) return {
+			name,
+			passed: false,
+			score: 0,
+			details: `Missing ${path}`
+		};
+		const normalized = content.trim();
+		const expectedNormalized = expected.trim();
+		const score = normalized === expectedNormalized ? 1 : normalized.includes(expectedNormalized) ? .75 : content.includes(expected) ? .5 : 0;
+		return {
+			name,
+			passed: score >= .75,
+			score,
+			details: score >= .75 ? void 0 : `Expected ${path} to closely match ${JSON.stringify(expectedNormalized)}`
+		};
+	};
+}
+/**
+* Aggregate a group of boolean scorers into a single `0..1` functionality
+* metric (fraction passed) and emit it. Useful for "N of M files present".
+*/
+function functionalityMetric(metricId, scorers, name = metricId) {
+	return async (ctx) => {
+		const results = await runScorers(ctx, scorers);
+		const passed = results.filter((r) => r.passed).length;
+		const fraction = results.length === 0 ? 1 : passed / results.length;
+		ctx.metric(metricId, fraction);
+		return {
+			name,
+			passed: passed === results.length,
+			score: fraction,
+			details: {
+				passed,
+				total: results.length,
+				checks: results.map((r) => ({
+					name: r.name,
+					passed: r.passed
+				}))
+			}
+		};
+	};
+}
+async function runScorers(ctx, scorers) {
+	const scores = [];
+	for (const scorer of scorers) try {
+		scores.push(await scorer(ctx));
+	} catch (err) {
+		if (err instanceof EvalMetricError) throw err;
+		scores.push({
+			name: scorer.name || "anonymous-scorer",
+			passed: false,
+			score: 0,
+			details: err instanceof Error ? err.message : String(err)
+		});
+	}
+	return scores;
+}
+function aggregateScore(scores) {
+	if (scores.length === 0) return 1;
+	return scores.reduce((sum, score) => sum + (score.score ?? (score.passed ? 1 : 0)), 0) / scores.length;
+}
+//#endregion
+//#region src/eval-trajectory.ts
+/**
+* Build an ordered, grouped trajectory from a transcript. Walks assistant
+* turns in order, emitting `think` / `text` / `tool:<name>` steps, grouping
+* only consecutive identical kinds. Per-step duration is approximated from the
+* owning turn's `createdAt` delta to the next turn.
+*/
+function buildTrajectory(transcript) {
+	const steps = [];
+	let total = 0;
+	const push = (kind, name, durationMs) => {
+		total++;
+		const last = steps[steps.length - 1];
+		if (last && last.kind === kind && last.name === name) {
+			last.count++;
+			if (durationMs > 0) last.durationMs = (last.durationMs ?? 0) + durationMs;
+			return;
+		}
+		steps.push({
+			kind,
+			...name ? { name } : {},
+			count: 1,
+			...durationMs > 0 ? { durationMs } : {}
+		});
+	};
+	for (let i = 0; i < transcript.length; i++) {
+		const turn = transcript[i];
+		if (turn.role !== "assistant") continue;
+		const next = transcript[i + 1];
+		const turnDuration = next ? Math.max(0, next.createdAt - turn.createdAt) : 0;
+		const blocks = turn.content.filter((b) => b.type === "thinking" || b.type === "redacted_thinking" || b.type === "text" || b.type === "tool_call");
+		const toolBlocks = blocks.filter((b) => b.type === "tool_call").length;
+		const perTool = toolBlocks > 0 ? Math.round(turnDuration / toolBlocks) : 0;
+		for (const block of blocks) if (block.type === "thinking" || block.type === "redacted_thinking") push("think", void 0, 0);
+		else if (block.type === "text") push("text", void 0, 0);
+		else if (block.type === "tool_call") push("tool", block.name, perTool);
+	}
+	return {
+		steps,
+		totalBlocks: total
+	};
+}
+/** Render a trajectory as a one-line timeline string (no color). */
+function formatTrajectoryLine(trajectory) {
+	return trajectory.steps.map(stepLabel).join(" → ");
+}
+function stepLabel(step) {
+	const base = step.kind === "tool" ? step.name ?? "tool" : step.kind;
+	return step.count > 1 ? `${base} ×${step.count}` : base;
+}
+//#endregion
+//#region src/eval-workspace.ts
+async function prepareWorkspace(execution, workspace) {
+	if (!workspace) return {
+		snapshot: () => void 0,
+		error: () => void 0,
+		cleanup: async () => {}
+	};
+	let tempDir;
+	let baseExecution = execution;
+	if (!baseExecution) {
+		const cwd = workspace.cwd ?? await mkdtemp(join(tmpdir(), "zidane-eval-"));
+		if (!workspace.cwd) tempDir = cwd;
+		baseExecution = createProcessContext({ cwd });
+	}
+	const captured = {};
+	return {
+		execution: withWorkspaceLifecycle(baseExecution, workspace, captured),
+		snapshot: () => captured.snapshot,
+		error: () => captured.error,
+		cleanup: async () => {
+			if (tempDir && !workspace.retain) await rm(tempDir, {
+				recursive: true,
+				force: true
+			});
+		}
+	};
+}
+function withWorkspaceLifecycle(execution, workspace, captured) {
+	return {
+		...execution,
+		async spawn(config) {
+			const handle = await execution.spawn({
+				...config,
+				...workspace.cwd && !config?.cwd ? { cwd: workspace.cwd } : {}
+			});
+			if (workspace.seedDir) await seedWorkspace(execution, handle, workspace);
+			return handle;
+		},
+		async destroy(handle) {
+			try {
+				captured.snapshot = await captureWorkspace(execution, handle, workspace);
+			} catch (err) {
+				captured.error = err instanceof Error ? err.message : String(err);
+			} finally {
+				await execution.destroy(handle);
+			}
+		}
+	};
+}
+async function seedWorkspace(execution, handle, workspace) {
+	const target = workspace.seedTarget ?? ".";
+	const result = await execution.exec(handle, `mkdir -p ${alwaysQuote(target)} && cp -R ${alwaysQuote(`${workspace.seedDir}/.`)} ${alwaysQuote(target)}`);
+	if (result.exitCode !== 0) throw new Error(`Failed to seed eval workspace: ${result.stderr || result.stdout}`);
+}
+async function captureWorkspace(execution, handle, workspace) {
+	const capturePaths = workspace.capture?.length ? workspace.capture : ["."];
+	const command = [
+		`cd ${alwaysQuote(handle.cwd)}`,
+		"&&",
+		"for p in",
+		capturePaths.map(alwaysQuote).join(" "),
+		"; do",
+		"if [ -e \"$p\" ]; then",
+		"find \"$p\" \\( -path \"*/node_modules/*\" -o -path \"*/.git/*\" \\) -prune -o -type f -print0",
+		"; fi",
+		"done"
+	].join(" ");
+	const listed = await execution.exec(handle, command);
+	if (listed.exitCode !== 0) throw new Error(`Failed to list eval workspace: ${listed.stderr || listed.stdout}`);
+	const maxFileChars = workspace.maxFileChars ?? 262144;
+	const files = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const rawPath of listed.stdout.split("\0").filter(Boolean)) {
+		const path = normalizeSnapshotPath(rawPath);
+		if (seen.has(path)) continue;
+		seen.add(path);
+		const content = await execution.readFile(handle, path);
+		const binary = content.includes("\0");
+		const truncated = content.length > maxFileChars;
+		files.push({
+			path,
+			size: content.length,
+			...binary ? { binary: true } : { content: truncated ? content.slice(0, maxFileChars) : content },
+			...truncated ? { truncated: true } : {}
+		});
+	}
+	return {
+		cwd: handle.cwd,
+		files
+	};
+}
+function normalizeSnapshotPath(path) {
+	const normalized = path.startsWith("./") ? path.slice(2) : path;
+	return normalized === "" ? "." : normalized;
+}
+//#endregion
+//#region src/eval-execution.ts
+/**
+* Wrap an execution context so repeated agent runs share one handle until the
+* caller disposes it. This is useful for eval fixtures that want per-test
+* Docker/process lifetime while still using `runHeadless()` per turn.
+*/
+function createReusableExecutionContext(base) {
+	let handle;
+	let spawnPromise;
+	let disposed = false;
+	let disposePromise;
+	const ensureHandle = (config) => {
+		if (disposed) throw new Error("Reusable execution context has been disposed");
+		if (handle) return Promise.resolve(handle);
+		spawnPromise ??= base.spawn(config).then((next) => {
+			handle = next;
+			return next;
+		}, (err) => {
+			spawnPromise = void 0;
+			throw err;
+		});
+		return spawnPromise;
+	};
+	return {
+		execution: {
+			...base,
+			async spawn(config) {
+				return ensureHandle(config);
+			},
+			async destroy() {}
+		},
+		dispose() {
+			disposePromise ??= (async () => {
+				disposed = true;
+				const active = handle ?? await spawnPromise?.catch(() => void 0);
+				handle = void 0;
+				spawnPromise = void 0;
+				if (active) await base.destroy(active);
+			})();
+			return disposePromise;
+		},
+		handle: () => handle
+	};
+}
+//#endregion
+//#region src/eval-agent.ts
+/**
+* Multi-turn eval agent over the low-level headless runner. It keeps session
+* and execution lifetime outside any Playwright-specific fixture layer,
+* so downstream platforms can wrap it with their own `agent.run(...)` shape.
+*/
+function createEvalAgent(options) {
+	const { cwd, execution, session: initialSession, store: initialStore, mcpServers: baseMcpServers, onEvent: baseOnEvent, ...headlessDefaults } = options;
+	const executionOwner = createReusableExecutionContext(execution ?? createProcessContext({ cwd }));
+	const store = initialStore ?? createMemoryStore();
+	let sessionPromise = initialSession ? Promise.resolve(initialSession) : void 0;
+	let disposed = false;
+	let runInFlight = false;
+	let disposePromise;
+	const stats = {
+		runs: 0,
+		input: 0,
+		output: 0,
+		cacheRead: 0,
+		cacheCreation: 0,
+		cost: 0,
+		turns: 0,
+		toolCalls: 0,
+		durationMs: 0
+	};
+	async function getSession() {
+		if (disposed) throw new Error("Eval agent has been disposed");
+		sessionPromise ??= createSession({ store });
+		return sessionPromise;
+	}
+	async function resolveMcpServers(runServers) {
+		const merged = [...(typeof baseMcpServers === "function" ? await baseMcpServers() : baseMcpServers) ?? [], ...runServers ?? []];
+		return merged.length > 0 ? [...merged] : void 0;
+	}
+	async function run(runOptions) {
+		if (runInFlight) throw new Error("Eval agent already has a run in progress");
+		runInFlight = true;
+		try {
+			const session = await getSession();
+			const { mcpServers: runMcpServers, onEvent: runOnEvent, ...restRunOptions } = runOptions;
+			const mcpServers = await resolveMcpServers(runMcpServers);
+			const result = await runHeadless({
+				...headlessDefaults,
+				...restRunOptions,
+				session,
+				execution: executionOwner.execution,
+				...mcpServers ? { mcpServers } : {},
+				onEvent(event) {
+					baseOnEvent?.(event);
+					runOnEvent?.(event);
+				}
+			});
+			const newTranscript = result.transcript.slice();
+			const runStats = {
+				input: result.usage.input,
+				output: result.usage.output,
+				cacheRead: result.usage.cacheRead,
+				cacheCreation: result.usage.cacheCreation,
+				cost: result.usage.cost ?? 0,
+				durationMs: result.durationMs,
+				turns: result.turns,
+				toolCalls: countToolCallsInTurns(newTranscript)
+			};
+			accumulateEvalAgentStats(stats, runStats);
+			return {
+				result,
+				stats: runStats,
+				newTranscript
+			};
+		} finally {
+			runInFlight = false;
+		}
+	}
+	function dispose() {
+		disposePromise ??= (async () => {
+			disposed = true;
+			await executionOwner.dispose();
+		})();
+		return disposePromise;
+	}
+	return {
+		run,
+		stats,
+		session: getSession,
+		dispose,
+		[Symbol.asyncDispose]: dispose
+	};
+}
+function accumulateEvalAgentStats(acc, turn) {
+	acc.runs += 1;
+	acc.input += turn.input;
+	acc.output += turn.output;
+	acc.cacheRead += turn.cacheRead;
+	acc.cacheCreation += turn.cacheCreation;
+	acc.cost += turn.cost;
+	acc.turns += turn.turns;
+	acc.toolCalls += turn.toolCalls;
+	acc.durationMs += turn.durationMs;
+}
+function countToolCallsInTurns(turns) {
+	let count = 0;
+	for (const turn of turns) {
+		if (turn.role !== "assistant") continue;
+		count += turn.content.filter((block) => block.type === "tool_call").length;
+	}
+	return count;
+}
+const UNPARSEABLE_REASONING = "Judge did not return a parseable score.";
+const DEFAULT_JUDGE_SYSTEM = [
+	"You are a strict software eval judge.",
+	"Grade only the supplied output against the rubric.",
+	"Call the `submit_grade` tool with your score and reasoning.",
+	"The score must be between 0 and 1."
+].join("\n");
+/** Forced-output tool the judge must call — the schema-enforcement path. */
+const JUDGE_TOOL = {
+	name: "submit_grade",
+	description: "Submit the grade for the output under evaluation.",
+	inputSchema: {
+		type: "object",
+		properties: {
+			score: {
+				type: "number",
+				minimum: 0,
+				maximum: 1,
+				description: "Quality score from 0 (worst) to 1 (best)."
+			},
+			reasoning: {
+				type: "string",
+				description: "Brief justification for the score."
+			},
+			feedback: {
+				type: "string",
+				description: "Optional actionable feedback."
+			}
+		},
+		required: ["score", "reasoning"],
+		additionalProperties: false
+	}
+};
+function llmJudge(options) {
+	const name = options.name ?? "llm.judge";
+	const model = options.model ?? options.provider.meta.defaultModel;
+	return async (ctx) => {
+		const userMessage = options.provider.userMessage(renderJudgePrompt({
+			rubric: options.rubric,
+			input: options.input?.(ctx) ?? defaultJudgeInput(ctx)
+		}));
+		let text = "";
+		const controller = new AbortController();
+		const timeoutMs = options.timeoutMs ?? 6e4;
+		let timedOut = false;
+		const timer = timeoutMs > 0 ? setTimeout(() => {
+			timedOut = true;
+			controller.abort(/* @__PURE__ */ new Error(`LLM judge timed out after ${timeoutMs}ms`));
+		}, timeoutMs) : void 0;
+		const timeoutPromise = timeoutMs > 0 ? new Promise((_resolve, reject) => {
+			controller.signal.addEventListener("abort", () => {
+				if (timedOut) reject(/* @__PURE__ */ new Error(`LLM judge timed out after ${timeoutMs}ms`));
+			}, { once: true });
+		}) : void 0;
+		timeoutPromise?.catch(() => {});
+		const grade = async (maxTokens) => {
+			text = "";
+			const streamPromise = options.provider.stream({
+				model,
+				system: options.system ?? DEFAULT_JUDGE_SYSTEM,
+				tools: options.provider.formatTools([JUDGE_TOOL]),
+				messages: [userMessage],
+				maxTokens,
+				cache: false,
+				signal: controller.signal,
+				toolChoice: {
+					type: "tool",
+					name: JUDGE_TOOL.name
+				}
+			}, { onText(delta) {
+				text += delta;
+			} });
+			return timeoutPromise ? await Promise.race([streamPromise, timeoutPromise]) : await streamPromise;
+		};
+		try {
+			const budget = options.maxTokens ?? 4e3;
+			let result = await grade(budget);
+			const submitted = (r) => r.toolCalls.some((tc) => tc.name === JUDGE_TOOL.name);
+			if (!submitted(result) && result.usage.finishReason === "length") result = await grade(budget * 2);
+			const parsed = readJudgeResult(result.toolCalls, text || result.text);
+			if (!submitted(result) && parsed.reasoning === UNPARSEABLE_REASONING && result.usage.finishReason) parsed.reasoning = `${UNPARSEABLE_REASONING} (finish reason: ${result.usage.finishReason})`;
+			if (options.metric) ctx.metric(options.metric, parsed.score);
+			return {
+				name,
+				passed: parsed.score >= .7,
+				score: parsed.score,
+				details: {
+					reasoning: parsed.reasoning,
+					...parsed.feedback ? { feedback: parsed.feedback } : {},
+					...parsed.raw ? { raw: parsed.raw } : {},
+					usage: result.usage
+				}
+			};
+		} finally {
+			if (timer) clearTimeout(timer);
+		}
+	};
+}
+function readJudgeResult(toolCalls, fallbackText) {
+	const call = toolCalls.find((tc) => tc.name === JUDGE_TOOL.name);
+	if (call) {
+		const input = call.input;
+		return {
+			score: clampScore(input.score),
+			reasoning: typeof input.reasoning === "string" ? input.reasoning : "",
+			...typeof input.feedback === "string" ? { feedback: input.feedback } : {}
+		};
+	}
+	return parseJudgeOutput(fallbackText);
+}
+function renderJudgePrompt(input) {
+	return [
+		"Rubric:",
+		input.rubric,
+		"",
+		"Output to grade:",
+		input.input,
+		"",
+		"Return JSON only."
+	].join("\n");
+}
+function defaultJudgeInput(ctx) {
+	const chunks = [`Final answer:\n${ctx.result.finalText}`];
+	if (ctx.workspace) chunks.push(`Workspace files:\n${ctx.workspace.files.map((file) => [`--- ${file.path}${file.truncated ? " (truncated)" : ""} ---`, file.binary ? "[binary omitted]" : file.content ?? ""].join("\n")).join("\n")}`);
+	return chunks.join("\n\n");
+}
+function parseJudgeOutput(raw) {
+	const trimmed = stripCodeFences(raw.trim());
+	const jsonText = extractJsonObject(trimmed);
+	if (jsonText) try {
+		const parsed = JSON.parse(jsonText);
+		if (parsed.score !== void 0 || parsed.reasoning !== void 0) return {
+			score: clampScore(parsed.score),
+			reasoning: typeof parsed.reasoning === "string" ? parsed.reasoning : "",
+			...typeof parsed.feedback === "string" ? { feedback: parsed.feedback } : {},
+			...jsonText === trimmed ? {} : { raw: trimmed }
+		};
+	} catch {}
+	const scoreFragment = trimmed.match(/score["']?\s*[:=]\s*(-?\d+(?:\.\d+)?)/i);
+	if (scoreFragment) return {
+		score: clampScore(Number(scoreFragment[1])),
+		reasoning: trimmed.slice(0, 500),
+		raw: trimmed
+	};
+	const bareFraction = trimmed.match(/^(-?\d+(?:\.\d+)?)\s*\/\s*(\d+(?:\.\d+)?)$/);
+	if (bareFraction) {
+		const denom = Number(bareFraction[2]);
+		return {
+			score: clampScore(denom === 0 ? 0 : Number(bareFraction[1]) / denom),
+			reasoning: trimmed,
+			raw: trimmed
+		};
+	}
+	if (trimmed.match(/^-?\d+(?:\.\d+)?$/)) return {
+		score: clampScore(Number(trimmed)),
+		reasoning: "",
+		raw: trimmed
+	};
+	return {
+		score: 0,
+		reasoning: UNPARSEABLE_REASONING,
+		raw: trimmed
+	};
+}
+/** Strip a leading/trailing markdown code fence (``` or ```json). */
+function stripCodeFences(text) {
+	const fence = text.match(/^```(?:json)?[ \t]*\n([\s\S]*?)\n```$/i);
+	return fence ? fence[1].trim() : text;
+}
+/** Extract the first balanced `{...}` object, ignoring braces inside strings. */
+function extractJsonObject(text) {
+	const start = text.indexOf("{");
+	if (start === -1) return void 0;
+	let depth = 0;
+	let inString = false;
+	let escaped = false;
+	for (let i = start; i < text.length; i++) {
+		const ch = text[i];
+		if (inString) {
+			if (escaped) escaped = false;
+			else if (ch === "\\") escaped = true;
+			else if (ch === "\"") inString = false;
+			continue;
+		}
+		if (ch === "\"") inString = true;
+		else if (ch === "{") depth++;
+		else if (ch === "}") {
+			depth--;
+			if (depth === 0) return text.slice(start, i + 1);
+		}
+	}
+}
+function clampScore(value) {
+	const n = typeof value === "number" ? value : Number(value);
+	if (!Number.isFinite(n)) return 0;
+	return Math.max(0, Math.min(1, n));
+}
+//#endregion
+//#region src/eval-registry.ts
+const evalRegistry = [];
+const STUB_PROVIDER = {
+	name: "stub",
+	meta: { defaultModel: "stub" },
+	formatTools: (tools) => tools,
+	userMessage: () => ({
+		role: "user",
+		content: []
+	}),
+	assistantMessage: () => ({
+		role: "assistant",
+		content: []
+	}),
+	toolResultsMessage: () => ({
+		role: "user",
+		content: []
+	}),
+	stream: async () => {
+		throw new Error("stub provider is not runnable");
+	}
+};
+/**
+* Register an eval definition. Eval files call this at module load; the harness
+* (`buildRegisteredEvals`) materializes them with a provider at run time. The
+* factory keeps definitions lazy so a single registry serves any provider/judge
+* pairing without re-importing eval files.
+*
+* Returns the same factory so a file can also `export default` it for direct,
+* registry-free use (e.g. hermetic unit tests).
+*/
+function defineEval(define) {
+	const id = safeRegistryId(define);
+	if (!evalRegistry.some((entry) => entry.id === id)) evalRegistry.push({
+		id,
+		define
+	});
+	return define;
+}
+/** Snapshot of every registered eval, materialized for one provider/judge pair. */
+function buildRegisteredEvals(ctx) {
+	return evalRegistry.map((entry) => entry.define(ctx));
+}
+/** Drop every registered eval — test isolation helper. */
+function clearRegisteredEvals() {
+	evalRegistry.length = 0;
+}
+function safeRegistryId(define) {
+	try {
+		const stub = define({
+			provider: STUB_PROVIDER,
+			judge: STUB_PROVIDER
+		});
+		return `${stub.suite ?? "eval"}/${stub.id}`;
+	} catch {
+		return `eval/${evalRegistry.length}`;
+	}
+}
+//#endregion
+//#region src/eval.ts
+async function runEvalCase(options) {
+	const { id, suite, variant, tags = [], artifactDir, workspace, scorers = [], onEvent, ...headless } = options;
+	const { metrics: declaredMetrics, sourceFile, ...headlessRest } = headless;
+	const events = [];
+	const caseArtifactDir = artifactDir ? join(artifactDir, ...variant ? [safeSegment(variant)] : [], safeSegment(suite ?? "eval"), safeSegment(id)) : void 0;
+	const workspaceState = await prepareWorkspace(headlessRest.execution, workspace);
+	const caseMetricIds = new Set(Object.keys(declaredMetrics ?? {}));
+	const hasCaseMetrics = caseMetricIds.size > 0;
+	const metricSpecs = {
+		...EFFICIENCY_METRICS,
+		...declaredMetrics
+	};
+	const rawMetrics = /* @__PURE__ */ new Map();
+	const emitMetric = (metricId, raw) => {
+		if (!(metricId in metricSpecs)) throw new EvalMetricError(`Eval ${id}: metric "${metricId}" emitted but not declared in defineMetrics`);
+		if (!Number.isFinite(raw)) throw new EvalMetricError(`Eval ${id}: metric "${metricId}" emitted a non-finite value (${raw}); scores require finite numbers`);
+		rawMetrics.set(metricId, raw);
+	};
+	try {
+		const result = await runHeadless({
+			...headlessRest,
+			...workspaceState.execution ? { execution: workspaceState.execution } : {},
+			onEvent(event) {
+				events.push(event);
+				onEvent?.(event);
+			}
+		});
+		emitEfficiencyMetrics(emitMetric, result);
+		const workspaceSnapshot = workspaceState.snapshot();
+		const workspaceError = workspaceState.error();
+		const scores = await runScorers({
+			id,
+			...suite ? { suite } : {},
+			tags,
+			result,
+			events,
+			metric: emitMetric,
+			...workspaceSnapshot ? { workspace: workspaceSnapshot } : {},
+			...caseArtifactDir ? { artifactDir: caseArtifactDir } : {}
+		}, scorers);
+		const metrics = finalizeEvalMetrics(metricSpecs, rawMetrics);
+		const tagScores = computeEvalTagScores(metrics);
+		const caseMetrics = metrics.filter((metric) => caseMetricIds.has(metric.id));
+		const score = hasCaseMetrics && caseMetrics.length > 0 ? meanNormalized(caseMetrics) : aggregateScore(scores);
+		const evalResult = {
+			id,
+			...suite ? { suite } : {},
+			...variant ? { variant } : {},
+			tags,
+			result,
+			score,
+			passed: result.status === "completed" && scores.every((score) => score.passed),
+			scores,
+			metrics,
+			tagScores,
+			trajectory: buildTrajectory(result.transcript),
+			events,
+			...sourceFile ? { sourceFile } : {},
+			...workspaceSnapshot ? { workspace: workspaceSnapshot } : {},
+			...workspaceError ? { workspaceError } : {}
+		};
+		const artifacts = caseArtifactDir ? await writeEvalArtifacts(caseArtifactDir, evalResult) : void 0;
+		return {
+			...evalResult,
+			...artifacts ? { artifacts } : {}
+		};
+	} finally {
+		await workspaceState.cleanup();
+	}
+}
+/**
+* Register one test per eval case against a test runner, funnel every result
+* into a shared reporter, and print the aggregated summary once after all cases.
+*
+* A failing test means the agent run itself broke (provider/tool/timeout). Low
+* scores are reported, not asserted — see {@link RegisterEvalTestsOptions.failOnIncomplete}.
+*/
+function registerEvalTests(options) {
+	const reporter = options.reporter ?? createEvalRunReporter();
+	const failOnIncomplete = options.failOnIncomplete ?? true;
+	const printSummary = options.printSummary ?? true;
+	const repeat = Math.max(1, options.repeat ?? 1);
+	const concurrency = Math.max(1, options.concurrency ?? 1);
+	options.runner.afterAll(async () => {
+		await reporter.flush();
+		if (printSummary) console.log(`\n${reporter.format()}\n`);
+		if (options.dispose) await options.dispose();
+	});
+	const work = [];
+	for (const evalCase of options.cases) {
+		const caseLabel = `${evalCase.suite ? `${evalCase.suite}/` : ""}${evalCase.id}`;
+		const baseLabel = evalCase.variant ? `[${evalCase.variant}] ${caseLabel}` : caseLabel;
+		for (let run = 1; run <= repeat; run++) {
+			const label = repeat > 1 ? `${baseLabel} #${run}` : baseLabel;
+			const caseId = repeat > 1 ? `${evalCase.id}-repeat-${run}` : evalCase.id;
+			work.push({
+				label,
+				run: () => runEvalCase({
+					...evalCase,
+					id: caseId,
+					...options.artifactDir ? { artifactDir: options.artifactDir } : {}
+				})
+			});
+		}
+	}
+	const schedule = createLazyPool(concurrency);
+	work.forEach((w) => {
+		options.runner.it(w.label, async () => {
+			const result = await schedule(w.run);
+			await reporter.record(result);
+			if (failOnIncomplete && result.result.status !== "completed") throw new Error(`Eval ${w.label} did not complete: ${JSON.stringify(result.result.error ?? { status: result.result.status }, null, 2)}`);
+		});
+	});
+	return reporter;
+}
+/**
+* Lazily run tasks with a bounded concurrency `limit`. Nothing starts until a
+* runner invokes the corresponding test callback, so filtered/skipped tests do
+* not leak model or Docker work in the background.
+*/
+function createLazyPool(limit) {
+	const queue = [];
+	let active = 0;
+	const drain = () => {
+		const available = Math.min(limit - active, queue.length);
+		for (let i = 0; i < available; i++) queue.shift()?.();
+	};
+	return (task) => new Promise((resolve, reject) => {
+		const run = async () => {
+			active++;
+			try {
+				resolve(await task());
+			} catch (err) {
+				reject(err);
+			} finally {
+				active--;
+				drain();
+			}
+		};
+		queue.push(() => void run());
+		drain();
+	});
+}
+function createEvalRunReporter(options = {}) {
+	const results = [];
+	return {
+		get results() {
+			return results;
+		},
+		async record(result) {
+			results.push(result);
+			if (options.outputDir) {
+				const segments = [
+					...result.variant ? [safeSegment(result.variant)] : [],
+					safeSegment(result.suite ?? "eval"),
+					safeSegment(result.id)
+				];
+				const casePath = join(options.outputDir, "cases", `${segments.join("--")}.json`);
+				await mkdir(join(options.outputDir, "cases"), { recursive: true });
+				await writeFile(casePath, `${JSON.stringify(result, null, 2)}\n`);
+			}
+		},
+		async flush() {
+			const summary = buildEvalRunSummary(results);
+			if (options.outputDir) {
+				await mkdir(options.outputDir, { recursive: true });
+				await writeFile(join(options.outputDir, "run-summary.json"), `${JSON.stringify(summary, null, 2)}\n`);
+			}
+			return summary;
+		},
+		format() {
+			return formatEvalRunSummaryWithOptions(results, { color: options.color });
+		}
+	};
+}
+//#endregion
+export { relativeArtifactPath as A, computeEvalTagScores as C, finalizeEvalMetrics as D, emitEfficiencyMetrics as E, normalizeMetric as O, EvalMetricError as S, efficiencyMetricValues as T, statusCompleted as _, clearRegisteredEvals as a, buildEvalRunSummary as b, createEvalAgent as c, formatTrajectoryLine as d, fileContains as f, functionalityMetric as g, fileExistsOneOf as h, buildRegisteredEvals as i, artifactPath as k, createReusableExecutionContext as l, fileExists as m, registerEvalTests as n, defineEval as o, fileContentQuality as p, runEvalCase as r, llmJudge as s, createEvalRunReporter as t, buildTrajectory as u, formatEvalCaseSummary as v, defineMetrics as w, EFFICIENCY_METRICS as x, formatEvalRunSummary as y };
+
+//# sourceMappingURL=eval-BNcvle_J.js.map