zidane 5.10.2 → 5.10.3

package/dist/eval.js

@@ -1,7 +1,7 @@
-import { _ as alwaysQuote } from "./tools-Bk9TqmCV.js";
+import { _ as alwaysQuote } from "./tools-0IqJRRj8.js";
 import { r as createProcessContext } from "./contexts-GKAWYq07.js";
-import { a as headlessEventToJsonl, c as runHeadless } from "./headless-Cn6XXmr3.js";
-import { i as createMemoryStore, t as createSession } from "./session-C0uGIWm_.js";
+import { a as headlessEventToJsonl, c as runHeadless } from "./headless-DHdHSA2s.js";
+import { i as createMemoryStore, t as createSession } from "./session-CZniOWFD.js";
 import { join, relative, resolve } from "node:path";
 import { tmpdir } from "node:os";
 import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
@@ -327,10 +327,10 @@ const JUDGE_TOOL = {
 	}
 };
 async function runEvalCase(options) {
-	const { id, suite, tags = [], artifactDir, workspace, scorers = [], onEvent, ...headless } = options;
+	const { id, suite, variant, tags = [], artifactDir, workspace, scorers = [], onEvent, ...headless } = options;
 	const { metrics: declaredMetrics, sourceFile, ...headlessRest } = headless;
 	const events = [];
-	const caseArtifactDir = artifactDir ? join(artifactDir, safeSegment(suite ?? "eval"), safeSegment(id)) : void 0;
+	const caseArtifactDir = artifactDir ? join(artifactDir, ...variant ? [safeSegment(variant)] : [], safeSegment(suite ?? "eval"), safeSegment(id)) : void 0;
 	const workspaceState = await prepareWorkspace(headlessRest.execution, workspace);
 	const caseMetricIds = new Set(Object.keys(declaredMetrics ?? {}));
 	const hasCaseMetrics = caseMetricIds.size > 0;
@@ -373,6 +373,7 @@ async function runEvalCase(options) {
 		const evalResult = {
 			id,
 			...suite ? { suite } : {},
+			...variant ? { variant } : {},
 			tags,
 			result,
 			score,
@@ -557,6 +558,7 @@ function functionalityMetric(metricId, scorers, name = metricId) {
 function formatEvalCaseSummary(result) {
 	const lines = [
 		`Eval ${result.suite ? `${result.suite}/` : ""}${result.id}`,
+		...result.variant ? [`variant: ${result.variant}`] : [],
 		`status: ${result.result.status}`,
 		`passed: ${result.passed}`,
 		`score: ${result.score.toFixed(2)}`,
@@ -602,7 +604,8 @@ function registerEvalTests(options) {
 	});
 	const work = [];
 	for (const evalCase of options.cases) {
-		const baseLabel = `${evalCase.suite ? `${evalCase.suite}/` : ""}${evalCase.id}`;
+		const caseLabel = `${evalCase.suite ? `${evalCase.suite}/` : ""}${evalCase.id}`;
+		const baseLabel = evalCase.variant ? `[${evalCase.variant}] ${caseLabel}` : caseLabel;
 		for (let run = 1; run <= repeat; run++) {
 			const label = repeat > 1 ? `${baseLabel} #${run}` : baseLabel;
 			const caseId = repeat > 1 ? `${evalCase.id}-repeat-${run}` : evalCase.id;
@@ -663,7 +666,12 @@ function createEvalRunReporter(options = {}) {
 		async record(result) {
 			results.push(result);
 			if (options.outputDir) {
-				const casePath = join(options.outputDir, "cases", `${safeSegment(result.suite ?? "eval")}--${safeSegment(result.id)}.json`);
+				const segments = [
+					...result.variant ? [safeSegment(result.variant)] : [],
+					safeSegment(result.suite ?? "eval"),
+					safeSegment(result.id)
+				];
+				const casePath = join(options.outputDir, "cases", `${segments.join("--")}.json`);
 				await mkdir(join(options.outputDir, "cases"), { recursive: true });
 				await writeFile(casePath, `${JSON.stringify(result, null, 2)}\n`);
 			}
@@ -681,14 +689,21 @@ function createEvalRunReporter(options = {}) {
 		}
 	};
 }
-/** Stable ordering by `suite/id` so parallel completion order doesn't shuffle output. */
+/** Stable ordering by `variant`, then `suite/id`, so parallel completion order doesn't shuffle output. */
 function sortCases(results) {
 	return [...results].sort((a, b) => {
+		const va = a.variant ?? "";
+		const vb = b.variant ?? "";
+		if (va !== vb) return va.localeCompare(vb);
 		const ka = `${a.suite ?? ""}/${a.id}`;
 		const kb = `${b.suite ?? ""}/${b.id}`;
 		return ka.localeCompare(kb);
 	});
 }
+/** Distinct variant labels across results, in sorted order. */
+function distinctVariants(results) {
+	return [...new Set(results.map((r) => r.variant).filter((v) => Boolean(v)))].sort();
+}
 function buildEvalRunSummary(input) {
 	const results = sortCases(input);
 	const usage = results.reduce((acc, result) => {
@@ -715,6 +730,7 @@ function buildEvalRunSummary(input) {
 		cases: results.map((result) => ({
 			id: result.id,
 			...result.suite ? { suite: result.suite } : {},
+			...result.variant ? { variant: result.variant } : {},
 			passed: result.passed,
 			score: result.score,
 			status: result.result.status,
@@ -725,9 +741,37 @@ function buildEvalRunSummary(input) {
 			trajectory: result.trajectory
 		})),
 		metrics: aggregateMetrics(results),
-		tagScores: aggregateTagScores(results)
+		tagScores: aggregateTagScores(results),
+		...distinctVariants(results).length > 0 ? { variants: aggregateVariants(results) } : {}
 	};
 }
+function aggregateVariants(results) {
+	return distinctVariants(results).map((variant) => {
+		const group = results.filter((r) => r.variant === variant);
+		const usage = group.reduce((acc, result) => {
+			acc.input += result.result.usage.input;
+			acc.output += result.result.usage.output;
+			acc.cacheRead += result.result.usage.cacheRead;
+			acc.cacheCreation += result.result.usage.cacheCreation;
+			acc.cost += result.result.usage.cost ?? 0;
+			return acc;
+		}, {
+			input: 0,
+			output: 0,
+			cacheRead: 0,
+			cacheCreation: 0,
+			cost: 0
+		});
+		return {
+			variant,
+			count: group.length,
+			passed: group.filter((r) => r.passed).length,
+			score: mean(group.map((r) => r.score)),
+			durationMs: group.reduce((sum, r) => sum + r.result.durationMs, 0),
+			usage
+		};
+	});
+}
 function aggregateMetrics(results) {
 	const byId = /* @__PURE__ */ new Map();
 	for (const result of results) for (const metric of result.metrics) {
@@ -776,8 +820,17 @@ function formatEvalRunSummaryWithOptions(input, options = {}) {
 	if (results.length === 0) return `${color.heading("Eval run summary")}\n${color.muted("no evals ran")}`;
 	const summary = buildEvalRunSummary(results);
 	const out = [];
+	const variants = distinctVariants(results);
 	out.push(color.heading("Eval run summary"));
-	out.push(renderResultsTable(results, summary, color));
+	if (variants.length > 1) {
+		out.push(renderComparisonTable(results, variants, color));
+		for (const variant of variants) {
+			const group = results.filter((r) => r.variant === variant);
+			out.push("");
+			out.push(color.heading(variant));
+			out.push(renderResultsTable(group, buildEvalRunSummary(group), color));
+		}
+	} else out.push(renderResultsTable(results, summary, color));
 	const tagTable = renderTagTable(summary.tagScores, color);
 	if (tagTable) {
 		out.push("");
@@ -790,7 +843,8 @@ function formatEvalRunSummaryWithOptions(input, options = {}) {
 		return hyperlinks ? oscLink(abs, abs) : abs;
 	};
 	for (const result of results) {
-		const label = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
+		const caseLabel = `${result.suite ? `${result.suite}/` : ""}${result.id}`;
+		const label = result.variant ? `[${result.variant}] ${caseLabel}` : caseLabel;
 		out.push(`${color.dot(result.passed)} ${color.caseStatus(result.passed, label)}`);
 		out.push(`${color.muted("score")} ${color.score(result.score)}`);
 		const tagEntries = Object.entries(result.tagScores);
@@ -848,6 +902,120 @@ function oscLink(target, text) {
 function absPath(target) {
 	return resolve(target.replace(/^file:\/\//, ""));
 }
+/**
+* Case key used to line up the same eval across variants in the comparison
+* matrix. Strips the `-repeat-N` suffix `registerEvalTests` appends so a
+* repeated case aggregates into one row (mean score, x/y passed).
+*/
+function comparisonCaseKey(result) {
+	const id = result.id.replace(/-repeat-\d+$/, "");
+	return `${result.suite ? `${result.suite}/` : ""}${id}`;
+}
+/**
+* Cross-variant comparison matrix: one row per eval case, one column per
+* variant (provider/model target), plus rollup rows (score / passed / cost /
+* time) so two models can be read side by side at a glance.
+*/
+function renderComparisonTable(input, variants, color) {
+	const results = sortCases(input);
+	const caseKeys = [...new Set(results.map(comparisonCaseKey))].sort();
+	const byCell = /* @__PURE__ */ new Map();
+	for (const result of results) {
+		const key = `${comparisonCaseKey(result)}\u0000${result.variant ?? ""}`;
+		const list = byCell.get(key) ?? [];
+		list.push(result);
+		byCell.set(key, list);
+	}
+	const cellFor = (caseKey, variant) => {
+		const group = byCell.get(`${caseKey}\u0000${variant}`);
+		if (!group || group.length === 0) return { text: "-" };
+		const score = mean(group.map((r) => r.score));
+		const passedCount = group.filter((r) => r.passed).length;
+		const allPassed = passedCount === group.length;
+		const status = group.length > 1 ? `${passedCount}/${group.length}` : allPassed ? "pass" : "fail";
+		return {
+			text: `${score.toFixed(2)} ${status}`,
+			score,
+			passed: allPassed
+		};
+	};
+	const variantSummaries = aggregateVariants(results);
+	const summaryOf = (variant) => variantSummaries.find((s) => s.variant === variant);
+	const rows = caseKeys.map((key) => ({
+		label: key,
+		cells: variants.map((variant) => cellFor(key, variant))
+	}));
+	const rollups = [
+		{
+			label: "SCORE",
+			rollup: true,
+			cells: variants.map((variant) => {
+				const s = summaryOf(variant);
+				return s ? {
+					text: s.score.toFixed(2),
+					score: s.score
+				} : { text: "-" };
+			})
+		},
+		{
+			label: "PASSED",
+			rollup: true,
+			cells: variants.map((variant) => {
+				const s = summaryOf(variant);
+				return s ? {
+					text: `${s.passed}/${s.count}`,
+					passed: s.passed === s.count
+				} : { text: "-" };
+			})
+		},
+		{
+			label: "COST",
+			rollup: true,
+			cells: variants.map((variant) => {
+				return { text: formatCost(summaryOf(variant)?.usage.cost ?? 0) };
+			})
+		},
+		{
+			label: "TIME",
+			rollup: true,
+			cells: variants.map((variant) => {
+				const s = summaryOf(variant);
+				return { text: s ? formatDuration(s.durationMs) : "-" };
+			})
+		}
+	];
+	const header = ["EVAL", ...variants];
+	const allRows = [...rows, ...rollups];
+	const widths = header.map((h, i) => Math.max(h.length, ...allRows.map((row) => (i === 0 ? row.label : row.cells[i - 1].text).length)));
+	const top = `┌${widths.map((w) => "─".repeat(w + 2)).join("┬")}┐`;
+	const mid = `├${widths.map((w) => "─".repeat(w + 2)).join("┼")}┤`;
+	const bot = `└${widths.map((w) => "─".repeat(w + 2)).join("┴")}┘`;
+	const paintComparisonCell = (cell, padded) => {
+		if (cell.score === void 0 && cell.passed === void 0) return color.muted(padded);
+		if (cell.score !== void 0) {
+			const scoreText = cell.score.toFixed(2);
+			const rest = cell.text.slice(scoreText.length);
+			const coloredRest = cell.passed === void 0 ? color.muted(rest) : cell.passed ? color.pass(rest) : color.fail(rest);
+			return padded.replace(cell.text, `${color.score(cell.score)}${coloredRest}`);
+		}
+		return padded.replace(cell.text, cell.passed ? color.pass(cell.text) : color.fail(cell.text));
+	};
+	const renderRow = (row) => {
+		const labelPadded = padEnd(row.label, widths[0]);
+		return `│ ${row.rollup ? color.heading(labelPadded) : color.muted(labelPadded)} │${row.cells.map((cell, i) => {
+			return ` ${paintComparisonCell(cell, padStart(cell.text, widths[i + 1]))} `;
+		}).join("│")}│`;
+	};
+	return [
+		top,
+		`│ ${color.heading(padEnd(header[0], widths[0]))} │${variants.map((v, i) => ` ${color.heading(padStart(v, widths[i + 1]))} `).join("│")}│`,
+		mid,
+		...rows.map(renderRow),
+		mid,
+		...rollups.map(renderRow),
+		bot
+	].join("\n");
+}
 function renderResultsTable(results, summary, color) {
 	const header = [
 		"EVAL",