reeval 0.10.0 → 0.12.0

package/dist/summary-0oQNGskv.js

@@ -0,0 +1,837 @@
+import { l as sha256 } from "./ids-Bp6ut7ZD.js";
+import { a as oneLine, i as formatDuration, n as clip, r as fenced } from "./text-CrVy5ioL.js";
+import { n as formatTokens } from "./usage-CleNKkly.js";
+import { D as runtimeLabel, F as checkLabels, R as commandLine, S as languageLabel, V as labelledJudgements, _ as harnessLabel, l as evalMarkOutcome, lt as formatSeconds, st as formatCount, x as isUngatedMiss } from "./model-CWlAqmbU.js";
+import { i as countedOf, l as suiteResult, n as cancelledOf } from "./suite-C8sw2pox.js";
+import { a as fromModel } from "./insights-B2Xmhz9o.js";
+import { o as warnings } from "./state-BmZ8QVZ8.js";
+import { t as colors } from "./colors-QV9dFVhN.js";
+import { r as remarkLines } from "./remarks-1521zwfW.js";
+//#region src/insights/hash.ts
+/** JSON with every object's keys sorted. */
+function canonicalJson(value) {
+	return JSON.stringify(value, (_key, v) => v && typeof v === "object" && !Array.isArray(v) ? Object.fromEntries(Object.entries(v).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)) : v);
+}
+const factsHash = (suite) => `sha256:${sha256(canonicalJson(suite))}`;
+//#endregion
+//#region src/ui/layout.ts
+/** The block for an eval: a pure function of its job and its shape. */
+function layout(job, across, isFamily, isCompare) {
+	if (isCompare) return {
+		kind: "regressions",
+		family: false
+	};
+	return {
+		kind: job === "choose" && across === "none" ? "choose" : across === "change" ? "measure" : across === "harness" ? "harnesses" : across === "both" ? "both" : "results",
+		family: isFamily
+	};
+}
+const moved = (c) => c.effect === "better" || c.effect === "worse";
+/** A cell the agent failed, or that errored; not one whose only fails the interrupt cancelled. */
+const failedCell = (c) => c.outcome === "fail" && c.failed > cancelledOf(c) || c.outcome === "infra_error";
+/**
+* The insight slots an eval's layout opens: which rows take a sentence (a failing case; an arm
+* that failed or moved; a failing harness), whether it has findings (a choose or finish eval that
+* measures a change), and whether it has named tools (a choose eval). An `other` eval takes only
+* its answer, unless it compares harnesses, which takes a sentence on any job. What the run's
+* interrupt cancelled opens nothing: the agent didn't fail it, and its effects aren't measured.
+*/
+function slotsOf(job, e) {
+	const known = job !== "other";
+	const measured = e.outcome !== "cancelled";
+	const harnesses = e.across === "harness" || e.across === "both";
+	const rows = [];
+	if (known || harnesses) {
+		if (e.cases) {
+			for (const k of e.cases) if (k.outcome !== "pass" && k.outcome !== "cancelled") rows.push(k.path);
+			for (const p of e.pooled ?? []) {
+				const pooledMoved = measured && (p.effect === "better" || p.effect === "worse");
+				if (p.outcome === "fail" || pooledMoved) rows.push(p.path);
+			}
+		} else if (e.across === "harness") {
+			for (const c of e.cells ?? []) if (failedCell(c)) rows.push(c.path);
+		} else if (e.across === "change" || e.across === "both") {
+			for (const c of e.cells ?? []) if (failedCell(c) || measured && moved(c)) rows.push(c.path);
+		}
+	}
+	return {
+		rows,
+		findings: known && e.across === "change" && measured,
+		namedTools: job === "choose"
+	};
+}
+/** The last line of a run's output while no analysis model writes its answers. */
+const NO_MODEL_HINT = "Set analysis.model in reeval.config.ts for answers to what each eval asks.";
+/** The narrowest a row's label column is, so short labels line up under the name. */
+const LABEL = 17;
+const ASSERTION_MAX = 160;
+const CHECK_LABEL_MAX = 48;
+/** A check's label on one line: an expected value can carry line breaks. */
+const checkLabel = (label) => clip(label.replace(/\s*\n\s*/g, " "), CHECK_LABEL_MAX);
+const VERDICT = {
+	pass: "pass",
+	fail: "fail",
+	infra_error: "infra error",
+	cancelled: "cancelled",
+	skipped: "skipped"
+};
+const EFFECT = {
+	better: "helped",
+	worse: "made it worse",
+	unclear: "can't tell yet",
+	same: ""
+};
+const t = (text, paint) => ({
+	text,
+	paint
+});
+const textOf = (cell) => typeof cell === "string" ? cell : cell.text;
+const show = (cell) => typeof cell === "string" ? cell : cell.paint ? cell.paint(cell.text) : cell.text;
+/**
+* Aligned rows: the first column is as wide as the widest label with its indent (at least
+* `LABEL`), the others as their widest cell, two spaces apart. Free lines print as they are.
+*/
+function table(items, width = LABEL) {
+	const rows = items.filter((i) => "cells" in i);
+	const first = Math.max(width, ...rows.filter((r) => r.cells.length > 1).map((r) => r.indent + textOf(r.cells[0] ?? "").length));
+	const widths = [];
+	for (const r of rows) r.cells.forEach((cell, i) => {
+		if (i > 0) widths[i] = Math.max(widths[i] ?? 0, textOf(cell).length);
+	});
+	return items.map((item) => {
+		const pad = " ".repeat(item.indent);
+		if ("free" in item) return `${pad}${show(item.free)}`;
+		return `${pad}${item.cells.map((cell, i) => {
+			const width = i === 0 ? first - item.indent : widths[i] ?? 0;
+			const fill = " ".repeat(Math.max(0, width - textOf(cell).length));
+			return i === item.cells.length - 1 ? show(cell) : `${show(cell)}${fill}`;
+		}).join("  ")}`.trimEnd();
+	});
+}
+const paintOutcome = (c, o) => o === "pass" ? c.green : o === "fail" ? c.red : o === "infra_error" || o === "cancelled" ? c.yellow : c.dim;
+/** A study (every cell `gate: 'none'`) passes whatever it saw, so it never reads `pass`. */
+function verdict(c, e) {
+	if (e.todo) return t("todo", c.dim);
+	const cells = e.cells ?? [];
+	if (e.outcome === "pass" && cells.length && cells.every((x) => !x.gated)) return t("not gated", c.dim);
+	return t(VERDICT[e.outcome], paintOutcome(c, e.outcome));
+}
+const plural = (n, one, many = `${one}s`) => `${n} ${n === 1 ? one : many}`;
+/**
+* `3 of 4`, with the attempts that didn't count named: `2 of 3 · 1 infra error · 1 skipped`. A
+* cancelled attempt is recorded as a fail, but the agent didn't fail it, so it's named apart.
+*/
+function cellCount(cell) {
+	const cancelled = cancelledOf(cell);
+	const counted = countedOf(cell);
+	const parts = [
+		counted > 0 ? `${cell.passed} of ${counted}` : "",
+		cell.infraError ? plural(cell.infraError, "infra error") : "",
+		cancelled ? `${cancelled} cancelled` : "",
+		cell.skipped ? `${cell.skipped} skipped` : ""
+	].filter(Boolean);
+	if (!parts.length) return "not run";
+	return parts.join(" · ");
+}
+const failing = (k) => k.outcome.status === "not run" || k.outcome.failed + k.outcome.errored > 0;
+/** A check's result in a cell: `not run`, `failed` for one try, else `1 of 4 · 2 not run`. */
+function checkCount(k, single) {
+	const o = k.outcome;
+	if (o.status === "not run") return "not run";
+	const answered = o.passed + o.failed + o.errored;
+	return [single && answered === 1 ? o.failed ? "failed" : o.errored ? "couldn't check" : "passed" : `${o.passed} of ${answered}`, ...[!single && o.errored ? `${o.errored} couldn't check` : "", o.notRun ? `${o.notRun} not run` : ""].filter(Boolean)].join(" · ");
+}
+/** The check's first failure, less the `label: ` its row already shows. */
+function assertionOf(k) {
+	if (k.outcome.status !== "ran" || !k.outcome.assertion) return "";
+	const line = k.outcome.assertion.trim().split("\n")[0]?.trim() ?? "";
+	return clip(line.startsWith(`${k.label}: `) ? line.slice(k.label.length + 2) : line, ASSERTION_MAX);
+}
+/** A cell passed outright: every counted attempt, every check, nothing errored. */
+const clean = (cell) => cell.outcome === "pass" && cell.failed === 0 && cell.infraError === 0 && cell.errors.length === 0 && !cell.checks.some(failing);
+/** An error no check explains, by kind: a throw in the eval's body reads `error`. */
+const ERROR_KIND = {
+	assertion: "error",
+	judge: "judge",
+	timeout: "timeout",
+	aborted: "cancelled",
+	infra: "infra error",
+	turn: "turn error"
+};
+const errorKind = (err) => ERROR_KIND[err.kind];
+const errorText = (err) => `${clip(err.message, ASSERTION_MAX)}${err.attempts > 1 ? ` (${plural(err.attempts, "try", "tries")})` : ""}`;
+/** The evidence under a failing cell: its failed and not-run checks, its errors, `+N more passed`. */
+function evidence(c, cell, indent, more = true) {
+	const single = cell.passed + cell.failed + cell.infraError <= 1;
+	const items = [];
+	for (const k of cell.checks.filter(failing)) {
+		const count = checkCount(k, single);
+		items.push({
+			indent,
+			cells: [checkLabel(k.label), count === "not run" ? t(count, c.dim) : t(count, c.red)]
+		});
+		const assertion = assertionOf(k);
+		if (assertion) items.push({
+			indent: indent + 2,
+			free: t(assertion, c.dim)
+		});
+	}
+	for (const err of cell.errors) items.push({
+		indent,
+		cells: [errorKind(err), t(errorText(err), c.dim)]
+	});
+	const passed = cell.checks.filter((k) => !failing(k)).length;
+	if (more && passed && items.length) items.push({
+		indent,
+		free: t(`+${passed} more passed`, c.dim)
+	});
+	return items;
+}
+/** A control with no environment of its own is the baseline the variants change. */
+const armOf = (cell) => cell.variant ?? (cell.environment === "-" ? "baseline" : cell.environment);
+/** Each cell's label: the parts that vary between them (harness, arm, sandbox). */
+function labeller(cells, parts) {
+	const vary = parts.filter((p) => {
+		const of = (cell) => p === "harness" ? cell.harness : p === "arm" ? armOf(cell) : cell.sandbox;
+		return new Set(cells.map(of)).size > 1;
+	});
+	const used = vary.length ? vary : parts.slice(0, 1);
+	return (cell) => used.map((p) => p === "harness" ? cell.harness : p === "arm" ? armOf(cell) : cell.sandbox).join(" · ");
+}
+const effectText = (c, effect) => {
+	if (!effect) return t("");
+	const paint = effect === "better" ? c.green : effect === "worse" ? c.red : c.yellow;
+	return t(EFFECT[effect], paint);
+};
+const signedPoints = (x) => `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
+/** `-v`: a variant's difference from its baseline and the interval its effect comes from. */
+function statsLine(cell, cells) {
+	const base = cells.find((x) => x.key === cell.vs);
+	if (!base || !cell.interval) return null;
+	const rate = (x) => x.passed + x.failed ? x.passed / (x.passed + x.failed) : 0;
+	return `difference ${signedPoints(rate(cell) - rate(base))} points, 95% interval ${signedPoints(cell.interval.low)}…${signedPoints(cell.interval.high)}`;
+}
+/** The checks that failed or didn't run in some cell, a column per cell; the rest pass everywhere. */
+function checkMatrix(c, cells, label, indent, width) {
+	const labels = [...new Set(cells.flatMap((x) => x.checks.map((k) => k.label)))];
+	const shown = labels.filter((l) => cells.some((x) => x.checks.some((k) => k.label === l && failing(k))));
+	if (!shown.length) return [];
+	const items = [{
+		indent,
+		cells: ["check", ...cells.map((x) => t(label(x), c.dim))]
+	}];
+	for (const l of shown) {
+		const row = cells.map((x) => {
+			const k = x.checks.find((y) => y.label === l);
+			if (!k) return t("–", c.dim);
+			const count = checkCount(k, false);
+			if (!failing(k)) return t(count);
+			return t(count, k.outcome.status === "not run" || !x.gated ? c.dim : c.red);
+		});
+		items.push({
+			indent,
+			cells: [checkLabel(l), ...row]
+		});
+		const assertion = cells.map((x) => x.checks.find((y) => y.label === l)).map((k) => k ? assertionOf(k) : "").find(Boolean);
+		if (assertion) items.push({
+			indent: indent + 2,
+			free: t(assertion, c.dim)
+		});
+	}
+	const passed = labels.length - shown.length;
+	if (passed) items.push({
+		indent,
+		free: t(`+${passed} more passed`, c.dim)
+	});
+	return table(items, width);
+}
+/** Tools, agent time and tokens per cell, when each has enough tries and it matters. */
+function effortTable(c, cells, label, indent, failed, width) {
+	if (cells.length < 2 || cells.some((x) => !x.effort)) return [];
+	if (!failed && !cells.some((x) => x.effortDiffers)) return [];
+	const tokens = cells.some((x) => x.effort?.tokens !== null);
+	const n = (v, f) => v == null ? "–" : f(v);
+	return table([{
+		indent,
+		cells: [
+			"effort",
+			t("tools", c.dim),
+			t("time", c.dim),
+			...tokens ? [t("tokens", c.dim)] : []
+		]
+	}, ...cells.map((x) => ({
+		indent,
+		cells: [
+			label(x),
+			n(x.effort?.tools, formatCount),
+			n(x.effort?.agentMs, formatSeconds),
+			...tokens ? [n(x.effort?.tokens, formatTokens)] : []
+		]
+	}))], width);
+}
+const NO_SLOTS = {
+	row: () => null,
+	named: () => [],
+	subject: null
+};
+const triesOf = (cells) => cells.reduce((n, x) => n + countedOf(x), 0);
+/** `named`, then a row per tool with the tries that named it. */
+function namedItems(c, slots, cells, indent) {
+	const named = slots.named(cells);
+	if (!named.length) return [];
+	const of = triesOf(cells);
+	return [{
+		indent,
+		free: t("named", c.dim)
+	}, ...named.map((n) => ({
+		indent: indent + 2,
+		cells: [n.tool, `${n.tries} of ${of}`]
+	}))];
+}
+/** The tries that named the subject, over the cells' tries: `0 of 4`; empty with no subject. */
+function subjectTries(slots, cells) {
+	if (!slots.subject) return "";
+	const subject = slots.subject.toLowerCase();
+	return `${slots.named(cells).find((x) => x.tool.toLowerCase() === subject)?.tries ?? 0} of ${triesOf(cells)}`;
+}
+/** Web searches or fetches over the tries, when every try reported a number. */
+function webItems(cells, indent) {
+	const row = (name, counts) => counts.length && counts.every((x) => x !== null) ? [{
+		indent,
+		cells: [name, String(counts.reduce((a, b) => (a ?? 0) + (b ?? 0), 0))]
+	}] : [];
+	return [...row("searches", cells.flatMap((x) => x.searches)), ...row("fetches", cells.flatMap((x) => x.fetches))];
+}
+/** A choose case that missed: its failed checks, the tools named, its searches, then `+N more passed`. */
+function chooseDetail(c, cells, indent, slots) {
+	const checks = cells.flatMap((cell) => evidence(c, cell, indent, false));
+	const passed = cells.flatMap((x) => x.checks.filter((k) => !failing(k))).length;
+	return [
+		...checks,
+		...namedItems(c, slots, cells, indent),
+		...webItems(cells, indent),
+		...passed ? [{
+			indent,
+			free: t(`+${passed} more passed`, c.dim)
+		}] : []
+	];
+}
+/** Choose a tool, on a single eval: the subject's count, and on a miss the detail. */
+function chooseBody(c, cells, indent, slots) {
+	const items = slots.subject ? [{
+		indent,
+		cells: [slots.subject, subjectTries(slots, cells)]
+	}] : [];
+	if (!cells.every(clean)) items.push(...chooseDetail(c, cells, indent, slots));
+	return table(items);
+}
+/** A row's sentence, as a line under it. */
+const rowLine = (slots, path, indent) => {
+	const text = slots.row(path);
+	return text ? [{
+		indent,
+		free: text
+	}] : [];
+};
+/** Measure a change: a row per arm with its effect, the check matrix and the effort table. */
+function measureBody(c, cells, indent, o) {
+	const label = labeller(cells, ["arm", "sandbox"]);
+	const arms = [{
+		indent,
+		cells: ["arm", t("pass", c.dim)]
+	}];
+	for (const cell of cells) {
+		const effect = o.effects ? gatedEffect(c, cell) : t("");
+		arms.push({
+			indent,
+			cells: [
+				label(cell),
+				cellCount(cell),
+				...effect.text ? [effect] : []
+			]
+		});
+		arms.push(...rowLine(o.slots, cell.path, indent + 2), ...namedItems(c, o.slots, [cell], indent + 2));
+		const stats = o.verbose && o.effects ? statsLine(cell, cells) : null;
+		if (stats) arms.push({
+			indent: indent + 2,
+			free: t(stats, c.dim)
+		});
+		for (const err of cell.errors) arms.push({
+			indent: indent + 2,
+			free: t(`${errorKind(err)}  ${errorText(err)}`, c.dim)
+		});
+	}
+	const matrix = o.failed || o.effects;
+	const labels = [...cells.map(label), ...matrix ? cells.flatMap((x) => x.checks.filter(failing).map((k) => checkLabel(k.label))) : []];
+	const width = Math.max(LABEL, ...labels.map((l) => indent + l.length));
+	return [
+		...table(arms, width),
+		...matrix ? checkMatrix(c, cells, label, indent, width) : [],
+		...effortTable(c, cells, label, indent, o.failed, width)
+	];
+}
+/** A row per cell (harness, or sandbox), the ones that didn't pass outright with their evidence. */
+function cellRows(c, cells, heading, indent, slots) {
+	const label = labeller(cells, heading === "harness" ? ["harness", "sandbox"] : ["sandbox"]);
+	const items = [{
+		indent,
+		cells: [heading, t("pass", c.dim)]
+	}];
+	for (const cell of cells) {
+		items.push({
+			indent,
+			cells: [label(cell), cellCount(cell)]
+		});
+		items.push(...rowLine(slots, cell.path, indent + 2));
+		if (!clean(cell)) items.push(...namedItems(c, slots, [cell], indent + 2), ...evidence(c, cell, indent + 2));
+	}
+	return table(items);
+}
+/** Compare harnesses with arms: a heading per harness, its arms as Measure a change under it. */
+function bothBody(c, cells, indent, o) {
+	const harnessOf = labeller(cells, ["harness", "sandbox"]);
+	const groups = /* @__PURE__ */ new Map();
+	for (const cell of cells) groups.set(harnessOf(cell), [...groups.get(harnessOf(cell)) ?? [], cell]);
+	return [...groups].flatMap(([harness, group]) => [`${" ".repeat(indent)}${harness}`, ...measureBody(c, group, indent + 2, o)]);
+}
+/** The rows under an eval's head (or a family case's), by what its cells compare. */
+function cellsBody(c, kind, cells, indent, o) {
+	if (kind === "measure") return measureBody(c, cells, indent, o);
+	if (kind === "harnesses") return cellRows(c, cells, "harness", indent, o.slots);
+	if (kind === "both") return bothBody(c, cells, indent, o);
+	if (kind === "choose") return chooseBody(c, cells, indent, o.slots);
+	if (cells.length > 1) return cellRows(c, cells, "sandbox", indent, o.slots);
+	return resultsBody(c, cells[0], indent);
+}
+/** A single eval with one cell: its tries, and on anything but a clean pass, the evidence. */
+function resultsBody(c, cell, indent) {
+	if (clean(cell)) return [];
+	const items = [];
+	if (cell.attempts > 1) items.push({
+		indent,
+		cells: [plural(cell.attempts, "try", "tries"), cellCount(cell)]
+	});
+	return table([...items, ...evidence(c, cell, indent)]);
+}
+function questionLines(c, questions, indent) {
+	return questions.flatMap((q) => {
+		const pad = " ".repeat(indent);
+		const head = `${pad}${c.dim("question")}  ${clip(q.question, 100)}`;
+		if (q.votes === 0 && q.error === null) return [head, `${pad}  ${c.dim("no pairs to judge: tries that failed their checks are left out")}`];
+		if (q.rates.length === 0 && q.error === null) return [head, `${pad}  ${c.dim(`${q.votes} ${q.votes === 1 ? "vote" : "votes"}, every one a tie`)}`];
+		const cuts = new Set(q.rates.map((r) => r.cut)).size > 1;
+		const rows = table(q.rates.map((r) => ({
+			indent: indent + 2,
+			cells: [cuts ? `${r.name} (${r.cut})` : r.name, `won ${r.wins} of ${r.n}`]
+		})));
+		const failed = q.error === null ? [] : [`${pad}  ${c.yellow(`the model call failed: ${clip(q.error, 200)}`)}`];
+		return [
+			head,
+			...rows,
+			...failed
+		];
+	});
+}
+/** A cell's counts summed over a family's cases. */
+function pooledCount(cases, key) {
+	let passed = 0;
+	let counted = 0;
+	for (const k of cases) {
+		const cell = k.cells.find((x) => x.key === key);
+		if (!cell) continue;
+		passed += cell.passed;
+		counted += cell.passed + cell.failed;
+	}
+	return counted ? `${passed} of ${counted}` : "not run";
+}
+/** A variant's effect, with the `minLift` its interval's low end fell short of, which fails it. */
+function gatedEffect(c, arm) {
+	const effect = effectText(c, arm.effect);
+	if (arm.minLift === null || !arm.interval || arm.interval.low >= arm.minLift) return effect;
+	const short = `not by minLift ${Math.round(arm.minLift * 100)}`;
+	return t(effect.text ? `${effect.text}, ${short}` : short, c.red);
+}
+function pooledStats(p, label) {
+	return `${label}: ${p.pairs} paired cases, 95% interval ${signedPoints(p.interval.low)}…${signedPoints(p.interval.high)}`;
+}
+/**
+* A family: its template, then a row per case. With one cell a case is a row with its count;
+* with more, the case is outer and its cells inner, laid out as a single eval's are, and a
+* pooled `all cases` row sums each cell's counts with its variant's pooled effect. A choose
+* family's case row carries the subject's count instead, and a missed case the tools named.
+*/
+function familyBlock(c, e, kind, verbose, slots) {
+	const cases = e.cases ?? [];
+	const columns = [...new Map(cases.flatMap((k) => k.cells).map((cell) => [cell.key, cell])).values()];
+	const pooled = e.pooled ?? [];
+	const pooledOutcome = pooled.some((p) => p.outcome === "fail") ? "fail" : "pass";
+	const caseWord = (k) => verdict(c, {
+		outcome: k.outcome,
+		todo: false,
+		cells: k.cells
+	});
+	const pooledWord = t(VERDICT[pooledOutcome], paintOutcome(c, pooledOutcome));
+	const words = [...cases.map(caseWord), ...pooled.length ? [pooledWord] : []];
+	const wordWidth = Math.max(...words.map((w) => w.text.length));
+	const lead = (word, name) => ({
+		text: `${word.text.padEnd(wordWidth)}  ${name}`,
+		paint: (s) => `${show(word)}${s.slice(word.text.length)}`
+	});
+	const head = `${" ".repeat(17)}${e.template ?? e.name}`;
+	const questions = questionLines(c, e.questions, 2);
+	if (kind === "choose") {
+		const subject = slots.subject ? [t(slots.subject, c.dim)] : [];
+		const items = [{
+			indent: 2,
+			cells: [
+				"case",
+				t("pass", c.dim),
+				...subject
+			]
+		}];
+		for (const k of cases) {
+			const passed = k.cells.reduce((n, x) => n + x.passed, 0);
+			const tries = triesOf(k.cells);
+			const count = k.cells.length === 1 && k.cells[0] ? cellCount(k.cells[0]) : tries ? `${passed} of ${tries}` : "not run";
+			const named = slots.subject && tries ? [subjectTries(slots, k.cells)] : [];
+			items.push({
+				indent: 2,
+				cells: [
+					lead(caseWord(k), k.label),
+					count,
+					...named
+				]
+			});
+			items.push(...rowLine(slots, k.path, 4));
+			if (!k.cells.every(clean)) items.push(...chooseDetail(c, k.cells, 4, slots));
+		}
+		return [
+			head,
+			...table(items),
+			...questions
+		];
+	}
+	if (columns.length <= 1) {
+		const items = [];
+		for (const k of cases) {
+			const cell = k.cells[0];
+			items.push({
+				indent: 2,
+				cells: [lead(caseWord(k), k.label), cell ? cellCount(cell) : ""]
+			});
+			items.push(...rowLine(slots, k.path, 4));
+			if (cell && !clean(cell)) items.push(...evidence(c, cell, 4));
+		}
+		return [
+			head,
+			...table(items),
+			...questions
+		];
+	}
+	const out = [head];
+	for (const k of cases) {
+		out.push(`  ${show(lead(caseWord(k), k.label))}`);
+		const sentence = slots.row(k.path);
+		if (sentence) out.push(`    ${sentence}`);
+		const failed = k.outcome !== "pass";
+		out.push(...cellsBody(c, kind, k.cells, 4, {
+			failed,
+			verbose,
+			effects: false,
+			slots
+		}));
+	}
+	if (pooled.length) {
+		const label = labeller(columns, [
+			"harness",
+			"arm",
+			"sandbox"
+		]);
+		const byKey = new Map(pooled.map((p) => [p.key, p]));
+		const items = [{
+			indent: 4,
+			cells: [kind === "harnesses" ? "harness" : "arm", t("pass", c.dim)]
+		}];
+		for (const col of columns) {
+			const p = byKey.get(col.key);
+			const effect = p ? gatedEffect(c, p) : t("");
+			items.push({
+				indent: 4,
+				cells: [
+					label(col),
+					pooledCount(cases, col.key),
+					...effect.text ? [effect] : []
+				]
+			});
+			if (p) items.push(...rowLine(slots, p.path, 6));
+			if (p && verbose) items.push({
+				indent: 6,
+				free: t(pooledStats(p, label(col)), c.dim)
+			});
+		}
+		out.push(`  ${show(lead(pooledWord, "all cases"))}`, ...table(items));
+	}
+	return [...out, ...questions];
+}
+/** The slots an eval's insight fills, or none without one. */
+function slotsFor(insight) {
+	if (!insight) return NO_SLOTS;
+	const rows = new Map((insight.rows ?? []).map((r) => [r.row, oneLine(r.text)]));
+	const choose = insight.intent.job === "choose";
+	const named = (cells) => {
+		if (!choose) return [];
+		const counts = /* @__PURE__ */ new Map();
+		for (const n of insight.namedTools ?? []) {
+			if (!cells.some((cell) => n.attempt.startsWith(`${cell.path}#`))) continue;
+			for (const tool of new Set(n.tools.map(oneLine))) counts.set(tool, (counts.get(tool) ?? 0) + 1);
+		}
+		return [...counts].map(([tool, tries]) => ({
+			tool,
+			tries
+		})).sort((a, b) => b.tries - a.tries || (a.tool < b.tool ? -1 : 1));
+	};
+	return {
+		row: (path) => rows.get(path) ?? null,
+		named,
+		subject: choose && insight.intent.subject.tool ? oneLine(insight.intent.subject.tool) : null
+	};
+}
+/** One eval's block, laid out as `layout()` says for its job, with its insight's sentences. */
+function evalBlock(c, e, verbose = false, insight = null) {
+	const shape = layout(insight?.intent.job ?? "other", e.across, e.family, false);
+	const slots = slotsFor(insight);
+	const answer = insight?.answer ? [`  ${oneLine(insight.answer.text)}`] : [];
+	const findings = (insight?.findings ?? []).map((f) => `  ${oneLine(f.text)}`);
+	if (shape.family) {
+		const [head = "", ...rest] = familyBlock(c, e, shape.kind, verbose, slots);
+		return [
+			head,
+			...answer,
+			...rest,
+			...findings
+		];
+	}
+	const v = verdict(c, e);
+	const name = `${e.group.length ? `${e.group.join(" › ")} › ` : ""}${e.name}`;
+	const head = `${show(v)}${" ".repeat(Math.max(2, 17 - v.text.length))}${name}`;
+	const cells = e.cells ?? [];
+	if (e.todo || e.outcome === "skipped" || !cells.length) return [head];
+	const failed = e.outcome !== "pass";
+	const body = cellsBody(c, shape.kind, cells, 2, {
+		failed,
+		verbose,
+		effects: true,
+		slots
+	});
+	return [
+		head,
+		...answer,
+		...body,
+		...findings,
+		...questionLines(c, e.questions, 2)
+	];
+}
+/**
+* `5 fail · 2 pass · 1 cancelled`: errors outside any eval and fails first, every case counting
+* one, zeros left out.
+*/
+function tallyLine(c, tally) {
+	const parts = [
+		tally.errors ? c.red(plural(tally.errors, "run error")) : "",
+		tally.fail ? c.red(`${tally.fail} fail`) : "",
+		tally.pass ? c.green(`${tally.pass} pass`) : "",
+		tally.infraError ? c.yellow(plural(tally.infraError, "infra error")) : "",
+		tally.cancelled ? c.yellow(`${tally.cancelled} cancelled`) : "",
+		tally.skipped ? c.dim(`${tally.skipped} skipped`) : ""
+	].filter(Boolean);
+	return parts.length ? parts.join(c.dim(" · ")) : "no evals ran";
+}
+/** Every eval's block in file-path order, then source order; a blank line around each longer one. */
+function renderBlocks(suite, opts, insights = null) {
+	const c = colors(opts.color);
+	const byEval = new Map((insights?.evals ?? []).map((x) => [x.evalId, x]));
+	const blocks = suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).map((e) => evalBlock(c, e, opts.verbose, byEval.get(e.path) ?? null)));
+	const out = [];
+	blocks.forEach((block, i) => {
+		const prev = blocks[i - 1];
+		if (prev && (prev.length > 1 || block.length > 1)) out.push("");
+		out.push(...block);
+	});
+	return out;
+}
+/** Insights for these facts: null when there are none, they failed, or they were written for other facts. */
+const freshInsights = (suite, insights) => insights && !insights.error && insights.factsHash === factsHash(suite) ? insights : null;
+/** Why a run's insights didn't print, or with `-v` what wrote them. */
+function insightNotes(c, suite, insights, fresh, verbose) {
+	if (!insights || !fromModel(insights)) return [];
+	if (insights.error) return [c.yellow(`insights failed (${clip(insights.error, 200)})`)];
+	if (!fresh) return [c.dim(`insights left out: insights.json was written for other facts than this run's; delete it and run reeval analyze ${suite.runId} --insights`)];
+	const skipped = insights.budget?.skipped.length ?? 0;
+	const failed = insights.failed ?? [];
+	const stopped = insights.stopped;
+	const finish = `reeval analyze ${suite.runId} --insights to finish`;
+	const budget = [
+		...skipped ? [c.yellow(`insights skipped for ${plural(skipped, "eval")}: analysis.maxTokens (${formatTokens(insights.budget?.maxTokens ?? 0)}) reached — raise it in reeval.config.ts, or ${finish}`)] : [],
+		...failed[0] ? [c.yellow(`insights failed for ${plural(failed.length, "eval")} (${clip(failed[0].error, 160)}), so they print their facts: ${finish}`)] : [],
+		...stopped ? [c.yellow(`insights stopped: ${plural(stopped.skipped.length, "eval")} skipped (${stopped.signal === "SIGTERM" ? "SIGTERM" : "Ctrl-C"}): ${finish}`)] : []
+	];
+	if (!verbose) return budget;
+	const { dropped, regenerated } = insights.validation;
+	return [...budget, c.dim([
+		`insights by ${insights.model} (${insights.promptVersion})`,
+		`${plural(dropped, "sentence")} removed`,
+		...regenerated ? [`${regenerated} asked again`] : [],
+		`${formatTokens(insights.usage.inputTokens)} in, ${formatTokens(insights.usage.outputTokens)} out${insights.usage.reasoningTokens ? ` (${formatTokens(insights.usage.reasoningTokens)} reasoning)` : ""}`
+	].join(" · "))];
+}
+/**
+* A run's result: its errors outside any eval, its blocks with their insights, the headline,
+* then the tally and, with no analysis model, the hint, then the remarks. Insights written for
+* other facts are left out with a note; remarks come from the record and history, not a model,
+* so they print either way.
+*/
+function render(suite, insights, opts) {
+	const c = colors(opts.color);
+	const fresh = freshInsights(suite, insights);
+	const blocks = renderBlocks(suite, opts, fresh);
+	return [
+		...suite.errors.flatMap((error) => [c.red(`error: ${error}`), ""]),
+		...blocks,
+		...blocks.length ? [""] : [],
+		...fresh?.headline ? [oneLine(fresh.headline.text)] : [],
+		...insightNotes(c, suite, insights, fresh, opts.verbose),
+		...opts.notes ?? [],
+		tallyLine(c, suite.tally),
+		...opts.hint ? [NO_MODEL_HINT] : [],
+		...remarkLines(c, suite, insights, opts.verbose)
+	];
+}
+//#endregion
+//#region src/history/summary.ts
+const OUTCOME_MARK = {
+	pass: "✓",
+	fail: "✗",
+	infra_error: "!",
+	skipped: "↓"
+};
+/** Enough to read the agent's conclusion; the record and transcript have it whole. */
+const LAST_MESSAGE_LIMIT = 4e3;
+const cell = (s) => s.replace(/\|/g, "\\|").replace(/\n/g, " ");
+/** `5 fail · 2 pass`, with the run's status when it didn't complete. */
+const runTally = (run) => `${tallyLine(colors(false), suiteResult(run).tally)}${run.status === "complete" ? "" : ` · ${run.status === "budget" ? "budget reached" : run.status}`}`;
+const short = (hash) => hash.replace(/^sha256:/, "").slice(0, 12);
+/**
+* The state a run executed in, as label/value pairs, shared by summary.md and `reeval show`. The
+* commit is where the run executed; the commit that adds its history file comes later.
+*/
+function stateRows(run) {
+	const { git, stack: env } = run;
+	const rows = [];
+	rows.push(["Executed at", git ? `${git.sha.slice(0, 12)}${git.branch ? ` on ${git.branch}` : ""}${git.dirty ? ", dirty" : ", clean"} (the commit that adds this record is later)` : "not a git repository"]);
+	if (git?.dirty) rows.push(["Uncommitted", git.worktreePatch ? `worktree.patch, ${git.worktreePatch.bytes} bytes, sha256 ${short(git.worktreePatch.sha256)}` : `not saved: ${git.worktreePatchNote ?? "unknown"}${git.diffHash ? `; sha256 ${short(git.diffHash)}` : ""}`]);
+	rows.push(["Lockfiles", run.dependencies.lockfiles.map((l) => `${l.path} ${short(l.sha256)}`).join(", ") || "none found"]);
+	rows.push(["Images", env.images.length ? env.images.map((i) => `${i.ref} · id ${short(i.id)} · ${i.repoDigests[0] ?? "no registry digest (built locally)"} · CLI ${i.agentCli ?? "n/a"}${i.dockerfile ? ` · dockerfile ${i.dockerfile.path} ${short(i.dockerfile.sha256)}` : ""}${i.install?.length ? ` · install ${i.install.length} on ${i.base}` : ""}`).join("; ") : "none (fake sandbox)"]);
+	rows.push(["Harnesses", env.harnesses.map(harnessLabel).join(", ") || "none"]);
+	rows.push(["Environments", run.environments.map((e) => `${e.name} ${short(e.digest)}${e.skills.length ? ` · skills ${e.skills.map((s) => `${s.name}${s.commit ? `@${s.commit.slice(0, 7)}` : ""}`).join(", ")}` : ""}${e.mcp.length ? ` · mcp ${e.mcp.join(", ")}` : ""}${e.secrets.length ? ` · secrets ${e.secrets.join(", ")}` : ""}`).join("; ") || "none"]);
+	rows.push(["Judge", `${env.judge.model ?? "none by default"}${env.judge.resolvedModels.length ? ` (served ${env.judge.resolvedModels.join(", ")})` : ""} · threshold ${env.judge.threshold} · version ${env.judge.version}`]);
+	rows.push(["Context", env.context.length ? `${env.context.length} file${env.context.length === 1 ? "" : "s"}: ${env.context.map((c) => `${c.path} ${short(c.sha256)}`).join(", ")}` : "no skills, rules or MCP config found in the sandbox"]);
+	rows.push(["Packages", Object.entries(env.packages).map(([name, v]) => `${name} ${v}`).join(" · ")]);
+	const language = languageLabel(env);
+	rows.push(["reeval", `${env.reeval} · ${runtimeLabel(env)}${language ? ` · ${language}` : ""} · ${env.os}/${env.arch}`]);
+	rows.push(["Runner", run.runner.kind === "ci" ? `CI (${run.runner.provider})${run.runner.runUrl ? ` ${run.runner.runUrl}` : ""}` : `local · ${run.author}`]);
+	return rows;
+}
+/** Markdown for people and agents landing on a run cold. Links are relative to the artifact dir. */
+function renderSummary(run, opts) {
+	const lines = [
+		`# reeval run ${run.runId}`,
+		"",
+		`${runTally(run)} · started ${run.startedAt} (${formatDuration(run.durationMs)})`,
+		"",
+		`Plan: ${run.profile ? `profile ${run.profile.name} (${run.profile.reason}) · ` : ""}${run.plan.evals} evals · ${run.plan.cells} cells · ${run.plan.attempts} attempts${run.budget ? ` · budget ${[run.budget.attempts !== void 0 ? `${run.budget.attempts} attempts` : "", run.budget.duration ?? ""].filter(Boolean).join(", ")}` : ""}`,
+		"",
+		...run.plan.sandboxWarnings.map((w) => `> **Warning:** ${w.message}\n`),
+		"## Results",
+		"",
+		...fenced(render(suiteResult(run), opts.insights ?? null, {
+			color: false,
+			verbose: false,
+			hint: opts.hint
+		}), "text"),
+		"",
+		"## State",
+		"",
+		"| | |",
+		"|---|---|",
+		...stateRows(run).map(([k, v]) => `| ${k} | ${cell(v)} |`),
+		"",
+		...warnings(run).map((w) => `> **Warning:** ${w}\n`),
+		`Record: [${run.runId}.json](../${run.runId}.json) · artifacts: ${run.artifacts}`,
+		"",
+		"## Attempts"
+	];
+	const rank = (o) => o === "fail" || o === "infra_error" ? 0 : o === "pass" ? 1 : 2;
+	for (const e of [...run.evals].sort((x, y) => rank(x.outcome) - rank(y.outcome))) {
+		if (e.outcome === "skipped") continue;
+		lines.push("", `### ${OUTCOME_MARK[evalMarkOutcome(e)]} ${e.id}`, "");
+		const multi = e.attempts.length > 1;
+		const inputs = (e.answers ?? []).filter((a) => a.input);
+		for (const a of inputs) lines.push(`- Question: ${cell(`"${clip(a.question, 80)}"`)} · [what the model saw](${a.input?.path})`);
+		if (inputs.length) lines.push("");
+		const attemptRank = (a) => isUngatedMiss(e, a) ? 3 : rank(a.outcome);
+		for (const a of [...e.attempts].sort((x, y) => attemptRank(x) - attemptRank(y))) {
+			if (a.outcome === "skipped") continue;
+			if (isUngatedMiss(e, a)) {
+				if (multi) lines.push(`#### ○ ${a.id} · missed, not gated`, "");
+				lines.push("<details><summary>Why it missed</summary>", "");
+				attemptDetail(lines, a);
+				lines.push("", "</details>");
+				continue;
+			}
+			if (multi) lines.push(`#### ${OUTCOME_MARK[a.outcome]} ${a.id}`, "");
+			attemptDetail(lines, a);
+		}
+	}
+	lines.push("");
+	return lines.join("\n");
+}
+/** A failed verdict already says why a judge error failed the attempt; its message repeats it. */
+const repeatsVerdict = (a) => a.error?.kind === "judge" && a.judgements.some((j) => j.verdict === "fail");
+function attemptDetail(lines, a) {
+	if (a.error && !repeatsVerdict(a)) lines.push(`**${a.error.kind}**`, "", ...fenced(a.error.message.split("\n").slice(0, 20)), "");
+	for (const turn of a.turns) {
+		lines.push(`- Prompt (${turn.status}, ${formatDuration(turn.durationMs)}): ${clip(cell(turn.prompt), 200)}`);
+		if (turn.error) lines.push(`  - Error: ${clip(cell(turn.error), 500)}`);
+	}
+	const t = a.timings;
+	lines.push(`- Time: setup ${formatDuration(t.setupMs)} · agent ${formatDuration(t.agentMs)} · checks ${formatDuration(t.checksMs)} · judge ${formatDuration(t.judgeMs)}`);
+	const act = a.activity;
+	const web = (n, what) => n === null ? `${what} not reported` : `${n} ${what}`;
+	lines.push(`- Agent: ${act.fileEdits} file edits · ${act.tools} other tool calls · ${web(act.webSearches, "web searches")} · ${web(act.webFetches, "fetches")} · ${formatDuration(act.reasoningMs)} reasoning${act.stalledTools ? ` · ${act.stalledTools} stalled tool${act.stalledTools === 1 ? "" : "s"} (≥2m)${act.maxToolMs ? `, longest ${formatDuration(act.maxToolMs)}` : ""}` : ""}`);
+	for (const { judgement: j, label, labelled } of labelledJudgements(a)) {
+		const retried = j.retries?.length ? `, after ${j.retries.length} ${j.retries.length === 1 ? "retry" : "retries"}` : "";
+		lines.push(`- Judge${labelled ? ` ${cell(label)}` : ""} **${j.verdict}** ${j.score.toFixed(2)} (threshold ${j.threshold}, ${j.model}${retried}): ${j.question}`);
+	}
+	const labels = checkLabels(a);
+	a.checks.forEach((c, i) => {
+		const label = labels[i];
+		lines.push(`- Check${label ? ` ${cell(label)}` : ""} exit ${c.exitCode}: \`${cell(commandLine(c.command))}\`${c.log ? ` ([log](${c.log.path}))` : ""}`);
+	});
+	if (a.changedFiles.length) lines.push(`- Changed: ${a.changedFiles.map((f) => f.count ? `\`${f.path}\` (${f.count} files)` : `${f.status} \`${f.path}\``).join(", ")}`);
+	const last = a.turns.at(-1)?.text;
+	if (last) {
+		const shown = last.length > LAST_MESSAGE_LIMIT ? `${last.slice(0, LAST_MESSAGE_LIMIT)}…` : last;
+		lines.push("", "Last message:", "", ...shown.trimEnd().split("\n").map((l) => `> ${l}`), "");
+	}
+	const links = [
+		a.files.setupLog && `[setup log](${a.files.setupLog.path})`,
+		a.files.transcript && `[transcript](${a.files.transcript.path})`,
+		a.files.diff && `[diff](${a.files.diff.path})`,
+		a.files.judgeInput && `[judge input](${a.files.judgeInput.path})`
+	].filter(Boolean);
+	if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
+}
+//#endregion
+export { t as _, assertionOf as a, errorKind as c, labeller as d, plural as f, slotsOf as g, show as h, stateRows as i, errorText as l, renderBlocks as m, renderSummary as n, checkCount as o, render as p, runTally as r, checkLabel as s, OUTCOME_MARK as t, evalBlock as u, table as v, factsHash as y };
+
+//# sourceMappingURL=summary-0oQNGskv.js.map