reeval 0.11.0 → 0.13.0

package/dist/generate-BnrvBAzF.js

@@ -0,0 +1,1646 @@
+import { l as sha256 } from "./ids-Bp6ut7ZD.js";
+import { l as ownFile } from "./root-DNuLhaA5.js";
+import { a as oneLine, i as formatDuration, n as clip } from "./text-CrVy5ioL.js";
+import { i as cannedModel, o as providerModel } from "./model-DpkclE4r.js";
+import { C as parseTranscript } from "./model-UgHDMhI1.js";
+import { c as suiteResult, r as countedOf, t as cancelledOf } from "./suite-DmuF_ApO.js";
+import { i as REMARK_KINDS, n as MODEL_REMARK_KINDS } from "./insights-B2Xmhz9o.js";
+import { g as slotsOf, y as factsHash } from "./summary-BFMHMTz2.js";
+import { n as readInsights, t as insightsFile } from "./files-Bs6H3HiG.js";
+import { t as unverifiedNumbers } from "./validate-DWjkshl1.js";
+import { t as analysisCacheDir } from "./files-D27nUk0K.js";
+import { r as quotedRemarkId } from "./rules-CcV3zLDR.js";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import { mkdir, readFile, readdir, rename, writeFile } from "node:fs/promises";
+import { NoObjectGeneratedError, Output, generateText } from "ai";
+//#region src/insights/classify.ts
+const classifyDir = (root) => join(analysisCacheDir(root), "classify");
+const RUN_FILE = /^\d{8}T\d{6}Z_.+\.json$/;
+/** The intents of the latest recorded run that has insights, by classification key. */
+async function readBack(historyDir) {
+	const runs = (await readdir(historyDir).catch(() => [])).filter((n) => RUN_FILE.test(n)).map((n) => n.slice(0, -5)).sort().reverse();
+	for (const runId of runs) {
+		const doc = await readInsights(insightsFile(historyDir, runId)).catch(() => null);
+		if (!doc) continue;
+		return new Map(doc.evals.map(({ intent: { key, job, question, subject, name } }) => [key, {
+			job,
+			question,
+			subject,
+			name
+		}]));
+	}
+	return /* @__PURE__ */ new Map();
+}
+/** The cache for a project: its local files first, then the latest recorded run's insights. */
+function classifyCache(root, historyDir) {
+	const dir = classifyDir(root);
+	let recorded;
+	return {
+		async get(key) {
+			const text = await readFile(join(dir, `${key}.json`), "utf8").catch(() => null);
+			if (text !== null) try {
+				const value = JSON.parse(text);
+				if (value && typeof value.job === "string" && typeof value.question === "string") return value;
+			} catch {}
+			recorded ??= readBack(historyDir);
+			return (await recorded).get(key) ?? null;
+		},
+		async set(key, value) {
+			await mkdir(dir, { recursive: true });
+			const path = join(dir, `${key}.json`);
+			const tmp = `${path}.${process.pid}.tmp`;
+			await writeFile(tmp, `${JSON.stringify(value, null, 2)}\n`);
+			await rename(tmp, path);
+		}
+	};
+}
+//#endregion
+//#region src/insights/budget.ts
+/** What an eval is estimated to use before any eval has finished, from gemini-3.8-flash runs. */
+const FIRST_ESTIMATE = 6e3;
+const ledger = (maxTokens) => ({
+	maxTokens,
+	used: 0,
+	reserved: 0,
+	finished: 0
+});
+/** What the next eval reserves: the finished evals' mean, else `FIRST_ESTIMATE`. */
+const estimate = (l) => l.finished ? Math.ceil(l.used / l.finished) : FIRST_ESTIMATE;
+/** Whether the next eval in priority order starts now, waits for one in flight, or is skipped. */
+function admit(l) {
+	if (l.used + l.reserved + estimate(l) <= l.maxTokens) return "start";
+	return l.reserved > 0 ? "wait" : "skip";
+}
+/** Reserves the next eval's estimate; returns it, to hand back to `settle`. */
+function reserve(l) {
+	const tokens = estimate(l);
+	l.reserved += tokens;
+	return tokens;
+}
+/**
+* Swaps an eval's reservation for what it used. One that used nothing (its calls failed or timed
+* out before any usage came back) says nothing of what the next will use, so it leaves the mean.
+*/
+function settle(l, reserved, used) {
+	l.reserved -= reserved;
+	l.used += used;
+	if (used > 0) l.finished++;
+}
+//#endregion
+//#region src/insights/input.ts
+/** The words an effect prints as, and the only ones a sentence may use for it. */
+const EFFECT_WORDS = {
+	better: "helped",
+	worse: "made it worse",
+	same: "no difference",
+	unclear: "can't tell yet"
+};
+const TEXT_CHARS = 1500;
+const LINE_CHARS = 200;
+const verdictOf = (o) => o === "infra_error" ? "infra error" : o;
+const armOf = (c) => c.variant ?? (c.environment === "-" ? "baseline" : c.environment);
+function checkFact(k, measured) {
+	const o = k.outcome;
+	if (o.status === "not run") return {
+		path: k.path,
+		kind: "check",
+		label: k.label,
+		result: "not run"
+	};
+	const answered = o.passed + o.failed + o.errored;
+	return {
+		path: k.path,
+		kind: "check",
+		label: k.label,
+		result: `${o.passed} of ${answered} passed`,
+		...o.notRun ? { notRun: o.notRun } : {},
+		...o.errored ? { couldNotCheck: o.errored } : {},
+		...o.assertion ? { assertion: clip(o.assertion.trim().split("\n")[0] ?? "", 240) } : {},
+		...k.effect && measured ? { effect: EFFECT_WORDS[k.effect] } : {},
+		...k.judge ? { judge: true } : {}
+	};
+}
+/**
+* A cell's facts. Its tries leave out the attempts the run's interrupt cancelled, as its row
+* does; under a cancelled eval or case (`measured` false) no effect is given, so none is claimed.
+*/
+function cellFacts(c, measured) {
+	const cancelled = cancelledOf(c);
+	const out = [{
+		path: c.path,
+		kind: "cell",
+		harness: c.harness,
+		arm: armOf(c),
+		sandbox: c.sandbox,
+		verdict: verdictOf(c.outcome),
+		passed: `${c.passed} of ${countedOf(c)} tries`,
+		...c.infraError ? { infraErrors: c.infraError } : {},
+		...cancelled ? { cancelled } : {},
+		...c.baseline ? { baseline: true } : {},
+		...c.effect && measured ? { effect: EFFECT_WORDS[c.effect] } : {},
+		...c.context.length ? { context: c.context } : {},
+		...c.errors.length ? { errors: c.errors.map((e) => `${e.kind}: ${clip(e.message, LINE_CHARS)}`) } : {}
+	}, ...c.checks.map((k) => checkFact(k, measured))];
+	if (c.effort) out.push({
+		path: `${c.path}/effort`,
+		kind: "effort",
+		toolCalls: c.effort.tools,
+		seconds: c.effort.agentMs === null ? null : Math.round(c.effort.agentMs / 100) / 10,
+		tokens: c.effort.tokens
+	});
+	return out;
+}
+function caseFact(k) {
+	const passed = k.cells.reduce((n, c) => n + c.passed, 0);
+	const counted = k.cells.reduce((n, c) => n + countedOf(c), 0);
+	return {
+		path: k.path,
+		kind: "case",
+		label: k.label,
+		verdict: verdictOf(k.outcome),
+		passed: `${passed} of ${counted} tries`
+	};
+}
+/** Every fact of the eval, its own node first. */
+function evalFacts(e) {
+	const out = [{
+		path: e.path,
+		kind: e.family ? "family" : "eval",
+		name: e.name,
+		verdict: verdictOf(e.outcome),
+		...e.cases ? { cases: e.cases.length } : {}
+	}];
+	const measured = e.outcome !== "cancelled";
+	for (const c of e.cells ?? []) out.push(...cellFacts(c, measured));
+	for (const k of e.cases ?? []) {
+		out.push(caseFact(k));
+		for (const c of k.cells) out.push(...cellFacts(c, measured && k.outcome !== "cancelled"));
+	}
+	for (const p of e.pooled ?? []) out.push({
+		path: p.path,
+		kind: "all cases",
+		arm: p.key,
+		...measured ? { effect: EFFECT_WORDS[p.effect] } : {},
+		verdict: p.outcome,
+		pairs: p.pairs
+	});
+	return out;
+}
+/** The record's evals behind a result: the eval, or each of a family's cases, with their cells' paths. */
+function entriesOf(record, e) {
+	const cells = (entry, results) => new Map(results.map((c) => [c.key, c]).filter(([k]) => entry.cells.some((x) => x.key === k)));
+	if (e.cases) return e.cases.flatMap((k) => {
+		const entry = record.evals.find((x) => x.id === k.id);
+		return entry ? [{
+			entry,
+			cells: cells(entry, k.cells)
+		}] : [];
+	});
+	const entry = record.evals.find((x) => x.id === e.id);
+	return entry ? [{
+		entry,
+		cells: cells(entry, e.cells ?? [])
+	}] : [];
+}
+/** An attempt's id in insights: its cell's path and its number. */
+const attemptId = (cell, a) => `${cell.path}#${a.n}`;
+/** A pass or a fail the agent made: not an infra error, a skip, or one the interrupt cancelled. */
+const counted = (a) => a.outcome === "pass" || a.outcome === "fail" && a.error?.kind !== "aborted";
+/** Each counted attempt's id and its whole final message, as the record keeps it. */
+function attemptTexts(record, e) {
+	const out = /* @__PURE__ */ new Map();
+	for (const { entry, cells } of entriesOf(record, e)) for (const a of entry.attempts) {
+		const cell = cells.get(a.cell);
+		if (cell && counted(a)) out.set(attemptId(cell, a), a.turns.at(-1)?.text ?? "");
+	}
+	return out;
+}
+function describeEvent(line) {
+	switch (line.type) {
+		case "text": return `text: ${line.text}`;
+		case "tool": return `tool ${line.tool}${line.ok ? "" : " (failed)"}: ${JSON.stringify(line.input)}`;
+		case "web-search": return `web search: ${line.query}`;
+		case "web-fetch": return `web fetch: ${line.url}`;
+		case "error": return `error: ${line.message}`;
+		default: return null;
+	}
+}
+/** A failed attempt's last tool calls and messages, numbered by their line in transcript.jsonl. */
+async function transcriptLines(runDir, a) {
+	if (a.outcome !== "fail" || !a.files.transcript) return [];
+	const text = await readFile(join(runDir, ...a.files.transcript.path.split("/")), "utf8").catch(() => null);
+	if (!text) return [];
+	const out = [];
+	text.split("\n").forEach((raw, i) => {
+		const [line] = raw.trim() ? parseTranscript(raw) : [];
+		const shown = line && describeEvent(line);
+		if (shown) out.push(`L${i + 1} ${clip(shown.replace(/\s+/g, " "), LINE_CHARS)}`);
+	});
+	return out.slice(-14);
+}
+async function attemptsOf(record, runDir, e) {
+	const out = [];
+	for (const { entry, cells } of entriesOf(record, e)) for (const a of entry.attempts) {
+		const cell = cells.get(a.cell);
+		if (!cell || !counted(a)) continue;
+		out.push({
+			id: attemptId(cell, a),
+			cell: cell.path,
+			outcome: a.outcome,
+			lastText: clip(a.turns.at(-1)?.text ?? "", TEXT_CHARS),
+			lines: runDir ? await transcriptLines(runDir, a) : []
+		});
+	}
+	return out;
+}
+const unique = (xs) => [...new Set(xs.filter(Boolean))];
+/** The prompts, judge questions and check labels classification reads after the name. */
+function declared(record, e) {
+	const entries = entriesOf(record, e).map((x) => x.entry);
+	return {
+		prompts: (entries.flatMap((x) => x.attempts).filter((a) => a.turns.length).sort((a, b) => (a.seq ?? Infinity) - (b.seq ?? Infinity))[0]?.turns ?? []).map((t) => clip(t.prompt, 1e3)),
+		judgeQuestions: unique(entries.flatMap((x) => x.attempts.flatMap((a) => a.judgements.map((j) => j.question)))),
+		checks: unique(entries.flatMap((x) => x.cells.flatMap((c) => (c.checks ?? []).map((k) => k.label)))),
+		sources: entries[0]?.sources ?? []
+	};
+}
+/** The eval's input to both of its calls; `call` and the insights call's fields are set by the caller. */
+async function evalInput(record, runDir, e) {
+	const { prompts, judgeQuestions, checks } = declared(record, e);
+	return {
+		eval: {
+			path: e.path,
+			name: e.template ?? e.name,
+			group: e.group,
+			file: e.id.split(" › ")[0] ?? "",
+			across: e.across,
+			family: e.family,
+			verdict: verdictOf(e.outcome)
+		},
+		prompts,
+		judgeQuestions,
+		checks,
+		facts: evalFacts(e),
+		attempts: await attemptsOf(record, runDir, e)
+	};
+}
+/** The most source lines an eval's input shows: its declaration, from its `evaluate` on. */
+const SOURCE_LINES = 80;
+/** A line that starts another eval in the same file. */
+const NEXT_EVAL = /\bevaluate(?:\.each|\.todo|\.skip|\.only)?\s*[(<]/;
+/**
+* Where the eval named `name` is declared in a file's lines: from the `evaluate` that names it (or
+* a family's template) up to the next eval's, at most `SOURCE_LINES`, as 0-based `[from, to)`.
+* Null when no line names it.
+*/
+function declarationOf(lines, name) {
+	const named = lines.findIndex((l) => [
+		`'${name}'`,
+		`"${name}"`,
+		`\`${name}\``
+	].some((q) => l.includes(q)));
+	if (named < 0) return null;
+	let from = named;
+	for (let i = named; i >= Math.max(0, named - 10); i--) if (NEXT_EVAL.test(lines[i] ?? "")) {
+		from = i;
+		break;
+	}
+	let to = Math.min(lines.length, from + SOURCE_LINES);
+	for (let i = named + 1; i < to; i++) if (NEXT_EVAL.test(lines[i] ?? "")) {
+		to = i;
+		break;
+	}
+	return {
+		from,
+		to
+	};
+}
+/**
+* The eval's declaration in its file, numbered as the file is (`declarationOf`), or the file's
+* first `SOURCE_LINES` when no line names it. Null when the file can't be read.
+*/
+async function evalSource(root, record, e) {
+	const entry = entriesOf(record, e)[0]?.entry;
+	if (!entry) return null;
+	const text = await readFile(join(root, entry.file), "utf8").catch(() => null);
+	if (text === null) return null;
+	const lines = text.split("\n");
+	const { from, to } = declarationOf(lines, entry.family?.name ?? entry.name) ?? {
+		from: 0,
+		to: Math.min(lines.length, SOURCE_LINES)
+	};
+	return {
+		file: entry.file,
+		lines: lines.slice(from, to).map((l, i) => `L${from + i + 1} ${clip(l, LINE_CHARS)}`.trimEnd())
+	};
+}
+/** The classification's cache key: what it read, the model and the prompt version. */
+function classificationKey(record, e, model, promptVersion) {
+	const { prompts, judgeQuestions, checks, sources } = declared(record, e);
+	return sha256(JSON.stringify({
+		id: e.id,
+		sources,
+		prompts,
+		judgeQuestions,
+		checks,
+		model,
+		promptVersion
+	}));
+}
+const TRANSCRIPT_ID = /^(A\d+)#L(\d+)$/;
+const ID = String.raw`\b[FA]\d+(?:#L\d+)?\b`;
+const ID_LIST = new RegExp(String.raw`\s*[([]${ID}(?:\s*(?:,|;|and)\s*${ID})*[)\]]`, "g");
+/** A short id left in a sentence's text, which the reader would see. */
+const ID_IN_TEXT = new RegExp(ID);
+/** The text without the bracketed id lists models append as citations (`… each (F2, F7).`). */
+const withoutIdLists = (text) => text.replace(ID_LIST, "");
+/**
+* The eval's input with a short id for each fact (`F1`…) and attempt (`A1`…), and the map back:
+* the model cites ids, not the long percent-encoded paths, and the validator sees paths again.
+*/
+function modelInput(input) {
+	const factIds = new Map(input.facts.map((f, i) => [f.path, `F${i + 1}`]));
+	const attemptIds = new Map(input.attempts.map((a, i) => [a.id, `A${i + 1}`]));
+	const facts = new Map([...factIds].map(([path, id]) => [id, path]));
+	const attempts = new Map([...attemptIds].map(([path, id]) => [id, path]));
+	const view = {
+		...input,
+		facts: input.facts.map((f) => ({
+			id: factIds.get(f.path),
+			...f
+		})),
+		attempts: input.attempts.map((a) => ({
+			...a,
+			id: attemptIds.get(a.id),
+			cell: factIds.get(a.cell) ?? a.cell
+		})),
+		...input.slots ? { slots: {
+			...input.slots,
+			rows: input.slots.rows.map((r) => factIds.get(r) ?? r)
+		} } : {}
+	};
+	const cite = (c) => {
+		const t = TRANSCRIPT_ID.exec(c.trim());
+		if (t) {
+			const attempt = attempts.get(t[1]);
+			return attempt ? `transcript:${attempt}#L${t[2]}` : c;
+		}
+		return facts.get(c.trim()) ?? c;
+	};
+	return {
+		input: view,
+		ids: {
+			cite,
+			row: (r) => facts.get(r.trim()) ?? r,
+			attempt: (a) => attempts.get(a.trim()) ?? a
+		}
+	};
+}
+/** The headline call's input with a short id for each eval, and the map back to their paths. */
+function modelHeadlineInput(input) {
+	const paths = new Map(input.evals.map((e, i) => [`F${i + 1}`, e.path]));
+	const cite = (c) => paths.get(c.trim()) ?? c;
+	return {
+		input: {
+			...input,
+			evals: input.evals.map((e, i) => ({
+				id: `F${i + 1}`,
+				...e
+			}))
+		},
+		ids: {
+			cite,
+			row: cite,
+			attempt: (a) => a
+		}
+	};
+}
+//#endregion
+//#region src/insights/priority.ts
+const cellsOf = (e) => [...e.cells ?? [], ...(e.cases ?? []).flatMap((k) => k.cells)];
+const madeWorse = (e) => (e.pooled ?? []).some((p) => p.effect === "worse") || cellsOf(e).some((c) => c.effect === "worse" || c.checks.some((k) => k.effect === "worse"));
+/** 0 is spent first. */
+function insightTier(e) {
+	if (e.outcome === "fail") return 0;
+	if (madeWorse(e)) return 1;
+	if (e.questions.length || e.across !== "none") return 2;
+	if (e.outcome === "infra_error") return 3;
+	if (e.outcome === "pass" && cellsOf(e).some((c) => c.failed > 0)) return 4;
+	return 5;
+}
+/** `evals` (in output order) in the order the budget reaches them. */
+const byPriority = (evals) => evals.map((e, i) => ({
+	e,
+	i,
+	tier: insightTier(e)
+})).sort((a, b) => a.tier - b.tier || a.i - b.i).map((x) => x.e);
+/** The most findings an eval prints. */
+const MAX_FINDINGS = 3;
+/** Words the output never uses: the statistics behind a verdict, and reeval's own terms for cells. */
+const BANNED = [
+	{
+		word: "within noise",
+		pattern: /\bwithin (the )?noise\b/i
+	},
+	{
+		word: "too close to call",
+		pattern: /\btoo close to call\b/i
+	},
+	{
+		word: "effect size",
+		pattern: /\beffect sizes?\b/i
+	},
+	{
+		word: "interval",
+		pattern: /\bintervals?\b/i
+	},
+	{
+		word: "significance",
+		pattern: /\b(statistical(ly)? )?significan(ce|t|tly)\b/i
+	},
+	{
+		word: "noise",
+		pattern: /\bnois(e|y)\b/i
+	},
+	{
+		word: "p-value",
+		pattern: /\bp-?values?\b|\bp\s*[<=>≤]\s*0?\.\d/i
+	},
+	{
+		word: "confidence",
+		pattern: /\bconfiden(ce|t)\b/i
+	},
+	{
+		word: "Newcombe",
+		pattern: /\bnewcombe\b/i
+	},
+	{
+		word: "Wilson",
+		pattern: /\bwilson\b/i
+	},
+	{
+		word: "pass@1",
+		pattern: /\bpass@1\b/i
+	},
+	{
+		word: "lift",
+		pattern: /\blifts?\b/i
+	},
+	{
+		word: "control",
+		pattern: /\bcontrols?\b/i
+	},
+	{
+		word: "variant",
+		pattern: /\bvariants?\b/i
+	},
+	{
+		word: "arm",
+		pattern: /\barms?\b/i
+	},
+	{
+		word: "cell",
+		pattern: /\bcells?\b/i
+	},
+	{
+		word: "discriminates",
+		pattern: /\bdiscriminat\w*/i
+	}
+];
+const TRANSCRIPT = /^transcript:(.+)#L(\d+)$/;
+/** A quoted span: double, curly or backtick quotes, or single quotes that aren't an apostrophe. */
+const QUOTE = /["“”]([^"“”]+)["“”]|‘(.+?)’(?!\w)|`([^`]+)`|(?<![\w'])'([^'\n]+?)'(?!\w)/g;
+const PERCENT = /\d\s*%|\bper ?cent\b/i;
+/**
+* Each effect's word and the claims that say it: the word itself and the plain words for it
+* (`helps`, `improved`, `worse`, `hurt`). A negated `help` (`didn't help`) claims no `helped`.
+*/
+const EFFECT_PHRASES = [
+	{
+		word: EFFECT_WORDS.better,
+		pattern: /(?<!(?:n['’]t|\bnot|\bnever)\s+)\b(?:help(?:s|ed|ing)?|improv(?:e|es|ed|ing))\b/gi
+	},
+	{
+		word: EFFECT_WORDS.worse,
+		pattern: /\b(?:(?:make|makes|made|making)\s+(?:it|things|them)\s+)?worse\b|\bhurt(?:s|ing)?\b/gi
+	},
+	{
+		word: EFFECT_WORDS.same,
+		pattern: /\bno difference\b/gi
+	},
+	{
+		word: EFFECT_WORDS.unclear,
+		pattern: /\bcan['’]t tell yet\b/gi
+	}
+];
+/**
+* A "can't tell yet if/whether …" clause, up to the end of its sentence, any comma, colon,
+* semicolon, dash or bracket, a conjunction, or an "and" that starts a new clause. It errs short:
+* a word past it is checked.
+*/
+const HEDGED = /\bcan['’]t tell yet (if|whether)\b.*?(?=[.,:;!?()–—]|\s-\s|\s+(but|while|though|although|whereas|yet|so|because)\b|\s+and\s+(it|they|the|this|that|its|their|with|plain)\b|$)/gi;
+const NUMBER_WORDS = [
+	"zero",
+	"one",
+	"two",
+	"three",
+	"four",
+	"five",
+	"six",
+	"seven",
+	"eight",
+	"nine",
+	"ten",
+	"eleven",
+	"twelve",
+	"thirteen",
+	"fourteen",
+	"fifteen",
+	"sixteen",
+	"seventeen",
+	"eighteen",
+	"nineteen",
+	"twenty"
+];
+const COUNT = String.raw`(\d+|${NUMBER_WORDS.join("|")})`;
+/** `N of M`, `N out of M` or `N of the M`, in digits or words. */
+const PAIR = new RegExp(String.raw`(?<![\w.])${COUNT}\s+(?:out\s+)?of\s+(?:the\s+|its\s+|their\s+)?${COUNT}\b`, "gi");
+const FACT_PAIR = /\b(\d+) of (\d+)\b/g;
+const OUTCOME_VERB = /\b(?:pass|fail|succeed|miss)\w*/gi;
+const PASSED_AFTER = /^\s+(?:tries\s+|checks\s+)?pass(?:ed|es|ing)?\b/i;
+const SMALL = /(?<![\w.#/:-])\d(?![\w/]|\.\d)/g;
+const escapeRegExp = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+const countOf = (raw) => {
+	const word = NUMBER_WORDS.indexOf(raw.toLowerCase());
+	return word >= 0 ? word : Number(raw);
+};
+/** Where a count's clause starts: a clause mark, a dash, or the end of the count before it. */
+const CLAUSE_END = /[.,:;!?()–—]|\s-\s/g;
+/** The tool a count's clause names after its last outcome verb, lower-cased, or null. */
+function toolBefore(clause, tools) {
+	const verbAt = [...clause.matchAll(OUTCOME_VERB)].at(-1)?.index ?? -1;
+	let found = null;
+	for (const tool of tools) {
+		const re = new RegExp(`(?<!\\w)${escapeRegExp(tool)}(?!\\w)`, "gi");
+		const at = [...clause.matchAll(re)].at(-1)?.index ?? -1;
+		if (at > verbAt && (!found || at > found.at)) found = {
+			tool,
+			at
+		};
+	}
+	return found ? found.tool.toLowerCase() : null;
+}
+/**
+* The counts and numbers in `text` that what it cites doesn't hold. An `N of M` must be a cited
+* fact's own count (a transcript line's text isn't a count). When the facts count passes and the
+* nearest verb before it is a fail, it counts failures: `failed 4 of 4` over `0 of 4 passed`, and
+* `failed 0 of 4` there is dropped, unless `passed` follows it. A count whose clause names a tool
+* after its last outcome verb (`recommended Genkit in 1 of 1 tries`) counts the tries that named
+* it, so it must be that tool's count over a cited fact, not a pass count. Any other number must
+* be in what it cites, as `unverifiedNumbers` reads them, and a digit from 0 to 9 exactly.
+*/
+function unheldNumbers(text, facts, all, passCounts, tools) {
+	const known = new Set(facts.flatMap((f) => [...f.matchAll(FACT_PAIR)].map((m) => m[0])));
+	const out = [];
+	let from = 0;
+	for (const m of text.matchAll(PAIR)) {
+		const at = m.index ?? 0;
+		const n = countOf(m[1]);
+		const of = countOf(m[2]);
+		const before = text.slice(0, at);
+		const passedAfter = PASSED_AFTER.test(text.slice(at + m[0].length));
+		const clauseAt = Math.max(from, ...[...before.matchAll(CLAUSE_END)].map((c) => c.index + 1));
+		from = at + m[0].length;
+		const tool = tools && !passedAfter ? toolBefore(before.slice(clauseAt), tools.names) : null;
+		if (tool !== null) {
+			if (!tools?.held(tool).has(`${n} of ${of}`)) out.push(m[0]);
+			continue;
+		}
+		const verb = [...before.matchAll(OUTCOME_VERB)].at(-1)?.[0] ?? "";
+		const failures = passCounts && /^(fail|miss)/i.test(verb) && !passedAfter;
+		if (known.has(failures ? `${of - n} of ${of}` : `${n} of ${of}`)) continue;
+		out.push(m[0]);
+	}
+	const rest = text.replace(PAIR, " ");
+	const digits = new Set(all.flatMap((t) => t.match(/\d+(?:\.\d+)?/g) ?? []));
+	const small = (rest.match(SMALL) ?? []).filter((d) => !digits.has(d));
+	return [.../* @__PURE__ */ new Set([
+		...out,
+		...small,
+		...unverifiedNumbers(rest, all)
+	])];
+}
+const normal = (s) => s.replace(/[’‘]/g, "'").replace(/[“”]/g, "\"").replace(/\s+/g, " ").trim().toLowerCase();
+/** Every string and number in a value, one a line: the text a citation stands for. */
+function flatText(value) {
+	if (value === null || value === void 0) return "";
+	if (typeof value === "string" || typeof value === "number") return String(value);
+	if (Array.isArray(value)) return value.map(flatText).join("\n");
+	if (typeof value === "object") return Object.values(value).map(flatText).join("\n");
+	return "";
+}
+function citableEval(input) {
+	const facts = new Map(input.facts.map((f) => [f.path, f]));
+	const attempts = new Map(input.attempts.map((a) => [a.id, a]));
+	const resolve = (cite) => {
+		const fact = facts.get(cite);
+		if (fact) return flatText({
+			...fact,
+			path: null
+		});
+		const t = TRANSCRIPT.exec(cite);
+		if (!t) return null;
+		const [, id = "", n = ""] = t;
+		return attempts.get(id)?.lines.find((l) => l.startsWith(`L${n} `)) ?? null;
+	};
+	const effects = (cite) => {
+		const effect = facts.get(cite)?.effect;
+		return typeof effect === "string" ? [effect] : [];
+	};
+	return {
+		resolve,
+		effects,
+		vocabulary: normal([
+			input.eval.name,
+			...input.eval.group,
+			input.eval.file,
+			...input.checks,
+			...input.facts.flatMap((f) => [
+				f.label,
+				f.arm,
+				f.harness,
+				f.sandbox
+			].map(flatText))
+		].join("\n")),
+		passCounts: true
+	};
+}
+function citableHeadline(input) {
+	const evals = new Map(input.evals.map((e) => [e.path, e]));
+	const { unit: _, total, errors: __, ...counts } = input.tally;
+	const ofTotal = Object.values(counts).map((n) => `${n} of ${total}`);
+	return {
+		resolve: (cite) => {
+			const e = evals.get(cite);
+			return e ? flatText({
+				...e,
+				path: null,
+				tally: input.tally,
+				ofTotal,
+				evals: input.evals.length
+			}) : null;
+		},
+		effects: () => [],
+		vocabulary: normal(input.evals.map((e) => e.name).join("\n")),
+		passCounts: false
+	};
+}
+const sentencesOf = (text) => text.split(/(?<=[.!?])\s+(?=\S)/).filter((s) => s.trim());
+const sameText = (text) => oneLine(text).toLowerCase();
+const wordsOf = (text) => text.match(/\S+/g)?.length ?? 0;
+/** The largest `of M` in the cited facts: how many tries the sentence's counts are over. */
+const triesOf = (texts) => Math.max(0, ...texts.flatMap((t) => [...t.matchAll(/\bof (\d+)\b/g)].map((m) => Number(m[1]))));
+/** One sentence against what it cites; `maxSentences` caps an answer at two. */
+function checkSentence(s, c, maxSentences = 1) {
+	const text = oneLine(s.text);
+	const cites = [...new Set(s.cites.map((x) => x.trim()).filter(Boolean))];
+	if (!text) return { dropped: "empty" };
+	if (!cites.length) return { dropped: "cites nothing" };
+	const texts = [];
+	const factTexts = [];
+	for (const cite of cites) {
+		const t = c.resolve(cite);
+		if (t === null) return { dropped: `cites ${cite}, which isn't a fact it was shown` };
+		texts.push(t);
+		if (!TRANSCRIPT.test(cite)) factTexts.push(t);
+	}
+	const id = ID_IN_TEXT.exec(text)?.[0];
+	if (id && !c.vocabulary.includes(normal(id))) return { banned: `names ${id} in its text; ids go in cites only` };
+	const parts = sentencesOf(text);
+	if (parts.length > maxSentences) return { dropped: `is ${parts.length} sentences, more than ${maxSentences}` };
+	const long = parts.find((p) => wordsOf(p) > 25);
+	if (long) return { dropped: `${parts.length > 1 ? `"${long}" ` : ""}is ${wordsOf(long)} words, more than 25` };
+	const banned = BANNED.find((b) => b.pattern.test(text) && !b.pattern.test(c.vocabulary));
+	if (banned) return { banned: `says "${banned.word}"; use plain words` };
+	if (PERCENT.test(text) && triesOf(texts) < 10) return { banned: `gives a percentage below 10 tries; say N of M tries` };
+	const haystack = normal(texts.join("\n"));
+	for (const m of text.matchAll(QUOTE)) {
+		const quote = m.slice(1).find((g) => g !== void 0) ?? "";
+		if (!haystack.includes(normal(quote))) return { dropped: `quotes "${quote}", which isn't in what it cites` };
+	}
+	const factCites = cites.filter((cite) => !TRANSCRIPT.test(cite));
+	const tools = c.tools && {
+		names: c.tools.names,
+		held: (tool) => new Set(factCites.map((cite) => c.tools?.count(tool, cite)).filter((x) => typeof x === "string"))
+	};
+	const numbers = unheldNumbers(text, factTexts, texts, c.passCounts, tools);
+	if (numbers.length) return { dropped: `has ${numbers.join(", ")}, which no cited fact has` };
+	const effects = new Set(cites.flatMap(c.effects));
+	const claims = effects.has(EFFECT_WORDS.unclear) ? text.replace(HEDGED, "") : text;
+	for (const e of EFFECT_PHRASES) {
+		if (effects.has(e.word)) continue;
+		const said = [...(e.word === EFFECT_WORDS.unclear ? text : claims).matchAll(e.pattern)].map((m) => m[0]).find((w) => !new RegExp(`\\b${w}\\b`, "i").test(c.vocabulary));
+		if (said) return { dropped: `says "${said}", but no cited fact's effect is` };
+	}
+	return { ok: {
+		text,
+		cites
+	} };
+}
+function tally(value, judged) {
+	const out = {
+		value,
+		dropped: [],
+		banned: []
+	};
+	for (const { slot, text, verdict: v } of judged) {
+		if ("dropped" in v) out.dropped.push({
+			slot,
+			text,
+			reason: v.dropped
+		});
+		if ("banned" in v) out.banned.push({
+			slot,
+			text,
+			reason: v.banned
+		});
+	}
+	return out;
+}
+/**
+* A choose eval's tool counts, as the layout prints them: under a fact's path, the counted tries
+* whose kept named tools include the tool (case aside), over the counted tries. The names are
+* every tool the model named, kept or not, and the subject, so a tool it named wrongly or never
+* counts 0.
+*/
+function toolCounts(output, input, kept, texts) {
+	const subject = input.intent?.subject.tool;
+	const names = [...new Set([...output.namedTools.flatMap((n) => n.tools), ...subject ? [subject] : []].map(oneLine).filter(Boolean))].sort((a, b) => b.length - a.length);
+	const named = new Map(kept.map((n) => [n.attempt, new Set(n.tools.map((t) => t.toLowerCase()))]));
+	const attempts = [...texts.keys()];
+	return {
+		names,
+		count: (tool, cite) => {
+			const under = attempts.filter((a) => a === cite || a.startsWith(`${cite}#`) || a.startsWith(`${cite}/`));
+			if (!under.length) return null;
+			return `${under.filter((a) => named.get(a)?.has(tool.toLowerCase())).length} of ${under.length}`;
+		}
+	};
+}
+/**
+* An eval's insights call, checked. `texts` holds each attempt's whole final message, which a
+* named tool must appear in verbatim. Each drop's slot names the eval and where the sentence was
+* going: `answer`, `finding 2`, `row <path under the eval>` or `named tools <attempt>`.
+*/
+function validateEvalInsights(output, input, slots, texts) {
+	const judged = [];
+	const under = (path) => path.startsWith(`${input.eval.path}/`) ? path.slice(input.eval.path.length + 1) : path;
+	const slot = (where) => `${input.eval.name} › ${where}`;
+	const drop = (where, text, reason) => judged.push({
+		slot: slot(where),
+		text,
+		verdict: { dropped: reason }
+	});
+	const namedTools = output.namedTools.flatMap((n) => {
+		const where = `named tools ${under(n.attempt)}`;
+		if (!slots.namedTools) {
+			drop(where, n.tools.join(", "), "isn't on a choose eval");
+			return [];
+		}
+		const text = texts.get(n.attempt);
+		if (text === void 0) {
+			drop(where, n.tools.join(", "), "isn't an attempt it was shown");
+			return [];
+		}
+		const tools = [...new Set(n.tools.map(oneLine).filter(Boolean))].filter((t) => {
+			if (new RegExp(`(?<!\\w)${escapeRegExp(t)}(?!\\w)`).test(text)) return true;
+			drop(where, t, "isn't in the attempt's final message");
+			return false;
+		});
+		return tools.length ? [{
+			attempt: n.attempt,
+			tools
+		}] : [];
+	});
+	const c = {
+		...citableEval(input),
+		...slots.namedTools ? { tools: toolCounts(output, input, namedTools, texts) } : {}
+	};
+	const keep = (s, where, max = 1) => {
+		const v = checkSentence(s, c, max);
+		judged.push({
+			slot: slot(where),
+			text: s.text,
+			verdict: v
+		});
+		return "ok" in v ? v.ok : null;
+	};
+	const answer = output.answer ? keep(output.answer, "answer", 2) : null;
+	const open = new Set(slots.rows);
+	const filled = /* @__PURE__ */ new Set();
+	const rows = output.rows.flatMap((r) => {
+		const where = `row ${under(r.row)}`;
+		if (!open.has(r.row)) {
+			drop(where, r.text, "isn't a row this eval's layout takes a sentence on");
+			return [];
+		}
+		if (filled.has(r.row)) {
+			drop(where, r.text, "is a second sentence on the row");
+			return [];
+		}
+		const kept = keep(r, where);
+		if (!kept) return [];
+		filled.add(r.row);
+		return [{
+			...kept,
+			row: r.row
+		}];
+	});
+	const shown = new Set([
+		answer?.text,
+		...answer ? sentencesOf(answer.text) : [],
+		...rows.map((r) => r.text)
+	].filter((t) => !!t).map(sameText));
+	let findingsKept = 0;
+	return tally({
+		answer,
+		findings: output.findings.flatMap((s, i) => {
+			if (!slots.findings) {
+				drop(`finding ${i + 1}`, s.text, "this eval's layout has no findings");
+				return [];
+			}
+			if (shown.has(sameText(s.text))) {
+				drop(`finding ${i + 1}`, s.text, "repeats a sentence already shown");
+				return [];
+			}
+			if (findingsKept === MAX_FINDINGS) {
+				drop(`finding ${i + 1}`, s.text, `is past the ${MAX_FINDINGS} findings an eval takes`);
+				return [];
+			}
+			const kept = keep(s, `finding ${i + 1}`);
+			if (kept) {
+				shown.add(sameText(kept.text));
+				findingsKept++;
+			}
+			return kept ? [kept] : [];
+		}),
+		...rows.length ? { rows } : {},
+		...namedTools.length ? { namedTools } : {}
+	}, judged);
+}
+const EVAL_COUNT = new RegExp(String.raw`\b(\d+|${NUMBER_WORDS.join("|")})\s+(?:[a-z-]+\s+)?evals?\b`, "gi");
+/**
+* Why a headline calls a tally number a count of evals, or undefined. The tally counts results
+* (each case of a family is one), so "22 evals" on a run of 13 evals is false.
+*/
+function evalMiscount(text, input) {
+	const evals = input.evals.length;
+	const { unit: _, ...counts } = input.tally;
+	const tallied = new Set(Object.values(counts));
+	for (const [, raw = ""] of text.matchAll(EVAL_COUNT)) {
+		const word = NUMBER_WORDS.indexOf(raw.toLowerCase());
+		const n = word >= 0 ? word : Number(raw);
+		if (n !== evals && tallied.has(n)) return `says ${raw} evals, but the run has ${evals}; the tally counts results, not evals`;
+	}
+}
+const ALL_OF_ONE = /\b(?:all|both)\s+(?:of\s+(?:the\s+)?)?(?:1|one)\b/i;
+/** Why a headline says "all" of a single thing ("All 1 result passed"), or undefined. */
+const allOfOne = (text) => ALL_OF_ONE.test(text) ? `says "${text.match(ALL_OF_ONE)?.[0]}"; say "1 result passed"` : void 0;
+function validateHeadline(output, input) {
+	if (!output.headline) return {
+		value: null,
+		dropped: [],
+		banned: []
+	};
+	const checked = checkSentence(output.headline, citableHeadline(input));
+	const miscounted = "ok" in checked ? evalMiscount(checked.ok.text, input) ?? allOfOne(checked.ok.text) : void 0;
+	const v = miscounted ? { dropped: miscounted } : checked;
+	return tally("ok" in v ? v.ok : null, [{
+		slot: "headline",
+		text: output.headline.text,
+		verdict: v
+	}]);
+}
+//#endregion
+//#region src/insights/remarks.ts
+const MIN_QUOTE_CHARS = 3;
+const LINE_AT = /^(?:source:(.+)#|#)?L(\d+)$/;
+const words = (s) => s.match(/\S+/g)?.length ?? 0;
+/** `source:<file>#L<n>`, `#L<n>` or `L<n>` (in the eval's file) as a path and a 1-based line. */
+function lineAt(at, file) {
+	const m = LINE_AT.exec(at.trim());
+	if (!m) return null;
+	return {
+		path: m[1] ?? file,
+		line: Number(m[2])
+	};
+}
+/**
+* The model's remarks on one eval, held to the gates: those kept, as candidates with the kind's
+* severity, and a drop for each refused, its slot `<eval> › remark <kind>`.
+*/
+function gateRemarks(output, gate) {
+	const { input, entries, files } = gate;
+	const evalFile = input.source?.file ?? input.eval.file;
+	const hashes = new Map(entries.flatMap((e) => e.sources.map((s) => [s.path, s.sha256])));
+	const shown = new Set((input.source?.lines ?? []).map((l) => Number(/^L(\d+)/.exec(l)?.[1])));
+	const kept = [];
+	const dropped = [];
+	const seen = /* @__PURE__ */ new Set();
+	const first = entries[0];
+	for (const r of output) {
+		const text = oneLine(r.text ?? "");
+		const why = check(r, text);
+		if (typeof why === "string") {
+			dropped.push({
+				slot: `${input.eval.name} › remark ${r.kind}`,
+				text,
+				reason: why
+			});
+			continue;
+		}
+		seen.add(why.kind);
+		kept.push(why);
+	}
+	return {
+		kept,
+		dropped
+	};
+	function line(at) {
+		const where = lineAt(at, evalFile);
+		if (!where) return `cites ${at}, which isn't a line (L<n>)`;
+		if (!hashes.has(where.path)) return `cites ${where.path}, which isn't one of the eval's files`;
+		const body = files.get(where.path);
+		if (body === null || body === void 0) return `can't be checked: ${where.path} can't be read`;
+		if (sha256(body) !== hashes.get(where.path)) return `can't be checked: ${where.path} changed since the run`;
+		const lines = body.split("\n");
+		if (where.line < 1 || where.line > lines.length) return `cites line ${where.line} of ${where.path}, which has ${lines.length}`;
+		if (where.path !== evalFile || !shown.has(where.line)) return `cites line ${where.line} of ${where.path}, which isn't in the eval's declaration`;
+		return {
+			...where,
+			text: lines[where.line - 1]
+		};
+	}
+	function check(r, text) {
+		const kind = r.kind;
+		if (!MODEL_REMARK_KINDS.includes(kind)) return `is kind "${r.kind}", which the model can't raise`;
+		if (seen.has(kind)) return `is a second ${kind} remark on this eval`;
+		if (!first) return "isn't on an eval the run recorded";
+		if (!text) return "is empty";
+		if (words(text) > 25) return `is ${words(text)} words, more than 25`;
+		if (ID_IN_TEXT.test(text)) return "names a fact id in its text";
+		if (!r.source) return "has no source quote";
+		const at = line(r.source.at);
+		if (typeof at === "string") return at;
+		const quote = r.source.quote?.trim() ?? "";
+		if (quote.length < MIN_QUOTE_CHARS) return "has no source quote";
+		if (!at.text.includes(quote)) return `quotes "${quote}", which isn't on line ${at.line} of ${at.path}`;
+		const source = `source:${at.path}#L${at.line}`;
+		const fixAt = r.fix ? line(r.fix.at) : null;
+		return {
+			id: quotedRemarkId(kind, input.eval.path, quote),
+			evalId: input.eval.path,
+			kind,
+			severity: REMARK_KINDS[kind].severity,
+			text,
+			source: {
+				at: source,
+				quote
+			},
+			evidence: [...new Set((r.evidence ?? []).map((c) => c.trim()).filter(Boolean))],
+			fix: {
+				at: fixAt && typeof fixAt !== "string" ? `source:${fixAt.path}#L${fixAt.line}` : source,
+				edit: oneLine(r.fix?.edit ?? "")
+			},
+			origin: "model"
+		};
+	}
+}
+//#endregion
+//#region src/insights/rubric.ts
+/** Where the rubric ships, in reeval's package. */
+const RUBRIC_FILE = "skills/eval-rubric/SKILL.md";
+let text;
+/** The rubric's text, read once. */
+const rubric = () => {
+	text ??= readFileSync(ownFile(RUBRIC_FILE), "utf8");
+	return text;
+};
+/**
+* The sections of the kinds the model may raise, as the insights call is given them: the rest of
+* the rubric (its gates, reeval's own rules) is enforced in code, so it isn't worth the tokens.
+*/
+const modelRubric = (kinds) => rubric().split(/^(?=## )/m).filter((section) => kinds.includes(/^## `([a-z-]+)`/.exec(section)?.[1] ?? "")).join("").trim();
+/** The first 8 hex characters of the rubric's sha256. */
+const rubricHash = () => sha256(rubric()).slice(0, 8);
+//#endregion
+//#region src/insights/schema.ts
+/** Bumped when a prompt or output schema changes: older insights were asked differently. */
+const PROMPT_REVISION = "insights-v6";
+/**
+* The prompts' version: their revision and the rubric's hash, since the rubric is part of the
+* insights prompt (`insights-v6-1a2b3c4d`).
+*/
+const insightsPromptVersion = () => `${PROMPT_REVISION}-${rubricHash()}`;
+const sentence = z.object({
+	text: z.string(),
+	cites: z.array(z.string()).describe("Fact ids from facts[].id (F1), or <attempt id>#L<n> (A2#L14) for a line in attempts[].lines")
+});
+const classificationSchema = z.object({
+	job: z.enum([
+		"choose",
+		"finish",
+		"other"
+	]),
+	question: z.string().describe("The eval's question in plain words, from its name"),
+	subject: z.object({
+		tool: z.string().nullable().describe("For choose: the tool the eval asks about; else null"),
+		change: z.string().nullable().describe("When the eval measures a change: what the change adds (a skill, a doc); else null")
+	}),
+	name: z.enum([
+		"states-intent",
+		"no-intent",
+		"contradicts"
+	]),
+	confidence: z.enum([
+		"high",
+		"medium",
+		"low"
+	])
+});
+const insightsSchema = z.object({
+	answer: sentence.nullable().describe("The answer to the question, at most two sentences; null when the facts give none"),
+	findings: z.array(sentence).describe("Only when slots.findings: up to three supporting points"),
+	rows: z.array(sentence.extend({ row: z.string().describe("A fact id from slots.rows") })).describe("At most one sentence per id in slots.rows; empty when there are none"),
+	namedTools: z.array(z.object({
+		attempt: z.string().describe("An attempts[].id (A1)"),
+		tools: z.array(z.string()).describe("Each copied exactly as it's written in lastText")
+	})).describe("Only when slots.namedTools: per attempt, the tools its reply recommended or used"),
+	remarks: z.array(z.object({
+		kind: z.string().describe(`One of: ${MODEL_REMARK_KINDS.join(", ")}`),
+		text: z.string().describe("One plain sentence: what's wrong with the eval"),
+		source: z.object({
+			at: z.string().describe("L<n> of a line in source.lines"),
+			quote: z.string().describe("Copied exactly from that line's text")
+		}).nullable(),
+		evidence: z.array(z.string()).describe("Fact ids (F1), or attempt lines (A2#L14), showing it mattered in this run"),
+		fix: z.object({
+			at: z.string().describe("L<n> of the line in source.lines to change"),
+			edit: z.string().describe("The concrete edit")
+		}).nullable()
+	})).describe("Remarks on the eval itself, by the rubric; almost always empty")
+});
+const headlineSchema = z.object({ headline: sentence.nullable() });
+const CLASSIFY_PROMPT = `You classify one reeval eval: what one attempt of it tests. reeval runs coding agents on a task and checks the result.
+
+You get the eval's NAME first. The name is the author's statement of what the eval asks: read it first and let it set the question and the subject. Then the prompts the agent was given, the judge questions and the check labels, which confirm the name or contradict it.
+
+- job: choose when the attempt asks the agent to pick or recommend a tool, library, framework or approach and the checks grade which one it picked; finish when it asks the agent to complete a task and the checks grade the result (files, builds, tests, output); other for anything else (explain, debug, answer).
+- question: the eval's question in plain words, as a reader would ask it ("Do agents pick Genkit for production AI features?").
+- subject.tool: for choose, the tool the eval asks about, as the name writes it; else null. subject.change: what the eval is about adding (a skill, a doc, an MCP server), as its name or its compare says, whether or not it compares tries with and without it; else null.
+- name: states-intent when the name says what the eval asks and the checks agree; no-intent when it's a topic, a ticket id or "test N"; contradicts when it names something no check or judge question looks at.
+- confidence: low when the name and the checks don't settle the job.
+When trimmed is set, reeval left that detail out to fit.`;
+const INSIGHTS_PROMPT = `You explain one reeval eval's result to the developer who wrote it, in plain words an average developer understands. reeval ran a coding agent on the eval's task, a number of tries per row, and checked each try.
+
+You get the eval (its name is its question), the classification (intent), the slots you may fill, its FACTS (each with an id, F1, and the path it stands for), its attempts (each with an id, A1, its cell's fact id, the final message and, for a failed try, transcript lines L<n>) and its source (the eval's declaration in its file, each line L<n>).
+
+Rules, checked by code: a sentence that breaks one is dropped.
+- Every sentence cites at least one fact by its id from facts[].id (F3), or <attempt id>#L<n> (A2#L14) for a line in that attempt's lines. Cite the facts whose numbers and words you use; never cite a path.
+- Ids go in cites only, never in the text: the reader never sees them.
+- Numbers come from the cited facts. Counts read "passed N of M tries", as the facts count passes; after "failed" a count is the failures ("failed 4 of 4 tries" when 0 of 4 passed). Never a percentage below 10 tries.
+- An effect is said with the cited fact's effect word only: helped, made it worse, no difference, can't tell yet. Never use an effect word the cited facts don't have.
+- Use the eval's own names ("with the skill", "plain", "on codex"), never: interval, significance, noise, p-value, confidence, Newcombe, Wilson, pass@1, lift, effect size, too close to call, within noise, control, variant, arm, cell, discriminates.
+- Each sentence is 25 words or fewer and leads with the answer. No stacked hedges. Never quote judge questions or thresholds; quote a reply or an assertion only when it is the evidence, in double quotes, copied exactly.
+- answer: the answer to intent.question, at most two sentences.
+- findings: only when slots.findings; up to three points on what moved, what broke or what the agent did.
+- rows: at most one sentence per fact id in slots.rows (row is that id), saying why that row failed or moved; none for other rows.
+- namedTools: only when slots.namedTools; for each attempt (attempt is its id), the tools, libraries or frameworks its final message recommended or used, each copied exactly as written there.
+- remarks: about the eval itself, not the agent, by the rubric below. Almost always none: raise one only when its kind's section says so and this run's facts show it mattered. Each has kind (only ${MODEL_REMARK_KINDS.join(", ")}), text (one plain sentence of 25 words or fewer), source (at: L<n> of a line in source.lines; quote: copied exactly from that line), evidence (the fact ids or attempt lines from this run that show it) and fix (at: L<n> of the line to change; edit: the concrete edit). One whose quote isn't on its line is dropped. At most one per kind.
+When trimmed is set, reeval left that detail out to fit: say nothing about what isn't in the input.
+When retry is set, your earlier sentences were refused for those reasons: write them again within the rules.`;
+/** The insights call's system prompt: the rules, then the rubric's sections on the model's kinds. */
+const insightsPrompt = () => `${INSIGHTS_PROMPT}\n\n<rubric>\n${modelRubric(MODEL_REMARK_KINDS)}\n</rubric>`;
+const HEADLINE_PROMPT = `You write the one-sentence headline of a reeval run for the developer who wrote its evals: what the run found, leading with what needs work. You get each eval's id (F1), path, name, verdict and answer, and the tally.
+
+- 25 words or fewer, plain words, citing the evals it's about by their ids from evals[].id, in cites only, never in the text.
+- Counts come from the evals and the tally. The tally counts results, not evals (tally.unit): each case of a family counts one and \`total\` is their sum ("9 of 22 results fail"). The number of evals is evals.length; never call a tally number evals. "All" needs more than one: a single result is "1 result passed", never "All 1 result passed". Never: interval, significance, noise, p-value, confidence, pass@1, lift, control, variant, arm, cell.
+When retry is set, your earlier headline was refused for those reasons: write it again within the rules.`;
+const size = (input) => JSON.stringify(input).length;
+const passing = (a) => a.outcome === "pass";
+const passedCheck = (f) => f.kind === "check" && typeof f.result === "string" && /^(\d+) of \1 passed$/.test(f.result) && f.effect === void 0;
+const attempts = (input, f) => ({
+	...input,
+	attempts: input.attempts.flatMap((a, i) => f(a, i) ?? [])
+});
+/** In the order they're applied: each runs only while the input is still over the cap. */
+const STEPS = [
+	{
+		cut: "passing attempts' final messages",
+		apply: (input) => attempts(input, (a) => passing(a) ? {
+			...a,
+			lastText: clip(a.lastText, 300)
+		} : a)
+	},
+	{
+		cut: "older transcript lines",
+		apply: (input) => attempts(input, (a) => ({
+			...a,
+			lines: a.lines.slice(-6)
+		}))
+	},
+	{
+		cut: "passing checks and effort",
+		apply: (input, keep) => ({
+			...input,
+			facts: input.facts.filter((f) => keep.has(f.path) || !(passedCheck(f) || f.kind === "effort"))
+		})
+	},
+	{
+		cut: "failing attempts' final messages",
+		apply: (input) => attempts(input, (a) => ({
+			...a,
+			lastText: clip(a.lastText, 400)
+		}))
+	},
+	{
+		cut: "transcript lines",
+		apply: (input) => attempts(input, (a) => ({
+			...a,
+			lines: a.lines.slice(-2)
+		}))
+	},
+	{
+		cut: "passing attempts after the first of each cell",
+		apply: (input) => {
+			const seen = /* @__PURE__ */ new Set();
+			return attempts(input, (a) => {
+				if (!passing(a)) return a;
+				if (seen.has(a.cell)) return null;
+				seen.add(a.cell);
+				return a;
+			});
+		}
+	},
+	{
+		cut: "source lines after the first 30",
+		apply: (input) => input.source ? {
+			...input,
+			source: {
+				...input.source,
+				lines: input.source.lines.slice(0, 30)
+			}
+		} : input
+	},
+	{
+		cut: "prompts",
+		apply: (input) => ({
+			...input,
+			prompts: input.prompts.map((p) => clip(p, 300))
+		})
+	},
+	{
+		cut: "passing cells",
+		apply: (input, keep) => ({
+			...input,
+			facts: input.facts.filter((f) => keep.has(f.path) || !(f.kind === "cell" && f.verdict === "pass" && !f.effect))
+		})
+	}
+];
+/**
+* `input` within `maxChars`, cut by `STEPS` in order until it fits; `keep` is the facts that must
+* stay whatever the size (the rows the layout opens a sentence on). What's still over after every
+* step goes as it is: the failing facts are never cut.
+*/
+function trimInput(input, maxChars, keep = []) {
+	if (size(input) <= maxChars) return input;
+	const kept = new Set(keep);
+	let out = input;
+	const cuts = [];
+	for (const step of STEPS) {
+		const next = step.apply(out, kept);
+		if (size(next) < size(out)) {
+			out = next;
+			cuts.push(step.cut);
+		}
+		if (size({
+			...out,
+			trimmed: cuts
+		}) <= maxChars) break;
+	}
+	return {
+		...out,
+		trimmed: cuts
+	};
+}
+/** The part of a fake's answer one call returns, as the model's JSON would have it. */
+function fakePart(call, answer, input) {
+	if (call === "classify") {
+		const i = answer.intent;
+		return {
+			job: i?.job ?? "other",
+			question: i?.question ?? (input.call === "headline" ? "" : input.eval.name),
+			subject: {
+				tool: i?.subject?.tool ?? null,
+				change: i?.subject?.change ?? null
+			},
+			name: i?.name ?? "states-intent",
+			confidence: i?.confidence ?? "high"
+		};
+	}
+	if (call === "headline") return { headline: answer.headline ?? null };
+	return {
+		answer: answer.answer ?? null,
+		findings: answer.findings ?? [],
+		rows: answer.rows ?? [],
+		namedTools: answer.namedTools ?? [],
+		remarks: (answer.remarks ?? []).map((r) => ({
+			kind: r.kind,
+			text: r.text,
+			source: r.source ?? null,
+			evidence: r.evidence ?? [],
+			fix: r.fix ?? null
+		}))
+	};
+}
+/** `work`, or the signal's reason as soon as it aborts. */
+function abortable(work, signal) {
+	if (!signal) return work;
+	if (signal.aborted) return Promise.reject(signal.reason);
+	return new Promise((resolve, reject) => {
+		const stop = () => reject(signal.reason);
+		signal.addEventListener("abort", stop, { once: true });
+		work.then(resolve, reject).finally(() => signal.removeEventListener("abort", stop));
+	});
+}
+async function languageModel(model, call, input, signal) {
+	if (!model.fake) return providerModel(model);
+	const insights = model.fake.insights;
+	if (insights === void 0) throw new Error("fakeAnalysis() has no insights, so it has none to return");
+	if (typeof insights === "object" && insights !== null && "error" in insights) return cannedModel({ error: insights.error });
+	const answer = typeof insights === "function" ? await abortable(Promise.resolve(insights(input)), signal) : insights;
+	return cannedModel({ text: JSON.stringify(fakePart(call, answer, input)) }, true);
+}
+async function ask(o, call, schema, system, input, usage) {
+	const timeoutMs = o.callTimeoutMs ?? 18e4;
+	const maxOutputTokens = o.maxOutputTokens ?? 2e4;
+	const clock = new AbortController();
+	const timer = setTimeout(() => clock.abort(), timeoutMs);
+	const timeout = clock.signal;
+	const signal = o.signal ? AbortSignal.any([o.signal, timeout]) : timeout;
+	const started = Date.now();
+	const count = (u) => {
+		const used = {
+			inputTokens: u?.inputTokens ?? 0,
+			outputTokens: u?.outputTokens ?? 0,
+			reasoningTokens: u?.outputTokenDetails?.reasoningTokens ?? 0
+		};
+		addUsage(usage, used);
+		const ms = Date.now() - started;
+		o.onUsage?.({
+			call,
+			eval: input.call === "headline" ? null : input.eval.path,
+			ms,
+			...used
+		});
+	};
+	const cutOff = /* @__PURE__ */ new Error(`the ${call} reply was cut off at ${maxOutputTokens} output tokens`);
+	let result;
+	try {
+		result = await generateText({
+			model: await languageModel(o.model, call, input, signal),
+			system,
+			prompt: JSON.stringify(input),
+			output: Output.object({
+				schema,
+				name: call
+			}),
+			maxOutputTokens,
+			maxRetries: 3,
+			abortSignal: signal
+		});
+	} catch (error) {
+		if (o.signal?.aborted) throw o.signal.reason;
+		if (timeout.aborted) throw new Error(`the ${call} call took longer than ${formatDuration(timeoutMs)}`);
+		if (NoObjectGeneratedError.isInstance(error)) {
+			count(error.usage);
+			if (error.finishReason === "length") throw cutOff;
+		}
+		throw error;
+	} finally {
+		clearTimeout(timer);
+	}
+	count(result.usage);
+	if (result.finishReason === "length") throw cutOff;
+	return result.output;
+}
+const sentenceFrom = (s, ids) => ({
+	...s,
+	text: withoutIdLists(s.text),
+	cites: s.cites.map(ids.cite)
+});
+/** The model's sentences with its short ids turned back into paths, for the validator and the file. */
+function fromModel(out, ids) {
+	return {
+		answer: out.answer && sentenceFrom(out.answer, ids),
+		findings: out.findings.map((s) => sentenceFrom(s, ids)),
+		rows: out.rows.map((r) => ({
+			...sentenceFrom(r, ids),
+			row: ids.row(r.row)
+		})),
+		namedTools: out.namedTools.map((n) => ({
+			...n,
+			attempt: ids.attempt(n.attempt)
+		})),
+		remarks: out.remarks.map((r) => ({
+			...r,
+			evidence: r.evidence.map(ids.cite)
+		}))
+	};
+}
+/** How a refused sentence is put to the model when it's asked again. */
+const retryLine = (d) => `${d.slot}: "${d.text}" ${d.reason}`;
+/** Checks an answer, and asks once more when the lint refused a sentence; what's refused twice is dropped. */
+async function checked(first, again, check, counts) {
+	const one = check(first);
+	if (!one.banned.length) {
+		counts.dropped += one.dropped.length;
+		counts.drops.push(...one.dropped);
+		return one.value;
+	}
+	counts.regenerated += one.banned.length;
+	const two = check(await again(one.banned.map(retryLine)));
+	const drops = [...two.dropped, ...two.banned];
+	counts.dropped += drops.length;
+	counts.drops.push(...drops);
+	return two.value;
+}
+/** One classification call; a low-confidence one falls back to `other`. */
+async function classify(o, base, usage) {
+	const out = await ask(o, "classify", classificationSchema, CLASSIFY_PROMPT, modelInput({
+		call: "classify",
+		...base
+	}).input, usage);
+	return {
+		job: out.confidence === "low" ? "other" : out.job,
+		question: out.question,
+		subject: {
+			...out.subject.tool ? { tool: out.subject.tool } : {},
+			...out.subject.change ? { change: out.subject.change } : {}
+		},
+		name: out.name
+	};
+}
+/** Each of the eval's source files as it is now, for the gates to hold a remark's quote to. */
+async function sourceFiles(root, record, e) {
+	const files = /* @__PURE__ */ new Map();
+	if (!root) return files;
+	for (const { entry } of entriesOf(record, e)) for (const s of entry.sources) if (!files.has(s.path)) files.set(s.path, await readFile(join(root, s.path), "utf8").catch(() => null));
+	return files;
+}
+async function evalInsight(o, e, usage, counts) {
+	const full = await evalInput(o.record, o.runDir, e);
+	const cap = o.maxInputChars ?? 24e3;
+	const key = classificationKey(o.record, e, o.model.name, insightsPromptVersion());
+	const classified = await o.cache?.get(key) ?? await classify(o, trimInput(full, cap), usage);
+	if (o.cache) await o.cache.set(key, classified).catch(() => {});
+	const { job, subject } = classified;
+	const slots = slotsOf(job, e);
+	const source = o.root ? await evalSource(o.root, o.record, e) : null;
+	const input = {
+		call: "insights",
+		...trimInput(source ? {
+			...full,
+			source
+		} : full, cap, slots.rows),
+		intent: {
+			job,
+			question: classified.question,
+			subject
+		},
+		slots
+	};
+	const model = modelInput(input);
+	const askInsights = async (retry) => fromModel(await ask(o, "insights", insightsSchema, insightsPrompt(), retry ? {
+		...model.input,
+		retry
+	} : model.input, usage), model.ids);
+	const texts = attemptTexts(o.record, e);
+	let last = null;
+	const sentences = await checked(await askInsights(), askInsights, (out) => {
+		last = out;
+		return validateEvalInsights(out, input, slots, texts);
+	}, counts);
+	const gated = gateRemarks(last?.remarks ?? [], {
+		input,
+		entries: entriesOf(o.record, e).map((x) => x.entry),
+		files: await sourceFiles(o.root, o.record, e)
+	});
+	counts.dropped += gated.dropped.length;
+	counts.drops.push(...gated.dropped);
+	return {
+		insight: {
+			evalId: e.path,
+			intent: {
+				job,
+				across: e.across,
+				question: classified.question,
+				subject,
+				name: classified.name,
+				key
+			},
+			...sentences
+		},
+		remarks: gated.kept
+	};
+}
+/** The evals that get an insight, in the order the output prints them: none for a todo or a skip. */
+const insightEvals = (suite) => suite.files.flatMap((f) => [...f.evals].sort((a, b) => a.order - b.order).filter((e) => !e.todo && e.outcome !== "skipped"));
+/**
+* An eval's insight, asked for once more when a call fails, is cut off or times out; its sentences
+* counted only once it lands. Ctrl-C isn't a failure: it rejects at once.
+*/
+async function evalInsightOnce(o, e, usage, validation) {
+	for (let tries = 1;; tries++) {
+		const counts = {
+			dropped: 0,
+			regenerated: 0,
+			drops: []
+		};
+		try {
+			const insight = await evalInsight(o, e, usage, counts);
+			validation.dropped += counts.dropped;
+			validation.regenerated += counts.regenerated;
+			validation.drops.push(...counts.drops);
+			return insight;
+		} catch (error) {
+			if (tries === 2 || o.signal?.aborted) throw error;
+		}
+	}
+}
+const spent = (u) => u.inputTokens + u.outputTokens;
+function addUsage(to, from) {
+	to.inputTokens += from.inputTokens;
+	to.outputTokens += from.outputTokens;
+	to.reasoningTokens += from.reasoningTokens;
+}
+const message = (error) => error instanceof Error ? error.message : String(error);
+/** Resolves on the signal's abort, so a wait on calls in flight ends with Ctrl-C. */
+const aborted = (signal) => new Promise((resolve) => {
+	if (!signal) return;
+	if (signal.aborted) resolve();
+	else signal.addEventListener("abort", () => resolve(), { once: true });
+});
+/**
+* Classifies each eval, writes and checks its sentences, then the run's headline. Evals start in
+* priority order and are kept in output order; with `maxTokens`, those it didn't reach are listed
+* in `budget.skipped`. An eval that fails twice is in `failed` and the others go on; on `signal`,
+* those that hadn't finished are in `stopped`. It rejects only when every eval it asked about
+* failed, with the first error.
+*/
+async function generateInsights(o) {
+	const suite = suiteResult(o.record);
+	const prior = o.prior;
+	const usage = {
+		inputTokens: prior?.usage.inputTokens ?? 0,
+		outputTokens: prior?.usage.outputTokens ?? 0,
+		reasoningTokens: prior?.usage.reasoningTokens ?? 0
+	};
+	const validation = {
+		dropped: prior?.validation.dropped ?? 0,
+		regenerated: prior?.validation.regenerated ?? 0,
+		drops: [...prior?.validation.drops ?? []]
+	};
+	const budget = o.maxTokens === void 0 ? null : ledger(o.maxTokens);
+	const all = insightEvals(suite);
+	const kept = new Map((prior?.evals ?? []).map((e) => [e.evalId, e]));
+	const pending = prior && unfinishedEvals(prior);
+	const ordered = pending ? all.filter((e) => pending.has(e.path)) : all;
+	const skipped = /* @__PURE__ */ new Set();
+	const done = /* @__PURE__ */ new Map();
+	const failed = /* @__PURE__ */ new Map();
+	const running = /* @__PURE__ */ new Set();
+	const concurrency = o.concurrency ?? 4;
+	const stop = aborted(o.signal);
+	const stopped = () => o.signal?.aborted === true;
+	const progress = () => o.onProgress?.(done.size + failed.size + skipped.size, ordered.length);
+	const settled = () => Promise.race([stop, ...running]);
+	for (const e of byPriority(ordered)) {
+		let decision = "start";
+		for (;;) {
+			if (stopped()) break;
+			if (running.size >= concurrency) {
+				await settled();
+				continue;
+			}
+			const next = budget ? admit(budget) : "start";
+			if (next !== "wait") {
+				decision = next;
+				break;
+			}
+			await settled();
+		}
+		if (stopped()) break;
+		if (decision === "skip") {
+			skipped.add(e.path);
+			progress();
+			continue;
+		}
+		const held = budget ? reserve(budget) : 0;
+		const used = {
+			inputTokens: 0,
+			outputTokens: 0,
+			reasoningTokens: 0
+		};
+		const call = evalInsightOnce(o, e, used, validation).then((insight) => void done.set(e.path, insight)).catch((error) => {
+			if (!stopped()) failed.set(e.path, error);
+		}).finally(() => {
+			if (budget) settle(budget, held, spent(used));
+			addUsage(usage, used);
+			running.delete(call);
+			if (!stopped()) progress();
+		});
+		running.add(call);
+	}
+	await Promise.all(running);
+	if (!stopped() && !done.size && failed.size) throw failed.values().next().value;
+	const left = ordered.flatMap((e) => done.has(e.path) || failed.has(e.path) || skipped.has(e.path) ? [] : [e.path]);
+	const insightOf = (path) => done.get(path)?.insight ?? kept.get(path);
+	const evals = all.flatMap((e) => insightOf(e.path) ?? []);
+	const headlineInput = {
+		call: "headline",
+		tally: {
+			...suite.tally,
+			total: Object.values(suite.tally).reduce((a, b) => a + b, 0) - suite.tally.errors,
+			unit: "results"
+		},
+		evals: all.map((e) => ({
+			path: e.path,
+			name: e.template ?? e.name,
+			verdict: e.outcome === "infra_error" ? "infra error" : e.outcome,
+			answer: insightOf(e.path)?.answer?.text ?? null
+		}))
+	};
+	const headlineModel = modelHeadlineInput(headlineInput);
+	const askHeadline = async (retry) => {
+		const out = await ask(o, "headline", headlineSchema, HEADLINE_PROMPT, retry ? {
+			...headlineModel.input,
+			retry
+		} : headlineModel.input, usage);
+		return { headline: out.headline && sentenceFrom(out.headline, headlineModel.ids) };
+	};
+	let headline = prior?.headline ?? null;
+	const wantsHeadline = !headline && evals.length && (budget === null || admit(budget) === "start");
+	for (let tries = 1; wantsHeadline && !stopped() && tries <= 2; tries++) try {
+		headline = await checked(await askHeadline(), askHeadline, (out) => validateHeadline(out, headlineInput), validation);
+		break;
+	} catch {}
+	const { reasoningTokens, ...tokens } = usage;
+	return {
+		kind: "reeval-insights",
+		schemaVersion: 1,
+		runId: o.record.runId,
+		model: o.model.name,
+		promptVersion: insightsPromptVersion(),
+		factsHash: factsHash(suite),
+		createdAt: (/* @__PURE__ */ new Date()).toISOString(),
+		usage: {
+			...tokens,
+			...reasoningTokens ? { reasoningTokens } : {},
+			costUsd: null
+		},
+		...o.maxTokens !== void 0 ? { budget: {
+			maxTokens: o.maxTokens,
+			skipped: ordered.flatMap((e) => skipped.has(e.path) ? [e.path] : [])
+		} } : {},
+		...failed.size ? { failed: ordered.flatMap((e) => failed.has(e.path) ? [{
+			evalId: e.path,
+			error: message(failed.get(e.path))
+		}] : []) } : {},
+		...left.length ? { stopped: {
+			signal: o.stopSignal?.() ?? "SIGINT",
+			skipped: left
+		} } : {},
+		headline,
+		evals,
+		remarks: [...prior?.remarks ?? [], ...all.flatMap((e) => (done.get(e.path)?.remarks ?? []).map((r) => ({
+			...r,
+			firstSeenRun: o.record.runId
+		})))],
+		validation
+	};
+}
+/** The evals a document lists as unfinished: stopped, failed or skipped by the budget. */
+function unfinishedEvals(doc) {
+	return /* @__PURE__ */ new Set([
+		...doc.stopped?.skipped ?? [],
+		...(doc.failed ?? []).map((f) => f.evalId),
+		...doc.budget?.skipped ?? []
+	]);
+}
+//#endregion
+export { classifyCache as a, insightsPromptVersion as i, insightEvals as n, unfinishedEvals as r, generateInsights as t };
+
+//# sourceMappingURL=generate-BnrvBAzF.js.map