reeval 0.8.0 → 0.10.0

package/dist/{command-BqJtFiPy.js → command-Bge8F7Ko.js}

@@ -1,31 +1,42 @@
-import { n as UsageError } from "./errors-yDAEwmZo.js";
+import { n as UsageError } from "./errors-B6BSMs03.js";
 import { a as fromCwd, l as sha256 } from "./ids-Bp6ut7ZD.js";
+import { r as questionShows, t as DEFAULT_QUESTION_STATE } from "./state-CeR6oz0k.js";
 import { n as clip } from "./text-gEcKjAyp.js";
-import { c as resolveAnalysisModel, l as SYSTEM_PROMPT, n as callAnalysisModel, t as analysisModelName } from "./model-Cl507trk.js";
-import { t as ANALYSIS_PROMPT_VERSION } from "./analysis-BTNnKh1w.js";
-import { ot as compare } from "./model-BndnmcLm.js";
-import { t as runHeader } from "./header-_F0EWob3.js";
-import { a as sampleReasons } from "./pairs-DDsjgm5F.js";
-import { o as parseRecord, p as scratchDir } from "./files-CMSUXAEg.js";
-import { l as writeCached, o as readCached, s as saveAnalysis } from "./files-B6aVVGrk.js";
-import { c as resolveRun, s as pickBaseline, t as RefError } from "./views-DZ1YDqey.js";
-import { c as lines, n as allRuns, t as CommandError, u as reachability } from "./commands-B618jg3C.js";
-import { a as commandSummary, i as checkCommand } from "./cli-spec-DSvKsOK4.js";
-import { s as openHistoryDb } from "./db-DFbtGIYy.js";
-import { t as author } from "./provenance-CsX6PHDR.js";
+import { c as resolveAnalysisModel, l as SYSTEM_PROMPT, n as callAnalysisModel, t as analysisModelName } from "./model-h_cDtcRr.js";
+import { n as formatTokens } from "./usage-CleNKkly.js";
+import { t as ANALYSIS_PROMPT_VERSION } from "./analysis-Bxub9LMT.js";
+import { V as labelledJudgements, yt as compare } from "./model-oqcDplJG.js";
+import { a as familyLiftLabel, o as familyRateLabel, r as familyEffortLabel } from "./family-D2b7WD10.js";
+import { t as runHeader } from "./header-hK-Z_Po9.js";
+import { n as headToHeadLines, r as pairTallies, s as sampleReasons, u as splitsLabel } from "./pairs-CnInRu-G.js";
+import { o as parseRecord, p as scratchDir } from "./files-BXUdY8tr.js";
+import { g as contrastsOf } from "./summary-BLAVLySU.js";
+import { l as writeCached, o as readCached, s as saveAnalysis } from "./files-DQmHE7pb.js";
+import { c as resolveRun, s as pickBaseline, t as RefError } from "./views-BmQF4UCI.js";
+import { c as lines, n as allRuns, t as CommandError, u as reachability } from "./commands-BFx16WB8.js";
+import { i as checkCommand, o as commandSummary } from "./cli-spec-BPZbQG2p.js";
+import { l as TOOL_CLASSES, s as openHistoryDb } from "./db-CdHizc4_.js";
+import { t as author } from "./provenance-DmE9uPVt.js";
 import { basename, join } from "node:path";
 import { readFile } from "node:fs/promises";
 //#region src/analysis/digest.ts
 const RULES = [
-	`Pass-rate lift is insufficient (not measured, never a results line) when either side has fewer than 2 counted attempts; otherwise it is an effect only when its 95% interval (Newcombe) excludes 0, and within noise when the interval spans 0, whatever the point estimate.`,
-	"When sources or the eval fingerprint differ across the compared runs, the pass-rate difference is not comparable: the eval changed. Never call it an effect. Do not put those mixed counts on a results line. The first line of the analysis says the eval changed. A lift is an effect only when sources match; the two-attempt floor still applies then.",
-	"Effort (tool calls, agent time, and tokens when both sides have usage samples) is compared with an exact Mann-Whitney test: \"fewer\"/\"more\" at p < 0.05; \"insufficient\" when the attempt counts can never reach it (2 vs 2 cannot) — never because a harness dropped usage. Attempts without usage are left out of the token sample only. If either side then has no token sample, the token metric is omitted (no line, not insufficient). A Goose cell does not blank tokens on Cursor cells in the same study. Pooled across evals, tokens are an effect only when at least 2/3 of the strata that reported usage agree.",
+	`Lift (variant pass@1 minus control pass@1) is insufficient (not measured, never a results line) when either side has fewer than 2 counted attempts; otherwise it is an effect only when its 95% interval (Newcombe) excludes 0, and within noise when the interval spans 0, whatever the point estimate.`,
+	"When sources or the eval fingerprint differ across the compared runs, the pass@1 difference is not comparable: the eval changed. Never call it an effect. Do not put those mixed counts on a results line. The first line of the analysis says the eval changed. A lift is an effect only when sources match; the two-attempt floor still applies then.",
+	`Effort (tool calls, agent time, and tokens when both sides have usage samples) is compared with an exact Mann-Whitney test: "fewer"/"more" at p < 0.05; "insufficient" when the attempt counts can never reach it (2 vs 2 cannot) — never because a harness dropped usage. Attempts without usage are left out of the token sample only. If either side then has no token sample, the token metric is omitted (no line, not insufficient). A Goose cell does not blank tokens on Cursor cells in the same study. Pooled across evals, tokens are an effect only when at least 2/3 of the strata that reported usage agree. Effort's separated (given from 3 vs 3) says, per metric, that the two sides' ranges do not overlap: every attempt on one side beat every attempt on the other. insufficient with separated true is a directional signal: report the ratio, the n and "no overlap, not significant" as directional, never as an effect. separated false is no direction at all.`,
+	"A family fact is one pooled evaluate.each: its cases are paired samples of one suite, so its lift (Newcombe paired interval and McNemar at one attempt a case, a t interval over cases otherwise) is the result for those cases; their own pass-rate facts carry no lift. insufficient (fewer than 6 discordant pairs or 6 cases) is not measured. A family's minLift gates the pooled lower bound.",
 	"Each factor claim has a tier: controlled (two cells in one run that differ in one factor), pooled (the same contrast over comparable runs, and across evals), observed (a change between runs, which is weaker; \"confounded\" when several parts changed), not_measured (one level only).",
 	`A cell with mixed outcomes is flaky only with at least 6 counted attempts over comparable runs; below that it is "insufficient", not flaky.`,
 	"Drift marked noise changed only an agent CLI's own runtime state (e.g. ~/.cursor/cli-config.json): not a real change, and those runs stay comparable.",
-	"An eval where every cell passes at least 95% is saturated: pass rate cannot tell cells apart there, effort can.",
+	"An eval where every cell passes at least 95% is saturated: pass@1 cannot tell cells apart there, effort can.",
 	"Judge verdicts carry no recorded reason; an explanation of a failed verdict is your reading of the cited output, and must say so.",
-	"Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check."
+	"A criteria fact lists each check (expect label, judges included) per cell: rows only for labels whose rate differs between cells, or is mixed within one; neverMet labels failed on every counted attempt of every cell, alwaysMet passed on all of them. A neverMet or alwaysMet label is the same in every arm: it can't explain a difference between arms.",
+	"frame study means every cell in scope is ungated (gate: 'none'): the run measures how its arms differ, so attempts that fail are data, not a failed gate.",
+	"setup, rules and cli give context and are not citable: cite facts, evidence and attempts.",
+	"A context-use fact counts each attempt's tool calls by class (context, dependency, web, verify, workspace) and lists which placed files each arm opened: how much an arm leaned on what its environment declared versus digging in installed packages. The classes are a heuristic over tool names and paths; cite them as counts, not as proof of intent.",
+	"A contrast fact (kind contrast) is one pair of arms on one harness, a against b, when an eval has two variants or more: pass rate and lift of a over b, the checks that tell them apart (discriminates or mixed; sameOnBoth counts the rest) and effort. It answers how two variants differ from each other, not only from the control; gating stays each variant against its control.",
+	"A check (kind checks) whose verdict is both-pass or both-fail passes, or fails, on both the variant and its control: it is no evidence for or against the variant. Only discriminates rows say which requirement separated them.",
+	"Compare-question win rates annotate the run: they never change pass, fail or the exit code. Cite them as annotation, never as a result that gates the run. A vote's reason is the model's wording: do not treat it as a check. Each pair is voted in both orders; splits counts pairs whose two orders picked opposite sides (the position decided, not the work), and rates count such a pair as one win each. With more than two arms, read headToHead (a–b wins per pair, splits apart), not the pooled rates. shows lists what each side was shown: a question that shows output or transcript can favour the better-written account rather than the better work, and one that shows artifacts rests on screenshots."
 ];
 const pts = (x) => x === null || x === void 0 ? null : `${x >= 0 ? "+" : "−"}${Math.round(Math.abs(x) * 100)}`;
 const liftText = (points, lo, hi) => `${pts(points)} pts (95% ${pts(lo)}…${pts(hi)})`;
@@ -53,7 +64,7 @@ function evalSourcesChanged(base, head) {
 /** Short tags for the evals in scope: the file's base name, numbered when a file has several. */
 function evalTags(evals) {
 	const tags = /* @__PURE__ */ new Map();
-	const bases = evals.map((e) => basename(e.file).replace(/\.(?:eval|es)\.ts$|\.[jt]s$/, ""));
+	const bases = evals.map((e) => basename(e.file).replace(/(?:\.\w+)?\.[jt]s$/, ""));
 	evals.forEach((e, i) => {
 		const b = bases[i];
 		const same = bases.filter((x) => x === b).length;
@@ -68,7 +79,91 @@ function cellTags(e) {
 	const prefix = (c) => harnesses.length < 2 ? "" : `${kinds.length === harnesses.length ? c.harnessKind : c.harness}:`;
 	return new Map(e.cells.map((c) => [c.key, `${prefix(c)}${c.variant ?? "control"}`]));
 }
+/**
+* Cell names for facts: each tag, with a compare's control named by its environment
+* (`no-skill (control)`) so a reading can name the arm. Attempt ids keep the tag.
+*/
+function cellNames(e, tags) {
+	const compare = e.cells.some((c) => c.role === "variant");
+	return new Map(e.cells.map((c) => {
+		const tag = tags.get(c.key) ?? c.key;
+		const named = compare && c.role === "control" && c.environment && c.environment !== "-";
+		return [c.key, named ? tag.replace(/control$/, `${c.environment} (control)`) : tag];
+	}));
+}
 const counted = (a) => a.outcome === "pass" || a.outcome === "fail";
+const LABEL_CHARS = 60;
+const round2 = (x) => Math.round(x * 100) / 100;
+const round1 = (x) => Math.round(x * 10) / 10;
+const ratioText = (r) => r == null ? "×–" : `×${Number(r).toFixed(2)}`;
+/** The labels of an attempt's assertions that failed or couldn't answer. */
+const failedLabels = (a) => (a.assertions ?? []).filter((x) => x.status !== "pass").map((x) => clip(x.label, LABEL_CHARS));
+/** Every counted attempt passed (`all`), none did, or some; undefined when none answered. */
+const sideOf = (t) => {
+	const answered = t.pass + t.fail + t.error + t.notRun;
+	if (answered === 0) return void 0;
+	return t.pass === answered ? "all" : t.pass === 0 ? "none" : "some";
+};
+/** Items interleaved by group, groups in order of first appearance: one from each, then the next. */
+function roundRobin(items, key) {
+	const groups = /* @__PURE__ */ new Map();
+	for (const item of items) groups.set(key(item), [...groups.get(key(item)) ?? [], item]);
+	const out = [];
+	for (let i = 0; out.length < items.length; i++) for (const g of groups.values()) if (i < g.length) out.push(g[i]);
+	return out;
+}
+/** `study` when every cell in scope is ungated, so the run compares arms rather than gating them. */
+const frameOf = (evals) => {
+	const cells = evals.flatMap((e) => e.cells);
+	return cells.length && cells.every((c) => c.gate === "none") ? "study" : "gate";
+};
+/**
+* One eval's checks per cell: a row with each cell's rate (and mean judge score) for each label
+* that differs between cells or is mixed within one, and the labels every cell failed, or passed,
+* on every counted attempt.
+*/
+function criteriaOf(e, tag) {
+	const cells = e.cells.filter((c) => c.checks?.length);
+	const labels = [...new Set(cells.flatMap((c) => (c.checks ?? []).map((k) => k.label)))];
+	if (!labels.length) return null;
+	const scores = /* @__PURE__ */ new Map();
+	for (const a of e.attempts) {
+		if (!counted(a)) continue;
+		for (const j of labelledJudgements(a)) {
+			if (j.check === void 0) continue;
+			const id = `${a.cell}\0${j.check}`;
+			scores.set(id, [...scores.get(id) ?? [], j.judgement.score]);
+		}
+	}
+	const rows = [];
+	const neverMet = [];
+	const alwaysMet = [];
+	for (const label of labels) {
+		const tallies = cells.flatMap((c) => {
+			const k = c.checks?.find((x) => x.label === label);
+			return k ? [{
+				c,
+				k
+			}] : [];
+		});
+		const sides = new Set(tallies.map(({ k }) => sideOf(k)));
+		if (sides.size === 1 && sides.has("none")) neverMet.push(clip(label, LABEL_CHARS));
+		else if (sides.size === 1 && sides.has("all")) alwaysMet.push(clip(label, LABEL_CHARS));
+		else rows.push({
+			label: clip(label, LABEL_CHARS),
+			cells: Object.fromEntries(tallies.map(({ c, k }) => {
+				const s = scores.get(`${c.key}\0${label}`);
+				const mean = s?.length ? ` · ${(s.reduce((x, y) => x + y, 0) / s.length).toFixed(2)}` : "";
+				return [tag(c.key), `${k.pass}/${k.pass + k.fail + k.error + k.notRun}${mean}`];
+			}))
+		});
+	}
+	return {
+		rows,
+		neverMet,
+		alwaysMet
+	};
+}
 /** The digest, and the evidence dropped from it to fit its cap, whole, for tools to read back. */
 async function buildDigestKeepingDropped(input) {
 	const { head, base, db } = input;
@@ -77,6 +172,7 @@ async function buildDigestKeepingDropped(input) {
 	const needle = input.evalFilter?.toLowerCase() ?? null;
 	const inScope = (e) => !needle || e.id.toLowerCase().includes(needle);
 	const scopeEvals = head.record.evals.filter(inScope);
+	const frame = frameOf(scopeEvals);
 	const tags = evalTags(scopeEvals);
 	const evalKeys = new Set(scopeEvals.map((e) => e.key));
 	const tagOf = (key) => tags.get(key) ?? key;
@@ -87,25 +183,30 @@ async function buildDigestKeepingDropped(input) {
 	for (const r of runs) for (const e of r.record.evals) {
 		if (!evalKeys.has(e.key)) continue;
 		const cells = cellTags(e);
-		for (const [key, tag] of cells) cellTagOf.set(`${r.record.runId}\0${e.key}\0${key}`, tag);
+		const names = cellNames(e, cells);
+		for (const [key, name] of names) cellTagOf.set(`${r.record.runId}\0${e.key}\0${key}`, name);
 		for (const a of e.attempts) {
 			const id = `${short.get(r.record.runId)}/${tagOf(e.key)}/${cells.get(a.cell)}#${a.n}`;
 			idOf.set(`${r.record.runId}\0${a.key}`, id);
-			const judge = a.judgements[0];
 			const any = a.files.transcript ?? a.files.judgeInput ?? a.checks.find((c) => c.log)?.log;
+			const assertions = a.assertions ?? [];
+			const failed = failedLabels(a);
+			const judges = labelledJudgements(a);
 			attempts[id] = {
 				run: short.get(r.record.runId),
 				runId: r.record.runId,
 				key: a.key,
 				eval: tagOf(e.key),
 				evalSlug: e.slug,
-				cell: cells.get(a.cell),
+				cell: names.get(a.cell),
 				attempt: a.id,
 				outcome: a.outcome,
 				tools: a.activity.tools,
 				agentS: Math.round(a.timings.agentMs / 1e3),
-				...judge ? { judge: Math.round(judge.score * 100) / 100 } : {},
-				checks: a.checks.length,
+				...assertions.length ? { checks: `${assertions.filter((x) => x.status === "pass").length}/${assertions.length}` } : {},
+				...failed.length ? { failed } : {},
+				...judges.length ? { judges: Object.fromEntries(judges.map((j) => [clip(j.label, LABEL_CHARS), round2(j.judgement.score)])) } : {},
+				execs: a.checks.length,
 				artifacts: any ? `${r.record.runId}/${any.path.split("/").slice(0, 3).join("/")}` : null
 			};
 		}
@@ -121,22 +222,51 @@ async function buildDigestKeepingDropped(input) {
 	const list = (values) => values.map((v) => `'${v.replace(/'/g, "''")}'`).join(", ");
 	const runList = list(runIds);
 	const evalList = list([...evalKeys]);
-	for (const r of runs) for (const e of r.record.evals) {
-		if (!evalKeys.has(e.key)) continue;
-		fact("pass-rate", {
-			run: short.get(r.record.runId),
-			eval: tagOf(e.key),
-			cells: e.cells.map((c) => ({
-				cell: cellTag(r.record.runId, e.key, c.key),
-				pass: `${c.pass}/${c.pass + c.fail}`,
-				...c.infraError ? { infraError: c.infraError } : {},
-				gated: c.gate !== "none",
-				...c.lift ? {
-					lift: digestLift(c.lift.variant.counted, c.lift.control.counted, c.lift.points, c.lift.interval.lo, c.lift.interval.hi),
-					verdict: digestLiftVerdict(c.lift.variant.counted, c.lift.control.counted, c.lift.withinNoise)
+	for (const r of runs) {
+		for (const e of r.record.evals) {
+			if (!evalKeys.has(e.key)) continue;
+			fact("pass@1", {
+				run: short.get(r.record.runId),
+				eval: tagOf(e.key),
+				cells: e.cells.map((c) => ({
+					cell: cellTag(r.record.runId, e.key, c.key),
+					pass: `${c.pass}/${c.pass + c.fail}`,
+					...c.infraError ? { infraError: c.infraError } : {},
+					gated: c.gate !== "none",
+					...c.lift && !e.family ? {
+						lift: digestLift(c.lift.variant.counted, c.lift.control.counted, c.lift.points, c.lift.interval.lo, c.lift.interval.hi),
+						verdict: digestLiftVerdict(c.lift.variant.counted, c.lift.control.counted, c.lift.withinNoise)
+					} : {}
+				})),
+				...e.family ? {
+					family: e.family.id,
+					case: e.family.case
 				} : {}
-			}))
-		});
+			});
+		}
+		for (const f of r.record.families ?? []) {
+			const member = r.record.evals.find((e) => e.family?.id === f.id && evalKeys.has(e.key));
+			if (!member) continue;
+			fact("family", {
+				run: short.get(r.record.runId),
+				family: f.id,
+				cases: f.ran === f.cases ? f.cases : `${f.ran} of ${f.cases}`,
+				rates: f.rates.map((x) => ({
+					cell: cellTag(r.record.runId, member.key, x.cell),
+					passAt1: familyRateLabel(x)
+				})),
+				lifts: f.lifts.map((l) => ({
+					cell: cellTag(r.record.runId, member.key, l.variant),
+					lift: familyLiftLabel(l),
+					verdict: l.verdict === "lift" || l.verdict === "drop" ? "effect" : l.verdict,
+					...l.minLift !== void 0 ? {
+						minLift: l.minLift,
+						gate: l.outcome
+					} : {},
+					...familyEffortLabel(l) ? { effort: familyEffortLabel(l) } : {}
+				}))
+			});
+		}
 	}
 	const effort = q(`SELECT * FROM effort_m WHERE run_id IN (${runList}) AND eval_key IN (${evalList}) ORDER BY run_id, rowid`);
 	const effortGroups = /* @__PURE__ */ new Map();
@@ -156,16 +286,27 @@ async function buildDigestKeepingDropped(input) {
 				medianToolCalls: row.tools_median,
 				medianAgentS: row.agent_s_median,
 				...row.tokens_median != null ? { medianTokens: row.tokens_median } : {},
+				...row.workspace_calls_median != null ? { medianCallsByClass: Object.fromEntries(TOOL_CLASSES.flatMap((k) => {
+					const m = row[`${k}_calls_median`];
+					return m ? [[k, m]] : [];
+				})) } : {},
 				...row.control_key ? { vsControl: {
-					toolCalls: `${row.tools_median} vs ${row.control_tools_median} (×${Number(row.tools_ratio).toFixed(2)})`,
-					agentS: `${row.agent_s_median} vs ${row.control_agent_s_median}`,
+					n: `${row.n} vs ${row.control_n}`,
+					toolCalls: `${row.tools_median} vs ${row.control_tools_median} (${ratioText(row.tools_ratio)})`,
+					agentS: `${row.agent_s_median} vs ${row.control_agent_s_median} (${ratioText(row.agent_s_ratio)})`,
 					p: row.tools_p,
 					verdict: row.verdict,
+					timeVerdict: row.time_verdict,
 					...row.tokens_verdict ? {
-						tokens: `${row.tokens_median} vs ${row.control_tokens_median} (×${Number(row.tokens_ratio).toFixed(2)})`,
+						tokens: `${formatTokens(row.tokens_median)} vs ${formatTokens(row.control_tokens_median)} (${ratioText(row.tokens_ratio)})`,
 						tokensP: row.tokens_p,
 						tokensVerdict: row.tokens_verdict
 					} : {},
+					...Math.min(Number(row.n), Number(row.control_n)) >= 3 ? { separated: {
+						tools: row.separated === 1,
+						agentS: row.time_separated === 1,
+						...row.tokens_separated != null ? { tokens: row.tokens_separated === 1 } : {}
+					} } : {},
 					...row.tokens_note ? { unreportedUsage: row.tokens_note } : {}
 				} } : {}
 			}))
@@ -183,7 +324,7 @@ async function buildDigestKeepingDropped(input) {
       OR (tier = 'observed' AND run_id IN (${runList}) AND eval_key IN (${evalList}))
       OR tier = 'not_measured'
      ORDER BY CASE tier WHEN 'pooled' THEN 0 WHEN 'controlled' THEN 1 WHEN 'observed' THEN 2 ELSE 3 END,
-       factor, level, eval_key`).filter((f) => f.tier !== "pooled" || f.factor !== "environment" || harnesses.includes(f.context));
+       factor, level, eval_key`).filter((f) => f.tier !== "pooled" || f.factor !== "environment" || f.context !== null && harnesses.includes(f.context));
 	for (const f of factors) {
 		if (!base && (f.tier === "pooled" || f.tier === "observed")) continue;
 		if (f.tier === "not_measured") {
@@ -263,6 +404,51 @@ async function buildDigestKeepingDropped(input) {
 			firstReadAtLines: [...c.lines].sort((a, b) => a - b)
 		}))
 	});
+	const classRows = q(`SELECT t.run_id, a.eval_key, a.cell_key, a.n, t.class, COUNT(*) AS calls
+     FROM tool_calls t JOIN attempts a ON a.run_id = t.run_id AND a.key = t.attempt_key
+     WHERE t.run_id IN (${runList}) AND a.eval_key IN (${evalList}) AND a.outcome IN ('pass', 'fail')
+     GROUP BY t.run_id, a.key, t.class ORDER BY t.run_id, a.eval_key, a.cell_key, a.n`);
+	const fileRows = q(`SELECT run_id, eval_key, cell_key, path, COUNT(*) AS attempts, SUM(reads > 0) AS read
+     FROM skill_files WHERE run_id IN (${runList}) AND eval_key IN (${evalList})
+     GROUP BY run_id, eval_key, cell_key, path ORDER BY run_id, eval_key, cell_key, path`);
+	const useOf = /* @__PURE__ */ new Map();
+	const useEntry = (row) => {
+		const id = `${row.run_id}\0${row.eval_key}\0${row.cell_key}`;
+		const entry = useOf.get(id) ?? {
+			classes: /* @__PURE__ */ new Map(),
+			files: []
+		};
+		useOf.set(id, entry);
+		return entry;
+	};
+	for (const r of classRows) {
+		const entry = useEntry(r);
+		const per = entry.classes.get(r.class) ?? /* @__PURE__ */ new Map();
+		per.set(r.n, r.calls);
+		entry.classes.set(r.class, per);
+	}
+	for (const r of fileRows) useEntry(r).files.push(`${r.path} ${r.read}/${r.attempts}`);
+	if (classRows.length) {
+		const attemptNs = /* @__PURE__ */ new Map();
+		for (const r of classRows) {
+			const id = `${r.run_id}\0${r.eval_key}\0${r.cell_key}`;
+			attemptNs.set(id, [.../* @__PURE__ */ new Set([...attemptNs.get(id) ?? [], r.n])]);
+		}
+		fact("context-use", {
+			note: "Tool calls per counted attempt by class (context: a placed skill, rules or MCP tool; dependency: node_modules, site-packages, vendor, --help; web; verify: a check command or test/build/lint; workspace: the rest), a heuristic. files: each placed file, opened in k/n attempts.",
+			cells: [...useOf].map(([id, u]) => {
+				const [runId, evalKey, cellKey] = id.split("\0");
+				const ns = (attemptNs.get(id) ?? []).sort((a, b) => a - b);
+				return {
+					run: short.get(runId),
+					eval: tagOf(evalKey),
+					cell: cellTag(runId, evalKey, cellKey),
+					perAttempt: Object.fromEntries([...u.classes].map(([k, per]) => [k, ns.map((n) => per.get(n) ?? 0).join(" ")])),
+					...u.files.length ? { files: u.files } : {}
+				};
+			})
+		});
+	}
 	const pooled = /* @__PURE__ */ new Map();
 	for (const r of runs) for (const e of r.record.evals) {
 		if (!evalKeys.has(e.key)) continue;
@@ -287,46 +473,59 @@ async function buildDigestKeepingDropped(input) {
 		})),
 		saturated
 	});
-	const judged = runs.flatMap((r) => r.record.evals.filter((e) => evalKeys.has(e.key)).flatMap((e) => e.attempts.flatMap((a) => a.judgements.map((j) => ({
-		r,
-		a,
-		j
-	})))));
+	const differing = /* @__PURE__ */ new Map();
+	for (const r of runs) for (const e of r.record.evals) {
+		if (!evalKeys.has(e.key)) continue;
+		const criteria = criteriaOf(e, (cell) => cellTag(r.record.runId, e.key, cell));
+		if (!criteria) continue;
+		differing.set(`${r.record.runId}\0${e.key}`, new Set(criteria.rows.map((x) => x.label)));
+		fact("criteria", {
+			run: short.get(r.record.runId),
+			eval: tagOf(e.key),
+			note: "Each check (expect label) per cell: passed/counted, then the mean judge score for a judge.",
+			...criteria
+		});
+	}
+	const judged = runs.flatMap((r) => r.record.evals.filter((e) => evalKeys.has(e.key)).flatMap((e) => e.attempts.flatMap((a) => a.judgements)));
 	if (judged.length) fact("judge", {
-		models: [...new Set(judged.map(({ j }) => j.resolvedModel ?? j.model))],
-		threshold: [...new Set(judged.map(({ j }) => j.threshold))],
+		models: [...new Set(judged.map((j) => j.resolvedModel ?? j.model))],
+		threshold: [...new Set(judged.map((j) => j.threshold))],
 		verdicts: judged.length,
-		pass: judged.filter(({ j }) => j.verdict === "pass").length,
-		scoresAtLeast098: judged.filter(({ j }) => j.score >= .98).length,
-		nearThreshold: judged.filter(({ j }) => Math.abs(j.score - j.threshold) <= .1).length,
-		failures: judged.filter(({ j }) => j.verdict === "fail").map(({ r, a, j }) => ({
-			attempt: idOf.get(`${r.record.runId}\0${a.key}`),
-			score: j.score
-		})),
+		pass: judged.filter((j) => j.verdict === "pass").length,
+		scoresAtLeast098: judged.filter((j) => j.score >= .98).length,
+		nearThreshold: judged.filter((j) => Math.abs(j.score - j.threshold) <= .1).length,
 		reasons: "not recorded"
 	});
-	const failures = runs.slice().reverse().flatMap((r) => r.record.evals.filter((e) => evalKeys.has(e.key)).flatMap((e) => e.attempts.filter((a) => a.outcome === "fail" || a.outcome === "infra_error").map((a) => ({
+	const failures = roundRobin(runs.slice().reverse().flatMap((r) => r.record.evals.filter((e) => evalKeys.has(e.key)).flatMap((e) => e.attempts.filter((a) => a.outcome === "fail" || a.outcome === "infra_error").map((a) => ({
 		r,
 		e,
 		a
-	}))));
-	for (const { r, e, a } of failures.slice(0, 8)) {
-		const j = a.judgements.find((x) => x.verdict === "fail");
+	})))), ({ r, e, a }) => `${r.record.runId}\0${e.key}\0${a.cell}`);
+	const cellsSeen = /* @__PURE__ */ new Set();
+	const failureFacts = frame === "study" ? failures.filter(({ r, e, a }) => {
+		const id = `${r.record.runId}\0${e.key}\0${a.cell}`;
+		return !cellsSeen.has(id) && !!cellsSeen.add(id);
+	}) : failures.slice(0, 8);
+	for (const { r, e, a } of failureFacts) {
+		const failed = failedLabels(a);
+		const judges = labelledJudgements(a).filter((j) => j.judgement.verdict === "fail");
 		fact("failure", {
 			attempt: idOf.get(`${r.record.runId}\0${a.key}`),
 			outcome: a.outcome,
 			errorKind: a.error?.kind ?? null,
 			message: a.error ? clip(a.error.message.split("\n")[0] ?? "", 300) : null,
 			gated: e.cells.find((c) => c.key === a.cell)?.gate !== "none",
-			...j ? { judge: {
-				question: j.question,
-				score: j.score,
-				threshold: j.threshold
-			} } : {},
-			failingChecks: a.checks.map((c, i) => ({
-				check: i + 1,
+			...failed.length ? { failed } : {},
+			...judges.length ? { judges: judges.map((j) => ({
+				label: clip(j.label, LABEL_CHARS),
+				question: clip(j.judgement.question, 300),
+				score: j.judgement.score,
+				threshold: j.judgement.threshold
+			})) } : {},
+			failingExecs: a.checks.map((c, i) => ({
+				exec: i + 1,
 				exit: c.exitCode,
-				command: c.command
+				command: clip(c.command, 300)
 			})).filter((c) => c.exit !== 0)
 		});
 	}
@@ -349,7 +548,7 @@ async function buildDigestKeepingDropped(input) {
 			eval: tagOf(e.key),
 			from: "r1",
 			to: "r2",
-			note: "not comparable: the eval changed; the pass-rate difference is not an effect of the thing under test"
+			note: "not comparable: the eval changed; the pass@1 difference is not an effect of the thing under test"
 		});
 	}
 	if (base) {
@@ -379,11 +578,14 @@ async function buildDigestKeepingDropped(input) {
 		for (const a of e.answers ?? []) {
 			const ties = a.votes.filter((v) => v.winner === "tie").length;
 			const reasons = sampleReasons(a.votes);
+			const splits = splitsLabel(pairTallies(a.votes));
+			const headToHead = headToHeadLines(a.votes);
 			fact("compare-question", {
 				run: short.get(r.record.runId),
 				eval: tagOf(e.key),
 				question: clip(a.question, 120),
 				model: a.resolvedModel ?? a.model,
+				shows: questionShows(a.stateOptions ?? DEFAULT_QUESTION_STATE, a.artifacts),
 				votes: a.votes.length,
 				ties,
 				...a.rates.length ? { rates: a.rates.map((x) => ({
@@ -392,12 +594,73 @@ async function buildDigestKeepingDropped(input) {
 					wins: x.wins,
 					n: x.n
 				})) } : {},
+				...splits ? { splits } : {},
+				...headToHead.length ? { headToHead } : {},
 				...a.error ? { error: clip(a.error, 200) } : {},
 				...a.votes.length === 0 && a.error === void 0 ? { note: "no pairs to judge: attempts that failed their checks are left out" } : {},
 				...reasons.length ? { reasons } : {}
 			});
 		}
 	}
+	for (const r of runs) for (const e of r.record.evals) {
+		if (!evalKeys.has(e.key)) continue;
+		const tag = (key) => cellTag(r.record.runId, e.key, key);
+		const contrasts = contrastsOf(e);
+		for (const x of contrasts) {
+			const apart = x.checks.filter((k) => k.verdict === "discriminates" || k.verdict === "mixed");
+			const ef = x.effort;
+			fact("contrast", {
+				run: short.get(r.record.runId),
+				eval: tagOf(e.key),
+				a: tag(x.a),
+				b: tag(x.b),
+				...x.lift ? {
+					passRate: `${x.lift.variant.pass}/${x.lift.variant.counted} vs ${x.lift.control.pass}/${x.lift.control.counted}`,
+					lift: digestLift(x.lift.variant.counted, x.lift.control.counted, x.lift.points, x.lift.interval.lo, x.lift.interval.hi),
+					liftVerdict: digestLiftVerdict(x.lift.variant.counted, x.lift.control.counted, x.lift.withinNoise)
+				} : {},
+				checks: apart.map((k) => ({
+					label: clip(k.label, LABEL_CHARS),
+					a: `${k.a.pass}/${k.a.pass + k.a.fail + k.a.notRun}`,
+					b: `${k.b.pass}/${k.b.pass + k.b.fail + k.b.notRun}`,
+					verdict: k.verdict
+				})),
+				sameOnBoth: x.checks.length - apart.length,
+				...ef ? { effort: {
+					n: `${ef.variant.n} vs ${ef.control.n}`,
+					toolCalls: `${ef.variant.tools.median} vs ${ef.control.tools.median} (${ratioText(ef.toolsRatio)})`,
+					agentS: `${round1(ef.variant.agentMs.median / 1e3)} vs ${round1(ef.control.agentMs.median / 1e3)} (${ratioText(ef.agentMsRatio)})`,
+					verdict: ef.verdict,
+					timeVerdict: ef.timeVerdict,
+					...ef.tokensVerdict && ef.variant.tokens && ef.control.tokens ? {
+						tokens: `${formatTokens(ef.variant.tokens.median)} vs ${formatTokens(ef.control.tokens.median)} (${ratioText(ef.tokensRatio)})`,
+						tokensVerdict: ef.tokensVerdict
+					} : {},
+					...Math.min(ef.variant.n, ef.control.n) >= 3 ? { separated: {
+						tools: ef.separated,
+						agentS: ef.timeSeparated,
+						...ef.tokensSeparated !== null ? { tokens: ef.tokensSeparated } : {}
+					} } : {}
+				} } : {}
+			});
+		}
+		if (contrasts.length) continue;
+		for (const c of e.cells) {
+			const rows = (c.checks ?? []).filter((k) => k.control && k.verdict);
+			if (!rows.length) continue;
+			fact("checks", {
+				run: short.get(r.record.runId),
+				eval: tagOf(e.key),
+				cell: c.key,
+				rows: rows.map((k) => ({
+					label: clip(k.label, 80),
+					variant: `${k.pass}/${k.pass + k.fail + k.notRun}`,
+					control: `${k.control?.pass}/${(k.control?.pass ?? 0) + (k.control?.fail ?? 0) + (k.control?.notRun ?? 0)}`,
+					verdict: k.verdict
+				}))
+			});
+		}
+	}
 	const evidence = [];
 	const add = (ref, why, lines, priority) => {
 		if (lines.length) evidence.push({
@@ -424,18 +687,29 @@ async function buildDigestKeepingDropped(input) {
 		} catch {}
 		return [];
 	};
-	for (const { r, e, a } of failures.slice(0, 3)) {
+	const discriminating = ({ r, e, a }) => failedLabels(a).some((l) => differing.get(`${r.record.runId}\0${e.key}`)?.has(l));
+	const perCell = /* @__PURE__ */ new Map();
+	for (const f of failures) {
+		const id = `${f.r.record.runId}\0${f.e.key}\0${f.a.cell}`;
+		const prev = perCell.get(id);
+		if (!prev || !discriminating(prev) && discriminating(f)) perCell.set(id, f);
+	}
+	for (const [i, { r, e, a }] of [...perCell.values()].slice(0, 6).entries()) {
 		const id = idOf.get(`${r.record.runId}\0${a.key}`);
 		const lines = await lastText(r.record.runId, a);
-		const j = a.judgements.find((x) => x.verdict === "fail");
-		if (j) lines.push(`judge question: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
-		for (const [i, c] of a.checks.entries()) {
+		const judges = labelledJudgements(a).filter((j) => j.judgement.verdict === "fail");
+		for (const { label, judgement: j } of judges) lines.push(`judge ${clip(label, LABEL_CHARS)}: ${j.question}`, `judge score: ${j.score} (threshold ${j.threshold})`);
+		for (const x of a.assertions ?? []) {
+			if (x.status === "pass" || /(^|\.)toPass$/.test(x.matcher)) continue;
+			lines.push(`check ${x.status}: ${clip(x.label, LABEL_CHARS)}: ${clip(x.message ?? "", 300)}`);
+		}
+		for (const [n, c] of a.checks.entries()) {
 			if (c.exitCode === 0 || !c.log) continue;
 			const log = await readFile(join(r.dir, r.record.runId, ...c.log.path.split("/")), "utf8").catch(() => "");
-			lines.push(`check ${i + 1} (exit ${c.exitCode}) ${c.command}:`, ...log.trimEnd().split("\n").slice(-15));
+			lines.push(`exec ${n + 1} (exit ${c.exitCode}) ${c.command}:`, ...log.trimEnd().split("\n").slice(-15));
 		}
-		add(`attempt:${id}`, `a failing attempt: its final message${j ? " and the judge question it failed" : ""}`, lines, 1);
-		if (j) {
+		add(`attempt:${id}`, `a failing attempt of ${cellTag(r.record.runId, e.key, a.cell)}: its final message${judges.length ? " and the judge questions it failed" : ""}`, lines, 1);
+		if (judges[0] && i < 3) {
 			const twin = e.attempts.find((x) => x.cell === a.cell && x.outcome === "pass");
 			if (twin) add(`attempt:${idOf.get(`${r.record.runId}\0${twin.key}`)}`, `the passing twin of ${id} (same cell): its final message`, await lastText(r.record.runId, twin), 2);
 		}
@@ -455,13 +729,14 @@ async function buildDigestKeepingDropped(input) {
 		}
 	}
 	const digest = {
-		digestVersion: 7,
+		digestVersion: 10,
 		scope: {
 			kind: "run",
 			head: short.get(head.record.runId),
 			base: base ? "r1" : null,
 			evals: input.evalFilter
 		},
+		frame,
 		runs: Object.fromEntries(runs.map((r) => [short.get(r.record.runId), {
 			runId: r.record.runId,
 			commit: r.record.git?.sha.slice(0, 7) ?? null,
@@ -590,7 +865,7 @@ function renderText(doc) {
 }
 //#endregion
 //#region src/analysis/validate.ts
-const CHECK = /^(.+):check(\d+)$/;
+const EXEC = /^(.+):exec(\d+)$/;
 const LINE = /^(E\d+):L(\d+)$/;
 const TRANSCRIPT = /^transcript:(.+)#L(\d+)$/;
 const normal = (s) => s.replace(/\s+/g, " ").trim().toLowerCase();
@@ -607,21 +882,21 @@ function citations(digest) {
 		if (ids.has(c)) return factText.get(c) ?? evidenceText.get(c) ?? JSON.stringify(digest.attempts[c]);
 		const line = LINE.exec(c);
 		if (line) {
-			const [, id, n] = line;
+			const [, id = "", n = ""] = line;
 			if (!lines.get(id)?.has(Number(n))) return null;
 			return digest.evidence.find((e) => e.id === id)?.lines.find((l) => l.startsWith(`L${n} `)) ?? null;
 		}
 		const transcript = TRANSCRIPT.exec(c);
 		if (transcript) {
-			const [, attempt, n] = transcript;
+			const [, attempt = "", n = ""] = transcript;
 			for (const id of evidenceOf(attempt)) if (lines.get(id)?.has(Number(n))) return resolve(`${id}:L${n}`);
 			return null;
 		}
-		const check = CHECK.exec(c);
-		if (check) {
-			const [, attempt, n] = check;
+		const exec = EXEC.exec(c);
+		if (exec) {
+			const [, attempt = "", n = ""] = exec;
 			const a = digest.attempts[attempt];
-			return a && Number(n) >= 1 && Number(n) <= a.checks ? JSON.stringify(a) : null;
+			return a && Number(n) >= 1 && Number(n) <= a.execs ? JSON.stringify(a) : null;
 		}
 		return null;
 	};
@@ -629,7 +904,7 @@ function citations(digest) {
 }
 const ANY_NUMBER = /[−-]?\d+(?:\.\d+)?/g;
 const FRACTION = /(\d+)\/(\d+)/g;
-const ATTEMPT_CITE = /:check\d+$|:L\d+$/;
+const ATTEMPT_CITE = /:exec\d+$|:L\d+$/;
 /** A decimal in [0, 1]: the form a judge score or threshold takes in the digest. */
 const looksLikeScore = (raw) => {
 	if (!raw.includes(".")) return false;
@@ -641,23 +916,24 @@ const addScore = (scores, n) => {
 };
 /**
 * Recorded judge scores (and thresholds) on the cited attempt, fact or evidence. The digest stores
-* them as `attempt.judge`, `judge: { score, threshold }` on a failure, and `judge score:` lines.
+* them as an attempt's `judges`, a failure's `judges: [{ score, threshold }]`, a criteria row's mean
+* scores, and `judge score:` lines.
 */
 function citedJudgeScores(cites, digest) {
 	const scores = [];
 	for (const cite of cites) {
 		const id = cite.trim().replace(/^\[|\]$/g, "").replace(ATTEMPT_CITE, "");
-		addScore(scores, digest.attempts[id]?.judge);
+		for (const s of Object.values(digest.attempts[id]?.judges ?? {})) addScore(scores, s);
 		const fact = digest.facts.find((f) => f.id === id);
 		if (fact) {
-			const j = fact.judge;
-			if (typeof j === "number") addScore(scores, j);
-			else if (j && typeof j === "object") {
-				addScore(scores, j.score);
-				addScore(scores, j.threshold);
+			if (Array.isArray(fact.judges)) for (const row of fact.judges) {
+				if (!row || typeof row !== "object") continue;
+				addScore(scores, row.score);
+				addScore(scores, row.threshold);
 			}
-			if (Array.isArray(fact.failures)) {
-				for (const row of fact.failures) if (row && typeof row === "object") addScore(scores, row.score);
+			if (Array.isArray(fact.rows)) for (const row of fact.rows) for (const v of Object.values(row?.cells ?? {})) {
+				const mean = / · (\d+(?:\.\d+)?)$/.exec(String(v));
+				if (mean) addScore(scores, Number(mean[1]));
 			}
 			if (Array.isArray(fact.threshold)) for (const t of fact.threshold) addScore(scores, t);
 		}
@@ -831,8 +1107,8 @@ async function load(config, entry) {
 		bytes
 	};
 }
-/** The model's input: the digest as JSON, after the project's own instructions. */
-const promptOf = (d) => `${d.instructions ? `PROJECT INSTRUCTIONS: ${d.instructions}\n\n` : ""}DIGEST:\n${JSON.stringify(d)}`;
+/** The model's input: the project's own instructions, which set the question, then the digest as JSON. */
+const promptOf = (d) => `${d.instructions ? `PROJECT INSTRUCTIONS (the question this analysis answers): ${d.instructions}\n\n` : ""}DIGEST:\n${JSON.stringify(d)}`;
 /** `--context`: the same digest and rules, for the user's own coding agent. */
 function contextMarkdown(d, head, base) {
 	return [
@@ -840,6 +1116,13 @@ function contextMarkdown(d, head, base) {
 		"",
 		"Analyse this reeval run for the engineer who wrote the evals. The digest below holds facts reeval computed (F…), evidence excerpts (E…, transcript lines as L<n>) and the attempts table. Cite those ids for every claim, keep to the verdicts in the facts, and suggest at most 5 concrete actions, each verified with a reeval command from the CLI list.",
 		"",
+		...d.frame === "study" ? ["This run is a study: every cell is ungated (gate: 'none'), so it measures how the arms differ. Failing attempts are data, not a failed gate.", ""] : [],
+		...d.instructions ? [
+			"## Project instructions (the question this analysis answers)",
+			"",
+			d.instructions,
+			""
+		] : [],
 		"## How reeval decided each verdict",
 		"",
 		...RULES.map((r) => `- ${r}`),
@@ -871,14 +1154,14 @@ function citedSummary(d, cites) {
 	for (const c of cites) {
 		const fact = d.facts.find((f) => f.id === c);
 		const evidence = d.evidence.find((e) => e.id === c || c.startsWith(`${e.id}:`));
-		const attempt = d.attempts[c.replace(/:check\d+$/, "")];
+		const attempt = d.attempts[c.replace(/:exec\d+$/, "")];
 		if (fact) {
 			const { id: _id, ...rest } = fact;
 			out[c] = clip(JSON.stringify(rest), 400);
 		} else if (evidence) {
 			const line = /:L(\d+)$/.exec(c);
 			out[c] = line ? clip(evidence.lines.find((l) => l.startsWith(`L${line[1]} `)) ?? evidence.ref, 400) : `${evidence.ref}: ${evidence.why}`;
-		} else if (attempt) out[c] = `${attempt.runId} · ${attempt.attempt} · ${attempt.outcome} · ${attempt.tools} tool calls`;
+		} else if (attempt) out[c] = `${attempt.runId} · ${attempt.attempt} · ${attempt.outcome} · ${attempt.tools} tool calls${attempt.checks ? ` · checks ${attempt.checks}` : ""}`;
 	}
 	return out;
 }
@@ -946,7 +1229,7 @@ async function prepareAnalysis(config, head, base, evalFilter) {
 	};
 }
 const docBase = async (config, p) => ({
-	kind: "es-analysis",
+	kind: "reeval-analysis",
 	schemaVersion: 1,
 	scope: p.scope,
 	anchor: p.head.runId,
@@ -1096,4 +1379,4 @@ async function analyzeCommand(ctx, args) {
 //#endregion
 export { failedAnalysis as a, defaultBase as i, analyzeCommand as n, prepareAnalysis as o, contextMarkdown as r, promptOf as s, analyseDigest as t };
 
-//# sourceMappingURL=command-BqJtFiPy.js.map
+//# sourceMappingURL=command-Bge8F7Ko.js.map