reeval 0.9.0 → 0.11.0

package/dist/{db-CfQFwJtq.js → db-DfhipHEF.js}

@@ -1,13 +1,224 @@
-import { n as toolTarget } from "./tool-target-BU1JnSE9.js";
-import { n as UsageError } from "./errors-yDAEwmZo.js";
-import { d as ownRequire, t as BUILD_ID } from "./root-CvdRlRZX.js";
+import { n as toolTarget, t as SKILL_LOAD_TOOLS } from "./tool-target-D2mcYlSQ.js";
+import { n as UsageError } from "./errors-B6BSMs03.js";
 import { t as RUN_ID } from "./paths-BmR-7tx3.js";
 import { c as relativeFile, l as sha256 } from "./ids-Bp6ut7ZD.js";
-import { n as isRuntimeState } from "./context-DbSMEVKs.js";
-import { G as contrastEffort, J as effortOf, lt as wilson, nt as liftInterval, st as round3 } from "./model-BOnlAPzs.js";
-import { m as warnUnreadable, p as scratchDir, s as readRecord, u as recordNames } from "./files-CSzVADHn.js";
+import { d as ownRequire, t as BUILD_ID } from "./root-DICDHCtp.js";
+import { n as isRuntimeState } from "./context-C1mxXh80.js";
+import { $ as contrastEffort, dt as liftInterval, gt as round3, mt as median, rt as effortOf, vt as wilson } from "./model-DaX4bNLZ.js";
+import { r as pairTallies } from "./pairs-CnInRu-G.js";
+import { m as warnUnreadable, p as scratchDir, s as readRecord, u as recordNames } from "./files-BpUYBwGu.js";
+import { n as contrastPairs, t as checkVerdict } from "./contrasts-5zsj32gk.js";
 import { dirname, join } from "node:path";
 import { mkdir, readFile, stat } from "node:fs/promises";
+//#region src/history/rows.ts
+/** A statement's rows as `Row`, the columns its SQL selects: SQLite checks none of them. */
+const rowsOf = (statement, ...params) => statement.all(...params);
+//#endregion
+//#region src/history/tool-class.ts
+const TOOL_CLASSES = [
+	"context",
+	"dependency",
+	"web",
+	"verify",
+	"workspace"
+];
+const SHELL_TOOL = /shell|bash|terminal|command|exec/i;
+const WEB_TOOL = /web|fetch|browse|url/i;
+const DEPENDENCY = /(^|\/)(node_modules|site-packages|dist-packages|vendor|\.venv|\.cargo\/registry|go\/pkg\/mod)(\/|$)/;
+const VERIFY = /\b(?:(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:test|build|lint|typecheck|check)|npx\s+(?:tsc|vitest|jest|eslint|biome)|tsc|vitest|jest|pytest|eslint|cargo\s+(?:test|build|check)|go\s+(?:test|build|vet))\b/;
+/** Words of a shell command, quotes removed; `;`, `|` and `&&` split words. */
+function shellWords(command) {
+	const words = [];
+	let word = "";
+	let quote = null;
+	let started = false;
+	const flush = () => {
+		if (started) words.push(word);
+		word = "";
+		started = false;
+	};
+	for (const ch of command) {
+		if (quote) {
+			if (ch === quote) quote = null;
+			else word += ch;
+			continue;
+		}
+		if (ch === "\"" || ch === "'") {
+			quote = ch;
+			started = true;
+		} else if (/\s/.test(ch)) flush();
+		else if (ch === ";" || ch === "|" || ch === "&") {
+			flush();
+			words.push(ch);
+		} else {
+			word += ch;
+			started = true;
+		}
+	}
+	flush();
+	return words;
+}
+/** A command's words, with a `bash -c '…'` (or `-lc`) script split into its own words. */
+function commandWords(command) {
+	const words = shellWords(command);
+	return words.flatMap((w, i) => i > 0 && /^-[a-z]*c$/.test(words[i - 1]) && /\s/.test(w) ? commandWords(w) : [w]);
+}
+const EXCLUDING_FLAG = /^--(?:exclude|exclude-dir|ignore|ignore-dir)(?:=|$)/;
+/** The words of a command that name something it reads, without the ones it excludes. */
+function readWords(words) {
+	const out = [];
+	let test = false;
+	for (let i = 0; i < words.length; i++) {
+		const w = words[i];
+		const prev = words[i - 1];
+		if (w === "[" || w === "[[" || w === "test") test = true;
+		else if (w === "]" || w === "]]" || w === ";" || w === "|" || w === "&") test = false;
+		if (test && /^-[defsrwx]$/.test(w)) {
+			i++;
+			continue;
+		}
+		if (EXCLUDING_FLAG.test(w)) {
+			if (!w.includes("=")) i++;
+			continue;
+		}
+		if ((w === "-path" || w === "-name" || w === "-ipath" || w === "-wholename") && i + 1 < words.length) {
+			const pruned = words[i + 2] === "-prune";
+			if (prev === "-not" || prev === "!" || pruned) {
+				i++;
+				continue;
+			}
+		}
+		if ((w === "-g" || w === "--glob" || w === "--iglob") && words[i + 1]?.startsWith("!")) {
+			i++;
+			continue;
+		}
+		if (/^--i?glob=!/.test(w)) continue;
+		if (w === "-v" && i > 0) {
+			i++;
+			continue;
+		}
+		out.push(w);
+	}
+	return out;
+}
+/** `/home/<user>/x` and `/root/x` as `~/x`; `./x` as `x`. */
+const normalise = (p) => p.replace(/^\/home\/[^/]+\//, "~/").replace(/^\/root\//, "~/").replace(/^\$HOME\//, "~/").replace(/^\.\//, "");
+const GLOB = /[*?[{]/;
+/** `*`, `**` and `?` as wildcards; every other character, `[` included, literally. */
+function wildcards(glob) {
+	let re = "";
+	for (let i = 0; i < glob.length; i++) {
+		const ch = glob[i];
+		if (ch === "*") {
+			if (glob[i + 1] === "*") {
+				re += ".*";
+				i++;
+			} else re += "[^/]*";
+		} else if (ch === "?") re += "[^/]";
+		else re += ch.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+	}
+	return re;
+}
+/** A glob as a regex, with `{a,b}` alternatives. Words come from transcripts, so it never throws. */
+function globRegex(glob) {
+	let re = "";
+	let from = 0;
+	for (let i = 0; i < glob.length; i++) {
+		if (glob[i] !== "{") continue;
+		const end = glob.indexOf("}", i);
+		if (end < 0) break;
+		re += wildcards(glob.slice(from, i));
+		re += `(?:${glob.slice(i + 1, end).split(",").map(wildcards).join("|")})`;
+		from = end + 1;
+		i = end;
+	}
+	return new RegExp(`^${re}${wildcards(glob.slice(from))}$`);
+}
+/** The declared files one word names: the file itself, one of a glob's matches, or a directory's. */
+function filesOf(word, files) {
+	const w = normalise(word).replace(/\/+$/, "");
+	if (!w) return {
+		files: [],
+		dir: false
+	};
+	const forms = (f) => f.startsWith("~/") ? [f, f.slice(2)] : [f];
+	const matches = (f, test) => forms(f).some((form) => test(form) || !w.startsWith("~/") && test(`~/${form}`));
+	if (GLOB.test(w)) {
+		const re = globRegex(w);
+		const fromWorkspace = /^[*?[{]/.test(w);
+		return {
+			files: files.filter((f) => fromWorkspace ? !f.startsWith("~/") && re.test(f) : matches(f, (form) => re.test(form))),
+			dir: false
+		};
+	}
+	const exact = files.filter((f) => matches(f, (form) => form === w || form.endsWith(`/${w}`)));
+	if (exact.length) return {
+		files: exact,
+		dir: false
+	};
+	return {
+		files: [],
+		dir: files.filter((f) => matches(f, (form) => form.startsWith(`${w}/`))).length > 0
+	};
+}
+/** A tool call's class, and the declared files it read (for `skill_files`). */
+function classifyCall(tool, target, ctx) {
+	if (SKILL_LOAD_TOOLS.has(tool)) {
+		if (!ctx.skills.includes(target)) return {
+			class: "workspace",
+			files: []
+		};
+		return {
+			class: "context",
+			files: ctx.files.filter((f) => f.endsWith(`skills/${target}/SKILL.md`))
+		};
+	}
+	if (ctx.mcp.some((s) => tool.startsWith(`mcp__${s}__`) || tool.startsWith(`${s}__`))) return {
+		class: "context",
+		files: []
+	};
+	const shell = SHELL_TOOL.test(tool);
+	const words = shell ? readWords(commandWords(target)) : [target];
+	const files = /* @__PURE__ */ new Set();
+	let context = false;
+	for (const w of words) {
+		const hit = filesOf(w, ctx.files);
+		for (const f of hit.files) files.add(f);
+		if (hit.files.length || hit.dir) context = true;
+		if (ctx.skills.some((s) => normalise(w).includes(`skills/${s}/`) || normalise(w).endsWith(`skills/${s}`))) context = true;
+	}
+	if (context) return {
+		class: "context",
+		files: [...files]
+	};
+	if (words.some((w) => DEPENDENCY.test(normalise(w))) || shell && words.includes("--help")) return {
+		class: "dependency",
+		files: []
+	};
+	if (!shell && WEB_TOOL.test(tool)) return {
+		class: "web",
+		files: []
+	};
+	if (shell && words.some((w) => w === "curl" || w === "wget") && /https?:\/\//.test(target)) return {
+		class: "web",
+		files: []
+	};
+	if (shell) {
+		const command = target.trim();
+		if (ctx.verify.some((v) => v.trim().length >= 4 && command.includes(v.trim()))) return {
+			class: "verify",
+			files: []
+		};
+		if (VERIFY.test(command)) return {
+			class: "verify",
+			files: []
+		};
+	}
+	return {
+		class: "workspace",
+		files: []
+	};
+}
 /** A failing verdict this close to its threshold points at the judge, not the agent. */
 const JUDGE_MARGIN = .1;
 const DERIVED_TABLES = `
@@ -20,7 +231,19 @@ CREATE TABLE effort_m (
   control_tokens_median REAL,
   tools_ratio REAL, agent_s_ratio REAL, agent_s_delta REAL, tokens_ratio REAL,
   tools_p REAL, agent_s_p REAL, tokens_p REAL,
-  separated INTEGER, verdict TEXT, time_verdict TEXT, tokens_verdict TEXT, tokens_note TEXT
+  separated INTEGER, time_separated INTEGER, tokens_separated INTEGER,
+  verdict TEXT, time_verdict TEXT, tokens_verdict TEXT, tokens_note TEXT,
+  context_calls_median REAL, dependency_calls_median REAL, web_calls_median REAL,
+  verify_calls_median REAL, workspace_calls_median REAL
+);
+CREATE TABLE contrasts_m (
+  run_id TEXT, eval_key TEXT, harness TEXT, sandbox TEXT, a_key TEXT, b_key TEXT,
+  a_environment TEXT, b_environment TEXT, a_pass INTEGER, a_counted INTEGER, b_pass INTEGER,
+  b_counted INTEGER, points REAL, lo REAL, hi REAL, pass_verdict TEXT,
+  a_tools_median REAL, b_tools_median REAL, tools_ratio REAL, tools_p REAL, separated INTEGER,
+  verdict TEXT, agent_s_ratio REAL, time_separated INTEGER, time_verdict TEXT,
+  tokens_ratio REAL, tokens_separated INTEGER, tokens_verdict TEXT,
+  checks INTEGER, discriminating TEXT
 );
 CREATE TABLE factor_effects_m (
   factor TEXT, level TEXT, baseline TEXT, context TEXT, tier TEXT, run_id TEXT, eval_key TEXT,
@@ -62,6 +285,7 @@ const FAMILY_COLUMNS = [
 ];
 const DERIVED = [
 	"effort_m",
+	"contrasts_m",
 	"factor_effects_m",
 	"flaky_m",
 	"drift_m"
@@ -177,18 +401,19 @@ const merge = (sides) => ({
 });
 function insertAll(db, table, rows) {
 	if (rows.length === 0) return;
-	const cols = db.prepare(`PRAGMA table_info("${table}")`).all().map((c) => c.name);
+	const cols = rowsOf(db.prepare(`PRAGMA table_info("${table}")`)).map((c) => c.name);
 	const statement = db.prepare(`INSERT INTO ${table} (${cols.join(", ")}) VALUES (${cols.map(() => "?").join(", ")})`);
 	for (const row of rows) statement.run(...cols.map((c) => row[c] ?? null));
 }
 /** Recomputes every derived table from the projected rows; run inside the sync's transaction. */
 function deriveFacts(db) {
 	for (const table of DERIVED) db.exec(`DELETE FROM ${table}`);
-	const cells = db.prepare("SELECT c.*, r.started_at FROM cells c JOIN runs r USING (run_id) ORDER BY c.run_id, c.rowid").all();
-	const attempts = db.prepare(`SELECT run_id, eval_key, cell_key, key, outcome, error_kind, tools, agent_ms, file_edits,
+	const cells = rowsOf(db.prepare("SELECT c.*, r.started_at FROM cells c JOIN runs r USING (run_id) ORDER BY c.run_id, c.rowid"));
+	const attempts = rowsOf(db.prepare(`SELECT run_id, eval_key, cell_key, key, outcome, error_kind, tools, agent_ms, file_edits,
         input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens
-       FROM attempts ORDER BY run_id, rowid`).all();
-	const nearThreshold = new Set(db.prepare(`SELECT DISTINCT run_id, attempt_key FROM judgements WHERE verdict = 'fail' AND ABS(score - threshold) <= ${JUDGE_MARGIN}`).all().map((r) => `${r.run_id}\0${r.attempt_key}`));
+       FROM attempts ORDER BY run_id, rowid`));
+	const nearThreshold = new Set(rowsOf(db.prepare(`SELECT DISTINCT run_id, attempt_key FROM judgements WHERE verdict = 'fail' AND ABS(score - threshold) <= ${JUDGE_MARGIN}`)).map((r) => `${r.run_id}\0${r.attempt_key}`));
+	const diffs = new Map(rowsOf(db.prepare(`SELECT run_id, attempt_key, sha256 FROM artifacts WHERE kind = 'diff' AND bytes > 0`)).flatMap((r) => r.sha256 ? [[`${r.run_id}\0${r.attempt_key}`, r.sha256]] : []));
 	const samples = /* @__PURE__ */ new Map();
 	const byCell = /* @__PURE__ */ new Map();
 	for (const a of attempts) {
@@ -209,6 +434,20 @@ function deriveFacts(db) {
 		samples: samples.get(cellId(c.run_id, c.eval_key, c.key)) ?? []
 	});
 	const find = (run, evalKey, key) => cells.find((c) => c.run_id === run && c.eval_key === evalKey && c.key === key);
+	const classCounts = /* @__PURE__ */ new Map();
+	for (const r of rowsOf(db.prepare("SELECT run_id, attempt_key, class, COUNT(*) AS n FROM tool_calls GROUP BY run_id, attempt_key, class"))) {
+		const id = `${r.run_id}\0${r.attempt_key}`;
+		const m = classCounts.get(id) ?? /* @__PURE__ */ new Map();
+		m.set(r.class, r.n);
+		classCounts.set(id, m);
+	}
+	const classMedians = (c) => {
+		const known = (byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []).filter((a) => a.outcome === "pass" || a.outcome === "fail").flatMap((a) => {
+			const m = classCounts.get(`${a.run_id}\0${a.key}`);
+			return m ? [m] : [];
+		});
+		return Object.fromEntries(TOOL_CLASSES.map((k) => [`${k}_calls_median`, known.length ? median(known.map((m) => m.get(k) ?? 0)) ?? null : null]));
+	};
 	insertAll(db, "effort_m", cells.map((c) => {
 		const own = effortOf(side(c).samples);
 		const control = c.lift_vs ? find(c.run_id, c.eval_key, c.lift_vs) : void 0;
@@ -245,12 +484,83 @@ function deriveFacts(db) {
 			agent_s_p: r4(vs?.agentMsP ?? null),
 			tokens_p: r4(vs?.tokensP ?? null),
 			separated: vs ? vs.separated ? 1 : 0 : null,
+			time_separated: vs ? vs.timeSeparated ? 1 : 0 : null,
+			tokens_separated: vs?.tokensSeparated == null ? null : vs.tokensSeparated ? 1 : 0,
 			verdict: vs?.verdict ?? null,
 			time_verdict: vs?.timeVerdict ?? null,
 			tokens_verdict: vs?.tokensVerdict ?? null,
-			tokens_note: vs?.tokensNote ?? null
+			tokens_note: vs?.tokensNote ?? null,
+			...classMedians(c)
 		};
 	}));
+	const checkRows = rowsOf(db.prepare("SELECT run_id, eval_key, cell_key, label, pass, fail, error, not_run FROM cell_checks"));
+	const checksOf = /* @__PURE__ */ new Map();
+	for (const k of checkRows) {
+		const id = cellId(k.run_id, k.eval_key, k.cell_key);
+		const m = checksOf.get(id) ?? /* @__PURE__ */ new Map();
+		m.set(k.label, {
+			pass: k.pass,
+			fail: k.fail,
+			error: k.error,
+			notRun: k.not_run
+		});
+		checksOf.set(id, m);
+	}
+	const evalCells = /* @__PURE__ */ new Map();
+	for (const c of cells) {
+		const id = `${c.run_id}\0${c.eval_key}`;
+		evalCells.set(id, [...evalCells.get(id) ?? [], c]);
+	}
+	const contrastRows = [];
+	for (const group of evalCells.values()) for (const [a, b] of contrastPairs(group)) {
+		const x = side(a);
+		const y = side(b);
+		const e = contrastEffort(x.samples, y.samples);
+		const ka = checksOf.get(cellId(a.run_id, a.eval_key, a.key)) ?? /* @__PURE__ */ new Map();
+		const kb = checksOf.get(cellId(b.run_id, b.eval_key, b.key)) ?? /* @__PURE__ */ new Map();
+		const verdicts = [...ka].flatMap(([label, t]) => {
+			const other = kb.get(label);
+			const verdict = other && checkVerdict(t, other);
+			return verdict ? [{
+				label,
+				verdict
+			}] : [];
+		});
+		const pass = passCols(x, y);
+		contrastRows.push({
+			run_id: a.run_id,
+			eval_key: a.eval_key,
+			harness: a.harness,
+			sandbox: a.sandbox,
+			a_key: a.key,
+			b_key: b.key,
+			a_environment: a.environment,
+			b_environment: b.environment,
+			a_pass: x.pass,
+			a_counted: x.counted,
+			b_pass: y.pass,
+			b_counted: y.counted,
+			points: pass.points ?? null,
+			lo: pass.lo ?? null,
+			hi: pass.hi ?? null,
+			pass_verdict: pass.pass_verdict ?? null,
+			a_tools_median: e?.variant.tools.median ?? null,
+			b_tools_median: e?.control.tools.median ?? null,
+			tools_ratio: r4(e?.toolsRatio ?? null),
+			tools_p: r4(e?.toolsP ?? null),
+			separated: e ? e.separated ? 1 : 0 : null,
+			verdict: e?.verdict ?? null,
+			agent_s_ratio: r4(e?.agentMsRatio ?? null),
+			time_separated: e ? e.timeSeparated ? 1 : 0 : null,
+			time_verdict: e?.timeVerdict ?? null,
+			tokens_ratio: r4(e?.tokensRatio ?? null),
+			tokens_separated: e?.tokensSeparated == null ? null : e.tokensSeparated ? 1 : 0,
+			tokens_verdict: e?.tokensVerdict ?? null,
+			checks: verdicts.length,
+			discriminating: verdicts.filter((v) => v.verdict === "discriminates").map((v) => v.label).join(" · ") || null
+		});
+	}
+	insertAll(db, "contrasts_m", contrastRows);
 	const pairs = [];
 	const groups = /* @__PURE__ */ new Map();
 	for (const c of cells) {
@@ -380,7 +690,7 @@ function deriveFacts(db) {
 			note: "pass@1: summed counts; effort: an effect in at least 2/3 of strata"
 		});
 	}
-	const familyLifts = db.prepare(`SELECT ${FAMILY_COLUMNS.join(", ")} FROM family_lift_m`).all();
+	const familyLifts = rowsOf(db.prepare(`SELECT ${FAMILY_COLUMNS.join(", ")} FROM family_lift_m`));
 	for (const f of familyLifts) factorRows.push({
 		factor: "environment",
 		level: f.variant,
@@ -414,7 +724,7 @@ function deriveFacts(db) {
 	const contextOf = db.prepare("SELECT path, sha256 FROM attempt_context WHERE run_id = ? AND attempt_key = ?");
 	const cellContext = (c) => {
 		const files = /* @__PURE__ */ new Map();
-		for (const a of byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []) for (const f of contextOf.all(c.run_id, a.key)) files.set(f.path, f.sha256);
+		for (const a of byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []) for (const f of rowsOf(contextOf, c.run_id, a.key)) files.set(f.path, f.sha256);
 		return files;
 	};
 	for (const list of series.values()) {
@@ -497,7 +807,12 @@ function deriveFacts(db) {
 		const infra = list.reduce((n, c) => n + c.infra_error, 0);
 		const failing = list.flatMap((c) => (byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []).filter((a) => a.outcome === "fail"));
 		const w = wilson(pass, pass + fail);
-		const cause = infra > 0 || failing.some((a) => a.error_kind === "infra") ? "infra" : failing.some((a) => nearThreshold.has(`${a.run_id}\0${a.key}`)) ? "judge" : "agent";
+		const passedDiffs = new Set(list.flatMap((c) => (byCell.get(cellId(c.run_id, c.eval_key, c.key)) ?? []).flatMap((a) => {
+			const sha = a.outcome === "pass" && diffs.get(`${a.run_id}\0${a.key}`);
+			return sha ? [sha] : [];
+		})));
+		const sameWork = failing.length > 0 && failing.every((a) => a.error_kind === "assertion" && passedDiffs.has(diffs.get(`${a.run_id}\0${a.key}`) ?? ""));
+		const cause = infra > 0 || failing.some((a) => a.error_kind === "infra") ? "infra" : failing.some((a) => nearThreshold.has(`${a.run_id}\0${a.key}`)) ? "judge" : sameWork ? "check" : "agent";
 		const starts = list.map((c) => c.started_at).sort();
 		flakyRows.push({
 			eval_key: first.eval_key,
@@ -524,7 +839,7 @@ function deriveFacts(db) {
 //#endregion
 //#region src/history/db.ts
 /** Bump when the tables or views change: a database from another version is rebuilt. */
-const DB_VERSION = 9;
+const DB_VERSION = 11;
 /** This build of reeval, like the header cache: another reeval re-derives every row. */
 const BUILD = BUILD_ID;
 const historyDbPath = (root) => join(root, ".reeval", "history.db");
@@ -556,7 +871,7 @@ CREATE TABLE files (run_id TEXT PRIMARY KEY, path TEXT NOT NULL, size INTEGER NO
 CREATE TABLE runs (
   run_id TEXT PRIMARY KEY, recorded INTEGER NOT NULL, status TEXT, started_at TEXT, ended_at TEXT,
   duration_ms INTEGER, author TEXT, git_sha TEXT, branch TEXT, dirty INTEGER, runner TEXT,
-  ci_provider TEXT, ci_url TEXT, pr_number INTEGER, pr_base TEXT, es TEXT, judge_model TEXT,
+  ci_provider TEXT, ci_url TEXT, pr_number INTEGER, pr_base TEXT, reeval TEXT, judge_model TEXT,
   judge_version TEXT, profile TEXT, evals INTEGER, attempts INTEGER, pass INTEGER, fail INTEGER,
   not_gated INTEGER, infra_error INTEGER, skipped INTEGER
 );
@@ -566,17 +881,32 @@ CREATE TABLE evals (
   case_index INTEGER,
   PRIMARY KEY (run_id, key)
 );
--- Each pooled evaluate.each per run: its cases, and each cell's pass rate over them.
+-- Each pooled evaluate.each per run: its cases, and each cell's pass@1 over them (the mean of the
+-- cases' pass@1).
 CREATE TABLE families (
   run_id TEXT NOT NULL, family_id TEXT NOT NULL, file TEXT, groups TEXT, name TEXT,
   cases INTEGER, ran INTEGER, cell_key TEXT NOT NULL, cell_cases INTEGER, pass INTEGER,
-  counted INTEGER, pass_rate REAL, lo REAL, hi REAL, method TEXT, outcome TEXT,
+  counted INTEGER, pass_at_1 REAL, lo REAL, hi REAL, method TEXT, outcome TEXT,
   PRIMARY KEY (run_id, family_id, cell_key)
 );
+-- Every compare-question vote: a pair of sides (arms within a harness, harnesses across an arm)
+-- on one run, shown in one order (swapped = 1: b first), and the side picked or 'tie'.
+CREATE TABLE votes (
+  run_id TEXT NOT NULL, eval_key TEXT NOT NULL, question TEXT NOT NULL, cut TEXT, held TEXT,
+  pair_run INTEGER, a TEXT, b TEXT, winner TEXT, swapped INTEGER, reason TEXT
+);
+-- Each pair of sides' head-to-head per question: a pair decides when both orders agree (or one
+-- ties); a split is a pair whose orders picked opposite sides, split_first those that picked
+-- the side shown first both times.
+CREATE TABLE question_pairs_m (
+  run_id TEXT NOT NULL, eval_key TEXT NOT NULL, question TEXT NOT NULL, cut TEXT, held TEXT,
+  a TEXT, b TEXT, pairs INTEGER, a_wins INTEGER, b_wins INTEGER, ties INTEGER, splits INTEGER,
+  splits_first INTEGER
+);
 CREATE TABLE family_lift_m (
   run_id TEXT NOT NULL, family_id TEXT NOT NULL, name TEXT, harness TEXT, variant TEXT,
   cell_key TEXT, control TEXT,
-  pairs INTEGER, variant_rate REAL, control_rate REAL, discordant_vc INTEGER,
+  pairs INTEGER, variant_pass_at_1 REAL, control_pass_at_1 REAL, discordant_vc INTEGER,
   discordant_cv INTEGER, points REAL, lo REAL, hi REAL, method TEXT, p REAL,
   within_noise INTEGER, verdict TEXT, min_lift REAL, outcome TEXT, tools_ratio REAL,
   tools_p REAL, effort_verdict TEXT, agent_s_ratio REAL, agent_s_p REAL, time_verdict TEXT,
@@ -619,14 +949,21 @@ CREATE TABLE artifacts (
   run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, kind TEXT NOT NULL, path TEXT NOT NULL,
   sha256 TEXT, bytes INTEGER
 );
--- What the agent could see, by content hash; runtime = 1 for an agent CLI's own session state.
+-- What the agent could see, by content hash; runtime = 1 for an agent CLI's own session state,
+-- declared = 1 for a file the environment declared (its skills' files, rules and config).
 CREATE TABLE attempt_context (
-  run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, path TEXT NOT NULL, sha256 TEXT, runtime INTEGER
+  run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, path TEXT NOT NULL, sha256 TEXT, runtime INTEGER,
+  declared INTEGER
 );
--- Every tool call in each attempt's transcript: its line in transcript.jsonl (from 1) and target.
+-- Every tool call in each attempt's transcript: its line in transcript.jsonl (from 1), target and
+-- class (context, dependency, web, verify or workspace: a heuristic, see tool-class.ts).
 CREATE TABLE tool_calls (
   run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, line INTEGER NOT NULL, turn INTEGER,
-  tool TEXT, target TEXT, ok INTEGER, duration_ms INTEGER
+  tool TEXT, target TEXT, ok INTEGER, duration_ms INTEGER, class TEXT
+);
+-- Each placed file a tool call named (directly or through a glob), by the call's line.
+CREATE TABLE context_reads (
+  run_id TEXT NOT NULL, attempt_key TEXT NOT NULL, line INTEGER NOT NULL, path TEXT NOT NULL
 );
 -- Every expect / expect.soft matcher call each attempt made, in call order (seq from 1).
 CREATE TABLE assertions (
@@ -645,10 +982,11 @@ CREATE INDEX assertions_by_attempt ON assertions (run_id, attempt_key);
 CREATE INDEX cells_by_eval ON cells (eval_key, key);
 CREATE INDEX tool_calls_by_attempt ON tool_calls (run_id, attempt_key);
 CREATE INDEX context_by_attempt ON attempt_context (run_id, attempt_key);
+CREATE INDEX context_reads_by_attempt ON context_reads (run_id, attempt_key);
 CREATE INDEX attempts_by_cell ON attempts (run_id, eval_key, cell_key);
 CREATE INDEX env_skills_by_digest ON env_skills (run_id, digest);
 `;
-/** The canned questions of docs/history-observability-review.md §2, as views to filter. */
+/** Canned questions about the history, as views to filter. */
 const VIEWS = `
 -- Variant against control, per harness, per run.
 CREATE VIEW lift AS
@@ -677,6 +1015,15 @@ CREATE VIEW drift AS SELECT * FROM drift_m;
 -- Tool calls, agent time and tokens per cell, and a variant's against its control in the same run.
 CREATE VIEW effort AS SELECT * FROM effort_m;
 
+-- Each pair of arms (cells a and b) on one harness and sandbox, when an eval has two variants or
+-- more: pass@1 lift of a over b, effort with its verdicts and separation, and the checks that
+-- tell them apart. A variant against its control is here too; gating stays variant vs control.
+CREATE VIEW contrasts AS SELECT * FROM contrasts_m;
+
+-- Each compare question's pairs of sides, head to head: wins when both orders agree, and splits
+-- when the order decided. The pooled win rate counts a split as a win for each side.
+CREATE VIEW question_pairs AS SELECT * FROM question_pairs_m;
+
 -- What a factor (environment, harness, model) did, by evidence tier: controlled (one run),
 -- pooled (comparable runs, and across evals), observed (over time), not_measured (one level).
 CREATE VIEW factor_effects AS SELECT * FROM factor_effects_m;
@@ -697,6 +1044,34 @@ CREATE VIEW skill_reads AS
   JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
   JOIN env_skills s ON s.run_id = c.run_id AND s.digest = c.environment_digest;
 
+-- Tool calls per attempt by class: context (a placed skill, rules file or other placed file, a
+-- skill load, an MCP tool of the environment), dependency (node_modules, site-packages, vendor,
+-- --help), web, verify (a command the eval's checks run, or test/build/lint/tsc) and workspace.
+-- A heuristic: shell commands are split into words and exclusions are not counted.
+CREATE VIEW context_use AS
+  SELECT a.run_id, a.eval_key, a.cell_key, c.environment, a.key AS attempt_key, a.id AS attempt_id,
+    t.class, COUNT(*) AS calls, MIN(t.line) AS first_line,
+    (SELECT x.target FROM tool_calls x WHERE x.run_id = a.run_id AND x.attempt_key = a.key
+      AND x.class = t.class ORDER BY x.line LIMIT 1) AS example
+  FROM tool_calls t
+  JOIN attempts a ON a.run_id = t.run_id AND a.key = t.attempt_key
+  JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
+  GROUP BY a.run_id, a.key, t.class;
+
+-- Each file the environment declared (its skills' files, rules), per attempt: how many tool calls
+-- named it, directly or through a glob, and the first one's transcript line (null: never read).
+CREATE VIEW skill_files AS
+  SELECT a.run_id, a.eval_key, a.cell_key, c.environment, a.key AS attempt_key,
+    a.id AS attempt_id, x.path,
+    (SELECT COUNT(*) FROM context_reads r WHERE r.run_id = a.run_id AND r.attempt_key = a.key
+      AND r.path = x.path) AS reads,
+    (SELECT MIN(r.line) FROM context_reads r WHERE r.run_id = a.run_id AND r.attempt_key = a.key
+      AND r.path = x.path) AS first_line
+  FROM attempt_context x
+  JOIN attempts a ON a.run_id = x.run_id AND a.key = x.attempt_key
+  JOIN cells c ON c.run_id = a.run_id AND c.eval_key = a.eval_key AND c.key = a.cell_key
+  WHERE x.declared = 1;
+
 -- Failed attempts and infra errors (an OOM kill among them) with the cell's harness, model, each
 -- of its environment's skills and the label of the first check that failed.
 CREATE VIEW failing_attempts AS
@@ -719,6 +1094,15 @@ CREATE VIEW check_discrimination AS
   JOIN cells c ON c.run_id = k.run_id AND c.eval_key = k.eval_key AND c.key = k.cell_key
   WHERE c.role = 'variant' AND k.verdict IS NOT NULL;
 
+-- Each check (expect label) per cell per run, on every cell (control and variant alike), over the
+-- cell's counted attempts. rate = pass / (pass + fail): an error (the matcher couldn't answer) and
+-- a not-run (a hard expect before it failed) are left out; null when neither passed nor failed.
+CREATE VIEW check_rates AS
+  SELECT k.run_id, k.eval_key, k.cell_key, c.harness, c.environment, c.role, k.label, k.pass,
+    k.fail, k.error, k.not_run, CAST(k.pass AS REAL) / NULLIF(k.pass + k.fail, 0) AS rate
+  FROM cell_checks k
+  JOIN cells c ON c.run_id = k.run_id AND c.eval_key = k.eval_key AND c.key = k.cell_key;
+
 -- One eval per run: outcome and pass@1, with the branch and pull request.
 CREATE VIEW eval_trend AS
   SELECT e.eval_key, r.run_id, r.started_at, r.branch, r.pr_number, r.recorded, e.outcome,
@@ -744,16 +1128,29 @@ const VIEW_NAMES = [
 	"lift",
 	"family_lift",
 	"effort",
+	"contrasts",
+	"question_pairs",
 	"factor_effects",
 	"skill_reads",
+	"context_use",
+	"skill_files",
 	"flaky_cells",
 	"drift",
 	"failing_attempts",
 	"eval_trend",
 	"verdict_flips",
-	"check_discrimination"
+	"check_discrimination",
+	"check_rates"
 ];
 const json = (value) => JSON.stringify(value);
+/** The agent CLI's home config, which its harness writes (MCP servers, settings), not the eval. */
+const HARNESS_CONFIG = /^~\/.*(^|\/)(config\.(toml|ya?ml|json)|settings(\.local)?\.json|mcp\.json)$/;
+/**
+* A placed file the environment declared: under one of its skills, or a rules file outside any
+* skills directory. Not an agent CLI's own state or home config, its bundled skills or the
+* adapter's skill index.
+*/
+const declaredContext = (path, skills) => !isRuntimeState(path) && !path.endsWith("/.ai-sdk-harness-skills.json") && !HARNESS_CONFIG.test(path) && (skills.some((s) => path.includes(`skills/${s}/`)) || !/(^|\/)skills(-[^/]+)?\//.test(path));
 /** `attempts.last_text` is the agent's final message, capped: most "why" questions start there. */
 const LAST_TEXT_LIMIT = 2048;
 /** The tool calls in a transcript.jsonl, numbered by line from 1; lines that aren't JSON are skipped. */
@@ -779,6 +1176,17 @@ function toolCalls(transcript) {
 	});
 	return calls;
 }
+/** `classifyCall`, but a call its heuristic can't read counts as workspace: one transcript line can't stop the sync. */
+function classifyOrWorkspace(tool, target, ctx) {
+	try {
+		return classifyCall(tool, target, ctx);
+	} catch {
+		return {
+			class: "workspace",
+			files: []
+		};
+	}
+}
 /**
 * What a cell's results can be pooled on across runs: its fingerprint parts, with the context
 * part replaced by the context files that aren't an agent CLI's own runtime state.
@@ -798,24 +1206,42 @@ function comparableFingerprint(cell, attempts) {
 function insertRun(db, record, recorded, transcripts) {
 	const run = (sql, ...values) => db.prepare(sql).run(...values);
 	const { runner, summary, git } = record;
-	run(`INSERT INTO runs VALUES (${Array(26).fill("?").join(", ")})`, record.runId, recorded ? 1 : 0, record.status, record.startedAt, record.endedAt, record.durationMs, record.author, git?.sha ?? null, git?.branch ?? null, git ? git.dirty ? 1 : 0 : null, runner.kind, runner.kind === "ci" ? runner.provider : null, runner.kind === "ci" ? runner.runUrl : null, runner.kind === "ci" ? runner.pr?.number ?? null : null, runner.kind === "ci" ? runner.pr?.baseRef ?? null : null, record.stack.es, record.stack.judge.model, record.stack.judge.version, record.profile?.name ?? null, summary.evals, summary.attempts, summary.pass, summary.fail, summary.notGated, summary.infraError, summary.skipped);
+	run(`INSERT INTO runs VALUES (${Array(26).fill("?").join(", ")})`, record.runId, recorded ? 1 : 0, record.status, record.startedAt, record.endedAt, record.durationMs, record.author, git?.sha ?? null, git?.branch ?? null, git ? git.dirty ? 1 : 0 : null, runner.kind, runner.kind === "ci" ? runner.provider : null, runner.kind === "ci" ? runner.runUrl : null, runner.kind === "ci" ? runner.pr?.number ?? null : null, runner.kind === "ci" ? runner.pr?.baseRef ?? null : null, record.stack.reeval, record.stack.judge.model, record.stack.judge.version, record.profile?.name ?? null, summary.evals, summary.attempts, summary.pass, summary.fail, summary.notGated, summary.infraError, summary.skipped);
 	for (const env of record.environments) {
 		run("INSERT OR IGNORE INTO environments VALUES (?, ?, ?, ?, ?)", record.runId, env.name, env.digest, env.parent, json(env.overrides));
 		for (const s of env.skills) run("INSERT INTO env_skills VALUES (?, ?, ?, ?, ?, ?, ?)", record.runId, env.digest, s.name, s.source, s.commit, s.sha256, s.placement);
 	}
+	const envOf = new Map(record.environments.map((env) => [env.digest, env]));
 	for (const e of record.evals) {
+		const cellEnv = new Map(e.cells.map((c) => [c.key, c.environmentDigest ?? ""]));
 		run(`INSERT INTO evals VALUES (${Array(14).fill("?").join(", ")})`, record.runId, e.key, e.id, e.slug, e.file, json(e.group), e.name, json(e.tags), e.fingerprint, e.outcome, e.passRate, json(e.sources), e.family?.id ?? null, e.family?.case ?? null);
 		for (const c of e.cells) {
 			const p = c.fingerprintParts;
 			run(`INSERT INTO cells VALUES (${Array(36).fill("?").join(", ")})`, record.runId, e.key, c.key, c.role, c.variant ?? null, c.harness, c.harnessKind, c.model, c.effort ?? null, c.environment, c.environmentDigest, c.sandbox, typeof c.gate === "string" ? c.gate : "passRate", typeof c.gate === "string" ? null : c.gate.passRate, c.fingerprint, p.sources, p.harness, p.environment, p.image, p.agentCli, p.judge, p.context, c.runs, c.pass, c.fail, c.infraError, c.skipped, c.passRate, c.outcome, c.lift?.vs ?? null, c.lift?.points ?? null, c.lift?.interval.lo ?? null, c.lift?.interval.hi ?? null, c.lift ? c.lift.withinNoise ? 1 : 0 : null, c.lift?.minLift ?? null, comparableFingerprint(c, e.attempts));
 			for (const k of c.checks ?? []) run(`INSERT INTO cell_checks VALUES (${Array(13).fill("?").join(", ")})`, record.runId, e.key, c.key, k.label, k.pass, k.fail, k.error, k.notRun, k.control?.pass ?? null, k.control?.fail ?? null, k.control?.error ?? null, k.control?.notRun ?? null, k.verdict ?? null);
 		}
+		for (const answer of e.answers ?? []) {
+			for (const v of answer.votes) run(`INSERT INTO votes VALUES (${Array(11).fill("?").join(", ")})`, record.runId, e.key, answer.question, v.cut, v.harness ?? v.arm ?? null, v.run, v.a, v.b, v.winner, v.swapped ? 1 : 0, v.reason ?? null);
+			for (const t of pairTallies(answer.votes)) run(`INSERT INTO question_pairs_m VALUES (${Array(13).fill("?").join(", ")})`, record.runId, e.key, answer.question, t.cut, t.held, t.a, t.b, t.pairs, t.aWins, t.bWins, t.ties, t.splits, t.splitsFirst);
+		}
 		for (const a of e.attempts) {
 			run(`INSERT INTO attempts VALUES (${Array(30).fill("?").join(", ")})`, record.runId, e.key, a.cell, a.key, a.id, a.n, a.seq, a.startedAt, a.outcome, a.error?.kind ?? null, a.error?.message ?? null, a.durationMs, a.imageId, a.turns.length, a.activity.tools, a.activity.fileEdits, a.timings.setupMs, a.timings.agentMs, a.timings.checksMs, a.timings.judgeMs, a.turns.at(-1)?.text.slice(0, 2048) ?? null, a.usage?.input ?? null, a.usage?.output ?? null, a.usage?.cacheRead ?? null, a.usage?.cacheWrite ?? null, a.usage?.reasoning ?? null, a.turns.findLast((t) => t.model)?.model ?? null, a.error?.resource?.cause === "oom" ? a.error.resource.kills : 0, a.effort ?? null, a.servedEffort ?? null);
-			for (const c of a.context) run("INSERT INTO attempt_context VALUES (?, ?, ?, ?, ?)", record.runId, a.key, c.path, c.sha256, isRuntimeState(c.path) ? 1 : 0);
+			const env = envOf.get(cellEnv.get(a.cell) ?? "");
+			const skills = env?.skills.map((x) => x.name) ?? [];
+			for (const c of a.context) run("INSERT INTO attempt_context VALUES (?, ?, ?, ?, ?, ?)", record.runId, a.key, c.path, c.sha256, isRuntimeState(c.path) ? 1 : 0, declaredContext(c.path, skills) ? 1 : 0);
 			for (const [i, x] of (a.assertions ?? []).entries()) run(`INSERT INTO assertions VALUES (${Array(12).fill("?").join(", ")})`, record.runId, a.key, e.key, a.cell, i + 1, x.label, x.labelled ? 1 : 0, x.matcher, x.status, x.soft ? 1 : 0, x.message ?? null, x.durationMs ?? null);
 			const transcript = transcripts.get(a.key);
-			for (const t of transcript ? toolCalls(transcript) : []) run("INSERT INTO tool_calls VALUES (?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, t.line, t.turn, t.tool, t.target, t.ok ? 1 : 0, t.durationMs);
+			const ctx = {
+				skills,
+				files: a.context.map((c) => c.path).filter((p) => declaredContext(p, skills)),
+				mcp: env?.mcp ?? [],
+				verify: a.checks.map((c) => c.command)
+			};
+			for (const t of transcript ? toolCalls(transcript) : []) {
+				const call = classifyOrWorkspace(t.tool, t.target, ctx);
+				run("INSERT INTO tool_calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, t.line, t.turn, t.tool, t.target, t.ok ? 1 : 0, t.durationMs, call.class);
+				for (const path of call.files) run("INSERT INTO context_reads VALUES (?, ?, ?, ?)", record.runId, a.key, t.line, path);
+			}
 			for (const j of a.judgements) run("INSERT OR IGNORE INTO judgements VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", record.runId, a.key, j.key, j.question, j.verdict, j.score, j.threshold, j.model, j.resolvedModel ?? null, a.files.judgeInput?.path ?? null);
 			for (const [kind, ref] of Object.entries(a.files)) {
 				if (ref == null || Array.isArray(ref) || typeof ref.path !== "string") continue;
@@ -841,6 +1267,8 @@ const DATA_TABLES = [
 	"evals",
 	"families",
 	"family_lift_m",
+	"votes",
+	"question_pairs_m",
 	"cells",
 	"attempts",
 	"judgements",
@@ -849,6 +1277,7 @@ const DATA_TABLES = [
 	"artifacts",
 	"attempt_context",
 	"tool_calls",
+	"context_reads",
 	"assertions",
 	"cell_checks"
 ];
@@ -857,7 +1286,7 @@ function removeRun(db, runId) {
 	db.prepare("DELETE FROM files WHERE run_id = ?").run(runId);
 }
 function freshSchema(db) {
-	const objects = db.prepare("SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view')").all();
+	const objects = rowsOf(db.prepare("SELECT type, name FROM sqlite_schema WHERE type IN ('table', 'view')"));
 	for (const o of objects) {
 		if (o.name.startsWith("sqlite_")) continue;
 		db.exec(`DROP ${o.type === "view" ? "VIEW" : "TABLE"} IF EXISTS "${o.name}"`);
@@ -868,7 +1297,7 @@ function freshSchema(db) {
 }
 function current(db) {
 	try {
-		const rows = db.prepare("SELECT key, value FROM meta").all();
+		const rows = rowsOf(db.prepare("SELECT key, value FROM meta"));
 		const meta = Object.fromEntries(rows.map((r) => [r.key, r.value]));
 		return meta.version === String(DB_VERSION) && meta.build === String(BUILD);
 	} catch {
@@ -902,7 +1331,7 @@ async function sync(db, root, historyDir) {
 			recorded
 		});
 	}
-	const known = new Map(db.prepare("SELECT run_id, path, size, mtime_ms FROM files").all().map((r) => [r.run_id, r]));
+	const known = new Map(rowsOf(db.prepare("SELECT run_id, path, size, mtime_ms FROM files")).map((r) => [r.run_id, r]));
 	let added = 0;
 	let removed = 0;
 	db.exec("BEGIN");
@@ -962,7 +1391,7 @@ async function openHistoryDb(root, historyDir, options = {}) {
 		query(sql, params = []) {
 			try {
 				const statement = db.prepare(sql);
-				const rows = statement.all(...params);
+				const rows = Array.isArray(params) ? statement.all(...params) : statement.all(params);
 				return {
 					columns: typeof statement.columns === "function" ? statement.columns().map((c) => c.name) : Object.keys(rows[0] ?? {}),
 					rows
@@ -973,15 +1402,15 @@ async function openHistoryDb(root, historyDir, options = {}) {
 			}
 		},
 		schema() {
-			return db.prepare(`SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT IN ('meta', 'files', 'family_lift_m', ${DERIVED.map((d) => `'${d}'`).join(", ")}) AND name NOT LIKE 'sqlite_%' ORDER BY type, name`).all().map((o) => ({
+			return rowsOf(db.prepare(`SELECT name, type FROM sqlite_schema WHERE type IN ('table', 'view') AND name NOT IN ('meta', 'files', 'family_lift_m', ${DERIVED.map((d) => `'${d}'`).join(", ")}) AND name NOT LIKE 'sqlite_%' ORDER BY type, name`)).map((o) => ({
 				...o,
-				columns: db.prepare(`PRAGMA table_info("${o.name}")`).all().map((c) => c.name)
+				columns: rowsOf(db.prepare(`PRAGMA table_info("${o.name}")`)).map((c) => c.name)
 			}));
 		},
 		close: () => db.close()
 	};
 }
 //#endregion
-export { historyDbPath as a, toolCalls as c, comparableFingerprint as i, SQLITE_NODE as n, nodeAtLeast as o, VIEW_NAMES as r, openHistoryDb as s, LAST_TEXT_LIMIT as t };
+export { historyDbPath as a, toolCalls as c, comparableFingerprint as i, TOOL_CLASSES as l, SQLITE_NODE as n, nodeAtLeast as o, VIEW_NAMES as r, openHistoryDb as s, LAST_TEXT_LIMIT as t };
 
-//# sourceMappingURL=db-CfQFwJtq.js.map
+//# sourceMappingURL=db-DfhipHEF.js.map