reeval 0.2.0 → 0.4.0

package/dist/studio-app/index.html

@@ -6,8 +6,8 @@
     <meta name="referrer" content="no-referrer" />
     <link rel="icon" type="image/svg+xml" href="./favicon.svg" />
     <title>reeval Studio</title>
-    <script type="module" crossorigin src="./assets/index-Da1XDzx8.js"></script>
-    <link rel="stylesheet" crossorigin href="./assets/index-xwzWdcE_.css">
+    <script type="module" crossorigin src="./assets/index-BhorHbrj.js"></script>
+    <link rel="stylesheet" crossorigin href="./assets/index-B1Pf3LAT.css">
   </head>
   <body>
     <div id="root"></div>

package/dist/{summary-D7Ttky0z.js → summary-CEMYhzXm.js}

@@ -1,6 +1,6 @@
 import { n as clip, r as formatDuration } from "./text-gEcKjAyp.js";
-import { N as effortLabel, O as cellContrast, m as harnessLabel, v as isUngatedMiss } from "./model-CDL98xue.js";
-import { o as warnings } from "./state-DZ7eqb1F.js";
+import { I as effortLabel, b as languageLabel, h as harnessLabel, j as cellContrast, w as runtimeLabel, y as isUngatedMiss } from "./model-ORtLm2UB.js";
+import { o as warnings } from "./state-DFbJrVi3.js";
 //#region src/history/summary.ts
 const OUTCOME_MARK = {
 	pass: "✓",
@@ -36,6 +36,29 @@ function liftLabel(lift) {
 	const counts = `${variant.pass}/${variant.counted} vs ${control.pass}/${control.counted}`;
 	return `lift ${signedPoints(lift.points)} pts (${range}, ${counts})${lift.withinNoise ? ", within noise" : ""}`;
 }
+const rateText = (r) => `${r.name} ${r.wins}/${r.n} (${Math.round(r.wins / r.n * 100)}%)`;
+/**
+* One compare question's answer: `"Which is cleaner?" within: with-skill 2/2 (100%), plain 0/2
+* (0%) · 2 votes, 0 ties · google/gemini-3.8-flash`, or why it has no votes.
+*/
+function answersLabel(answer) {
+	const q = `"${clip(answer.question, 80)}"`;
+	const ties = answer.votes.filter((v) => v.winner === "tie").length;
+	const cuts = ["within", "across"].flatMap((cut) => {
+		const rates = answer.rates.filter((r) => r.cut === cut);
+		return rates.length ? [`${cut}: ${rates.map(rateText).join(", ")}`] : [];
+	});
+	const votes = answer.votes.length;
+	const tally = `${votes} vote${votes === 1 ? "" : "s"}, ${ties} tie${ties === 1 ? "" : "s"}`;
+	const by = answer.resolvedModel ?? answer.model;
+	const failed = answer.error !== void 0 ? ` · failed: ${clip(answer.error, 200)}` : "";
+	if (votes === 0 && answer.error === void 0) return `${q} no pairs to judge: attempts that failed their checks are left out`;
+	return `${q} ${[
+		...cuts,
+		tally,
+		by
+	].join(" · ")}${failed}`;
+}
 /** `fake/-/fake 0/2 (control, not gated) · fake/with-skill/fake 2/2 (lift +100 pts …)`. */
 function cellCounts(e) {
 	return e.cells.map((c) => {
@@ -73,7 +96,8 @@ function stateRows(run) {
 	rows.push(["Judge", `${env.judge.model ?? "none by default"}${env.judge.resolvedModels.length ? ` (served ${env.judge.resolvedModels.join(", ")})` : ""} · threshold ${env.judge.threshold} · version ${env.judge.version}`]);
 	rows.push(["Context", env.context.length ? `${env.context.length} file${env.context.length === 1 ? "" : "s"}: ${env.context.map((c) => `${c.path} ${short(c.sha256)}`).join(", ")}` : "no skills, rules or MCP config found in the sandbox"]);
 	rows.push(["Packages", Object.entries(env.packages).map(([name, v]) => `${name} ${v}`).join(" · ")]);
-	rows.push(["reeval", `${env.es} · node ${env.node} · ${env.os}/${env.arch}`]);
+	const language = languageLabel(env);
+	rows.push(["reeval", `${env.es} · ${runtimeLabel(env)}${language ? ` · ${language}` : ""} · ${env.os}/${env.arch}`]);
 	rows.push(["Runner", run.runner.kind === "ci" ? `CI (${run.runner.provider})${run.runner.runUrl ? ` ${run.runner.runUrl}` : ""}` : `local · ${run.author}`]);
 	return rows;
 }
@@ -111,6 +135,10 @@ function renderSummary(run) {
 		lines.push("", `### ${OUTCOME_MARK[e.outcome]} ${e.id}`, "");
 		const multi = e.attempts.length > 1;
 		if (multi) lines.push(`Cells: ${cellCounts(e)}`, "");
+		if (e.answers?.length) {
+			for (const a of e.answers) lines.push(`- Question: ${cell(answersLabel(a))}`);
+			lines.push("");
+		}
 		const attemptRank = (a) => isUngatedMiss(e, a) ? 3 : rank(a.outcome);
 		for (const a of [...e.attempts].sort((x, y) => attemptRank(x) - attemptRank(y))) {
 			if (a.outcome === "skipped") continue;
@@ -161,6 +189,6 @@ function attemptDetail(lines, a) {
 	if (links.length) lines.push(`- Files: ${links.join(" · ")}`);
 }
 //#endregion
-export { renderSummary as a, summaryCounts as c, liftLabel as i, cellCounts as n, repeatsVerdict as o, gateLabel as r, stateRows as s, OUTCOME_MARK as t };
+export { liftLabel as a, stateRows as c, gateLabel as i, summaryCounts as l, answersLabel as n, renderSummary as o, cellCounts as r, repeatsVerdict as s, OUTCOME_MARK as t };
 
-//# sourceMappingURL=summary-D7Ttky0z.js.map
+//# sourceMappingURL=summary-CEMYhzXm.js.map

package/dist/summary-CEMYhzXm.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary-CEMYhzXm.js","names":[],"sources":["../src/history/summary.ts"],"mappings":";;;;AAOA,MAAa,eAAwC;CACnD,MAAM;CACN,MAAM;CACN,aAAa;CACb,SAAS;AACX;;AAGA,MAAM,qBAAqB;AAE3B,MAAM,QAAQ,MAAc,EAAE,QAAQ,OAAO,KAAK,CAAC,CAAC,QAAQ,OAAO,GAAG;AAEtE,SAAgB,cAAc,GAAiC;CAC7D,MAAM,QAAQ;EACZ,EAAE,QAAQ,GAAG,EAAE,KAAK;EACpB,EAAE,OAAO,EAAE,WAAW,KAAK,GAAG,EAAE,OAAO,EAAE,SAAS;EAClD,EAAE,WAAW,KAAK,GAAG,EAAE,SAAS;EAChC,EAAE,cAAc,GAAG,EAAE,WAAW;EAChC,EAAE,WAAW,GAAG,EAAE,QAAQ;CAC5B,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,MAAM,WAAW,GAAG,OAAO;CAC/B,MAAM,QAAQ,EAAE,aAAa,EAAE,QAAQ,KAAK,KAAK,EAAE,SAAS,aAAa,EAAE,MAAM;CACjF,OAAO,GAAG,MAAM,KAAK,KAAK,IAAI;AAChC;AAEA,MAAM,gBAAgB,MAAc,GAAG,KAAK,IAAI,MAAM,MAAM,KAAK,MAAM,KAAK,IAAI,CAAC,IAAI,GAAG;;AAGxF,MAAa,aAAa,SACxB,SAAS,SAAS,SAAS,SAAS,OAAO,YAAY,KAAK;;;;;AAM9D,SAAgB,UAAU,MAA+C;CACvE,MAAM,EAAE,UAAU,SAAS,YAAY;CACvC,MAAM,QAAQ,OAAO,aAAa,SAAS,EAAE,EAAE,GAAG,aAAa,SAAS,EAAE;CAC1E,MAAM,SAAS,GAAG,QAAQ,KAAK,GAAG,QAAQ,QAAQ,MAAM,QAAQ,KAAK,GAAG,QAAQ;CAChF,OAAO,QAAQ,aAAa,KAAK,MAAM,EAAE,QAAQ,MAAM,IAAI,OAAO,GAAG,KAAK,cAAc,mBAAmB;AAC7G;AAEA,MAAM,YAAY,MAChB,GAAG,EAAE,KAAK,GAAG,EAAE,KAAK,GAAG,EAAE,EAAE,IAAI,KAAK,MAAO,EAAE,OAAO,EAAE,IAAK,GAAG,EAAE;;;;;AAMlE,SAAgB,aAAa,QAA+B;CAC1D,MAAM,IAAI,IAAI,KAAK,OAAO,UAAU,EAAE,EAAE;CACxC,MAAM,OAAO,OAAO,MAAM,QAAQ,MAAM,EAAE,WAAW,KAAK,CAAC,CAAC;CAC5D,MAAM,OAAQ,CAAC,UAAU,QAAQ,CAAC,CAAW,SAAS,QAAQ;EAC5D,MAAM,QAAQ,OAAO,MAAM,QAAQ,MAAM,EAAE,QAAQ,GAAG;EACtD,OAAO,MAAM,SAAS,CAAC,GAAG,IAAI,IAAI,MAAM,IAAI,QAAQ,CAAC,CAAC,KAAK,IAAI,GAAG,IAAI,CAAC;CACzE,CAAC;CACD,MAAM,QAAQ,OAAO,MAAM;CAC3B,MAAM,QAAQ,GAAG,MAAM,OAAO,UAAU,IAAI,KAAK,IAAI,IAAI,KAAK,MAAM,SAAS,IAAI,KAAK;CACtF,MAAM,KAAK,OAAO,iBAAiB,OAAO;CAC1C,MAAM,SAAS,OAAO,UAAU,KAAA,IAAY,cAAc,KAAK,OAAO,OAAO,GAAG,MAAM;CACtF,IAAI,UAAU,KAAK,OAAO,UAAU,KAAA,GAClC,OAAO,GAAG,EAAE;CAEd,OAAO,GAAG,EAAE,GAAG;EAAC,GAAG;EAAM;EAAO;CAAE,CAAC,CAAC,KAAK,KAAK,IAAI;AACpD;;AAGA,SAAgB,WAAW,GAAsB;CAC/C,OAAO,EAAE,MACN,KAAK,MAAM;EACV,MAAM,QAAQ;GACZ,EAAE,SAAS,aAAa,EAAE,MAAM,MAAM,MAAM,EAAE,SAAS,SAAS,IAAI,YAAY;GAChF,EAAE,SAAS,SAAS,cAAc;GAClC,EAAE,SAAS,SAAS,EAAE,SAAS,SAAS,QAAQ,UAAU,EAAE,IAAI,MAAM;GACtE,EAAE,OAAO,UAAU,EAAE,IAAI,IAAI;GAC7B,GAAG,CAAC,aAAa,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,SAAS,WACnC,SAAS,CAAC,UAAU,YAAY,MAAM,GAAG,IAAI,CAAC,CAChD;GACA,EAAE,UAAU,SAAS,aAAa,EAAE,UAAU,KAAK,IAAI,MAAM;EAC/D,CAAC,CAAC,OAAO,OAAO;EAChB,OAAO,GAAG,EAAE,IAAI,GAAG,EAAE,KAAK,GAAG,EAAE,OAAO,EAAE,OAAO,MAAM,SAAS,KAAK,MAAM,KAAK,IAAI,EAAE,KAAK;CAC3F,CAAC,CAAC,CACD,KAAK,KAAK;AACf;AAEA,SAAS,UAAU,GAAsB;CACvC,MAAM,aAAa,EAAE,SAAS,SAAS,MAAM,EAAE,UAAU;CACzD,IAAI,WAAW,WAAW,GAAG,OAAO;CACpC,OAAO,WAAW,KAAK,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,GAAG,EAAE,WAAW,CAAC,CAAC,KAAK,IAAI;AAC7F;;AAGA,MAAM,SAAS,SAAiB,KAAK,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;;;;;AAMxE,SAAgB,UAAU,KAAyC;CACjE,MAAM,EAAE,KAAK,OAAO,QAAQ;CAC5B,MAAM,OAAgC,CAAC;CACvC,KAAK,KAAK,CACR,eACA,MACI,GAAG,IAAI,IAAI,MAAM,GAAG,EAAE,IAAI,IAAI,SAAS,OAAO,IAAI,WAAW,KAAK,IAAI,QAAQ,YAAY,UAAU,gDACpG,sBACN,CAAC;CACD,IAAI,KAAK,OACP,KAAK,KAAK,CACR,eACA,IAAI,gBACA,mBAAmB,IAAI,cAAc,MAAM,iBAAiB,MAAM,IAAI,cAAc,MAAM,MAC1F,cAAc,IAAI,qBAAqB,YAAY,IAAI,WAAW,YAAY,MAAM,IAAI,QAAQ,MAAM,IAC5G,CAAC;CAEH,KAAK,KAAK,CACR,aACA,IAAI,aAAa,UAAU,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,KAC7E,YACJ,CAAC;CACD,KAAK,KAAK,CACR,UACA,IAAI,OAAO,SACP,IAAI,OACD,KACE,MACC,GAAG,EAAE,IAAI,QAAQ,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,YAAY,MAAM,qCAAqC,SAAS,EAAE,YAAY,QAAQ,EAAE,aAAa,iBAAiB,EAAE,WAAW,KAAK,GAAG,MAAM,EAAE,WAAW,MAAM,MAAM,KAAK,EAAE,SAAS,SAAS,cAAc,EAAE,QAAQ,OAAO,MAAM,EAAE,SAAS,IAC3R,CAAC,CACA,KAAK,IAAI,IACZ,qBACN,CAAC;CACD,KAAK,KAAK,CAAC,aAAa,IAAI,UAAU,IAAI,YAAY,CAAC,CAAC,KAAK,IAAI,KAAK,MAAM,CAAC;CAC7E,KAAK,KAAK,CACR,gBACA,IAAI,aACD,KACE,MACC,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,IAAI,EAAE,OAAO,SAAS,aAAa,EAAE,OAAO,KAAK,MAAM,GAAG,EAAE,OAAO,EAAE,SAAS,IAAI,EAAE,OAAO,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,IAAI,MAAM,KAAK,EAAE,IAAI,SAAS,UAAU,EAAE,IAAI,KAAK,IAAI,MAAM,KAAK,EAAE,QAAQ,SAAS,cAAc,EAAE,QAAQ,KAAK,IAAI,MAAM,IACjR,CAAC,CACA,KAAK,IAAI,KAAK,MACnB,CAAC;CACD,KAAK,KAAK,CACR,SACA,GAAG,IAAI,MAAM,SAAS,oBAAoB,IAAI,MAAM,eAAe,SAAS,YAAY,IAAI,MAAM,eAAe,KAAK,IAAI,EAAE,KAAK,GAAG,eAAe,IAAI,MAAM,UAAU,aAAa,IAAI,MAAM,SAChM,CAAC;CACD,KAAK,KAAK,CACR,WACA,IAAI,QAAQ,SACR,GAAG,IAAI,QAAQ,OAAO,OAAO,IAAI,QAAQ,WAAW,IAAI,KAAK,IAAI,IAAI,IAAI,QAAQ,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,MAAM,EAAE,MAAM,GAAG,CAAC,CAAC,KAAK,IAAI,MACrI,qDACN,CAAC;CACD,KAAK,KAAK,CACR,YACA,OAAO,QAAQ,IAAI,QAAQ,CAAC,CACzB,KAAK,CAAC,MAAM,OAAO,GAAG,KAAK,GAAG,GAAG,CAAC,CAClC,KAAK,KAAK,CACf,CAAC;CACD,MAAM,WAAW,cAAc,GAAG;CAClC,KAAK,KAAK,CACR,UACA,GAAG,IAAI,GAAG,KAAK,aAAa,GAAG,IAAI,WAAW,MAAM,aAAa,GAAG,KAAK,IAAI,GAAG,GAAG,IAAI,MACzF,CAAC;CACD,KAAK,KAAK,CACR,UACA,IAAI,OAAO,SAAS,OAChB,OAAO,IAAI,OAAO,SAAS,GAAG,IAAI,OAAO,SAAS,IAAI,IAAI,OAAO,WAAW,OAC5E,WAAW,IAAI,QACrB,CAAC;CACD,OAAO;AACT;AAEA,SAAgB,cAAc,KAAwB;CACpD,MAAM,QAAQ;EACZ,gBAAgB,IAAI;EACpB;EACA,GAAG,cAAc,IAAI,OAAO,IAAI,IAAI,WAAW,aAAa,KAAK,QAAQ,IAAI,WAAW,WAAW,mBAAmB,IAAI,OAAO,IAAI,aAAa,IAAI,UAAU,IAAI,eAAe,IAAI,UAAU,EAAE;EACnM;EACA,SAAS,IAAI,UAAU,WAAW,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,OAAO,QAAQ,KAAK,IAAI,KAAK,MAAM,WAAW,IAAI,KAAK,MAAM,WAAW,IAAI,KAAK,SAAS,WAAW,IAAI,SAAS,aAAa,CAAC,IAAI,OAAO,aAAa,KAAA,IAAY,GAAG,IAAI,OAAO,SAAS,aAAa,IAAI,IAAI,OAAO,YAAY,EAAE,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,KAAK,IAAI,MAAM;EACzU;EACA,GAAG,IAAI,KAAK,gBAAgB,KAAK,MAAM,kBAAkB,EAAE,QAAQ,GAAG;EACtE;EACA;EACA;EACA;EACA,GAAG,UAAU,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,OAAO,KAAK,EAAE,KAAK,KAAK,CAAC,EAAE,GAAG;EACzD;EACA,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,MAAM,kBAAkB,EAAE,GAAG;EACnD,YAAY,IAAI,MAAM,YAAY,IAAI,MAAM,sBAAsB,IAAI;EACtE;EACA;EACA;EACA;EACA;CACF;CACA,KAAK,MAAM,KAAK,IAAI,OAAO;EACzB,MAAM,IAAI,EAAE,SAAS;EACrB,MAAM,OAAO,EAAE,OACX,SACA,EAAE,SAAS,SAAS,IAClB,WAAW,CAAC,IACZ,IACE,eAAe,EAAE,UAAU,IAC3B;EACR,MAAM,KAAK,KAAK,aAAa,EAAE,SAAS,KAAK,KAAK,EAAE,EAAE,EAAE,KAAK,KAAK,IAAI,EAAE,KAAK,UAAU,CAAC,EAAE,GAAG;CAC/F;CAGA,MAAM,QAAQ,MAAgB,MAAM,UAAU,MAAM,gBAAgB,IAAI,MAAM,SAAS,IAAI;CAC3F,KAAK,MAAM,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC,MAAM,GAAG,MAAM,KAAK,EAAE,OAAO,IAAI,KAAK,EAAE,OAAO,CAAC,GAAG;EAChF,IAAI,EAAE,YAAY,WAAW;EAC7B,MAAM,KAAK,IAAI,OAAO,aAAa,EAAE,SAAS,GAAG,EAAE,MAAM,EAAE;EAC3D,MAAM,QAAQ,EAAE,SAAS,SAAS;EAClC,IAAI,OAAO,MAAM,KAAK,UAAU,WAAW,CAAC,KAAK,EAAE;EACnD,IAAI,EAAE,SAAS,QAAQ;GACrB,KAAK,MAAM,KAAK,EAAE,SAAS,MAAM,KAAK,eAAe,KAAK,aAAa,CAAC,CAAC,GAAG;GAC5E,MAAM,KAAK,EAAE;EACf;EACA,MAAM,eAAe,MAAsB,cAAc,GAAG,CAAC,IAAI,IAAI,KAAK,EAAE,OAAO;EACnF,KAAK,MAAM,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,YAAY,CAAC,CAAC,GAAG;GAC/E,IAAI,EAAE,YAAY,WAAW;GAC7B,IAAI,cAAc,GAAG,CAAC,GAAG;IACvB,IAAI,OAAO,MAAM,KAAK,UAAU,EAAE,GAAG,uBAAuB,EAAE;IAC9D,MAAM,KAAK,6CAA6C,EAAE;IAC1D,cAAc,OAAO,CAAC;IACtB,MAAM,KAAK,IAAI,YAAY;IAC3B;GACF;GACA,IAAI,OAAO,MAAM,KAAK,QAAQ,aAAa,EAAE,SAAS,GAAG,EAAE,MAAM,EAAE;GACnE,cAAc,OAAO,CAAC;EACxB;CACF;CACA,MAAM,KAAK,EAAE;CACb,OAAO,MAAM,KAAK,IAAI;AACxB;;AAGA,MAAa,kBAAkB,MAC7B,EAAE,OAAO,SAAS,WAAW,EAAE,WAAW,MAAM,MAAM,EAAE,YAAY,MAAM;AAE5E,SAAS,cAAc,OAAiB,GAAwB;CAC9D,IAAI,EAAE,SAAS,CAAC,eAAe,CAAC,GAC9B,MAAM,KACJ,KAAK,EAAE,MAAM,KAAK,KAClB,IACA,OACA,EAAE,MAAM,QAAQ,MAAM,IAAI,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,KAAK,IAAI,GAClD,OACA,EACF;CAEF,KAAK,MAAM,QAAQ,EAAE,OAAO;EAC1B,MAAM,KACJ,aAAa,KAAK,OAAO,IAAI,eAAe,KAAK,UAAU,EAAE,KAAK,KAAK,KAAK,KAAK,MAAM,GAAG,GAAG,GAC/F;EACA,IAAI,KAAK,OAAO,MAAM,KAAK,cAAc,KAAK,KAAK,KAAK,KAAK,GAAG,GAAG,GAAG;CACxE;CACA,MAAM,IAAI,EAAE;CACZ,MAAM,KACJ,iBAAiB,eAAe,EAAE,OAAO,EAAE,WAAW,eAAe,EAAE,OAAO,EAAE,YAAY,eAAe,EAAE,QAAQ,EAAE,WAAW,eAAe,EAAE,OAAO,GAC5J;CACA,MAAM,MAAM,EAAE;CACd,MAAM,OAAO,GAAkB,SAC7B,MAAM,OAAO,GAAG,KAAK,iBAAiB,GAAG,EAAE,GAAG;CAChD,MAAM,KACJ,YAAY,IAAI,UAAU,gBAAgB,IAAI,MAAM,sBAAsB,IAAI,IAAI,aAAa,cAAc,EAAE,KAAK,IAAI,IAAI,YAAY,SAAS,EAAE,KAAK,eAAe,IAAI,WAAW,EAAE,WAC1L;CACA,KAAK,MAAM,KAAK,EAAE,YAAY;EAC5B,MAAM,UAAU,EAAE,SAAS,SACvB,WAAW,EAAE,QAAQ,OAAO,GAAG,EAAE,QAAQ,WAAW,IAAI,UAAU,cAClE;EACJ,MAAM,KACJ,aAAa,EAAE,QAAQ,KAAK,EAAE,MAAM,QAAQ,CAAC,EAAE,cAAc,EAAE,UAAU,IAAI,EAAE,QAAQ,QAAQ,KAAK,EAAE,UACxG;CACF;CACA,KAAK,MAAM,KAAK,EAAE,QAChB,MAAM,KACJ,gBAAgB,EAAE,SAAS,MAAM,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,EAAE,IAAI,KAAK,MAAM,IAC3F;CAEF,IAAI,EAAE,aAAa,QACjB,MAAM,KACJ,cAAc,EAAE,aAAa,KAAK,MAAO,EAAE,QAAQ,KAAK,EAAE,KAAK,MAAM,EAAE,MAAM,WAAW,GAAG,EAAE,OAAO,KAAK,EAAE,KAAK,GAAI,CAAC,CAAC,KAAK,IAAI,GACjI;CAEF,MAAM,OAAO,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE;CAC7B,IAAI,MAAM;EACR,MAAM,QAAQ,KAAK,SAAS,qBAAqB,GAAG,KAAK,MAAM,GAAG,kBAAkB,EAAE,KAAK;EAC3F,MAAM,KACJ,IACA,iBACA,IACA,GAAG,MACA,QAAQ,CAAC,CACT,MAAM,IAAI,CAAC,CACX,KAAK,MAAM,KAAK,GAAG,GACtB,EACF;CACF;CACA,MAAM,QAAQ;EACZ,EAAE,MAAM,YAAY,eAAe,EAAE,MAAM,SAAS,KAAK;EACzD,EAAE,MAAM,cAAc,gBAAgB,EAAE,MAAM,WAAW,KAAK;EAC9D,EAAE,MAAM,QAAQ,UAAU,EAAE,MAAM,KAAK,KAAK;EAC5C,EAAE,MAAM,cAAc,iBAAiB,EAAE,MAAM,WAAW,KAAK;CACjE,CAAC,CAAC,OAAO,OAAO;CAChB,IAAI,MAAM,QAAQ,MAAM,KAAK,YAAY,MAAM,KAAK,KAAK,GAAG;AAC9D"}

package/dist/summary-WR5qLj_z.js

@@ -0,0 +1,192 @@
+import { l as writeCached } from "./files-CMoTBEul.js";
+import { a as providerModel } from "./model-DUZtNaYR.js";
+import { r as fromRecorded, t as attemptSummary } from "./tools-DN-cJcfi.js";
+import { n as allRuns } from "./commands-CPrrZ2mI.js";
+import { a as failedAnalysis, i as defaultBase, o as prepareAnalysis, s as promptOf, t as analyseDigest } from "./command-CQiFfn-C.js";
+import { join } from "node:path";
+import { z } from "zod";
+import { generateText, stepCountIs, tool } from "ai";
+/** Items the fake reads of each kind, as a careful model would: a few, not all of them. */
+const FAKE_READS = 3;
+const SUMMARY_TOOLS_PROMPT = `You are about to analyse a reeval run from the DIGEST in the prompt. First, read what the digest leaves out, with the tools:
+- read_evidence: an evidence item dropped from the digest to fit its size cap (the ids in "dropped").
+- open_attempt: one attempt's turns, steps, checks, verdicts and diff, by its id in the attempts table (e.g. r1/skill/control#1).
+
+Open failing attempts, and attempts whose facts look odd; at most six items in all, never every attempt. Then reply with one line saying what you read, and stop: you write the analysis itself in the next step.`;
+/** The tools over a prepared digest, with every result counted against `maxChars`. */
+function summaryTools(p, maxChars) {
+	let left = maxChars;
+	const spend = (value) => {
+		const size = JSON.stringify(value).length;
+		if (size > left) return { error: "The input budget (analysis.maxInputTokens) is spent: write the analysis from what you have." };
+		left -= size;
+		return value;
+	};
+	const runs = [p.headRun, p.baseRun].flatMap((r) => r ? [r] : []);
+	return {
+		read_evidence: tool({
+			description: "An evidence item dropped from the digest to fit its cap: its lines (transcript lines keep their L<n>), what it is and why it was chosen.",
+			inputSchema: z.object({ id: z.string().describe("An evidence id from the digest’s dropped") }),
+			execute: async ({ id }) => {
+				const item = p.dropped.find((e) => e.id === id);
+				if (item) {
+					const { priority: _priority, ...rest } = item;
+					return spend(rest);
+				}
+				return { error: p.digest.evidence.some((e) => e.id === id) ? `${id} is in the digest already.` : `No dropped evidence ${id}. Dropped: ${p.digest.dropped.join(", ") || "none"}.` };
+			}
+		}),
+		open_attempt: tool({
+			description: "One attempt: each turn's prompt, steps (tool calls, edits, errors) and final message, its checks, verdicts and diff.",
+			inputSchema: z.object({ id: z.string().describe("An attempt id from the digest’s attempts table") }),
+			execute: async ({ id }) => {
+				const a = p.digest.attempts[id];
+				const run = a && runs.find((r) => r.record.runId === a.runId);
+				const record = run?.record.evals.flatMap((e) => e.attempts).find((x) => x.key === a?.key);
+				if (!a || !run || !record) return { error: `No attempt ${id}. The attempts are ${Object.keys(p.digest.attempts).join(", ")}.` };
+				const view = await fromRecorded(join(run.dir, run.record.runId), record);
+				return spend({
+					...attemptSummary(view),
+					id
+				});
+			}
+		})
+	};
+}
+const isError = (v) => typeof v === "object" && v !== null && "error" in v;
+/** What the tools returned, as evidence to cite and as the text the structured call is shown. */
+function toolReads(reads) {
+	const ok = reads.filter((r) => !isError(r.output));
+	const evidence = ok.filter((r) => r.tool === "read_evidence").map((r) => ({
+		...r.output,
+		priority: 1
+	}));
+	if (!ok.length) return {
+		evidence,
+		text: ""
+	};
+	return {
+		evidence,
+		text: `\n\nREAD WITH TOOLS (cite a dropped evidence item read here by its id, as you would the digest's):\n${ok.map((r) => `${r.tool} ${JSON.stringify(r.input)}: ${JSON.stringify(r.output)}`).join("\n")}`
+	};
+}
+/** Lets the model read with the tools; what it read, for the structured call. */
+async function investigate(p, model, config) {
+	const result = await generateText({
+		model: model.fake ? fakeSummaryModel(p) : await providerModel(model),
+		system: SUMMARY_TOOLS_PROMPT,
+		prompt: promptOf(p.digest),
+		tools: summaryTools(p, config.analysis.maxInputTokens * 4),
+		stopWhen: stepCountIs(4),
+		prepareStep: ({ stepNumber }) => stepNumber === 3 ? { toolChoice: "none" } : void 0,
+		temperature: .2,
+		maxOutputTokens: config.analysis.maxOutputTokens,
+		maxRetries: 3
+	});
+	return {
+		reads: result.steps.flatMap((s) => s.toolResults.map((r) => ({
+			tool: r.toolName,
+			input: r.input,
+			output: r.output
+		}))),
+		usage: {
+			inputTokens: result.totalUsage?.inputTokens ?? 0,
+			outputTokens: result.totalUsage?.outputTokens ?? 0
+		}
+	};
+}
+/**
+* Writes the run's summary into the cache. A failed call is cached as a document with `error`, never
+* a throw; only reading the run (before any call) can throw.
+*/
+async function writeRunSummary(options) {
+	const { config, runId, model } = options;
+	const entries = await allRuns(config);
+	const head = entries.find((e) => e.runId === runId);
+	if (!head) throw new Error(`${runId} isn't in .reeval/runs`);
+	const prepared = await prepareAnalysis(config, head, await defaultBase(config, entries, head), null);
+	const started = Date.now();
+	let doc;
+	try {
+		const { reads, usage } = await investigate(prepared, model, config);
+		doc = await analyseDigest(config, prepared, model, toolReads(reads));
+		const plus = (a, b) => a === null ? null : a + b;
+		doc.usage = {
+			inputTokens: plus(doc.usage.inputTokens, usage.inputTokens),
+			outputTokens: plus(doc.usage.outputTokens, usage.outputTokens)
+		};
+		doc.latencyMs = Date.now() - started;
+	} catch (error) {
+		const message = error instanceof Error ? error.message : String(error);
+		doc = await failedAnalysis(config, prepared, model, message, Date.now() - started);
+	}
+	const path = await writeCached(config.root, prepared.key, doc);
+	return {
+		doc,
+		key: prepared.key,
+		path
+	};
+}
+/**
+* A language model that reads as the prompt asks a real one to: the dropped evidence and the
+* failing attempts, a few of each, in one round, then says it's done, through the AI SDK's own tool
+* loop, so the tools and the budget run as they would.
+*/
+function fakeSummaryModel(p) {
+	const usage = {
+		inputTokens: {
+			total: 0,
+			noCache: 0,
+			cacheRead: 0,
+			cacheWrite: 0
+		},
+		outputTokens: {
+			total: 0,
+			text: 0,
+			reasoning: 0
+		}
+	};
+	const answer = (content, finish) => ({
+		content,
+		finishReason: {
+			unified: finish,
+			raw: finish === "stop" ? "stop" : "tool_calls"
+		},
+		usage,
+		warnings: [],
+		response: { modelId: "fake-analysis" }
+	});
+	return {
+		specificationVersion: "v4",
+		provider: "fake",
+		modelId: "fake-analysis",
+		supportedUrls: {},
+		doGenerate: async (options) => {
+			const readSome = options.prompt.some((m) => m.role === "tool" && Array.isArray(m.content) && m.content.some((c) => c.type === "tool-result"));
+			const calls = [...p.digest.dropped.slice(0, FAKE_READS).map((id) => ({
+				name: "read_evidence",
+				input: { id }
+			})), ...Object.entries(p.digest.attempts).filter(([, a]) => a.outcome === "fail").slice(0, FAKE_READS).map(([id]) => ({
+				name: "open_attempt",
+				input: { id }
+			}))];
+			if (readSome || !calls.length) return answer([{
+				type: "text",
+				text: `Read ${readSome ? "what I needed" : "nothing"}.`
+			}], "stop");
+			return answer(calls.map((c, i) => ({
+				type: "tool-call",
+				toolCallId: `fake_${c.name}_${i}`,
+				toolName: c.name,
+				input: JSON.stringify(c.input)
+			})), "tool-calls");
+		},
+		doStream: async () => {
+			throw new Error("the fake analysis model does not stream");
+		}
+	};
+}
+//#endregion
+export { writeRunSummary };
+
+//# sourceMappingURL=summary-WR5qLj_z.js.map

package/dist/summary-WR5qLj_z.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"summary-WR5qLj_z.js","names":[],"sources":["../src/analysis/agent/summary.ts"],"mappings":";;;;;;;;;AA4BA,MAAM,aAAa;AAEnB,MAAa,uBAAuB;;;;;;AAOpC,SAAgB,aAAa,GAAqB,UAAkB;CAClE,IAAI,OAAO;CACX,MAAM,SAAY,UAAoC;EACpD,MAAM,OAAO,KAAK,UAAU,KAAK,CAAC,CAAC;EACnC,IAAI,OAAO,MACT,OAAO,EACL,OACE,8FACJ;EAEF,QAAQ;EACR,OAAO;CACT;CACA,MAAM,OAAO,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,SAAS,MAAO,IAAI,CAAC,CAAC,IAAI,CAAC,CAAE;CACjE,OAAO;EACL,eAAe,KAAK;GAClB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,0CAA+C,EACzE,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,OAAO,EAAE,QAAQ,MAAM,MAAM,EAAE,OAAO,EAAE;IAC9C,IAAI,MAAM;KACR,MAAM,EAAE,UAAU,WAAW,GAAG,SAAS;KACzC,OAAO,MAAM,IAAI;IACnB;IACA,OAAO,EACL,OAAO,EAAE,OAAO,SAAS,MAAM,MAAM,EAAE,OAAO,EAAE,IAC5C,GAAG,GAAG,8BACN,uBAAuB,GAAG,aAAa,EAAE,OAAO,QAAQ,KAAK,IAAI,KAAK,OAAO,GACnF;GACF;EACF,CAAC;EACD,cAAc,KAAK;GACjB,aACE;GACF,aAAa,EAAE,OAAO,EACpB,IAAI,EAAE,OAAO,CAAC,CAAC,SAAS,gDAAqD,EAC/E,CAAC;GACD,SAAS,OAAO,EAAE,SAAS;IACzB,MAAM,IAAI,EAAE,OAAO,SAAS;IAC5B,MAAM,MAAM,KAAK,KAAK,MAAM,MAAM,EAAE,OAAO,UAAU,EAAE,KAAK;IAC5D,MAAM,SAAS,KAAK,OAAO,MAAM,SAAS,MAAM,EAAE,QAAQ,CAAC,CAAC,MAAM,MAAM,EAAE,QAAQ,GAAG,GAAG;IACxF,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QACjB,OAAO,EACL,OAAO,cAAc,GAAG,qBAAqB,OAAO,KAAK,EAAE,OAAO,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,GACzF;IAEF,MAAM,OAAO,MAAM,aAAa,KAAK,IAAI,KAAK,IAAI,OAAO,KAAK,GAAG,MAAM;IACvE,OAAO,MAAM;KAAE,GAAG,eAAe,IAAI;KAAG;IAAG,CAAC;GAC9C;EACF,CAAC;CACH;AACF;AAIA,MAAM,WAAW,MAAe,OAAO,MAAM,YAAY,MAAM,QAAQ,WAAW;;AAGlF,SAAgB,UAAU,OAA8B;CACtD,MAAM,KAAK,MAAM,QAAQ,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC;CACjD,MAAM,WAAW,GACd,QAAQ,MAAM,EAAE,SAAS,eAAe,CAAC,CACzC,KAAK,OAAO;EAAE,GAAI,EAAE;EAA6C,UAAU;CAAE,EAAE;CAClF,IAAI,CAAC,GAAG,QAAQ,OAAO;EAAE;EAAU,MAAM;CAAG;CAC5C,OAAO;EACL;EACA,MAAM,uGAAuG,GAC1G,KAAK,MAAM,GAAG,EAAE,KAAK,GAAG,KAAK,UAAU,EAAE,KAAK,EAAE,IAAI,KAAK,UAAU,EAAE,MAAM,GAAG,CAAC,CAC/E,KAAK,IAAI;CACd;AACF;;AAGA,eAAsB,YACpB,GACA,OACA,QACsF;CACtF,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,MAAM,OAAO,iBAAiB,CAAC,IAAI,MAAM,cAAc,KAAK;EACnE,QAAQ;EACR,QAAQ,SAAS,EAAE,MAAM;EACzB,OAAO,aAAa,GAAG,OAAO,SAAS,iBAAiB,CAAC;EACzD,UAAU,YAAA,CAAyB;EACnC,cAAc,EAAE,iBACd,eAAe,IAAoB,EAAE,YAAY,OAAgB,IAAI,KAAA;EACvE,aAAa;EACb,iBAAiB,OAAO,SAAS;EACjC,YAAY;CACd,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,SAAS,MAC3B,EAAE,YAAY,KAAK,OAAO;GAAE,MAAM,EAAE;GAAU,OAAO,EAAE;GAAO,QAAQ,EAAE;EAAO,EAAE,CACnF;EACA,OAAO;GACL,aAAa,OAAO,YAAY,eAAe;GAC/C,cAAc,OAAO,YAAY,gBAAgB;EACnD;CACF;AACF;;;;;AAQA,eAAsB,gBAAgB,SAId;CACtB,MAAM,EAAE,QAAQ,OAAO,UAAU;CACjC,MAAM,UAAU,MAAM,QAAQ,MAAM;CACpC,MAAM,OAAO,QAAQ,MAAM,MAAM,EAAE,UAAU,KAAK;CAClD,IAAI,CAAC,MAAM,MAAM,IAAI,MAAM,GAAG,MAAM,uBAAuB;CAC3D,MAAM,WAAW,MAAM,gBACrB,QACA,MACA,MAAM,YAAY,QAAQ,SAAS,IAAI,GACvC,IACF;CACA,MAAM,UAAU,KAAK,IAAI;CACzB,IAAI;CACJ,IAAI;EACF,MAAM,EAAE,OAAO,UAAU,MAAM,YAAY,UAAU,OAAO,MAAM;EAClE,MAAM,MAAM,cAAc,QAAQ,UAAU,OAAO,UAAU,KAAK,CAAC;EACnE,MAAM,QAAQ,GAAkB,MAAe,MAAM,OAAO,OAAO,IAAI;EACvE,IAAI,QAAQ;GACV,aAAa,KAAK,IAAI,MAAM,aAAa,MAAM,WAAW;GAC1D,cAAc,KAAK,IAAI,MAAM,cAAc,MAAM,YAAY;EAC/D;EACA,IAAI,YAAY,KAAK,IAAI,IAAI;CAC/B,SAAS,OAAO;EACd,MAAM,UAAU,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;EACrE,MAAM,MAAM,eAAe,QAAQ,UAAU,OAAO,SAAS,KAAK,IAAI,IAAI,OAAO;CACnF;CACA,MAAM,OAAO,MAAM,YAAY,OAAO,MAAM,SAAS,KAAK,GAAG;CAC7D,OAAO;EAAE;EAAK,KAAK,SAAS;EAAK;CAAK;AACxC;;;;;;AASA,SAAS,iBAAiB,GAAoC;CAC5D,MAAM,QAAQ;EACZ,aAAa;GAAE,OAAO;GAAG,SAAS;GAAG,WAAW;GAAG,YAAY;EAAE;EACjE,cAAc;GAAE,OAAO;GAAG,MAAM;GAAG,WAAW;EAAE;CAClD;CACA,MAAM,UAAU,SAAoB,YAAmC;EACrE;EACA,cAAc;GAAE,SAAS;GAAQ,KAAK,WAAW,SAAS,SAAS;EAAa;EAChF;EACA,UAAU,CAAC;EACX,UAAU,EAAE,SAAS,gBAAgB;CACvC;CACA,OAAO;EACL,sBAAsB;EACtB,UAAU;EACV,SAAS;EACT,eAAe,CAAC;EAChB,YAAY,OAAO,YAAmE;GACpF,MAAM,WAAW,QAAQ,OAAO,MAC7B,MACC,EAAE,SAAS,UACX,MAAM,QAAQ,EAAE,OAAO,KACtB,EAAE,QAAyB,MAAM,MAAM,EAAE,SAAS,aAAa,CACpE;GACA,MAAM,QAAQ,CACZ,GAAG,EAAE,OAAO,QACT,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,QAAQ;IAAE,MAAM;IAAiB,OAAO,EAAE,GAAG;GAAE,EAAE,GACzD,GAAG,OAAO,QAAQ,EAAE,OAAO,QAAQ,CAAC,CACjC,QAAQ,GAAG,OAAO,EAAE,YAAY,MAAM,CAAC,CACvC,MAAM,GAAG,UAAU,CAAC,CACpB,KAAK,CAAC,SAAS;IAAE,MAAM;IAAgB,OAAO,EAAE,GAAG;GAAE,EAAE,CAC5D;GACA,IAAI,YAAY,CAAC,MAAM,QACrB,OAAO,OACL,CAAC;IAAE,MAAM;IAAQ,MAAM,QAAQ,WAAW,kBAAkB,UAAU;GAAG,CAAC,GAC1E,MACF;GAEF,OAAO,OACL,MAAM,KAAK,GAAG,OAAO;IACnB,MAAM;IACN,YAAY,QAAQ,EAAE,KAAK,GAAG;IAC9B,UAAU,EAAE;IACZ,OAAO,KAAK,UAAU,EAAE,KAAK;GAC/B,EAAE,GACF,YACF;EACF;EACA,UAAU,YAAY;GACpB,MAAM,IAAI,MAAM,yCAAyC;EAC3D;CACF;AACF"}

package/dist/tools-DN-cJcfi.js

@@ -0,0 +1,249 @@
+import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
+import { n as clip } from "./text-gEcKjAyp.js";
+import { x as parseTranscript } from "./model-ORtLm2UB.js";
+import { s as mergeOutcome } from "./assemble-BmVT26kf.js";
+import { join } from "node:path";
+import { readFile } from "node:fs/promises";
+//#region src/analysis/agent/tools.ts
+const INPUT_KEYS = [
+	"command",
+	"cmd",
+	"path",
+	"file_path",
+	"filePath",
+	"target_file",
+	"pattern",
+	"query",
+	"url"
+];
+/** A tool call's input in a few words: its command or path when it has one. */
+function inputSummary(input) {
+	if (typeof input === "string") return clip(input.replace(/\s+/g, " "), 120);
+	if (typeof input !== "object" || input === null) return "";
+	const fields = input;
+	for (const key of INPUT_KEYS) {
+		const value = fields[key];
+		if (typeof value === "string" && value) return clip(value.replace(/\s+/g, " "), 120);
+		if (Array.isArray(value) && value.every((v) => typeof v === "string")) return clip(value.join(" "), 120);
+	}
+	return clip(JSON.stringify(input), 120);
+}
+/** One transcript event as a step, or null for text and reasoning (a turn's text is kept whole). */
+function stepOf(event) {
+	switch (event.type) {
+		case "tool": {
+			const input = inputSummary(event.input);
+			return `${event.tool}${input ? `: ${input}` : ""}${event.ok ? "" : " (failed)"}`;
+		}
+		case "file-edit": return `edit (${event.kind}): ${event.path}`;
+		case "web-search": return `web-search: ${clip(event.query, 120)}`;
+		case "web-fetch": return `web-fetch: ${event.url}${event.ok ? "" : " (failed)"}`;
+		case "error": return `error: ${clip(event.message, 200)}`;
+		case "retry": return `retry: ${clip(event.message, 200)}`;
+		default: return null;
+	}
+}
+const steps = (events) => events.flatMap((e) => stepOf(e) ?? []);
+const tail = (text, max) => text.length <= max ? text : `…${text.slice(-max)}`;
+const head = (text, max) => text.length <= max ? text : `${text.slice(0, max)}\n… ${text.length - max} more characters`;
+/** A worker's staged record, read the way the run will assemble it. */
+function fromStaged(result, record) {
+	const attempt = result.attempt;
+	if (!attempt) return null;
+	const { outcome, error } = mergeOutcome(result, record);
+	return {
+		id: `${attempt.cell.key}#${attempt.n}`,
+		cell: attempt.cell.key,
+		n: attempt.n,
+		outcome,
+		error: error ? {
+			kind: error.kind,
+			message: error.message
+		} : null,
+		checks: (record?.checks ?? []).map((c) => ({
+			command: c.command,
+			exitCode: c.exitCode,
+			output: tail(`${c.stdout}\n${c.stderr}`.trim(), 600)
+		})),
+		judgements: (record?.judgements ?? []).map((j) => ({
+			question: j.question,
+			verdict: j.verdict,
+			score: j.score
+		})),
+		turns: (record?.turns ?? []).map((t) => ({
+			prompt: t.prompt,
+			text: t.text,
+			status: t.status,
+			steps: steps(t.transcript)
+		})),
+		files: (record?.diff?.files ?? []).map((f) => f.path),
+		patch: record?.diff?.patch ?? ""
+	};
+}
+/** The staged attempts of one eval on one harness, from `.reeval/staging/<run>/evals`. */
+async function stagedAttempts(stagingDir, results) {
+	const views = [];
+	for (const result of results) {
+		const path = join(stagingDir, "evals", recordFileName(result.key));
+		let record = null;
+		try {
+			record = JSON.parse(await readFile(path, "utf8"));
+		} catch {}
+		const view = fromStaged(result, record);
+		if (view) views.push(view);
+	}
+	return views;
+}
+/** A recorded attempt, read from its run's artifact directory (`<history>/<run>/`, or `.reeval/runs/<run>/`). */
+async function fromRecorded(runDir, attempt) {
+	const read = (path) => path ? readFile(join(runDir, path), "utf8").catch(() => "") : Promise.resolve("");
+	const lines = parseTranscript(await read(attempt.files.transcript?.path));
+	return {
+		id: attempt.id,
+		cell: attempt.cell,
+		n: attempt.n,
+		outcome: attempt.outcome,
+		error: attempt.error ? {
+			kind: attempt.error.kind,
+			message: attempt.error.message
+		} : null,
+		checks: await Promise.all(attempt.checks.map(async (c) => ({
+			command: c.command,
+			exitCode: c.exitCode,
+			output: tail((await read(c.log?.path)).trim(), 600)
+		}))),
+		judgements: attempt.judgements.map((j) => ({
+			question: j.question,
+			verdict: j.verdict,
+			score: j.score
+		})),
+		turns: attempt.turns.map((t, i) => ({
+			prompt: t.prompt,
+			text: t.text,
+			status: t.status,
+			steps: steps(lines.filter((l) => l.turn === i && l.type !== "prompt" && l.type !== "turn-end"))
+		})),
+		files: attempt.changedFiles.map((f) => f.path),
+		patch: await read(attempt.files.diff?.path)
+	};
+}
+/** Each failed attempt beside the nearest passing run of its cell, else of another cell. */
+function failedVsPassed(attempts) {
+	const passed = attempts.filter((a) => a.outcome === "pass");
+	const near = (a) => (b, c) => Math.abs(b.n - a.n) - Math.abs(c.n - a.n);
+	return attempts.filter((a) => a.outcome === "fail").flatMap((a) => {
+		const sameCell = passed.filter((p) => p.cell === a.cell).sort(near(a))[0];
+		const other = passed.filter((p) => p.cell !== a.cell).sort(near(a))[0];
+		const match = sameCell ?? other;
+		return match ? [{
+			failed: a.id,
+			passed: match.id,
+			match: sameCell ? "cell" : "harness"
+		}] : [];
+	});
+}
+const stepCount = (a) => a.turns.reduce((n, t) => n + t.steps.length, 0);
+/** `list_attempts`: every attempt's outcome in a line, and the failed-vs-passed pairs to open. */
+function listOutcomes(attempts) {
+	return {
+		attempts: attempts.map((a) => ({
+			id: a.id,
+			outcome: a.outcome,
+			error: a.error ? `${a.error.kind}: ${clip(a.error.message, 300)}` : null,
+			failedChecks: a.checks.filter((c) => c.exitCode !== 0).map((c) => c.command),
+			judgements: a.judgements.map((j) => `${j.verdict} ${j.score.toFixed(2)}: ${clip(j.question, 120)}`),
+			turns: a.turns.length,
+			steps: stepCount(a),
+			files: a.files.length
+		})),
+		pairs: failedVsPassed(attempts)
+	};
+}
+/** `open_attempt`: one attempt's turns, steps (the first 40 and last 20), checks, verdicts and diff. */
+function attemptSummary(a) {
+	return {
+		id: a.id,
+		outcome: a.outcome,
+		error: a.error ? {
+			kind: a.error.kind,
+			message: clip(a.error.message, 1e3)
+		} : null,
+		turns: a.turns.map((t, i) => ({
+			turn: i + 1,
+			prompt: clip(t.prompt, 400),
+			status: t.status,
+			steps: t.steps.length > 60 ? [
+				...t.steps.slice(0, 40),
+				`… ${t.steps.length - 60} steps …`,
+				...t.steps.slice(-20)
+			] : t.steps,
+			text: clip(t.text, 800)
+		})),
+		checks: a.checks.map((c) => ({
+			command: c.command,
+			exitCode: c.exitCode,
+			...c.exitCode !== 0 ? { output: c.output } : {}
+		})),
+		judgements: a.judgements,
+		files: a.files.slice(0, 50),
+		diff: head(a.patch, 3e3)
+	};
+}
+/**
+* `compare_attempts`: where a failed attempt's steps first differ from a passing one's, a few
+* steps either side, the checks, verdicts and files that differ, and the start of each diff.
+*/
+function divergence(failed, passed, window = 6) {
+	const a = failed.turns.flatMap((t) => t.steps);
+	const b = passed.turns.flatMap((t) => t.steps);
+	let same = 0;
+	while (same < a.length && same < b.length && a[same] === b[same]) same++;
+	const diverged = same < a.length || same < b.length;
+	const from = Math.max(0, same - 2);
+	const commands = [...new Set([...failed.checks, ...passed.checks].map((c) => c.command))];
+	const exit = (v, command) => v.checks.find((c) => c.command === command)?.exitCode ?? null;
+	const verdict = (v, question) => v.judgements.find((j) => j.question === question)?.verdict ?? null;
+	const questions = [...new Set([...failed.judgements, ...passed.judgements].map((j) => j.question))];
+	const last = (v) => clip(v.turns.at(-1)?.text ?? "", 600);
+	return {
+		failed: failed.id,
+		passed: passed.id,
+		error: failed.error ? `${failed.error.kind}: ${clip(failed.error.message, 600)}` : null,
+		stepsInCommon: same,
+		/** The first step (from 1) where they differ; null when the steps are the same. */
+		divergedAt: diverged ? same + 1 : null,
+		steps: {
+			failed: a.length,
+			passed: b.length
+		},
+		failedSteps: a.slice(from, same + window),
+		passedSteps: b.slice(from, same + window),
+		checks: commands.filter((c) => exit(failed, c) !== exit(passed, c)).map((c) => ({
+			command: c,
+			failed: exit(failed, c),
+			passed: exit(passed, c),
+			failedOutput: failed.checks.find((x) => x.command === c)?.output ?? null
+		})),
+		judgements: questions.filter((q) => verdict(failed, q) !== verdict(passed, q)).map((q) => ({
+			question: clip(q, 200),
+			failed: verdict(failed, q),
+			passed: verdict(passed, q)
+		})),
+		files: {
+			onlyFailed: failed.files.filter((f) => !passed.files.includes(f)).slice(0, 30),
+			onlyPassed: passed.files.filter((f) => !failed.files.includes(f)).slice(0, 30)
+		},
+		finalText: {
+			failed: last(failed),
+			passed: last(passed)
+		},
+		diff: {
+			failed: head(failed.patch, 1500),
+			passed: head(passed.patch, 1500)
+		}
+	};
+}
+//#endregion
+export { stagedAttempts as a, listOutcomes as i, divergence as n, fromRecorded as r, attemptSummary as t };
+
+//# sourceMappingURL=tools-DN-cJcfi.js.map

package/dist/tools-DN-cJcfi.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tools-DN-cJcfi.js","names":[],"sources":["../src/analysis/agent/tools.ts"],"mappings":";;;;;;;AA8BA,MAAM,aAAa;CACjB;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;AACF;;AAGA,SAAS,aAAa,OAAwB;CAC5C,IAAI,OAAO,UAAU,UAAU,OAAO,KAAK,MAAM,QAAQ,QAAQ,GAAG,GAAG,GAAG;CAC1E,IAAI,OAAO,UAAU,YAAY,UAAU,MAAM,OAAO;CACxD,MAAM,SAAS;CACf,KAAK,MAAM,OAAO,YAAY;EAC5B,MAAM,QAAQ,OAAO;EACrB,IAAI,OAAO,UAAU,YAAY,OAAO,OAAO,KAAK,MAAM,QAAQ,QAAQ,GAAG,GAAG,GAAG;EACnF,IAAI,MAAM,QAAQ,KAAK,KAAK,MAAM,OAAO,MAAM,OAAO,MAAM,QAAQ,GAClE,OAAO,KAAK,MAAM,KAAK,GAAG,GAAG,GAAG;CACpC;CACA,OAAO,KAAK,KAAK,UAAU,KAAK,GAAG,GAAG;AACxC;;AAGA,SAAgB,OAAO,OAAuC;CAC5D,QAAQ,MAAM,MAAd;EACE,KAAK,QAAQ;GACX,MAAM,QAAQ,aAAa,MAAM,KAAK;GACtC,OAAO,GAAG,MAAM,OAAO,QAAQ,KAAK,UAAU,KAAK,MAAM,KAAK,KAAK;EACrE;EACA,KAAK,aACH,OAAO,SAAS,MAAM,KAAK,KAAK,MAAM;EACxC,KAAK,cACH,OAAO,eAAe,KAAK,MAAM,OAAO,GAAG;EAC7C,KAAK,aACH,OAAO,cAAc,MAAM,MAAM,MAAM,KAAK,KAAK;EACnD,KAAK,SACH,OAAO,UAAU,KAAK,MAAM,SAAS,GAAG;EAC1C,KAAK,SACH,OAAO,UAAU,KAAK,MAAM,SAAS,GAAG;EAC1C,SACE,OAAO;CACX;AACF;AAEA,MAAM,SAAS,WAA8B,OAAO,SAAS,MAAM,OAAO,CAAC,KAAK,CAAC,CAAC;AAElF,MAAM,QAAQ,MAAc,QAAiB,KAAK,UAAU,MAAM,OAAO,IAAI,KAAK,MAAM,CAAC,GAAG;AAC5F,MAAM,QAAQ,MAAc,QAC1B,KAAK,UAAU,MAAM,OAAO,GAAG,KAAK,MAAM,GAAG,GAAG,EAAE,MAAM,KAAK,SAAS,IAAI;;AAG5E,SAAgB,WAAW,QAAoB,QAA+C;CAC5F,MAAM,UAAU,OAAO;CACvB,IAAI,CAAC,SAAS,OAAO;CACrB,MAAM,EAAE,SAAS,UAAU,aAAa,QAAQ,MAAM;CACtD,OAAO;EACL,IAAI,GAAG,QAAQ,KAAK,IAAI,GAAG,QAAQ;EACnC,MAAM,QAAQ,KAAK;EACnB,GAAG,QAAQ;EACX;EACA,OAAO,QAAQ;GAAE,MAAM,MAAM;GAAM,SAAS,MAAM;EAAQ,IAAI;EAC9D,SAAS,QAAQ,UAAU,CAAC,EAAA,CAAG,KAAK,OAAO;GACzC,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,QAAQ,KAAK,GAAG,EAAE,OAAO,IAAI,EAAE,SAAS,KAAK,GAAG,GAAG;EACrD,EAAE;EACF,aAAa,QAAQ,cAAc,CAAC,EAAA,CAAG,KAAK,OAAO;GACjD,UAAU,EAAE;GACZ,SAAS,EAAE;GACX,OAAO,EAAE;EACX,EAAE;EACF,QAAQ,QAAQ,SAAS,CAAC,EAAA,CAAG,KAAK,OAAO;GACvC,QAAQ,EAAE;GACV,MAAM,EAAE;GACR,QAAQ,EAAE;GACV,OAAO,MAAM,EAAE,UAAU;EAC3B,EAAE;EACF,QAAQ,QAAQ,MAAM,SAAS,CAAC,EAAA,CAAG,KAAK,MAAM,EAAE,IAAI;EACpD,OAAO,QAAQ,MAAM,SAAS;CAChC;AACF;;AAGA,eAAsB,eACpB,YACA,SACwB;CACxB,MAAM,QAAuB,CAAC;CAC9B,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,OAAO,KAAK,YAAY,SAAS,eAAe,OAAO,GAAG,CAAC;EACjE,IAAI,SAA4B;EAChC,IAAI;GACF,SAAS,KAAK,MAAM,MAAM,SAAS,MAAM,MAAM,CAAC;EAClD,QAAQ,CAER;EACA,MAAM,OAAO,WAAW,QAAQ,MAAM;EACtC,IAAI,MAAM,MAAM,KAAK,IAAI;CAC3B;CACA,OAAO;AACT;;AAGA,eAAsB,aAAa,QAAgB,SAA8C;CAC/F,MAAM,QAAQ,SACZ,OAAO,SAAS,KAAK,QAAQ,IAAI,GAAG,MAAM,CAAC,CAAC,YAAY,EAAE,IAAI,QAAQ,QAAQ,EAAE;CAClF,MAAM,QAAQ,gBAAgB,MAAM,KAAK,QAAQ,MAAM,YAAY,IAAI,CAAC;CACxE,OAAO;EACL,IAAI,QAAQ;EACZ,MAAM,QAAQ;EACd,GAAG,QAAQ;EACX,SAAS,QAAQ;EACjB,OAAO,QAAQ,QAAQ;GAAE,MAAM,QAAQ,MAAM;GAAM,SAAS,QAAQ,MAAM;EAAQ,IAAI;EACtF,QAAQ,MAAM,QAAQ,IACpB,QAAQ,OAAO,IAAI,OAAO,OAAO;GAC/B,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,QAAQ,MAAM,MAAM,KAAK,EAAE,KAAK,IAAI,EAAA,CAAG,KAAK,GAAG,GAAG;EACpD,EAAE,CACJ;EACA,YAAY,QAAQ,WAAW,KAAK,OAAO;GACzC,UAAU,EAAE;GACZ,SAAS,EAAE;GACX,OAAO,EAAE;EACX,EAAE;EACF,OAAO,QAAQ,MAAM,KAAK,GAAG,OAAO;GAClC,QAAQ,EAAE;GACV,MAAM,EAAE;GACR,QAAQ,EAAE;GACV,OAAO,MACL,MAAM,QACH,MACC,EAAE,SAAS,KAAK,EAAE,SAAS,YAAY,EAAE,SAAS,UACtD,CACF;EACF,EAAE;EACF,OAAO,QAAQ,aAAa,KAAK,MAAM,EAAE,IAAI;EAC7C,OAAO,MAAM,KAAK,QAAQ,MAAM,MAAM,IAAI;CAC5C;AACF;;AAMA,SAAgB,eAAe,UAAuC;CACpE,MAAM,SAAS,SAAS,QAAQ,MAAM,EAAE,YAAY,MAAM;CAC1D,MAAM,QAAQ,OAAoB,GAAgB,MAChD,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,EAAE,IAAI,EAAE,CAAC;CAC1C,OAAO,SACJ,QAAQ,MAAM,EAAE,YAAY,MAAM,CAAC,CACnC,SAAS,MAAM;EACd,MAAM,WAAW,OAAO,QAAQ,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC;EACvE,MAAM,QAAQ,OAAO,QAAQ,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC;EACpE,MAAM,QAAQ,YAAY;EAC1B,OAAO,QACH,CACE;GACE,QAAQ,EAAE;GACV,QAAQ,MAAM;GACd,OAAO,WAAY,SAAoB;EACzC,CACF,IACA,CAAC;CACP,CAAC;AACL;AAEA,MAAM,aAAa,MAAmB,EAAE,MAAM,QAAQ,GAAG,MAAM,IAAI,EAAE,MAAM,QAAQ,CAAC;;AAGpF,SAAgB,aAAa,UAAyB;CACpD,OAAO;EACL,UAAU,SAAS,KAAK,OAAO;GAC7B,IAAI,EAAE;GACN,SAAS,EAAE;GACX,OAAO,EAAE,QAAQ,GAAG,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,MAAM,SAAS,GAAG,MAAM;GACpE,cAAc,EAAE,OAAO,QAAQ,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,KAAK,MAAM,EAAE,OAAO;GAC3E,YAAY,EAAE,WAAW,KACtB,MAAM,GAAG,EAAE,QAAQ,GAAG,EAAE,MAAM,QAAQ,CAAC,EAAE,IAAI,KAAK,EAAE,UAAU,GAAG,GACpE;GACA,OAAO,EAAE,MAAM;GACf,OAAO,UAAU,CAAC;GAClB,OAAO,EAAE,MAAM;EACjB,EAAE;EACF,OAAO,eAAe,QAAQ;CAChC;AACF;;AAGA,SAAgB,eAAe,GAAgB;CAC7C,OAAO;EACL,IAAI,EAAE;EACN,SAAS,EAAE;EACX,OAAO,EAAE,QAAQ;GAAE,MAAM,EAAE,MAAM;GAAM,SAAS,KAAK,EAAE,MAAM,SAAS,GAAI;EAAE,IAAI;EAChF,OAAO,EAAE,MAAM,KAAK,GAAG,OAAO;GAC5B,MAAM,IAAI;GACV,QAAQ,KAAK,EAAE,QAAQ,GAAG;GAC1B,QAAQ,EAAE;GACV,OACE,EAAE,MAAM,SAAS,KACb;IAAC,GAAG,EAAE,MAAM,MAAM,GAAG,EAAE;IAAG,KAAK,EAAE,MAAM,SAAS,GAAG;IAAW,GAAG,EAAE,MAAM,MAAM,GAAG;GAAC,IACnF,EAAE;GACR,MAAM,KAAK,EAAE,MAAM,GAAG;EACxB,EAAE;EACF,QAAQ,EAAE,OAAO,KAAK,OAAO;GAC3B,SAAS,EAAE;GACX,UAAU,EAAE;GACZ,GAAI,EAAE,aAAa,IAAI,EAAE,QAAQ,EAAE,OAAO,IAAI,CAAC;EACjD,EAAE;EACF,YAAY,EAAE;EACd,OAAO,EAAE,MAAM,MAAM,GAAG,EAAE;EAC1B,MAAM,KAAK,EAAE,OAAO,GAAI;CAC1B;AACF;;;;;AAMA,SAAgB,WAAW,QAAqB,QAAqB,SAAS,GAAG;CAC/E,MAAM,IAAI,OAAO,MAAM,SAAS,MAAM,EAAE,KAAK;CAC7C,MAAM,IAAI,OAAO,MAAM,SAAS,MAAM,EAAE,KAAK;CAC7C,IAAI,OAAO;CACX,OAAO,OAAO,EAAE,UAAU,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO;CAClE,MAAM,WAAW,OAAO,EAAE,UAAU,OAAO,EAAE;CAC7C,MAAM,OAAO,KAAK,IAAI,GAAG,OAAO,CAAC;CACjC,MAAM,WAAW,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,OAAO,QAAQ,GAAG,OAAO,MAAM,CAAC,CAAC,KAAK,MAAM,EAAE,OAAO,CAAC,CAAC;CACxF,MAAM,QAAQ,GAAgB,YAC5B,EAAE,OAAO,MAAM,MAAM,EAAE,YAAY,OAAO,CAAC,EAAE,YAAY;CAC3D,MAAM,WAAW,GAAgB,aAC/B,EAAE,WAAW,MAAM,MAAM,EAAE,aAAa,QAAQ,CAAC,EAAE,WAAW;CAChE,MAAM,YAAY,CAChB,GAAG,IAAI,IAAI,CAAC,GAAG,OAAO,YAAY,GAAG,OAAO,UAAU,CAAC,CAAC,KAAK,MAAM,EAAE,QAAQ,CAAC,CAChF;CACA,MAAM,QAAQ,MAAmB,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC,EAAE,QAAQ,IAAI,GAAG;CACrE,OAAO;EACL,QAAQ,OAAO;EACf,QAAQ,OAAO;EACf,OAAO,OAAO,QAAQ,GAAG,OAAO,MAAM,KAAK,IAAI,KAAK,OAAO,MAAM,SAAS,GAAG,MAAM;EACnF,eAAe;;EAEf,YAAY,WAAW,OAAO,IAAI;EAClC,OAAO;GAAE,QAAQ,EAAE;GAAQ,QAAQ,EAAE;EAAO;EAC5C,aAAa,EAAE,MAAM,MAAM,OAAO,MAAM;EACxC,aAAa,EAAE,MAAM,MAAM,OAAO,MAAM;EACxC,QAAQ,SACL,QAAQ,MAAM,KAAK,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAClD,KAAK,OAAO;GACX,SAAS;GACT,QAAQ,KAAK,QAAQ,CAAC;GACtB,QAAQ,KAAK,QAAQ,CAAC;GACtB,cAAc,OAAO,OAAO,MAAM,MAAM,EAAE,YAAY,CAAC,CAAC,EAAE,UAAU;EACtE,EAAE;EACJ,YAAY,UACT,QAAQ,MAAM,QAAQ,QAAQ,CAAC,MAAM,QAAQ,QAAQ,CAAC,CAAC,CAAC,CACxD,KAAK,OAAO;GACX,UAAU,KAAK,GAAG,GAAG;GACrB,QAAQ,QAAQ,QAAQ,CAAC;GACzB,QAAQ,QAAQ,QAAQ,CAAC;EAC3B,EAAE;EACJ,OAAO;GACL,YAAY,OAAO,MAAM,QAAQ,MAAM,CAAC,OAAO,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE;GAC7E,YAAY,OAAO,MAAM,QAAQ,MAAM,CAAC,OAAO,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE;EAC/E;EACA,WAAW;GAAE,QAAQ,KAAK,MAAM;GAAG,QAAQ,KAAK,MAAM;EAAE;EACxD,MAAM;GAAE,QAAQ,KAAK,OAAO,OAAO,IAAI;GAAG,QAAQ,KAAK,OAAO,OAAO,IAAI;EAAE;CAC7E;AACF"}