reeval 0.6.3 → 0.8.0

package/dist/{questions-ZdT3JDyQ.js → questions-BEPj98fZ.js}

@@ -1,122 +1,25 @@
 import { o as redactSecrets } from "./redact-DJ9JCShu.js";
 import { s as recordFileName } from "./ids-Bp6ut7ZD.js";
-import { l as judgeRoute } from "./judge-7WjNdU9z.js";
+import { l as judgeRoute } from "./judge-CeYPN8P_.js";
+import { a as providerModel, i as isFake, r as cannedModel } from "./model-Cl507trk.js";
 import { r as patchForJudge } from "./patch-CGCVipPu.js";
-import { a as providerModel, i as isFake, r as cannedModel } from "./model-Cp6NoAT3.js";
+import { c as voteReason, i as ratesOf, n as pairsOf, o as slots, s as voteOf, t as acrossOf } from "./pairs-DDsjgm5F.js";
 import { join } from "node:path";
 import { readFile } from "node:fs/promises";
-import { z } from "zod";
 import { Output, generateText } from "ai";
-//#region src/compare/pairs.ts
-/** The side a pair names: an arm within a harness, a harness across them. */
-const sideName = (cut, side) => cut === "within" ? side.arm : side.harness;
-/** Every two-element combination, in the order given. */
-function combinations(items) {
-	const out = [];
-	for (let i = 0; i < items.length; i++) for (let j = i + 1; j < items.length; j++) out.push([items[i], items[j]]);
-	return out;
-}
-/** Distinct values in first-seen order. */
-const distinct = (values) => [...new Set(values)];
-/**
-* The pairs to judge. Within a harness, for each run, every two arms; across harnesses, for each
-* arm and run, every two harnesses. Attempts that failed their deterministic checks are left out
-* (they still count in the pass rate), so a pair missing a side isn't judged. The control arm
-* comes first, then arms and harnesses in the order they first appear.
-*/
-function pairsOf(attempts) {
-	const eligible = attempts.filter((a) => a.checksPassed);
-	const find = (harness, arm, run) => eligible.find((a) => a.harness === harness && a.arm === arm && a.run === run);
-	const harnesses = distinct(attempts.map((a) => a.harness));
-	const arms = distinct([...attempts.filter((a) => a.control).map((a) => a.arm), ...attempts.map((a) => a.arm)]);
-	const runs = [...new Set(attempts.map((a) => a.run))].sort((x, y) => x - y);
-	const pairs = [];
-	for (const harness of harnesses) for (const run of runs) for (const [x, y] of combinations(arms)) {
-		const a = find(harness, x, run);
-		const b = find(harness, y, run);
-		if (a && b) pairs.push({
-			cut: "within",
-			harness,
-			run,
-			a,
-			b
-		});
-	}
-	for (const arm of arms) for (const run of runs) for (const [x, y] of combinations(harnesses)) {
-		const a = find(x, arm, run);
-		const b = find(y, arm, run);
-		if (a && b) pairs.push({
-			cut: "across",
-			arm,
-			run,
-			a,
-			b
-		});
-	}
-	return pairs;
-}
-/** The two sides in the order the model sees them: `a` first, or `b` first when swapped. */
-const slots = (pair, swapped) => swapped ? [pair.b, pair.a] : [pair.a, pair.b];
-/** Maps a blinded pick (`A` is the first slot) back to the side's name. */
-function winnerOf(pair, swapped, pick) {
-	if (pick === "tie") return "tie";
-	const [first, second] = slots(pair, swapped);
-	return sideName(pair.cut, pick === "A" ? first : second);
-}
-/** The vote a pick makes, with the pair's names. */
-function voteOf(question, pair, swapped, pick) {
-	return {
-		question,
-		cut: pair.cut,
-		...pair.cut === "within" ? { harness: pair.harness } : { arm: pair.arm },
-		run: pair.run,
-		a: sideName(pair.cut, pair.a),
-		b: sideName(pair.cut, pair.b),
-		winner: winnerOf(pair, swapped, pick),
-		swapped
-	};
-}
-/**
-* Each side's wins over the decided votes it was in, per cut. A tie decides nothing, so it counts
-* for neither side; both orders of a pair count, since each is its own judgement.
-*/
-function ratesOf(votes) {
-	const rates = /* @__PURE__ */ new Map();
-	const rate = (cut, name) => {
-		const key = `${cut}\0${name}`;
-		let found = rates.get(key);
-		if (!found) {
-			found = {
-				cut,
-				name,
-				wins: 0,
-				n: 0
-			};
-			rates.set(key, found);
-		}
-		return found;
-	};
-	for (const v of votes) {
-		if (v.winner === "tie") continue;
-		for (const name of [v.a, v.b]) {
-			const r = rate(v.cut, name);
-			r.n++;
-			if (v.winner === name) r.wins++;
-		}
-	}
-	const order = ["within", "across"];
-	return [...rates.values()].sort((x, y) => order.indexOf(x.cut) - order.indexOf(y.cut));
-}
-//#endregion
+import { z } from "zod";
 //#region src/compare/questions.ts
 const QUESTION_SYSTEM = `You compare two attempts by coding agents at the same task and answer one question about them.
 Each attempt is shown as the changes it made to the project, as a unified diff, labelled A or B. The labels are assigned at random: which attempt comes first says nothing about it.
-Judge only by the question, from the changes shown. Answer A or B for the attempt the question favours, or tie when the changes don't separate them on it.`;
-const voteSchema = z.object({ winner: z.enum([
-	"A",
-	"B",
-	"tie"
-]) });
+Judge only by the question, from the changes shown. Answer A or B for the attempt the question favours, or tie when the changes don't separate them on it. Give a short reason for the pick.`;
+const voteSchema = z.object({
+	winner: z.enum([
+		"A",
+		"B",
+		"tie"
+	]),
+	reason: z.string().optional()
+});
 /** Roughly four characters a token: each side gets half of `maxInputTokens`. */
 const CHARS_PER_TOKEN = 4;
 /** A diff as the model sees it: dependency and binary noise dropped, then cut to `chars`. */
@@ -167,9 +70,14 @@ function blindPair(question, pair, swapped, maxInputTokens) {
 }
 function fakeAnswer(fake, pair) {
 	const vote = fake.vote ?? "tie";
-	if (typeof vote === "object") return cannedModel({ error: vote.error });
+	if (typeof vote === "object" && "error" in vote) return cannedModel({ error: vote.error });
 	const pick = typeof vote === "function" ? vote(pair) : vote;
-	return cannedModel({ text: JSON.stringify({ winner: pick }) });
+	const winner = typeof pick === "string" ? pick : pick.winner;
+	const reason = typeof pick === "string" ? void 0 : pick.reason;
+	return cannedModel({ text: JSON.stringify({
+		winner,
+		...reason !== void 0 ? { reason } : {}
+	}) });
 }
 async function ask(model, pair, maxOutputTokens, signal) {
 	const result = await generateText({
@@ -188,9 +96,10 @@ async function ask(model, pair, maxOutputTokens, signal) {
 		maxRetries: 3,
 		...signal ? { abortSignal: signal } : {}
 	});
-	const { winner } = result.output;
+	const { winner, reason } = result.output;
 	return {
 		pick: winner,
+		reason: voteReason(reason),
 		resolvedModel: isFake(model) ? null : result.response?.modelId ?? null
 	};
 }
@@ -224,9 +133,9 @@ async function answerQuestion(options) {
 		if (error !== void 0) return null;
 		const blind = blindPair(question, pair, swapped, options.maxInputTokens);
 		try {
-			const { pick, resolvedModel } = await ask(model, blind, options.maxOutputTokens, options.signal);
+			const { pick, reason, resolvedModel } = await ask(model, blind, options.maxOutputTokens, options.signal);
 			if (resolvedModel) served.add(resolvedModel);
-			return voteOf(question, pair, swapped, pick);
+			return voteOf(question, pair, swapped, pick, reason);
 		} catch (e) {
 			error ??= errorText(e);
 			return null;
@@ -272,12 +181,12 @@ function questionRunner(options) {
 		if (!e || started.has(id)) return;
 		const attempts = ended.get(id) ?? [];
 		const answers = (async () => {
-			const pairs = pairsOf((await Promise.all(attempts.map(async (a) => stagedAttempt(a, await readStaged(options.stagingDir, a.key))))).filter((s) => s !== null).sort((x, y) => (x.id ?? "") < (y.id ?? "") ? -1 : 1));
+			const staged = (await Promise.all(attempts.map(async (a) => stagedAttempt(a, await readStaged(options.stagingDir, a.key))))).filter((s) => s !== null).sort((x, y) => (x.id ?? "") < (y.id ?? "") ? -1 : 1);
 			return Promise.all(e.questions.map(async (q) => {
 				const answer = await answerQuestion({
 					question: q.question,
 					model: q.model,
-					pairs,
+					pairs: pairsOf(staged, acrossOf(q.across)),
 					maxInputTokens: options.maxInputTokens,
 					maxOutputTokens: options.maxOutputTokens,
 					signal: controller.signal
@@ -322,4 +231,4 @@ function questionRunner(options) {
 //#endregion
 export { questionRunner };
 
-//# sourceMappingURL=questions-ZdT3JDyQ.js.map
+//# sourceMappingURL=questions-BEPj98fZ.js.map

package/dist/questions-BEPj98fZ.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"questions-BEPj98fZ.js","names":[],"sources":["../src/compare/questions.ts"],"mappings":";;;;;;;;;;;AAkCA,MAAa,kBAAkB;;;AAI/B,MAAM,aAAa,EAAE,OAAO;CAC1B,QAAQ,EAAE,KAAK;EAAC;EAAK;EAAK;CAAK,CAAC;CAChC,QAAQ,EAAE,OAAO,CAAC,CAAC,SAAS;AAC9B,CAAC;;AAGD,MAAM,kBAAkB;;AAGxB,SAAS,UAAU,MAAc,OAAuB;CACtD,MAAM,QAAQ,cAAc,IAAI;CAChC,IAAI,CAAC,MAAM,KAAK,GAAG,OAAO;CAC1B,IAAI,MAAM,UAAU,OAAO,OAAO;CAClC,OAAO,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE,yBAAyB,MAAM,MAAM,MAAM,OAAO;AACpF;;AAGA,SAAS,OAAO,OAAoB,QAAoC;CACtE,MAAM,IAAI,MAAM,WAAW,CAAC;CAC5B,MAAM,IAAI,OAAO,WAAW,CAAC;CAC7B,IAAI,CAAC,EAAE,UAAU,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,GAAG,MAAM,MAAM,EAAE,EAAE,GAAG,OAAO;CAC/E,OAAO,EAAE,KAAK,MAAM;AACtB;;AAGA,SAAgB,UACd,UACA,MACA,SACA,gBACW;CACX,MAAM,CAAC,OAAO,UAAU,MAAM,MAAM,OAAO;CAC3C,MAAM,QAAQ,KAAK,MAAO,iBAAiB,kBAAmB,CAAC;CAC/D,MAAM,IAAI,UAAU,MAAM,MAAM,KAAK;CACrC,MAAM,IAAI,UAAU,OAAO,MAAM,KAAK;CACtC,MAAM,OAAO,OAAO,OAAO,MAAM;CACjC,MAAM,SAAS;EACb,aAAa;EACb,GAAI,OAAO;GAAC;GAAI;GAAsC;GAAU;GAAM;EAAS,IAAI,CAAC;EACpF;EACA;EACA;EACA;EACA;EACA;EACA;EACA;CACF,CAAC,CAAC,KAAK,IAAI;CACX,OAAO;EAAE;EAAU,QAAQ,cAAc,MAAM;EAAG;EAAG;CAAE;AACzD;AAIA,SAAS,WAAW,MAAyB,MAAiB;CAC5D,MAAM,OAAO,KAAK,QAAQ;CAC1B,IAAI,OAAO,SAAS,YAAY,WAAW,MAAM,OAAO,YAAY,EAAE,OAAO,KAAK,MAAM,CAAC;CACzF,MAAM,OAAO,OAAO,SAAS,aAAa,KAAK,IAAI,IAAI;CACvD,MAAM,SAAS,OAAO,SAAS,WAAW,OAAO,KAAK;CACtD,MAAM,SAAS,OAAO,SAAS,WAAW,KAAA,IAAY,KAAK;CAC3D,OAAO,YAAY,EACjB,MAAM,KAAK,UAAU;EAAE;EAAQ,GAAI,WAAW,KAAA,IAAY,EAAE,OAAO,IAAI,CAAC;CAAG,CAAC,EAC9E,CAAC;AACH;AAIA,eAAe,IACb,OACA,MACA,iBACA,QACe;CACf,MAAM,SAAS,MAAM,aAAa;EAChC,OAAO,OAAO,KAAK,IACf,WAAW,OAAO,IAAI,IACtB,MAAM,cAAc;GAAE,MAAM;GAAO,OAAO,WAAW,KAAK,CAAC,CAAC;EAAK,CAAC;EACtE,QAAQ;EACR,QAAQ,KAAK;EACb,QAAQ,OAAO,OAAO;GAAE,QAAQ;GAAY,MAAM;EAAO,CAAC;EAC1D,aAAa;EACb;EACA,YAAY;EACZ,GAAI,SAAS,EAAE,aAAa,OAAO,IAAI,CAAC;CAC1C,CAAC;CACD,MAAM,EAAE,QAAQ,WAAW,OAAO;CAClC,OAAO;EACL,MAAM;EACN,QAAQ,WAAW,MAAM;EACzB,eAAe,OAAO,KAAK,IAAI,OAAQ,OAAO,UAAU,WAAW;CACrE;AACF;;AAGA,eAAe,OAAa,OAAY,OAAe,MAA+B;CACpF,MAAM,MAAW,IAAI,MAAM,MAAM,MAAM;CACvC,IAAI,OAAO;CACX,MAAM,QAAQ,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,OAAO,MAAM,MAAM,EAAE,GAAG,YAAY;EAC9E,OAAO,OAAO,MAAM,QAAQ;GAC1B,MAAM,IAAI;GACV,IAAI,KAAK,MAAM,KAAK,MAAM,EAAO;EACnC;CACF,CAAC;CACD,MAAM,QAAQ,IAAI,KAAK;CACvB,OAAO;AACT;AAaA,MAAM,aAAa,UACjB,eAAe,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAA,CAAG,MAAM,IAAI,CAAC,CAAC,MAAM,EAAE;;;;;AAM7F,eAAsB,eAAe,SAAgD;CACnF,MAAM,EAAE,UAAU,OAAO,UAAU;CACnC,MAAM,QAAQ,MAAM,SAAS,SAAS,CAAC,OAAO,IAAI,CAAC,CAAC,KAAK,aAAa;EAAE;EAAM;CAAQ,EAAE,CAAC;CACzF,MAAM,yBAAS,IAAI,IAAY;CAC/B,IAAI;CAkBJ,MAAM,QAAO,MAjBO,OAAO,OAAO,QAAQ,eAAe,GAAG,OAAO,EAAE,MAAM,cAAc;EACvF,IAAI,UAAU,KAAA,GAAW,OAAO;EAChC,MAAM,QAAQ,UAAU,UAAU,MAAM,SAAS,QAAQ,cAAc;EACvE,IAAI;GACF,MAAM,EAAE,MAAM,QAAQ,kBAAkB,MAAM,IAC5C,OACA,OACA,QAAQ,iBACR,QAAQ,MACV;GACA,IAAI,eAAe,OAAO,IAAI,aAAa;GAC3C,OAAO,OAAO,UAAU,MAAM,SAAS,MAAM,MAAM;EACrD,SAAS,GAAG;GACV,UAAU,UAAU,CAAC;GACrB,OAAO;EACT;CACF,CAAC,EAAA,CACkB,QAAQ,MAAqB,MAAM,IAAI;CAC1D,OAAO;EACL;EACA,OAAO,OAAO,KAAK,IAAI,SAAS;EAChC,GAAI,OAAO,OAAO,EAAE,eAAe,CAAC,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,EAAE,IAAI,CAAC;EACtE,OAAO;EACP,OAAO,QAAQ,IAAI;EACnB,GAAI,UAAU,KAAA,IAAY,EAAE,MAAM,IAAI,CAAC;CACzC;AACF;;AAGA,SAAgB,cAAc,GAAe,QAA+C;CAC1F,MAAM,OAAO,EAAE,SAAS;CACxB,IAAI,CAAC,MAAM,OAAO;CAClB,MAAM,eACJ,WAAW,SACV,EAAE,UAAU,YAAa,EAAE,UAAU,YAAY,OAAO,OAAO,SAAS;CAC3E,OAAO;EACL,SAAS,KAAK;EACd,KAAK,KAAK;EACV,KAAK,EAAE,SAAS,KAAK;EACrB,MAAM,QAAQ,MAAM,SAAS;EAC7B;EACA,SAAS,KAAK,SAAS;EACvB,SAAS,QAAQ,MAAM,KAAK,MAAM,EAAE,MAAM,KAAK,CAAC;EAChD,IAAI,GAAG,KAAK,IAAI,GAAG,EAAE,SAAS,KAAK;CACrC;AACF;AAEA,MAAM,aAAa,OAAO,YAAoB,QAC5C,KAAK,MACH,MAAM,SAAS,KAAK,YAAY,SAAS,eAAe,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,YAAY,MAAM,CAC3F;;;;;AAqBF,SAAgB,eAAe,SAAgC;CAC7D,MAAM,OAAO,IAAI,IAAI,QAAQ,MAAM,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;CAC5D,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,MAAM,0BAAU,IAAI,IAAsC;CAC1D,MAAM,aAAa,IAAI,gBAAgB;CAEvC,MAAM,SAAS,OAAe;EAC5B,MAAM,IAAI,KAAK,IAAI,EAAE;EACrB,IAAI,CAAC,KAAK,QAAQ,IAAI,EAAE,GAAG;EAC3B,MAAM,WAAW,MAAM,IAAI,EAAE,KAAK,CAAC;EACnC,MAAM,WAAW,YAAY;GAK3B,MAAM,UAAS,MAJK,QAAQ,IAC1B,SAAS,IAAI,OAAO,MAAM,cAAc,GAAG,MAAM,WAAW,QAAQ,YAAY,EAAE,GAAG,CAAC,CAAC,CACzF,EAAA,CAGG,QAAQ,MAAwB,MAAM,IAAI,CAAC,CAC3C,MAAM,GAAG,OAAQ,EAAE,MAAM,OAAO,EAAE,MAAM,MAAM,KAAK,CAAE;GACxD,OAAO,QAAQ,IACb,EAAE,UAAU,IAAI,OAAO,MAAM;IAC3B,MAAM,SAAS,MAAM,eAAe;KAClC,UAAU,EAAE;KACZ,OAAO,EAAE;KACT,OAAO,QAAQ,QAAQ,SAAS,EAAE,MAAM,CAAC;KACzC,gBAAgB,QAAQ;KACxB,iBAAiB,QAAQ;KACzB,QAAQ,WAAW;IACrB,CAAC;IACD,QAAQ,WAAW,IAAI,MAAM;IAC7B,OAAO;GACT,CAAC,CACH;EACF,EAAA,CAAG,CAAC,CAAC,OAAO,UACV,EAAE,UAAU,KAAK,MAAM;GACrB,MAAM,SAAwB;IAC5B,UAAU,EAAE;IACZ,OAAO,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE;IACpC,OAAO,CAAC;IACR,OAAO,CAAC;IACR,OAAO,UAAU,KAAK;GACxB;GACA,QAAQ,WAAW,IAAI,MAAM;GAC7B,OAAO;EACT,CAAC,CACH;EACA,QAAQ,IAAI,IAAI,OAAO;CACzB;CAEA,OAAO;EACL,UAAU,QAA0B;GAClC,MAAM,IAAI,KAAK,IAAI,OAAO,EAAE;GAC5B,IAAI,CAAC,GAAG;GACR,MAAM,OAAO,MAAM,IAAI,OAAO,EAAE,KAAK,CAAC;GACtC,KAAK,KAAK,MAAM;GAChB,MAAM,IAAI,OAAO,IAAI,IAAI;GACzB,IAAI,KAAK,UAAU,EAAE,IAAI,KAAK,UAAU,MAAM,OAAO,EAAE;EACzD;;EAEA,SAAe;GACb,WAAW,MAAM;EACnB;EACA,MAAM,SAAgD;GACpD,KAAK,MAAM,MAAM,KAAK,KAAK,GAAG,IAAI,MAAM,IAAI,EAAE,GAAG,MAAM,EAAE;GACzD,MAAM,sBAAM,IAAI,IAA6B;GAC7C,KAAK,MAAM,CAAC,IAAI,YAAY,SAAS,IAAI,IAAI,IAAI,MAAM,OAAO;GAC9D,OAAO;EACT;CACF;AACF"}

package/dist/{record-CbuQH4l1.js → record-BRP9vZrj.js}

@@ -9,9 +9,29 @@ const READ_DEFAULTS = {
 			path: "stack.harnesses[].resolvedModels",
 			empty: () => []
 		},
+		{
+			path: "stack.harnesses[].effort",
+			empty: () => null
+		},
+		{
+			path: "stack.harnesses[].resolvedEffort",
+			empty: () => []
+		},
 		{
 			path: "evals[].todo",
 			empty: () => false
+		},
+		{
+			path: "evals[].cells[].effort",
+			empty: () => null
+		},
+		{
+			path: "evals[].attempts[].effort",
+			empty: () => null
+		},
+		{
+			path: "evals[].attempts[].servedEffort",
+			empty: () => null
 		}
 	],
 	"transcript-line": [],
@@ -100,4 +120,4 @@ const readableCliInvocation = (entry) => fillDefaults("cli-invocation", entry);
 //#endregion
 export { readableCliInvocation as a, readableTranscriptLine as c, readableAnalysisNote as i, fillDefaults as n, readableDebugRequest as o, readableAnalysis as r, readableRecord as s, NewerSchemaError as t };
 
-//# sourceMappingURL=record-CbuQH4l1.js.map
+//# sourceMappingURL=record-BRP9vZrj.js.map

package/dist/record-BRP9vZrj.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"record-BRP9vZrj.js","names":[],"sources":["../src/view/record.ts"],"mappings":";AA6BA,MAAa,gBAAkD;CAC7D,cAAc;EACZ;GAAE,MAAM;GAAwB,aAAa,CAAC;EAAE;EAChD;GAAE,MAAM;GAAoC,aAAa,CAAC;EAAE;EAC5D;GAAE,MAAM;GAA4B,aAAa;EAAK;EACtD;GAAE,MAAM;GAAoC,aAAa,CAAC;EAAE;EAC5D;GAAE,MAAM;GAAgB,aAAa;EAAM;EAC3C;GAAE,MAAM;GAA0B,aAAa;EAAK;EACpD;GAAE,MAAM;GAA6B,aAAa;EAAK;EACvD;GAAE,MAAM;GAAmC,aAAa;EAAK;CAC/D;CACA,mBAAmB,CAAC;CACpB,UAAU,CAAC;CACX,iBAAiB,CAAC;CAClB,iBAAiB,CAAC;CAClB,iBAAiB,CAAC;CAClB,kBAAkB,CAAC;AACrB;;;;;AAMA,MAAa,aAAoF,CAAC;;AAGlG,MAAa,kBAAkB;CAC7B,cAAA;CACA,UAAU;CACV,iBAAiB;AACnB;;AAGA,IAAa,mBAAb,cAAsC,MAAM;CAC1C,OAAgB;;CAEhB;CACA,YAAY,MAAc,SAAiB,QAAgB;EACzD,MAAM,SAAS,4CAA4C,QAAQ,4BAA4B,OAAO;EACtG,MAAM,GAAG,KAAK,OAAO,QAAQ;EAC7B,KAAK,SAAS;CAChB;AACF;AAGA,MAAM,YAAY,MAAyB,OAAO,MAAM,YAAY,MAAM,QAAQ,CAAC,MAAM,QAAQ,CAAC;AAElG,SAAS,KAAK,OAAgB,UAAoB,OAA4B;CAC5E,IAAI,CAAC,SAAS,KAAK,GAAG;CACtB,MAAM,CAAC,OAAO,IAAI,GAAG,QAAQ;CAC7B,MAAM,SAAS,iBAAiB,KAAK,IAAI;CACzC,IAAI,QAAQ;EACV,IAAI,OAAO,MAAM,OAAO,GAAa,MAAM,OAAO,IAAI,KAAK,OAAO,MAAM,KAAK;EAC7E;CACF;CACA,MAAM,GAAG,MAAM,MAAM,QAAQ,sBAAsB,KAAK,IAAI,KAAK,CAAC;CAClE,IAAI,CAAC,KAAK,UAAU,CAAC,MAAM;EACzB,IAAI,MAAM,SAAS,KAAA,GAAW,MAAM,OAAO,MAAM;EACjD;CACF;CACA,MAAM,QAAQ,MAAM;CACpB,MAAM,WACJ,SAAS,OACL,MAAM,QAAQ,KAAK,IACjB,QACA,CAAC,IACH,SAAS,OACP,SAAS,KAAK,IACZ,OAAO,OAAO,KAAK,IACnB,CAAC,IACH,CAAC,KAAK;CACd,KAAK,MAAM,KAAK,UAAU,KAAK,GAAG,MAAM,KAAK;AAC/C;;AAGA,SAAgB,aAAgB,OAAkB,OAAa;CAC7D,KAAK,MAAM,KAAK,cAAc,QAAQ,KAAK,OAAO,EAAE,KAAK,MAAM,GAAG,GAAG,EAAE,KAAK;CAC5E,OAAO;AACT;AAEA,SAAS,QAAW,OAAqC,OAAU,MAAiB;CAClF,MAAM,SAAS,gBAAgB;CAC/B,IAAI,UAAW,MAAc;CAC7B,IAAI,OAAO,YAAY,UAAU,OAAO;CACxC,IAAI,UAAU,QAAQ,MAAM,IAAI,iBAAiB,MAAM,SAAS,MAAM;CACtE,IAAI,IAAa;CACjB,OAAO,UAAU,QAAQ,WAAW;EAClC,MAAM,UAAU,WAAW,MAAM,GAAG;EACpC,IAAI,CAAC,SAAS;EACd,IAAI,QAAQ,CAAU;CACxB;CACA,OAAO;AACT;;;;;;AAOA,SAAgB,eAAe,QAAmB,OAAO,OAAO,OAAO,SAAoB;CACzF,OAAO,aAAa,cAAc,QAAQ,cAAc,QAAQ,IAAI,CAAC;AACvE;;AAGA,MAAa,0BAA0B,SACrC,aAAa,mBAAmB,IAAI;;AAGtC,MAAa,oBAAoB,KAAkB,OAAO,oBACxD,aAAa,YAAY,QAAQ,YAAY,KAAK,IAAI,CAAC;;AAGzD,MAAa,wBACX,MACA,OAAO,yBACU,aAAa,iBAAiB,QAAQ,iBAAiB,MAAM,IAAI,CAAC;;AAGrF,MAAa,wBAAwB,YACnC,aAAa,iBAAiB,OAAO;;AAGvC,MAAa,yBAAyB,UACpC,aAAa,kBAAkB,KAAK"}

package/dist/results-BcSHWbgB.js

@@ -0,0 +1,52 @@
+import { h as writeRun, p as scratchDir, s as readRecord } from "./files-CMSUXAEg.js";
+import { o as renderSummary } from "./summary-k3mhWWXv.js";
+import { n as assembleRun } from "./assemble-BbFVjHeS.js";
+import { join } from "node:path";
+import { readdir, rm, stat } from "node:fs/promises";
+//#region src/results.ts
+const STALE_STAGING_MS = 864e5;
+/** Where workers write raw per-eval records before the run is assembled. */
+const stagingDir = (root, runId) => join(root, ".reeval", "staging", runId);
+/** Removes staging left behind by runs that were killed before they could clean up. */
+async function sweepStaging(root, keep) {
+	const dir = join(root, ".reeval", "staging");
+	const entries = await readdir(dir).catch(() => []);
+	for (const name of entries) {
+		if (name === keep) continue;
+		if (await stat(join(dir, name)).then((s) => Date.now() - s.mtimeMs, () => 0) > STALE_STAGING_MS) await rm(join(dir, name), {
+			recursive: true,
+			force: true
+		});
+	}
+}
+/** Assembles the run into `.reeval/runs/`, with every artifact and a summary.md. */
+async function writeScratchRun(input, landed) {
+	const { record, artifacts } = await assembleRun(input);
+	artifacts.set("summary.md", Buffer.from(renderSummary(record)));
+	const scratch = scratchDir(input.ctx.config.root);
+	const existing = join(scratch, `${record.runId}.json`);
+	try {
+		const old = await readRecord(existing);
+		if (old.status === "interrupted" || old.status === "budget") {
+			await rm(existing, { force: true });
+			await rm(join(scratch, record.runId), {
+				recursive: true,
+				force: true
+			});
+		}
+	} catch {}
+	const written = await writeRun(scratch, record, artifacts);
+	landed?.();
+	await rm(input.ctx.stagingDir, {
+		recursive: true,
+		force: true
+	});
+	return {
+		...written,
+		record
+	};
+}
+//#endregion
+export { stagingDir, sweepStaging, writeScratchRun };
+
+//# sourceMappingURL=results-BcSHWbgB.js.map

package/dist/results-BcSHWbgB.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"results-BcSHWbgB.js","names":[],"sources":["../src/results.ts"],"mappings":";;;;;;AAOA,MAAM,mBAAmB;;AAGzB,MAAa,cAAc,MAAc,UAAkB,KAAK,MAAM,WAAW,WAAW,KAAK;;AAGjG,eAAsB,aAAa,MAAc,MAA8B;CAC7E,MAAM,MAAM,KAAK,MAAM,WAAW,SAAS;CAC3C,MAAM,UAAU,MAAM,QAAQ,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;CACjD,KAAK,MAAM,QAAQ,SAAS;EAC1B,IAAI,SAAS,MAAM;EAKnB,IAAI,MAJc,KAAK,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,MACrC,MAAM,KAAK,IAAI,IAAI,EAAE,eAChB,CACR,IACU,kBAAkB,MAAM,GAAG,KAAK,KAAK,IAAI,GAAG;GAAE,WAAW;GAAM,OAAO;EAAK,CAAC;CACxF;AACF;;AAGA,eAAsB,gBACpB,OACA,QAC6C;CAC7C,MAAM,EAAE,QAAQ,cAAc,MAAM,YAAY,KAAK;CACrD,UAAU,IAAI,cAAc,OAAO,KAAK,cAAc,MAAM,CAAC,CAAC;CAC9D,MAAM,UAAU,WAAW,MAAM,IAAI,OAAO,IAAI;CAChD,MAAM,WAAW,KAAK,SAAS,GAAG,OAAO,MAAM,MAAM;CACrD,IAAI;EACF,MAAM,MAAM,MAAM,WAAW,QAAQ;EACrC,IAAI,IAAI,WAAW,iBAAiB,IAAI,WAAW,UAAU;GAC3D,MAAM,GAAG,UAAU,EAAE,OAAO,KAAK,CAAC;GAClC,MAAM,GAAG,KAAK,SAAS,OAAO,KAAK,GAAG;IAAE,WAAW;IAAM,OAAO;GAAK,CAAC;EACxE;CACF,QAAQ,CAAC;CACT,MAAM,UAAU,MAAM,SAAS,SAAS,QAAQ,SAAS;CACzD,SAAS;CACT,MAAM,GAAG,MAAM,IAAI,YAAY;EAAE,WAAW;EAAM,OAAO;CAAK,CAAC;CAC/D,OAAO;EAAE,GAAG;EAAS;CAAO;AAC9B"}