reeval 0.9.0 → 0.10.0

package/dist/{images-BMgdpMAX.js → images-0On0cztu.js}

@@ -1,18 +1,46 @@
-import { I as LABEL, U as imageFor, X as imageInfo, c as dockerfileTag, d as installLayerTag, h as usesEsBase, it as isTransient, m as resolveFrom, n as REEVAL_BASE_ARG, s as contextHash, u as harnessImageFor } from "./src-8rRXy4u9.js";
-import { d as sandboxId } from "./options-CKqyPGxX.js";
-import { n as prepareAll } from "./preflight-BkoSxf4C.js";
-import { i as installSteps, o as projectPath, r as imageHarness } from "./prepare-CUAWHebF.js";
-import { n as createProviders, r as dockerKey } from "./providers-B1DzMvmH.js";
+import { f as sandboxId, p as sandboxProvider } from "./options-Bh2W63dK.js";
+import { I as LABEL, U as imageFor, X as imageInfo, at as isTransient, c as dockerfileTag, d as installLayerTag, h as usesReevalBase, m as resolveFrom, n as REEVAL_BASE_ARG, s as contextHash, u as harnessImageFor, ut as engineSpec } from "./src-B7treWa8.js";
+import { n as prepareAll } from "./preflight-D7iFPKeK.js";
+import { a as projectPath, n as imageHarness, r as installSteps } from "./prepare-doGxaAGR.js";
+import { i as dockerKey, r as createProviders, t as attemptImage } from "./providers-tAOD1NQN.js";
 //#region src/sandbox/images.ts
+/**
+* What `reeval image build` builds and `reeval doctor` checks: the harness and sandbox of each
+* attempt the evals plan (collected as `--dry-run` does), or the config's harnesses on its sandbox
+* when they plan none.
+*/
+async function plannedImageUses(run, filters = [], unmatched) {
+	const { runEngine } = await import("./node-B3ydHo6t.js");
+	let planned = [];
+	await runEngine({
+		run: JSON.parse(JSON.stringify(run)),
+		filters,
+		onPlan: async (collected) => {
+			if (unmatched && collected.unmatched.length) throw unmatched(collected.unmatched);
+			planned = collected.planned.filter((p) => !p.attempt.plan.skip);
+			return false;
+		}
+	});
+	return planned.length ? planned.map((p) => ({
+		id: p.id,
+		harness: p.attempt.harness,
+		sandbox: p.attempt.sandbox
+	})) : run.config.harness.map((harness) => ({
+		id: "",
+		harness,
+		sandbox: run.config.sandbox
+	}));
+}
 /** The distinct Docker sandbox and harness pairs, in first-use order, with how many evals use each. */
 function imageTargets(uses) {
 	const targets = /* @__PURE__ */ new Map();
 	for (const { id, sandbox, ...use } of uses) {
 		if (sandbox.kind !== "docker") continue;
 		const harness = imageHarness(use.harness);
-		const { image, from, dockerfile, install } = sandbox;
+		const { engine, image, from, dockerfile, install } = sandbox;
 		const key = JSON.stringify({
 			harness,
+			engine,
 			image: {
 				image,
 				from,
@@ -21,7 +49,7 @@ function imageTargets(uses) {
 			}
 		});
 		const target = targets.get(key) ?? {
-			id: harness === "cursor" ? sandboxId(sandbox) : `${sandboxId(sandbox)} · ${harness}`,
+			id: `${sandboxId(sandbox)} · ${harness}`,
 			harness,
 			sandbox,
 			evals: 0,
@@ -45,10 +73,18 @@ function imageTargets(uses) {
 	});
 }
 const short = (digest) => digest.replace(/^sha256:/, "").slice(0, 12);
+const look = (ref, engine) => imageInfo(ref, engine).then((info) => ({ info }), (error) => ({
+	info: null,
+	error: error.message
+}));
 /** `from`'s image, checked against what the tag names now (a registry lookup, no pull). */
-async function fromStatus(from, ref) {
-	const [cached, now] = await Promise.all([imageInfo(ref), resolveFrom(from)]);
-	if (!cached) {
+async function fromStatus(from, ref, engine) {
+	const [cached, now] = await Promise.all([look(ref, engine), resolveFrom(from, void 0, engine)]);
+	if (cached.error) return {
+		status: "unknown",
+		detail: cached.error
+	};
+	if (!cached.info) {
 		if (!("error" in now)) return {
 			status: "build",
 			detail: `on ${from} (${short(now.digest)})`
@@ -61,7 +97,7 @@ async function fromStatus(from, ref) {
 			detail: `doesn't resolve: ${now.error}`
 		};
 	}
-	const was = cached.labels[LABEL.fromDigest];
+	const was = cached.info.labels[LABEL.fromDigest];
 	if ("error" in now) return {
 		status: "ready",
 		detail: `built; ${from} couldn't be checked`
@@ -75,20 +111,34 @@ async function fromStatus(from, ref) {
 		detail: `built on ${short(now.digest)}`
 	};
 }
-/** How each target's image is made, in build order, and how each step stands, building nothing. */
+/**
+* How each target's image is made, in build order, and how each step stands, building nothing. A
+* step whose image Docker couldn't say is there is `unknown`, with Docker's error.
+*/
 async function imageStages(target, root) {
 	const { sandbox, harness } = target;
+	const { engine } = sandbox;
 	const stages = [];
 	const exists = async (ref) => {
-		const info = await imageInfo(ref);
+		const { info, error } = await look(ref, engine);
+		if (error) return {
+			info,
+			status: "unknown",
+			detail: error
+		};
 		return {
 			info,
 			status: info ? "ready" : "build",
 			detail: info ? `built (${short(info.id)})` : "not built yet"
 		};
 	};
-	const esBase = async () => {
-		const info = await imageInfo(imageFor(harness));
+	const reevalBase = async () => {
+		const { info, error } = await look(imageFor(harness), engine);
+		if (error) return {
+			info,
+			status: "unknown",
+			detail: error
+		};
 		if (!info) return {
 			info,
 			status: "build",
@@ -105,16 +155,21 @@ async function imageStages(target, root) {
 		};
 	};
 	let parent = null;
+	let unanswered = null;
 	if (sandbox.image) {
-		const found = await imageInfo(sandbox.image);
-		parent = found?.id ?? null;
+		const found = await look(sandbox.image, engine);
+		parent = found.info?.id ?? null;
+		unanswered = found.error ?? null;
 		stages.push({
 			what: "image",
 			ref: sandbox.image,
 			action: "used as is",
-			...found ? {
+			...found.error ? {
+				status: "unknown",
+				detail: found.error
+			} : found.info ? {
 				status: "ready",
-				detail: `found (${short(found.id)})`
+				detail: `found (${short(found.info.id)})`
 			} : {
 				status: "missing",
 				detail: "not found: pull or build it yourself"
@@ -122,8 +177,9 @@ async function imageStages(target, root) {
 		});
 	} else if (sandbox.from) {
 		const ref = harnessImageFor(harness, sandbox.from);
-		const status = await fromStatus(sandbox.from, ref);
-		parent = status.status === "ready" ? (await imageInfo(ref))?.id ?? null : null;
+		const status = await fromStatus(sandbox.from, ref, engine);
+		parent = status.status === "ready" ? (await look(ref, engine)).info?.id ?? null : null;
+		if (status.status === "unknown") unanswered = status.detail;
 		stages.push({
 			what: `from ${sandbox.from}`,
 			ref,
@@ -131,30 +187,32 @@ async function imageStages(target, root) {
 			...status
 		});
 	} else if (sandbox.dockerfile) {
-		const onEs = usesEsBase(sandbox.dockerfile);
-		const es = onEs ? await esBase() : null;
-		const real = dockerfileTag(sandbox.dockerfile, harness, es?.info?.id ?? null).tag;
-		const tag = onEs ? real.replace(/-[0-9a-f]{16}$/, "-<context + reeval image id>") : real;
+		const onReeval = usesReevalBase(sandbox.dockerfile);
+		const base = onReeval ? await reevalBase() : null;
+		const real = dockerfileTag(sandbox.dockerfile, harness, base?.info?.id ?? null).tag;
+		const tag = onReeval ? real.replace(/-[0-9a-f]{16}$/, "-<context + reeval image id>") : real;
 		const built = await exists(real);
 		stages.push({
 			what: `dockerfile ${projectPath(root, sandbox.dockerfile)}`,
 			ref: tag,
-			action: "docker build (layer cache), tagged by content hash",
+			action: `${engineSpec(engine).bin} build (layer cache), tagged by content hash`,
 			status: built.status,
-			detail: built.info || !onEs ? built.detail : `${built.detail} (${real})`
+			detail: built.info || !onReeval || built.status === "unknown" ? built.detail : `${built.detail} (${real})`
 		});
-		if (onEs) {
+		if (onReeval) {
 			parent = built.info?.id ?? null;
+			if (built.status === "unknown") unanswered = built.detail;
 			stages.push({
 				what: "harness",
 				ref: tag,
 				action: `inherited: FROM \${${REEVAL_BASE_ARG}}, which is ${imageFor(harness)}`,
-				status: es?.status ?? "build",
-				detail: `${imageFor(harness)} ${es?.detail ?? ""}`.trim()
+				status: base?.status ?? "build",
+				detail: base?.status === "unknown" ? base.detail : `${imageFor(harness)} ${base?.detail ?? ""}`.trim()
 			});
 		} else {
 			const baked = await exists(harnessImageFor(harness, real));
 			parent = baked.info?.id ?? null;
+			if (baked.status === "unknown") unanswered = baked.detail;
 			stages.push({
 				what: "harness",
 				ref: harnessImageFor(harness, tag),
@@ -164,33 +222,42 @@ async function imageStages(target, root) {
 			});
 		}
 	} else {
-		const es = await esBase();
-		parent = es.status === "ready" ? es.info?.id ?? null : null;
+		const base = await reevalBase();
+		parent = base.status === "ready" ? base.info?.id ?? null : null;
+		if (base.status === "unknown") unanswered = base.detail;
 		stages.push({
 			what: "base",
 			ref: imageFor(harness),
 			action: "reeval's image: built when missing or when its base changed",
-			status: es.status,
-			detail: es.detail
+			status: base.status,
+			detail: base.detail
 		});
 	}
 	const steps = installSteps(sandbox);
 	if (steps.length) {
-		const layer = parent ? await exists(installLayerTag({ id: parent }, steps)) : null;
+		const ref = parent ? installLayerTag({ id: parent }, steps) : null;
+		const layer = ref ? await exists(ref) : unanswered ? {
+			status: "unknown",
+			detail: unanswered
+		} : {
+			status: "build",
+			detail: "not built yet"
+		};
 		stages.push({
 			what: `install ${steps.length} command${steps.length === 1 ? "" : "s"}`,
-			ref: "reeval-install:<base id + commands>",
+			ref: ref ?? "reeval-install:<base id + commands>",
 			action: "build unless cached",
-			status: layer?.status ?? "build",
-			detail: layer?.detail ?? "not built yet"
+			status: layer.status,
+			detail: layer.detail
 		});
 	}
 	return stages;
 }
 /** reeval's harness images the targets need: used directly, or as a Dockerfile's `REEVAL_BASE`. */
-const esImages = (targets) => [...new Set(targets.filter(({ sandbox: s }) => !s.image && !s.from && !s.dockerfile || s.dockerfile && usesEsBase(s.dockerfile)).map((t) => t.harness))];
+const reevalImages = (targets) => [...new Set(targets.filter(({ sandbox: s }) => !s.image && !s.from && !s.dockerfile || s.dockerfile && usesReevalBase(s.dockerfile)).map((t) => t.harness))];
 const WORST = [
 	"missing",
+	"unknown",
 	"stale",
 	"build",
 	"ready"
@@ -199,13 +266,15 @@ const MARKS = {
 	ready: "✓",
 	build: "+",
 	stale: "~",
-	missing: "✗"
+	missing: "✗",
+	unknown: "?"
 };
 const WORDS = {
 	ready: "ready",
 	build: "to build",
 	stale: "stale",
-	missing: "missing"
+	missing: "missing",
+	unknown: "unchecked"
 };
 /** A target's standing: its worst step. */
 const targetStatus = (stages) => WORST.find((w) => stages.some((s) => s.status === w)) ?? "ready";
@@ -246,7 +315,7 @@ async function imageStatusLines(targets, root) {
 async function buildImages(targets, root, options) {
 	const providers = createProviders({ refresh: options.force ? "force" : "check" });
 	const prepared = targets.map((t) => ({
-		kind: "docker",
+		kind: sandboxProvider(t.sandbox),
 		key: dockerKey(t.sandbox, harnessFor(t.harness)),
 		label: t.id,
 		sandbox: t.sandbox,
@@ -265,7 +334,7 @@ async function buildImages(targets, root, options) {
 		return o.ok ? {
 			...t,
 			ok: true,
-			image: o.handle,
+			image: attemptImage(o.handle),
 			ms: o.ms
 		} : {
 			...t,
@@ -281,6 +350,6 @@ const harnessFor = (harness) => ({
 	layout: harness
 });
 //#endregion
-export { buildImages, esImages, imagePlanLines, imageStages, imageStatusLines, imageTargets };
+export { buildImages, imagePlanLines, imageStages, imageStatusLines, imageTargets, plannedImageUses, reevalImages };
 
-//# sourceMappingURL=images-BMgdpMAX.js.map
+//# sourceMappingURL=images-0On0cztu.js.map

package/dist/images-0On0cztu.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"images-0On0cztu.js","names":[],"sources":["../src/sandbox/images.ts"],"mappings":";;;;;;;;;;;AAiDA,eAAsB,iBACpB,KAIA,UAAoB,CAAC,GACrB,WACqB;CACrB,MAAM,EAAE,cAAc,MAAM,OAAO;CACnC,IAAI,UAAyB,CAAC;CAC9B,MAAM,UAAU;EACd,KAAK,KAAK,MAAM,KAAK,UAAU,GAAG,CAAC;EACnC;EACA,QAAQ,OAAO,cAAc;GAC3B,IAAI,aAAa,UAAU,UAAU,QAAQ,MAAM,UAAU,UAAU,SAAS;GAChF,UAAU,UAAU,QAAQ,QAAQ,MAAM,CAAC,EAAE,QAAQ,KAAK,IAAI;GAC9D,OAAO;EACT;CACF,CAAC;CACD,OAAO,QAAQ,SACX,QAAQ,KAAK,OAAO;EAAE,IAAI,EAAE;EAAI,SAAS,EAAE,QAAQ;EAAS,SAAS,EAAE,QAAQ;CAAQ,EAAE,IACzF,IAAI,OAAO,QAAQ,KAAK,aAAa;EAAE,IAAI;EAAI;EAAS,SAAS,IAAI,OAAO;CAAQ,EAAE;AAC5F;;AAGA,SAAgB,aAAa,MAAkB;CAC7C,MAAM,0BAAU,IAAI,IAAiD;CACrE,KAAK,MAAM,EAAE,IAAI,SAAS,GAAG,SAAS,MAAM;EAC1C,IAAI,QAAQ,SAAS,UAAU;EAC/B,MAAM,UAAU,aAAa,IAAI,OAAO;EAExC,MAAM,EAAE,QAAQ,OAAO,MAAM,YAAY,YAAY;EACrD,MAAM,MAAM,KAAK,UAAU;GAAE;GAAS;GAAQ,OAAO;IAAE;IAAO;IAAM;IAAY;GAAQ;EAAE,CAAC;EAC3F,MAAM,SAAS,QAAQ,IAAI,GAAG,KAAK;GACjC,IAAI,GAAG,UAAU,OAAO,EAAE,KAAK;GAC/B;GACA;GACA,OAAO;GACP,sBAAM,IAAI,IAAI;EAChB;EACA,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,GAAG;GACxB,OAAO,KAAK,IAAI,EAAE;GAClB,OAAO;EACT;EACA,QAAQ,IAAI,KAAK,MAAM;CACzB;CAEA,MAAM,yBAAS,IAAI,IAAoB;CACvC,OAAO,CAAC,GAAG,QAAQ,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,QAAQ;EACtD,MAAM,QAAQ,GAAG,EAAE,KAAK,EAAE,QAAQ,SAAS,SAAS,eAAe;EACnE,MAAM,KAAK,OAAO,IAAI,KAAK,KAAK,KAAK;EACrC,OAAO,IAAI,OAAO,CAAC;EACnB,OAAO;GAAE,GAAG;GAAG,IAAI,MAAM,IAAI,QAAQ,GAAG,MAAM,IAAI,EAAE;EAAG;CACzD,CAAC;AACH;AAcA,MAAM,SAAS,WAAmB,OAAO,QAAQ,YAAY,EAAE,CAAC,CAAC,MAAM,GAAG,EAAE;AAK5E,MAAM,QAAQ,KAAa,WACzB,UAAU,KAAK,MAAM,CAAC,CAAC,MACpB,UAAU,EAAE,KAAK,KACjB,WAAkB;CAAE,MAAM;CAAM,OAAO,MAAM;AAAQ,EACxD;;AAGF,eAAe,WACb,MACA,KACA,QAC2C;CAC3C,MAAM,CAAC,QAAQ,OAAO,MAAM,QAAQ,IAAI,CAAC,KAAK,KAAK,MAAM,GAAG,YAAY,MAAM,KAAA,GAAW,MAAM,CAAC,CAAC;CACjG,IAAI,OAAO,OAAO,OAAO;EAAE,QAAQ;EAAW,QAAQ,OAAO;CAAM;CACnE,IAAI,CAAC,OAAO,MAAM;EAChB,IAAI,EAAE,WAAW,MAAM,OAAO;GAAE,QAAQ;GAAS,QAAQ,MAAM,KAAK,IAAI,MAAM,IAAI,MAAM,EAAE;EAAG;EAE7F,OAAO,YAAY,IAAI,KAAK,IACxB;GAAE,QAAQ;GAAS,QAAQ,GAAG,KAAK,wBAAwB,IAAI;EAAQ,IACvE;GAAE,QAAQ;GAAW,QAAQ,oBAAoB,IAAI;EAAQ;CACnE;CACA,MAAM,MAAM,OAAO,KAAK,OAAO,MAAM;CACrC,IAAI,WAAW,KAAK,OAAO;EAAE,QAAQ;EAAS,QAAQ,UAAU,KAAK;CAAsB;CAC3F,IAAI,OAAO,QAAQ,IAAI,QACrB,OAAO;EACL,QAAQ;EACR,QAAQ,GAAG,KAAK,mBAAmB,MAAM,GAAG,EAAE,KAAK,MAAM,IAAI,MAAM,EAAE;CACvE;CAEF,OAAO;EAAE,QAAQ;EAAS,QAAQ,YAAY,MAAM,IAAI,MAAM;CAAI;AACpE;;;;;AAMA,eAAsB,YAAY,QAAqB,MAAgC;CACrF,MAAM,EAAE,SAAS,YAAY;CAC7B,MAAM,EAAE,WAAW;CACnB,MAAM,SAAkB,CAAC;CAEzB,MAAM,SAAS,OAAO,QAAgC;EACpD,MAAM,EAAE,MAAM,UAAU,MAAM,KAAK,KAAK,MAAM;EAC9C,IAAI,OAAO,OAAO;GAAE;GAAM,QAAQ;GAAW,QAAQ;EAAM;EAC3D,OAAO;GACL;GACA,QAAQ,OAAO,UAAU;GACzB,QAAQ,OAAO,UAAU,MAAM,KAAK,EAAE,EAAE,KAAK;EAC/C;CACF;CACA,MAAM,aAAa,YAA4B;EAC7C,MAAM,EAAE,MAAM,UAAU,MAAM,KAAK,SAAS,OAAO,GAAG,MAAM;EAC5D,IAAI,OAAO,OAAO;GAAE;GAAM,QAAQ;GAAW,QAAQ;EAAM;EAC3D,IAAI,CAAC,MAAM,OAAO;GAAE;GAAM,QAAQ;GAAS,QAAQ;EAAgB;EACnE,OAAO,KAAK,OAAO,MAAM,aAAa,YAAY,IAC9C;GAAE;GAAM,QAAQ;GAAS,QAAQ,UAAU,MAAM,KAAK,EAAE,EAAE;EAAG,IAC7D;GACE;GACA,QAAQ;GACR,QAAQ;EACV;CACN;CACA,IAAI,SAAwB;CAE5B,IAAI,aAA4B;CAChC,IAAI,QAAQ,OAAO;EACjB,MAAM,QAAQ,MAAM,KAAK,QAAQ,OAAO,MAAM;EAC9C,SAAS,MAAM,MAAM,MAAM;EAC3B,aAAa,MAAM,SAAS;EAC5B,OAAO,KAAK;GACV,MAAM;GACN,KAAK,QAAQ;GACb,QAAQ;GACR,GAAI,MAAM,QACN;IAAE,QAAQ;IAAW,QAAQ,MAAM;GAAM,IACzC,MAAM,OACJ;IAAE,QAAQ;IAAS,QAAQ,UAAU,MAAM,MAAM,KAAK,EAAE,EAAE;GAAG,IAC7D;IAAE,QAAQ;IAAW,QAAQ;GAAuC;EAC5E,CAAC;CACH,OAAO,IAAI,QAAQ,MAAM;EACvB,MAAM,MAAM,gBAAgB,SAAS,QAAQ,IAAI;EACjD,MAAM,SAAS,MAAM,WAAW,QAAQ,MAAM,KAAK,MAAM;EACzD,SAAS,OAAO,WAAW,WAAY,MAAM,KAAK,KAAK,MAAM,EAAA,CAAG,MAAM,MAAM,OAAQ;EACpF,IAAI,OAAO,WAAW,WAAW,aAAa,OAAO;EACrD,OAAO,KAAK;GACV,MAAM,QAAQ,QAAQ;GACtB;GACA,QACE;GACF,GAAG;EACL,CAAC;CACH,OAAO,IAAI,QAAQ,YAAY;EAC7B,MAAM,WAAW,eAAe,QAAQ,UAAU;EAClD,MAAM,OAAO,WAAW,MAAM,WAAW,IAAI;EAC7C,MAAM,OAAO,cAAc,QAAQ,YAAY,SAAS,MAAM,MAAM,MAAM,IAAI,CAAC,CAAC;EAEhF,MAAM,MAAM,WAAW,KAAK,QAAQ,kBAAkB,8BAA8B,IAAI;EACxF,MAAM,QAAQ,MAAM,OAAO,IAAI;EAC/B,OAAO,KAAK;GACV,MAAM,cAAc,YAAY,MAAM,QAAQ,UAAU;GACxD,KAAK;GACL,QAAQ,GAAG,WAAW,MAAM,CAAC,CAAC,IAAI;GAClC,QAAQ,MAAM;GACd,QACE,MAAM,QAAQ,CAAC,YAAY,MAAM,WAAW,YACxC,MAAM,SACN,GAAG,MAAM,OAAO,IAAI,KAAK;EACjC,CAAC;EACD,IAAI,UAAU;GACZ,SAAS,MAAM,MAAM,MAAM;GAC3B,IAAI,MAAM,WAAW,WAAW,aAAa,MAAM;GACnD,OAAO,KAAK;IACV,MAAM;IACN,KAAK;IACL,QAAQ,sBAAsB,gBAAgB,cAAc,SAAS,OAAO;IAC5E,QAAQ,MAAM,UAAU;IACxB,QACE,MAAM,WAAW,YACb,KAAK,SACL,GAAG,SAAS,OAAO,EAAE,GAAG,MAAM,UAAU,KAAK,KAAK;GAC1D,CAAC;EACH,OAAO;GACL,MAAM,QAAQ,MAAM,OAAO,gBAAgB,SAAS,IAAI,CAAC;GACzD,SAAS,MAAM,MAAM,MAAM;GAC3B,IAAI,MAAM,WAAW,WAAW,aAAa,MAAM;GACnD,OAAO,KAAK;IACV,MAAM;IACN,KAAK,gBAAgB,SAAS,GAAG;IACjC,QAAQ;IACR,QAAQ,MAAM;IACd,QAAQ,MAAM;GAChB,CAAC;EACH;CACF,OAAO;EACL,MAAM,OAAO,MAAM,WAAW;EAC9B,SAAS,KAAK,WAAW,UAAW,KAAK,MAAM,MAAM,OAAQ;EAC7D,IAAI,KAAK,WAAW,WAAW,aAAa,KAAK;EACjD,OAAO,KAAK;GACV,MAAM;GACN,KAAK,SAAS,OAAO;GACrB,QAAQ;GACR,QAAQ,KAAK;GACb,QAAQ,KAAK;EACf,CAAC;CACH;CACA,MAAM,QAAQ,aAAa,OAAO;CAClC,IAAI,MAAM,QAAQ;EAEhB,MAAM,MAAM,SAAS,gBAAgB,EAAE,IAAI,OAAO,GAAG,KAAK,IAAI;EAC9D,MAAM,QAA0C,MAC5C,MAAM,OAAO,GAAG,IAChB,aACE;GAAE,QAAQ;GAAW,QAAQ;EAAW,IACxC;GAAE,QAAQ;GAAS,QAAQ;EAAgB;EACjD,OAAO,KAAK;GACV,MAAM,WAAW,MAAM,OAAO,UAAU,MAAM,WAAW,IAAI,KAAK;GAClE,KAAK,OAAO;GACZ,QAAQ;GACR,QAAQ,MAAM;GACd,QAAQ,MAAM;EAChB,CAAC;CACH;CACA,OAAO;AACT;;AAGA,MAAa,gBAAgB,YAA2C,CACtE,GAAG,IAAI,IACL,QACG,QACE,EAAE,SAAS,QACT,CAAC,EAAE,SAAS,CAAC,EAAE,QAAQ,CAAC,EAAE,cAAgB,EAAE,cAAc,eAAe,EAAE,UAAU,CAC1F,CAAC,CACA,KAAK,MAAM,EAAE,OAAO,CACzB,CACF;AAEA,MAAM,QAAuB;CAAC;CAAW;CAAW;CAAS;CAAS;AAAO;AAC7E,MAAM,QAAqC;CACzC,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,SAAS;AACX;AACA,MAAM,QAAqC;CACzC,OAAO;CACP,OAAO;CACP,OAAO;CACP,SAAS;CACT,SAAS;AACX;;AAGA,MAAa,gBAAgB,WAC3B,MAAM,MAAM,MAAM,OAAO,MAAM,MAAM,EAAE,WAAW,CAAC,CAAC,KAAK;;AAG3D,eAAsB,eAAe,SAAwB,MAAiC;CAC5F,MAAM,QAAQ,CACZ,WAAW,QAAQ,OAAO,gBAAgB,QAAQ,WAAW,IAAI,KAAK,IAAI,4BAC5E;CACA,KAAK,MAAM,UAAU,SAAS;EAC5B,MAAM,OAAO,OAAO,QAAQ,MAAM,OAAO,MAAM,OAAO,OAAO,UAAU,IAAI,KAAK,QAAQ;EACxF,MAAM,KAAK,GAAG,OAAO,KAAK,MAAM;EAChC,MAAM,SAAS,MAAM,YAAY,QAAQ,IAAI;EAC7C,MAAM,QAAQ,KAAK,IAAI,GAAG,OAAO,KAAK,MAAM,EAAE,KAAK,MAAM,CAAC;EAC1D,KAAK,MAAM,KAAK,QACd,MAAM,KACJ,KAAK,MAAM,EAAE,QAAQ,GAAG,EAAE,KAAK,OAAO,KAAK,EAAE,IAAI,EAAE,IAAI,KAAK,MAAM,EAAE,QAAQ,IAAI,EAAE,OAAO,EAC3F;CAEJ;CACA,OAAO;AACT;;AAGA,eAAsB,iBAAiB,SAAwB,MAAiC;CAC9F,IAAI,CAAC,QAAQ,QAAQ,OAAO,CAAC;CAC7B,MAAM,OAAO,MAAM,QAAQ,IACzB,QAAQ,IAAI,OAAO,MAAM;EACvB,MAAM,SAAS,MAAM,YAAY,GAAG,IAAI;EACxC,MAAM,SAAS,aAAa,MAAM;EAClC,MAAM,QAAQ,OAAO,MAAM,MAAM,EAAE,WAAW,MAAM;EACpD,OAAO;GAAE;GAAG;GAAQ,QAAQ,WAAW,UAAU,KAAK,KAAK,OAAO,KAAK,IAAI,OAAO;EAAS;CAC7F,CAAC,CACH;CACA,MAAM,SAAS,MAAmB,KAAK,QAAQ,MAAM,EAAE,WAAW,CAAC,CAAC,CAAC;CACrE,MAAM,UAAU,MAAM,QAAQ,MAAM,MAAM,CAAC,CAAC,CAAC,CAC1C,QAAQ,CAAC,CACT,KAAK,MAAM,GAAG,MAAM,CAAC,EAAE,GAAG,MAAM,IAAI,CAAC,CACrC,KAAK,IAAI;CACZ,MAAM,QAAQ,KAAK,IAAI,GAAG,KAAK,KAAK,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC;CACxD,OAAO,CACL,WAAW,QAAQ,OAAO,gBAAgB,QAAQ,WAAW,IAAI,KAAK,IAAI,IAAI,WAC9E,GAAG,KAAK,KACL,MAAM,KAAK,MAAM,EAAE,QAAQ,GAAG,EAAE,EAAE,GAAG,OAAO,KAAK,EAAE,IAAI,MAAM,EAAE,UAAU,EAAE,QAC9E,CACF;AACF;;;;;AAYA,eAAsB,YACpB,SACA,MACA,SACuB;CACvB,MAAM,YAAY,gBAAgB,EAAE,SAAS,QAAQ,QAAQ,UAAU,QAAQ,CAAC;CAChF,MAAM,WAAW,QAAQ,KAAK,OAAO;EACnC,MAAM,gBAAgB,EAAE,OAAO;EAC/B,KAAK,UAAU,EAAE,SAAS,WAAW,EAAE,OAAO,CAAC;EAC/C,OAAO,EAAE;EACT,SAAS,EAAE;EACX,SAAS,WAAW,EAAE,OAAO;EAC7B,OAAO,CAAC;EACR,SAAS;CACX,EAAE;CACF,MAAM,WAAW,MAAM,WAAW,UAAU;EAC1C;EACA;EACA,KAAK,QAAQ;EACb,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;CACrD,CAAC;CACD,OAAO,QAAQ,KAAK,GAAG,MAAM;EAC3B,MAAM,IAAI,SAAS;EACnB,OAAO,EAAE,KACL;GAAE,GAAG;GAAG,IAAI;GAAM,OAAO,aAAa,EAAE,MAAM;GAAG,IAAI,EAAE;EAAG,IAC1D;GAAE,GAAG;GAAG,IAAI;GAAO,SAAS,EAAE;GAAS,IAAI,EAAE;EAAG;CACtD,CAAC;AACH;;AAGA,MAAM,cAAc,aAA8C;CAAE,MAAM;CAAQ,QAAQ;AAAQ"}

package/dist/index.d.ts

@@ -1,4 +1,5 @@
-import { $ as HarnessData, A as DirectoryOptions, At as Turn, B as EsConfig, Bt as EvalEntry, C as CompareCut, Ct as Secret, Dt as SkillSource, E as ConfigHelpers, Et as SetupStep, F as EnvVars, Ft as Wait, H as EvalOptions, I as Environment, It as WebEventType, L as EnvironmentHelpers, M as Duration, N as EgressPolicy, Nt as Verdict, P as EgressRecord, Pt as Viewport, Q as GroupOptions, R as EnvironmentInput, Rt as AttemptRecord, S as CompareAnswer, T as CompareQuestion, Tt as ServeHandle, U as ExecResult, Ut as FamilyRecord, V as EvalFn, Yt as RunRecord, _ as Cell, _t as PinnableKey, a as BackgroundHandle, at as JudgeArtifactRef, bt as ProfileLimit, c as Capture, d as CaptureOptions, dt as McpServer, f as CaptureUse, g as CaptureWindow, h as CaptureWait, ht as PairVote, i as AttemptContext, it as HostFile, jt as TurnData, k as Diff, kt as TranscriptEvent, l as CaptureCall, lt as JudgeOptions, m as CaptureUseScript, mt as PairFactor, nt as HarnessOption, ot as JudgeCallOptions, p as CaptureUseAction, rt as HistoryConfig, s as Budget, st as JudgeFailure, tt as HarnessEdit, u as CaptureInstruction, v as CellRole, vt as Profile, w as CompareOptions, wt as SecretTemplate, yt as ProfileDefaults, z as EnvironmentSpec, zt as CellRecord } from "./types-D8RLfkFP.js";
+import { $t as RunRecord, A as DirectoryOptions, At as SetupStep, B as EvalFn, Bt as WebEventType, C as CompareCut, Ct as ProfileLimit, Dt as Secret, E as ConfigHelpers, F as EnvVars, Ft as TurnData, Ht as AttemptRecord, I as Environment, L as EnvironmentHelpers, Lt as Verdict, M as Duration, N as EgressPolicy, Nt as TranscriptEvent, Ot as SecretTemplate, P as EgressRecord, Pt as Turn, R as EnvironmentInput, Rt as Viewport, S as CompareAnswer, St as ProfileDefaults, T as CompareQuestion, U as ExecResult, Ut as CellRecord, V as EvalOptions, Wt as EvalEntry, _ as Cell, _t as PairFactor, a as BackgroundHandle, at as HistoryConfig, bt as PinnableKey, c as Capture, ct as JudgeCallOptions, d as CaptureOptions, dt as JudgeOptions, et as GroupOptions, f as CaptureUse, g as CaptureWindow, h as CaptureWait, i as AttemptContext, it as HarnessOption, jt as SkillSource, k as Diff, kt as ServeHandle, l as CaptureCall, lt as JudgeFailure, m as CaptureUseScript, mt as McpServer, ot as HostFile, p as CaptureUseAction, pt as JudgeStateOptions, qt as FamilyRecord, rt as HarnessEdit, s as Budget, st as JudgeArtifactRef, tt as HarnessData, u as CaptureInstruction, v as CellRole, vt as PairVote, w as CompareOptions, wt as ReevalConfig, xt as Profile, z as EnvironmentSpec, zt as Wait } from "./types-g17bRAzt.js";
+import { i as group, r as evaluate, t as EachOptions } from "./evaluate-C1UVL21l.js";
 import { Duplex } from "node:stream";
 import { ExpectStatic, Tester } from "@vitest/expect";
 import "@ai-sdk/harness";
@@ -9,7 +10,7 @@ export declare class HarnessCapabilityError extends Error {
 }
 //#endregion
 //#region src/config.d.ts
-type ConfigInput = EsConfig | ((helpers: ConfigHelpers) => EsConfig);
+type ConfigInput = ReevalConfig | ((helpers: ConfigHelpers) => ReevalConfig);
 /**
  * The config: a plain object, or a callback that destructures the helpers it needs, e.g.
  * `defineConfig(({ secret }) => ({ env: { KEY: secret.fromEnv('KEY') } }))`.
@@ -42,6 +43,11 @@ declare module '@vitest/expect' {
     hasAssertions: () => void;
   }
   interface Matchers<R extends void | Promise<void> = void | Promise<void>, T = unknown> {
+    toBe(expected: unknown): void;
+    toEqual(expected: unknown): void;
+    toBeNull(): void;
+    toBeTruthy(): void;
+    toContain(needle: unknown): void;
     toHaveFile(path: string, pattern?: RegExp | string): Promise<void>;
     toSucceed(): Promise<void>;
     toHaveSearched(query?: RegExp | string): void;
@@ -87,39 +93,5 @@ export declare const DEFAULT_DIRECTORY_IGNORES: readonly string[];
  */
 export declare function defineEnvironment(name: string, input: EnvironmentInput): Environment;
 //#endregion
-//#region src/evaluate.d.ts
-type GroupFn = {
-  (name: string, fn: () => void, options?: GroupOptions): void;
-  (name: string, options: GroupOptions, fn: () => void): void;
-};
-export declare const group: GroupFn & {
-  skip: GroupFn;
-  only: GroupFn;
-  todo: (name: string) => void;
-};
-type EvaluateFn<F = EvalFn> = {
-  (name: string, fn: F, options?: EvalOptions): void;
-  (name: string, options: EvalOptions, fn: F): void;
-};
-/** How an `evaluate.each` treats its cases. */
-type EachOptions = {
-  /**
-   * Pool the cases into one suite: a pass rate over them and, with `compare`, one paired lift
-   * that `minLift` gates. On by default; `false` keeps them independent evals.
-   */
-  pool?: boolean;
-};
-/**
- * One eval per case, pooled into one suite:
- * `evaluate.each(cases)('adds $a and $b', async (c, { a, b }) => …)`.
- */
-declare const evalEach: <T>(cases: readonly T[], options?: EachOptions) => EvaluateFn<(c: AttemptContext, item: T) => Promise<void>>;
-export declare const evaluate: EvaluateFn & {
-  skip: EvaluateFn;
-  only: EvaluateFn;
-  todo: (name: string) => void;
-  each: typeof evalEach;
-};
-//#endregion
-export type { AttemptContext, AttemptRecord, BackgroundHandle, Budget, Capture, CaptureCall, CaptureInstruction, CaptureOptions, CaptureUse, CaptureUseAction, CaptureUseScript, CaptureWait, CaptureWindow, Cell, CellRecord, CellRole, CompareAnswer, CompareCut, CompareOptions, CompareQuestion, ConfigHelpers, Diff, DirectoryOptions, Duration, EachOptions, EgressPolicy, EgressRecord, EnvVars, Environment, EnvironmentHelpers, EnvironmentInput, EnvironmentSpec, EsConfig, EvalEntry, EvalFn, EvalOptions, ExecResult, FamilyRecord, GroupOptions, HarnessData, HarnessEdit, HarnessOption, HistoryConfig, HostFile, JudgeArtifactRef, JudgeCallOptions, JudgeFailure, JudgeOptions, McpServer, PairFactor, PairVote, PinnableKey, Profile, ProfileDefaults, ProfileLimit, RunRecord, Secret, SecretTemplate, ServeHandle, SetupStep, SkillSource, TranscriptEvent, Turn, TurnData, Verdict, Viewport, Wait, WebEventType };
+export { type AttemptContext, type AttemptRecord, type BackgroundHandle, type Budget, type Capture, type CaptureCall, type CaptureInstruction, type CaptureOptions, type CaptureUse, type CaptureUseAction, type CaptureUseScript, type CaptureWait, type CaptureWindow, type Cell, type CellRecord, type CellRole, type CompareAnswer, type CompareCut, type CompareOptions, type CompareQuestion, type ConfigHelpers, type Diff, type DirectoryOptions, type Duration, type EachOptions, type EgressPolicy, type EgressRecord, type EnvVars, type Environment, type EnvironmentHelpers, type EnvironmentInput, type EnvironmentSpec, type EvalEntry, type EvalFn, type EvalOptions, type ExecResult, type FamilyRecord, type GroupOptions, type HarnessData, type HarnessEdit, type HarnessOption, type HistoryConfig, type HostFile, type JudgeArtifactRef, type JudgeCallOptions, type JudgeFailure, type JudgeOptions, type JudgeStateOptions, type McpServer, type PairFactor, type PairVote, type PinnableKey, type Profile, type ProfileDefaults, type ProfileLimit, type ReevalConfig, type RunRecord, type Secret, type SecretTemplate, type ServeHandle, type SetupStep, type SkillSource, type TranscriptEvent, type Turn, type TurnData, type Verdict, type Viewport, type Wait, type WebEventType, evaluate, group };
 //# sourceMappingURL=index.d.ts.map