reeval 0.0.1

package/dist/assemble-B-nmz8oH.js

@@ -0,0 +1,695 @@
+import { n as isTimeout } from "./errors-CSxfJ4qu.js";
+import "./environment-Mz7wvrtD.js";
+import { i as redactSecrets } from "./redact-C4fL_bZP.js";
+import { i as evalSlug, l as sha256, n as attemptKey, s as recordFileName, t as attemptId } from "./ids-Bp6ut7ZD.js";
+import { s as judgeModelName } from "./judge-D8Uf4dA9.js";
+import { s as harnessId, u as sandboxId } from "./options-CAQhu--C.js";
+import { n as collapseDependencies, r as patchForJudge } from "./patch-BkKje4cp.js";
+import { o as round3, t as liftInterval } from "./stats-BNAHlNMD.js";
+import { n as promptTokens, r as sumUsage } from "./usage-DsYUqIUM.js";
+import { a as packageVersions, r as judgeVersion, t as FINGERPRINT_SCOPE } from "./state-DZ7eqb1F.js";
+import { dirname, join, relative, resolve, sep } from "node:path";
+import { readFile, stat } from "node:fs/promises";
+//#region src/history/sources.ts
+/** `from '…'`, `import '…'`, `import('…')` and `export … from '…'`, with a literal specifier. */
+const SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\bimport\s+)(['"])((?:\.{1,2}\/|\/)[^'"]+)\1/g;
+const CANDIDATES = (path) => {
+	return [
+		path,
+		.../\.(m?js|cjs)$/.exec(path) ? [path.replace(/\.(m?js|cjs)$/, ".ts"), path.replace(/\.(m?js|cjs)$/, ".tsx")] : [],
+		`${path}.ts`,
+		`${path}.tsx`,
+		`${path}.js`,
+		`${path}.mjs`,
+		join(path, "index.ts"),
+		join(path, "index.js")
+	];
+};
+async function isFile(path) {
+	return (await stat(path).catch(() => null))?.isFile() ?? false;
+}
+async function resolveImport(from, specifier) {
+	const base = specifier.startsWith("/") ? specifier : resolve(dirname(from), specifier);
+	for (const candidate of CANDIDATES(base)) if (await isFile(candidate)) return candidate;
+	return null;
+}
+/**
+* The eval file and every local module it imports, transitively, by path from `root` and
+* content hash, sorted by path. Only relative and absolute specifiers are followed: packages
+* (`reeval`, `node:*`, anything in `node_modules`) aren't traced.
+*/
+async function traceSources(root, file) {
+	const seen = /* @__PURE__ */ new Map();
+	const queue = [resolve(root, file)];
+	while (queue.length) {
+		const path = queue.shift();
+		if (seen.has(path) || path.split(sep).includes("node_modules")) continue;
+		const bytes = await readFile(path).catch(() => null);
+		if (!bytes) continue;
+		seen.set(path, {
+			path: relative(root, path).split(sep).join("/"),
+			sha256: sha256(bytes)
+		});
+		for (const match of bytes.toString("utf8").matchAll(SPECIFIER)) {
+			const found = await resolveImport(path, match[2]);
+			if (found && !seen.has(found)) queue.push(found);
+		}
+	}
+	return [...seen.values()].sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0);
+}
+//#endregion
+//#region src/history/assemble.ts
+/** Final messages are cut at this size in the record; `transcript.jsonl` keeps them whole. */
+const TURN_TEXT_LIMIT = 8192;
+const TURN_ERROR_LIMIT = 1024;
+/** A turn's first error event, such as the provider's "Credit balance is too low". */
+function turnError(transcript) {
+	const found = transcript?.find((e) => e.type === "error");
+	if (found?.type !== "error") return null;
+	const message = found.message.trim();
+	return message.length > TURN_ERROR_LIMIT ? `${message.slice(0, TURN_ERROR_LIMIT)}…` : message;
+}
+/** Tokens across every turn that reported them. */
+function costOf(attempts) {
+	const usage = sumUsage(attempts.map((a) => a.usage));
+	if (!usage) return {
+		inputTokens: null,
+		outputTokens: null,
+		usage: null,
+		source: "unreported"
+	};
+	return {
+		inputTokens: promptTokens(usage),
+		outputTokens: usage.output,
+		usage,
+		source: "reported"
+	};
+}
+/** A single tool output, check stream or file diff bigger than this keeps its head and tail. */
+const OUTPUT_CAP = 131072;
+const HEAD = 98304;
+const TAIL = 16384;
+/** Keeps evidence readable without letting one `cat` of a lockfile dominate the record. */
+function capText(text, cap = OUTPUT_CAP) {
+	if (text.length <= cap) return text;
+	const omitted = text.length - HEAD - TAIL;
+	return `${text.slice(0, HEAD)}\n[reeval: truncated ${omitted} of ${text.length} characters, sha256 ${sha256(text)}]\n${text.slice(-16384)}`;
+}
+const readRecord = async (path) => JSON.parse(await readFile(path, "utf8").catch(() => "null"));
+/** The engine and the worker's record can disagree, e.g. when a timeout fails the eval first. */
+function mergeOutcome(e, record) {
+	if (e.state === "skipped" && record && record.outcome !== "pass") return {
+		outcome: record.outcome,
+		error: record.error
+	};
+	if (e.state === "skipped") return {
+		outcome: "skipped",
+		error: null
+	};
+	if (e.state === "passed") return {
+		outcome: "pass",
+		error: null
+	};
+	const message = e.errors[0] ?? "Eval failed";
+	const error = record?.error ?? (isTimeout(message) ? {
+		kind: "timeout",
+		message
+	} : {
+		kind: "assertion",
+		message
+	});
+	return {
+		outcome: record?.outcome === "infra_error" ? "infra_error" : "fail",
+		error
+	};
+}
+/** Any failure fails; then infra errors; then all-skipped; else pass. */
+function worstOutcome(outcomes) {
+	if (outcomes.includes("fail")) return "fail";
+	if (outcomes.includes("infra_error")) return "infra_error";
+	if (outcomes.length > 0 && outcomes.every((o) => o === "skipped")) return "skipped";
+	return outcomes.length ? "pass" : "skipped";
+}
+const passRateOf = (outcomes) => {
+	const pass = outcomes.filter((o) => o === "pass").length;
+	const counted = pass + outcomes.filter((o) => o === "fail").length;
+	return counted ? pass / counted : null;
+};
+/** Concurrency caps change how fast a run goes, not what it measures. */
+/** Without the operational limits (`maxConcurrent`, `startTimeout`), which don't change results. */
+const withoutCaps = ({ maxConcurrent: _, startTimeout: __, ...rest }) => rest;
+const harnessModel = (h) => h.kind === "fake" ? null : h.model ?? null;
+/** A cell fingerprint: the hash of its parts, so each part can be compared on its own. */
+const fingerprintOf = (parts) => sha256(JSON.stringify(parts));
+/** `<index>:<question hash>`, which follows one verdict across runs. */
+const judgementKey = (index, question) => `${index}:${sha256(question).slice(0, 12)}`;
+/** Collects artifact files, redacting before anything is hashed. */
+var Artifacts = class {
+	files = /* @__PURE__ */ new Map();
+	add(path, content) {
+		const bytes = Buffer.from(redactSecrets(content));
+		this.files.set(path, bytes);
+		return {
+			path,
+			sha256: sha256(bytes),
+			bytes: bytes.length
+		};
+	}
+	json(path, value) {
+		return this.add(path, `${JSON.stringify(value, null, 2)}\n`);
+	}
+};
+const endOf = (startedAt, durationMs) => new Date(Date.parse(startedAt) + durationMs).toISOString();
+/** One JSON object per line: each turn's prompt, its events, then a `turn-end`. */
+function transcriptLines(record) {
+	const lines = [];
+	record.turns.forEach((turn, index) => {
+		lines.push(JSON.stringify({
+			ts: turn.startedAt,
+			atMs: 0,
+			turn: index,
+			type: "prompt",
+			text: turn.prompt
+		}));
+		for (const event of turn.transcript) {
+			if (event.type === "tool") {
+				const raw = typeof event.output === "string" ? event.output : JSON.stringify(event.output, null, 2);
+				if (raw !== void 0 && raw.length > 131072) {
+					lines.push(JSON.stringify({
+						turn: index,
+						...event,
+						output: capText(raw)
+					}));
+					continue;
+				}
+			}
+			lines.push(JSON.stringify({
+				turn: index,
+				...event
+			}));
+		}
+		lines.push(JSON.stringify({
+			ts: endOf(turn.startedAt, turn.durationMs),
+			atMs: turn.durationMs,
+			turn: index,
+			type: "turn-end",
+			status: turn.status,
+			durationMs: turn.durationMs
+		}));
+	});
+	return `${lines.join("\n")}\n`;
+}
+/** Every stream part the harness sent, for debugging the normaliser (`REEVAL_RAW_TRANSCRIPT=1`). */
+function rawLines(record) {
+	if (!record.turns.some((t) => t.raw)) return null;
+	return `${record.turns.flatMap((turn, index) => (turn.raw ?? []).map((part) => JSON.stringify({
+		turn: index,
+		...part
+	}))).join("\n")}\n`;
+}
+function activity(record) {
+	const events = record.turns.flatMap((t) => t.transcript);
+	const count = (type) => events.filter((e) => e.type === type).length;
+	const web = (type) => {
+		const n = count(type);
+		return n === 0 && record.turns.some((t) => t.unreported?.includes(type)) ? null : n;
+	};
+	const reasoningMs = events.reduce((n, e) => n + (e.type === "reasoning" ? e.durationMs ?? 0 : 0), 0);
+	return {
+		tools: count("tool"),
+		fileEdits: count("file-edit"),
+		webSearches: web("web-search"),
+		webFetches: web("web-fetch"),
+		reasoningMs
+	};
+}
+const block = (label, text) => {
+	const body = capText(text);
+	return `--- ${label}\n${body}${body.endsWith("\n") || !body ? "" : "\n"}`;
+};
+const checkLog = (c) => `$ ${c.command}\nexit ${c.exitCode}\n${block("stdout", c.stdout)}${block("stderr", c.stderr)}`;
+/** Dependency and binary noise dropped, and each remaining file's diff capped. */
+function historyPatch(patch) {
+	return patchForJudge(patch).split(/(?=^diff --git )/m).map((part) => capText(part)).join("");
+}
+function changedFiles(files) {
+	const { kept, dirs } = collapseDependencies(files, (f) => f.path);
+	return [...kept, ...dirs.map((d) => ({
+		path: d.dir,
+		status: "added",
+		count: d.count
+	}))];
+}
+/** Judge inputs whose diff is the recorded diff point at `diff.patch` instead of repeating it. */
+function judgeInputs(record, diff, diffPath) {
+	return (record.judgeInputs ?? []).map((input) => {
+		if (diff === null || input.state.diff !== diff) return input;
+		return {
+			...input,
+			state: {
+				...input.state,
+				diff: {
+					$ref: diffPath,
+					sha256: sha256(diff)
+				}
+			}
+		};
+	});
+}
+/** What the run's attempts have in common, gathered while their records are read. */
+var Seen = class {
+	sources = /* @__PURE__ */ new Map();
+	byEval = /* @__PURE__ */ new Map();
+	contexts = /* @__PURE__ */ new Map();
+	resolvedJudges = /* @__PURE__ */ new Set();
+	images = /* @__PURE__ */ new Map();
+	environments = /* @__PURE__ */ new Map();
+	harnesses = /* @__PURE__ */ new Map();
+	resolvedHarnesses = /* @__PURE__ */ new Map();
+	/** Registry digests found by the preflight, by image id. */
+	digests;
+	constructor(sandbox) {
+		this.digests = new Map(sandbox.images.map((i) => [i.id, i.repoDigests]));
+	}
+	add(record, cell, e) {
+		for (const c of record?.context ?? []) this.contexts.set(`${c.path}\0${c.sha256}`, c);
+		for (const j of record?.judgements ?? []) if (j.resolvedModel) this.resolvedJudges.add(j.resolvedModel);
+		if (record?.image) {
+			const repoDigests = this.digests.get(record.image.id) ?? [];
+			this.images.set(record.image.id, {
+				...record.image,
+				repoDigests
+			});
+		}
+		if (record?.environment && record.environment.name !== "-") {
+			const key = `${record.environment.name}\0${record.environment.digest}`;
+			this.environments.set(key, record.environment);
+		}
+		if (e.attempt) {
+			const h = e.attempt.harness;
+			this.harnesses.set(cell.harness, {
+				id: cell.harness,
+				kind: h.kind,
+				model: harnessModel(h)
+			});
+		}
+		const resolved = this.resolvedHarnesses.get(cell.harness) ?? /* @__PURE__ */ new Set();
+		for (const t of record?.turns ?? []) if (t.model) resolved.add(t.model);
+		this.resolvedHarnesses.set(cell.harness, resolved);
+	}
+	entry(e, sources) {
+		let entry = this.byEval.get(e.id);
+		if (!entry) {
+			const key = e.id;
+			entry = {
+				id: e.id,
+				key,
+				slug: evalSlug(key),
+				file: e.file,
+				group: e.group,
+				name: e.name,
+				tags: e.attempt?.tags ?? [],
+				fingerprint: "",
+				sources,
+				outcome: "skipped",
+				todo: e.todo === true,
+				passRate: null,
+				cells: [],
+				attempts: []
+			};
+			this.byEval.set(e.id, entry);
+		}
+		return entry;
+	}
+};
+const UNKNOWN_CELL = {
+	key: "unknown",
+	harness: "unknown",
+	environment: "unknown",
+	sandbox: "unknown",
+	role: "control"
+};
+/** minLift per cell, kept aside while attempts are added; see `rollUp`. */
+const minLifts = /* @__PURE__ */ new WeakMap();
+const EMPTY_PARTS = {
+	sources: "",
+	harness: "",
+	environment: null,
+	image: null,
+	agentCli: null,
+	judge: "",
+	context: ""
+};
+function cellOf(entry, cell, gate, harness) {
+	let found = entry.cells.find((c) => c.key === cell.key);
+	if (!found) {
+		found = {
+			...cell,
+			fingerprint: "",
+			fingerprintParts: EMPTY_PARTS,
+			environmentDigest: null,
+			harnessKind: harness?.kind ?? "unknown",
+			model: harness ? harnessModel(harness) : null,
+			gate,
+			overrides: [],
+			runs: 0,
+			pass: 0,
+			fail: 0,
+			infraError: 0,
+			skipped: 0,
+			passRate: null,
+			outcome: "skipped"
+		};
+		entry.cells.push(found);
+	}
+	return found;
+}
+/** Reads one attempt's worker record, files its artifacts and adds it to its eval and cell. */
+async function addAttempt(e, ctx, seen, out, judge) {
+	const cell = e.attempt?.cell ?? UNKNOWN_CELL;
+	const n = e.attempt?.n ?? 1;
+	const record = await readRecord(join(ctx.stagingDir, "evals", recordFileName(e.key)));
+	const { outcome, error } = mergeOutcome(e, record);
+	if (!seen.sources.has(e.file)) seen.sources.set(e.file, await traceSources(ctx.config.root, e.file));
+	const sources = seen.sources.get(e.file) ?? [];
+	const context = record?.context ?? [];
+	seen.add(record, cell, e);
+	const entry = seen.entry(e, sources);
+	const cellRecord = cellOf(entry, cell, e.attempt?.gate ?? "all", e.attempt?.harness);
+	if (e.attempt?.minLift !== void 0) minLifts.set(cellRecord, e.attempt.minLift);
+	if (record?.environment?.overrides.length && cellRecord.overrides.length === 0) cellRecord.overrides = [...record.environment.overrides];
+	const environment = record?.environment;
+	if (environment && environment.name !== "-") cellRecord.environmentDigest = environment.digest;
+	if (!cellRecord.fingerprint || record) {
+		const parts = {
+			sources: sha256(JSON.stringify(sources)),
+			harness: sha256(JSON.stringify(e.attempt ? withoutCaps(e.attempt.harness) : null)),
+			environment: environment?.digest ?? null,
+			image: record?.image?.id ?? null,
+			agentCli: record?.image?.agentCli ?? null,
+			judge,
+			context: sha256(JSON.stringify(context))
+		};
+		cellRecord.fingerprintParts = parts;
+		cellRecord.fingerprint = fingerprintOf(parts);
+	}
+	entry.attempts.push({
+		key: attemptKey(entry.key, cell.key, n),
+		id: attemptId(cell.key, n),
+		cell: cell.key,
+		n,
+		seq: null,
+		startedAt: record?.startedAt ?? null,
+		imageId: record?.image?.id ?? null,
+		env: record?.env ?? {
+			vars: [],
+			secrets: [],
+			brokered: []
+		},
+		context,
+		egress: record?.egress ?? null,
+		servedModels: record?.servedModels ?? null,
+		outcome,
+		error,
+		durationMs: e.durationMs,
+		...attemptEvidence(record, `${entry.slug}/${evalSlug(cell.key)}/${n}`, out)
+	});
+}
+/** Turns, timings, activity, verdicts, checks and changed files, with their artifact files. */
+function attemptEvidence(record, dir, out) {
+	const files = {};
+	if (record?.setup?.length) {
+		const log = record.setup.map((s) => `$ ${s.run}\n${s.exitCode === null ? "timed out" : `exit ${s.exitCode}`} after ${s.durationMs}ms\n${capText(s.output)}${s.output.endsWith("\n") || !s.output ? "" : "\n"}`).join("\n");
+		files.setupLog = out.add(`${dir}/setup.log`, log);
+	}
+	if (record && record.turns.length > 0) {
+		files.transcript = out.add(`${dir}/transcript.jsonl`, transcriptLines(record));
+		const raw = rawLines(record);
+		if (raw) files.rawTranscript = out.add(`${dir}/transcript.raw.jsonl`, raw);
+	}
+	const patch = record?.diff ? historyPatch(record.diff.patch) : null;
+	if (patch !== null) files.diff = out.add(`${dir}/diff.patch`, patch);
+	if (record?.judgeInputs?.length) files.judgeInput = out.json(`${dir}/judge-input.json`, judgeInputs(record, patch, "diff.patch"));
+	const judgements = (record?.judgements ?? []).map((j, i) => ({
+		...j,
+		key: judgementKey(i, j.question)
+	}));
+	return {
+		turns: (record?.turns ?? []).map((t) => ({
+			prompt: t.prompt,
+			status: t.status,
+			startedAt: t.startedAt,
+			durationMs: t.durationMs,
+			usage: t.usage ?? null,
+			model: t.model ?? null,
+			unreported: t.unreported ?? [],
+			error: turnError(t.transcript),
+			text: t.text.length > TURN_TEXT_LIMIT ? `${t.text.slice(0, TURN_TEXT_LIMIT)}… [reeval: truncated ${t.text.length} characters; the whole message is the turn's last text event in transcript.jsonl]` : t.text
+		})),
+		usage: sumUsage((record?.turns ?? []).map((t) => t.usage)),
+		timings: record?.timings ?? {
+			setupMs: 0,
+			agentMs: 0,
+			diffMs: 0,
+			checksMs: 0,
+			judgeMs: 0
+		},
+		activity: record ? activity(record) : {
+			tools: 0,
+			fileEdits: 0,
+			webSearches: 0,
+			webFetches: 0,
+			reasoningMs: 0
+		},
+		judgements,
+		...record?.startRetries?.length ? { startRetries: record.startRetries } : {},
+		checks: (record?.checks ?? []).map((c, i) => ({
+			command: c.command,
+			exitCode: c.exitCode,
+			durationMs: c.durationMs,
+			log: out.add(`${dir}/checks/${i + 1}.log`, checkLog(c))
+		})),
+		changedFiles: changedFiles(record?.diff?.files ?? []),
+		files
+	};
+}
+/** Lift of a variant cell over the control on the same harness and sandbox, with its interval. */
+function liftOf(control, variant) {
+	if (control.passRate === null || variant.passRate === null) return void 0;
+	const minLift = minLifts.get(variant);
+	const c = {
+		pass: control.pass,
+		counted: control.pass + control.fail
+	};
+	const v = {
+		pass: variant.pass,
+		counted: variant.pass + variant.fail
+	};
+	const { points, low, high } = liftInterval({
+		pass: v.pass,
+		n: v.counted
+	}, {
+		pass: c.pass,
+		n: c.counted
+	});
+	return {
+		vs: control.key,
+		control: c,
+		variant: v,
+		points: round3(points),
+		interval: {
+			lo: round3(low),
+			hi: round3(high),
+			level: .95,
+			method: "newcombe-wilson"
+		},
+		withinNoise: low < 0 && high > 0,
+		...minLift !== void 0 ? { minLift } : {}
+	};
+}
+/**
+* A cell's outcome by its gate. `'all'` and `'none'` take the worst attempt; `{ passRate }` passes
+* when enough of the passes and fails are passes, and is an infra error only when nothing counted.
+*/
+function cellOutcome(outcomes, gate) {
+	if (gate === "all" || gate === "none") return worstOutcome(outcomes);
+	const rate = passRateOf(outcomes);
+	if (rate === null) return worstOutcome(outcomes);
+	return rate >= gate.passRate ? "pass" : "fail";
+}
+/** Per-cell and per-eval counts, pass rates, lift, outcomes and fingerprints, once every attempt is in. */
+function rollUp(evals) {
+	for (const entry of evals) {
+		for (const c of entry.cells) {
+			const outcomes = entry.attempts.filter((a) => a.cell === c.key).map((a) => a.outcome);
+			c.runs = outcomes.length;
+			c.pass = outcomes.filter((o) => o === "pass").length;
+			c.fail = outcomes.filter((o) => o === "fail").length;
+			c.infraError = outcomes.filter((o) => o === "infra_error").length;
+			c.skipped = outcomes.filter((o) => o === "skipped").length;
+			c.passRate = passRateOf(outcomes);
+			c.outcome = cellOutcome(outcomes, c.gate);
+		}
+		for (const c of entry.cells) {
+			if (c.role !== "variant") continue;
+			const control = entry.cells.find((x) => x.role === "control" && x.harness === c.harness && x.sandbox === c.sandbox);
+			const lift = control && liftOf(control, c);
+			if (!lift) continue;
+			c.lift = lift;
+			if (lift.minLift !== void 0 && lift.interval.lo < lift.minLift && c.outcome === "pass") c.outcome = "fail";
+		}
+		entry.outcome = worstOutcome(entry.cells.map((c) => c.gate === "none" && c.outcome === "fail" ? "pass" : c.outcome));
+		const gated = new Set(entry.cells.filter((c) => c.gate !== "none").map((c) => c.key));
+		entry.passRate = passRateOf(entry.attempts.filter((a) => gated.has(a.cell)).map((a) => a.outcome));
+		entry.fingerprint = sha256(JSON.stringify(entry.cells.map((c) => [c.key, c.fingerprint]).sort()));
+	}
+}
+/** Start order across the run, by when each attempt's worker began it. */
+function sequence(evals) {
+	evals.flatMap((e) => e.attempts).filter((a) => a.startedAt !== null).sort((a, b) => a.startedAt < b.startedAt ? -1 : a.startedAt > b.startedAt ? 1 : a.key < b.key ? -1 : 1).forEach((a, i) => {
+		a.seq = i + 1;
+	});
+}
+/** The run's git state, with the uncommitted changes kept as `worktree.patch` when they fit. */
+function gitState(input, out) {
+	const git = input.git;
+	if (!git?.dirty) return git;
+	const patch = input.worktreePatch;
+	const state = {
+		...git,
+		diffHash: patch ? sha256(patch) : null
+	};
+	if (!patch) state.worktreePatchNote = "git could not produce a patch";
+	else if (patch.length > 5242880) state.worktreePatchNote = `too large to keep (${patch.length} bytes)`;
+	else {
+		out.files.set("worktree.patch", patch);
+		state.worktreePatch = {
+			path: "worktree.patch",
+			sha256: sha256(patch),
+			bytes: patch.length
+		};
+	}
+	return state;
+}
+const byName = (key) => (a, b) => key(a) < key(b) ? -1 : key(a) > key(b) ? 1 : 0;
+function stackOf(input, seen, judgeModel) {
+	const { config } = input.ctx;
+	const packages = packageVersions(input.enginePackages);
+	return {
+		es: input.es,
+		node: process.versions.node,
+		os: process.platform,
+		arch: process.arch,
+		images: [...seen.images.values()].sort(byName((i) => i.ref)),
+		packages,
+		harnesses: [...seen.harnesses.values()].map((h) => ({
+			...h,
+			resolvedModels: [...seen.resolvedHarnesses.get(h.id) ?? []].sort()
+		})).sort(byName((h) => h.id)),
+		judge: {
+			model: judgeModel,
+			resolvedModels: [...seen.resolvedJudges].sort(),
+			threshold: config.judge.threshold,
+			version: judgeVersion(input.es, packages, judgeModel ?? "")
+		},
+		context: [...seen.contexts.values()].sort(byName((c) => c.path)),
+		fingerprintScope: FINGERPRINT_SCOPE
+	};
+}
+async function assembleRun(input) {
+	const { ctx, engine } = input;
+	const { config, runId } = ctx;
+	const out = new Artifacts();
+	const judgeModel = config.judge.model === void 0 ? null : judgeModelName(config.judge.model);
+	const configHash = sha256(JSON.stringify({
+		include: config.include,
+		harness: config.harness.map(withoutCaps),
+		sandbox: withoutCaps(config.sandbox),
+		environment: config.environment ?? null,
+		runs: config.runs,
+		env: config.env,
+		judge: withoutCaps(config.judge),
+		timeoutMs: config.timeoutMs
+	}));
+	const seen = new Seen(input.sandbox);
+	const judge = sha256(JSON.stringify({
+		model: judgeModel,
+		threshold: config.judge.threshold,
+		version: judgeVersion(input.es, packageVersions(input.enginePackages), judgeModel ?? "")
+	}));
+	for (const e of engine.evals) await addAttempt(e, ctx, seen, out, judge);
+	const evals = [...seen.byEval.values()];
+	rollUp(evals);
+	sequence(evals);
+	const attempts = evals.flatMap((e) => e.attempts);
+	const count = (o) => attempts.filter((a) => a.outcome === o).length;
+	const notGated = evals.flatMap((e) => e.attempts.filter((a) => e.cells.find((c) => c.key === a.cell)?.gate === "none")).filter((a) => a.outcome === "fail").length;
+	const { profile } = ctx;
+	const ids = (list) => list === void 0 ? void 0 : [list].flat().map(harnessId);
+	return {
+		record: {
+			kind: "es-run",
+			schemaVersion: 1,
+			runId,
+			status: input.interrupted ? "interrupted" : input.budgetReached ? "budget" : "complete",
+			startedAt: input.startedAt.toISOString(),
+			endedAt: input.endedAt.toISOString(),
+			durationMs: input.endedAt.getTime() - input.startedAt.getTime(),
+			author: input.author,
+			git: gitState(input, out),
+			runner: input.runner,
+			dependencies: { lockfiles: input.lockfiles },
+			stack: stackOf(input, seen, judgeModel),
+			environments: [...seen.environments.values()].sort(byName((e) => e.name)),
+			profile: profile ? {
+				name: profile.name,
+				reason: profile.reason,
+				defaults: JSON.parse(JSON.stringify({
+					harness: ids(profile.defaults.harness),
+					environment: profile.defaults.environment?.name,
+					sandbox: profile.defaults.sandbox && sandboxId(profile.defaults.sandbox),
+					runs: profile.defaults.runs,
+					timeout: profile.defaults.timeout,
+					env: profile.defaults.env && Object.keys(profile.defaults.env).sort()
+				})),
+				limit: profile.limit
+			} : null,
+			plan: {
+				evals: input.plan.evals,
+				cells: input.plan.cells,
+				attempts: input.plan.attempts,
+				estimateMs: Math.round(input.plan.estimateMs),
+				sandboxWarnings: input.plan.sandboxWarnings
+			},
+			budget: Object.keys(input.plan.budget).length ? input.plan.budget : null,
+			config: {
+				sandbox: sandboxId(config.sandbox),
+				environment: config.environment?.name ?? null,
+				runs: config.runs,
+				judgeModel,
+				threshold: config.judge.threshold,
+				timeoutMs: config.timeoutMs,
+				concurrency: input.ctx.concurrency,
+				configHash
+			},
+			cost: costOf(attempts),
+			artifacts: "all",
+			redaction: { version: 1 },
+			summary: {
+				evals: evals.length,
+				attempts: attempts.length,
+				pass: count("pass"),
+				fail: count("fail"),
+				notGated,
+				infraError: count("infra_error"),
+				skipped: count("skipped")
+			},
+			evals
+		},
+		artifacts: out.files
+	};
+}
+//#endregion
+export { fingerprintOf as a, worstOutcome as c, cellOutcome as i, assembleRun as n, judgementKey as o, capText as r, transcriptLines as s, OUTPUT_CAP as t };
+
+//# sourceMappingURL=assemble-B-nmz8oH.js.map