reeval 0.9.0 → 0.11.0

package/dist/options-C0d0AzeR.js

@@ -0,0 +1,1118 @@
+import { t as isObjectLike } from "./object-CLf2-tNq.js";
+import { n as UsageError } from "./errors-B6BSMs03.js";
+import { c as jsonValueSchema, d as refusal, l as nonEmpty, p as parseDuration, t as closest, u as oneOf } from "./shared-CA61BuPJ.js";
+import { f as FACTORY, m as harnessProblem, p as harnessDescriptorSchema, u as opencodeModelError } from "./harness-CEuntrla.js";
+import { A as workspaceLabel, L as gateProblem, T as sandboxOf, _ as isEnvironment, i as applied, j as workspaceOf, k as unbrokeredSecrets, l as emptyEnvironment, o as checkEgress, r as EMPTY_BASELINE, s as checkEnv, u as environmentDataProblem, w as plannedSecrets, y as isMovingRef, z as sandboxProblem } from "./environment-Cca6a4wB.js";
+import { l as sha256 } from "./ids-Bp6ut7ZD.js";
+import { i as checkModelString } from "./judge-DW_GDOmB.js";
+import { g as isAgentKind, i as plannedEgress, l as HARNESS_EXAMPLE, r as egressLabel } from "./egress-2-qzPvGi.js";
+import { a as showsNothing, n as mergeQuestion, o as withQuestionDefaults } from "./state-CeR6oz0k.js";
+import { z } from "zod";
+import { createHash } from "node:crypto";
+//#region src/contract/compare.ts
+const pairFactorSchema = z.enum(["environment", "harness"]).describe("A matrix factor a compare question varies when pairing; the others are held.");
+const compareGateSchema = z.enum([
+	"variants",
+	"all",
+	"none"
+]);
+/** Evaluation models score yes/no questions only, so they can't pick between two attempts. */
+const isEvaluationModel = (model) => model.startsWith("typesafe-ai/");
+const questionStateSchema = z.strictObject({
+	task: z.boolean().optional().describe("The prompts the attempt was sent, once for both sides when they match. Default true."),
+	output: z.boolean().optional().describe("The final message: the agent's own account of its work. Default false."),
+	files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default false."),
+	diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
+	transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. It names each harness's tools, so a pair of two harnesses isn't blind. Default false."),
+	turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn under `turns`, each with its task, output and (with `transcript`) transcript; `'last'`: the last turn only. Default `'all'`."),
+	checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results; a list shows only those commands, as passed to `c.exec`. Default false.")
+}).describe("What each side of a compare question shows: the keys of `judge.state` but `contents` (the attempts' workspaces are gone when questions are asked). An unset key keeps its default, which is what a question shows with no `state` at all: the task and the diff.");
+const questionArtifactsSchema = z.union([z.literal(true), z.array(nonEmpty()).min(1)]);
+const questionModelSchema = z.string().refine((m) => m.includes("/"), { params: { provider: true } }).refine((m) => !isEvaluationModel(m), { params: { evaluation: true } });
+const questionOptionsSchema = z.strictObject({
+	model: questionModelSchema.optional().describe("`'provider/model'` that answers every question below it without its own `model`."),
+	state: questionStateSchema.optional().describe("What each side shows."),
+	artifacts: questionArtifactsSchema.optional().describe("Each side's own captures, by name; `true` sends all of them. Default none.")
+}).describe("Defaults for every compare question: `model` and `artifacts` replaced, `state` merged key by key.");
+const questionObjectSchema = z.strictObject({
+	question: nonEmpty(),
+	model: questionModelSchema.optional().describe("`'provider/model'` that answers it, over the cascaded `question.model`."),
+	across: pairFactorSchema.optional().describe("The factor it varies when pairing."),
+	state: questionStateSchema.optional().describe("What each side shows, over the cascaded `question.state`, key by key."),
+	artifacts: questionArtifactsSchema.optional().describe("Each side's own captures, by name; `true` sends all of them. Replaces `question.artifacts`.")
+});
+const compareQuestionSchema = z.union([nonEmpty(), questionObjectSchema]).describe("A question for pairwise judgement: its text, or the text and the `'provider/model'` that answers it. Without a model, the cascaded `question.model` answers, else `analysis.model`, else the judge's when it's a language model. `across` is the factor it varies when pairing; without it (and without a one-factor `compare.across`) both pair kinds run. `state` and `artifacts` choose what each side shows.");
+/** The first label two items share. */
+const twice = (labels) => labels.find((k, i) => labels.indexOf(k) !== i);
+function unique(label) {
+	return (list, ctx) => {
+		if (twice(list.map(label)) !== void 0) ctx.addIssue({
+			code: "custom",
+			message: "twice"
+		});
+	};
+}
+const questionText = (q) => typeof q === "string" ? q : q.question;
+/** The compare object, with environments checked by `environment`. */
+const compareSchema = (environment) => z.strictObject({
+	baseline: environment.optional(),
+	variants: z.array(environment).min(1),
+	gate: compareGateSchema.optional(),
+	minLift: z.number().min(-1).max(1).optional(),
+	harness: z.array(z.union([z.string(), harnessDescriptorSchema])).optional(),
+	questions: z.array(compareQuestionSchema).min(1).superRefine(unique(questionText)).optional(),
+	across: z.union([pairFactorSchema, z.array(pairFactorSchema).min(1).superRefine(unique(String))]).optional()
+});
+compareSchema(z.custom());
+const QUESTION_EXAMPLE = `questions: ['Which handles state more cleanly?']`;
+const ACROSS_HINT = `'environment' (vary the environment, same harness) or 'harness' (vary the harness, same environment)`;
+const ACROSS_MEAN = " Did you mean \"across\"? It is the factor you vary: across: 'environment'.";
+const QUESTION_KEYS = Object.keys(questionObjectSchema.shape);
+const QUESTION_SHAPE = "{ question, model?, across?, state?, artifacts? }";
+const STATE_KEYS$1 = Object.keys(questionStateSchema.shape);
+const STATE_VALUES = {
+	task: "true or false",
+	output: "true or false",
+	files: "true or false",
+	diff: "true, false or a non-empty list of patterns",
+	transcript: "true, false, 'tools' or 'full'",
+	turns: "'last' or 'all'",
+	checks: "true, false or a list of commands"
+};
+const describe$1 = (value) => {
+	if (Array.isArray(value)) return `a list of ${value.length}`;
+	if (typeof value === "object" && value !== null) {
+		const keys = Object.keys(value);
+		return keys.length ? `an object with ${keys.map((k) => `"${k}"`).join(", ")}` : "an object";
+	}
+	return JSON.stringify(value) ?? String(value);
+};
+/**
+* What's wrong with a question's `state` or `artifacts`, as one sentence, or undefined. `name`
+* turns a key into how the sentence names it: `question.state` on the option,
+* `compare.questions "…" state` on an entry.
+*/
+function questionPartsProblem(value, name, where) {
+	const { state, artifacts } = value;
+	if (state !== void 0) {
+		if (!isObjectLike(state) || Array.isArray(state)) return `${name("state")} in ${where} must be an object such as { transcript: 'tools' } (got ${describe$1(state)})`;
+		for (const [key, v] of Object.entries(state)) {
+			if (key === "contents") return `${name("state.contents")} in ${where} can't be set: questions are asked after the attempts, when their workspaces are gone. Narrow state.diff instead.`;
+			const schema = questionStateSchema.shape[key];
+			if (!schema) {
+				const hint = closest(key, STATE_KEYS$1);
+				return `Unknown ${name("state")} key "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""}`;
+			}
+			if (!schema.safeParse(v).success) return `${name(`state.${key}`)} in ${where} must be ${STATE_VALUES[key]} (got ${describe$1(v)})`;
+		}
+	}
+	if (artifacts !== void 0 && !questionArtifactsSchema.safeParse(artifacts).success) return `${name("artifacts")} in ${where} must be true or a non-empty list of capture names (got ${describe$1(artifacts)})`;
+}
+/** What's wrong with the `question` option set in `where`, as one sentence, or undefined. */
+function questionOptionsProblem(value, where) {
+	if (!isObjectLike(value) || Array.isArray(value)) return `question in ${where} is { model?, state?, artifacts? } (got ${describe$1(value)}), e.g. question: { model: 'google/gemini-3.8-flash' }`;
+	const keys = Object.keys(questionOptionsSchema.shape);
+	for (const key of Object.keys(value)) {
+		if (keys.includes(key)) continue;
+		if (key === "across") return `Unknown question option "across" in ${where}. across goes on each compare.questions entry, { question, across }, or on compare.across.`;
+		const hint = closest(key, keys);
+		return `Unknown question option "${key}" in ${where}.${hint ? ` Did you mean "${hint}"?` : ""} question takes model, state and artifacts.`;
+	}
+	const { model } = value;
+	if (model !== void 0 && typeof model === "string" && isEvaluationModel(model)) return `question.model in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't pick between two attempts. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
+	if (model !== void 0 && !questionModelSchema.safeParse(model).success) return `question.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$1(model)})`;
+	return questionPartsProblem(value, (key) => `question.${key}`, where);
+}
+/** One question's sentence: the union refuses it whole, so the branch it meant says why. */
+function questionProblem(item, where) {
+	if (typeof item === "string") return `compare.questions in ${where} is an empty string. Ask something, e.g. ${QUESTION_EXAMPLE}.`;
+	if (typeof item !== "object" || item === null || Array.isArray(item)) return `compare.questions in ${where} takes strings or ${QUESTION_SHAPE} (got ${describe$1(item)}), e.g. ${QUESTION_EXAMPLE}.`;
+	const unknown = Object.keys(item).filter((k) => !QUESTION_KEYS.includes(k));
+	if (unknown.length) {
+		const hint = unknown.some((k) => k === "cut" || k === "cuts") ? ACROSS_MEAN : "";
+		return `compare.questions in ${where} takes ${QUESTION_SHAPE} (got ${unknown.map((k) => `"${k}"`).join(", ")}).${hint}`;
+	}
+	const { question, model, across } = item;
+	const issues = questionObjectSchema.safeParse(item).error?.issues ?? [];
+	const at = (key) => issues.find((i) => i.path[0] === key);
+	if (at("question")) return `compare.questions in ${where} has an entry without its question${question === void 0 ? "" : ` (got ${describe$1(question)})`}, e.g. { question: 'Which is cleaner?', model: 'google/gemini-3.8-flash' }.`;
+	if (at("across")) return `compare.questions "${question}" in ${where}: across is ${ACROSS_HINT} (got ${describe$1(across)}). Leave it out to vary both.`;
+	const parts = questionPartsProblem(item, (key) => `compare.questions "${question}" ${key}`, where);
+	if (parts) return parts;
+	if (at("model")?.params?.evaluation) return `compare.questions "${question}" in ${where} names ${model}, an evaluation model: it scores yes/no questions and can't pick between two attempts. Use a language model, e.g. 'google/gemini-3.8-flash'.`;
+	return `compare.questions "${question}" in ${where}: model is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${describe$1(model)})`;
+}
+function explain(where) {
+	return (issue, value) => {
+		const spec = value;
+		if (issue.code === "unrecognized_keys" && issue.path.length === 0) {
+			const key = issue.keys[0];
+			return `Unknown compare option "${key}" in ${where}${key === "question" ? `. Did you mean "questions"? It's a list: ${QUESTION_EXAMPLE}.` : key === "cut" || key === "cuts" ? `.${ACROSS_MEAN}` : `. compare takes ${Object.keys(compareSchema(z.any()).shape).join(", ")}.`}`;
+		}
+		const [key, index] = issue.path;
+		const v = spec[key];
+		switch (key) {
+			case "questions":
+				if (issue.code === "custom" && index === void 0) return `compare.questions in ${where} asks "${twice(v.map(questionText))}" twice. Ask each question once.`;
+				if (index !== void 0) return questionProblem(v[index], where);
+				return Array.isArray(v) ? `compare.questions in ${where} is an empty list. Ask at least one, e.g. ${QUESTION_EXAMPLE}, or leave questions out.` : `compare.questions in ${where} is a list (got ${describe$1(v)}), e.g. ${QUESTION_EXAMPLE}.`;
+			case "across": {
+				const list = Array.isArray(v) ? v : [v];
+				if (list.length === 0) return `compare.across in ${where} is ${ACROSS_HINT}, or a list of those (got ${describe$1(v)}). Leave it out to vary both.`;
+				const bad = list.findIndex((f) => !pairFactorSchema.safeParse(f).success);
+				return bad === -1 ? `compare.across in ${where} lists "${twice(list)}" twice. List each factor once.` : `compare.across in ${where} is ${ACROSS_HINT}, or a list of those (got ${describe$1(list[bad])}).`;
+			}
+			case "baseline": return `compare.baseline in ${where} must be an environment from defineEnvironment (got ${describe$1(v)}). It is the control; variants are not layered on it or on the eval's environment. Leave it out to use the eval's environment as the control.`;
+			case "gate": return `compare.gate in ${where} is ${oneOf(compareGateSchema.options)} (got ${describe$1(v)})`;
+			case "minLift": return `compare.minLift in ${where} is points from -1 to 1, e.g. minLift: 0.1 (got ${describe$1(v)})`;
+			case "harness": {
+				const item = index === void 0 ? void 0 : v[index];
+				const kind = item?.kind;
+				const problem = kind && kind in FACTORY ? harnessProblem(item) : void 0;
+				return problem ? `compare.harness in ${where}: ${problem}` : `compare.harness in ${where} must be a list of harness ids, e.g. harness: ['cursor:composer-2.5'] (got ${describe$1(index === void 0 ? v : item)})`;
+			}
+			default: return index === void 0 ? `compare has no variants in ${where}. Give at least one environment.` : `compare.variants in ${where} must be environments from defineEnvironment (got ${describe$1(v[index])}). Variants share the eval's prompt and checks; if a variant needs different checks, write a separate eval.`;
+		}
+	};
+}
+/** The order the hand-written checks ran in, so the first refusal stays the same. */
+const ORDER = [
+	"questions",
+	"across",
+	"baseline",
+	"gate",
+	"minLift",
+	"harness",
+	"variants"
+];
+/**
+* What's wrong with a compare object whose environments `environment` checks, or undefined.
+* `where` names the group or eval it's set on.
+*/
+function compareProblem(value, where, environment) {
+	if (Array.isArray(value)) return `compare in ${where} must be an object, not a list. Use compare: { baseline, variants: [...] }.`;
+	if (!isObjectLike(value) || !Array.isArray(value.variants)) return `compare in ${where} takes { variants, baseline? } (got ${describe$1(value)}). Use compare: { variants: [a, b], baseline?, gate?, minLift?, harness?, questions?, across? }.`;
+	return refusal(compareSchema(environment), value, explain(where), { order: ORDER });
+}
+//#endregion
+//#region src/judge/artifacts.ts
+/** A capture handle: a name plus the file the attempt wrote. */
+function isCaptureHandle(value) {
+	return isObjectLike(value) && typeof value.name === "string" && isObjectLike(value.file) && typeof value.file.path === "string";
+}
+/** The first `artifacts` value that isn't `true` or a list of capture handles or names. */
+function badJudgeArtifacts(artifacts) {
+	if (artifacts === void 0 || artifacts === true) return void 0;
+	if (!Array.isArray(artifacts)) return `must be true or a list of capture handles or names (got ${JSON.stringify(artifacts)})`;
+	for (const item of artifacts) {
+		if (typeof item === "string" && item.trim()) continue;
+		if (isCaptureHandle(item)) continue;
+		return `must be true or a list of capture handles or names (got ${JSON.stringify(item)})`;
+	}
+}
+const nameOf = (item) => typeof item === "string" ? item.trim() : item.name;
+/**
+* Resolves call `artifacts` (capture handles or their names, or `true` for every capture so far,
+* in capture order) to PNG bytes this attempt captured. Unknown names throw: a mistake in the
+* call, before the provider is asked.
+*/
+function resolveJudgeArtifacts(artifacts, blobs) {
+	const items = artifacts === true ? [...blobs.keys()] : artifacts;
+	if (!items?.length) return [];
+	return items.map((item) => {
+		const name = nameOf(item);
+		const data = blobs.get(name);
+		if (!data) throw new Error(`c.judge() artifacts names a capture this attempt doesn't have: ${JSON.stringify(name)}`);
+		return {
+			name,
+			path: `artifacts/${name}.png`,
+			sha256: sha256(data),
+			bytes: data.byteLength,
+			mediaType: "image/png",
+			data
+		};
+	});
+}
+const filePartsOf = (resolved) => resolved.map((a) => ({
+	type: "file",
+	mediaType: a.mediaType,
+	data: a.data
+}));
+const artifactRefsOf = (resolved) => resolved.map(({ data: _data, ...ref }) => ref);
+/**
+* JSON-only state is a plain object, as TypeSafe and other JSON judges read it.
+* File parts are an ordered decide-state array: one JSON part, then each capture.
+*/
+function decideState(json, files) {
+	if (files.length === 0) return json;
+	return [{
+		type: "json",
+		value: json
+	}, ...files];
+}
+//#endregion
+//#region src/contract/judge.ts
+const judgeStateSchema = z.strictObject({
+	task: z.boolean().optional().describe("The graded turn's prompt. Default true."),
+	output: z.boolean().optional().describe("The graded turn's final message, or `on`'s value. Default true."),
+	files: z.boolean().optional().describe("`status path` lines for the changed files, under `diff`'s patterns. Default true."),
+	diff: z.union([z.boolean(), z.array(nonEmpty()).min(1)]).optional().describe("The diff since the eval started. A list is gitignore-style patterns over workspace paths: the last matching pattern wins and `!` excludes. Default true."),
+	contents: z.array(nonEmpty()).optional().describe("Workspace files shown whole, as they are now, changed or not: exact paths or gitignore-style patterns (`'**'`, `'src/**'`; the last match wins, `!` excludes) over the files git sees in the workspace. A missing exact path is `null`; a pattern that matches nothing adds nothing. These files leave the diff. Default none."),
+	transcript: z.union([z.boolean(), z.enum(["tools", "full"])]).optional().describe("The graded turn's transcript as text lines: `true` is messages and tool calls with tool output clipped, `'tools'` tool calls only, `'full'` adds reasoning and whole tool output. Default false."),
+	turns: z.enum(["last", "all"]).optional().describe("`'all'`: every turn up to the graded one under `turns`, each with its task, output and (with `transcript`) transcript. Default `'last'`."),
+	checks: z.union([z.boolean(), z.array(nonEmpty())]).optional().describe("`c.exec` results so far; a list shows only those commands, as passed to `c.exec`. Default false.")
+}).describe("What the judge is given: each key is a key of the state in `judge-input.json`. An unset key keeps its default, which is what the judge gets with no `state` option at all.");
+const judgeFields = {
+	threshold: z.number().min(0, "must be between 0 and 1").max(1, "must be between 0 and 1").optional().describe("Minimum score (0–1) for a `pass` verdict."),
+	providerOptions: z.record(z.string(), z.record(z.string(), jsonValueSchema)).optional().describe("AI SDK provider options for the judge's calls, keyed by provider."),
+	maxConcurrent: z.number().int().min(1).optional().describe("The most judge calls to this model's provider at once, across the run."),
+	state: judgeStateSchema.optional().describe("The judge's state: what the judge is given beside the question, not the workspace's or the attempt's state. Merged key by key down the cascade.")
+};
+const fakeJudgeDataSchema = z.strictObject({
+	kind: z.literal("fake-judge"),
+	score: z.number().min(0).max(1).describe("The score (0–1) every call gets.")
+}).describe("`fakeJudge()` as data: the score every call gets.");
+z.string().regex(/\//, "must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash'");
+const configJudgeDataSchema = z.strictObject({
+	model: z.union([z.string(), fakeJudgeDataSchema]).optional().describe("`'provider/model'`, or a fake judge."),
+	...judgeFields
+}).describe("`ConfigJudgeOptions` as data: the model by name, or a fake judge with a fixed score.");
+const judgeOptionsSchema = z.strictObject({
+	model: z.custom().optional(),
+	...judgeFields
+});
+const judgeCallSchema = judgeOptionsSchema.extend({
+	on: z.custom().optional(),
+	artifacts: z.union([z.literal(true), z.array(z.custom())]).optional()
+});
+const STATE_KEYS = Object.keys(judgeStateSchema.shape);
+const JUDGE_HINTS = {
+	see: "state",
+	shows: "state",
+	show: "state",
+	input: "state"
+};
+const VALUES = {
+	task: "true or false",
+	output: "true or false",
+	files: "true or false",
+	diff: "true, false or a non-empty list of patterns",
+	contents: "a list of paths or patterns",
+	transcript: "true, false, 'tools' or 'full'",
+	turns: "'last' or 'all'",
+	checks: "true, false or a list of commands"
+};
+function distance(a, b) {
+	const row = Array.from({ length: b.length + 1 }, (_, i) => i);
+	for (let i = 1; i <= a.length; i++) {
+		let prev = row[0] ?? 0;
+		row[0] = i;
+		for (let j = 1; j <= b.length; j++) {
+			const here = row[j] ?? 0;
+			row[j] = Math.min(here + 1, (row[j - 1] ?? 0) + 1, prev + (a[i - 1] === b[j - 1] ? 0 : 1));
+			prev = here;
+		}
+	}
+	return row[b.length] ?? 0;
+}
+const nearest = (key, keys) => keys.find((k) => distance(key.toLowerCase(), k.toLowerCase()) <= 2);
+const shown = (value) => JSON.stringify(value) ?? String(value);
+/**
+* The first problem with a judge options object, as one sentence, or undefined. The checks run in
+* the order each site has always run them, so a value with several problems names the same one.
+*/
+function judgeProblem(judge, site) {
+	/** A judge field and where it was set: `judge.threshold in evals/a.eval.ts › e`. */
+	const field = (key) => {
+		if (site.at === "call") return `c.judge() ${key ?? "options"}`;
+		const base = site.at === "config" ? site.path : "judge";
+		return `${key ? `${base}.${key}` : base} in ${site.at === "config" ? site.file : site.where}`;
+	};
+	const loc = site.at === "call" ? " in c.judge()" : site.at === "options" ? ` in ${site.where}` : ` in ${site.path === "judge" ? "" : `${site.path.replace(/\.judge$/, "")} in `}${site.file}`;
+	if (typeof judge !== "object" || judge === null || Array.isArray(judge)) return `${field()} must be an object such as { threshold: 0.8 } (got ${shown(judge)})`;
+	const value = judge;
+	const model = value.model;
+	if (site.at === "config" && typeof model === "string") {
+		const problem = checkModelString(model, field("model"));
+		if (problem) return problem;
+	}
+	const schema = site.at === "call" ? judgeCallSchema : site.at === "config" ? configJudgeDataSchema : judgeOptionsSchema;
+	const parsed = schema.safeParse(judge);
+	if (parsed.success) return void 0;
+	const numericFake = site.at === "config" && typeof model === "object" && model?.kind === "fake-judge" && typeof model.score === "number";
+	const issues = parsed.error.issues.filter((i) => !(numericFake && i.path[0] === "model"));
+	if (!issues.length) return void 0;
+	const at = (key) => issues.some((i) => i.path[0] === key);
+	const order = [];
+	const modelProblem = () => {
+		if (site.at !== "config" || !at("model")) return void 0;
+		if (typeof model === "object" && model?.kind === "fake-judge") return `${field("model")} must be fakeJudge({ score }) with a number. Pass a score function per group, eval or call instead.`;
+		return `${field("model")} must be a "provider/model" string or fakeJudge(). Pass a model object per group, eval or call instead.`;
+	};
+	const maxConcurrentProblem = () => {
+		if (site.at === "config" || !at("maxConcurrent")) return void 0;
+		return `${field("maxConcurrent")} must be a whole number of at least 1 (got ${shown(value.maxConcurrent)})`;
+	};
+	const thresholdProblem = () => {
+		if (!at("threshold")) return void 0;
+		return `${field("threshold")} must be between 0 and 1 (got ${shown(value.threshold)})`;
+	};
+	const keyProblem = () => {
+		const name = issues.find((i) => i.code === "unrecognized_keys" && i.path.length === 0)?.keys[0];
+		if (name === void 0) return void 0;
+		const allowed = Object.keys(schema.shape);
+		const hint = JUDGE_HINTS[name] ?? (STATE_KEYS.includes(name) ? `state.${name}` : nearest(name, allowed));
+		const why = site.at !== "call" && (name === "on" || name === "artifacts") ? ` ${name} is per call: c.judge(question, { ${name} }).` : hint ? ` Did you mean "${hint}"?` : "";
+		return `Unknown judge option "${name}"${loc}.${why}`;
+	};
+	const stateProblem = () => {
+		const issue = issues.find((i) => i.path[0] === "state");
+		if (!issue) return void 0;
+		const { state } = value;
+		if (typeof state !== "object" || state === null || Array.isArray(state)) return `${field("state")} must be an object such as { transcript: 'tools' } (got ${shown(state)})`;
+		if (issue.code === "unrecognized_keys" && issue.path.length === 1) {
+			const key = issue.keys[0] ?? "";
+			const hint = nearest(key, STATE_KEYS);
+			return `Unknown judge.state key "${key}"${loc}.${hint ? ` Did you mean "${hint}"?` : ""}`;
+		}
+		const key = issue.path[1];
+		return `${field(`state.${String(key)}`)} must be ${VALUES[key]} (got ${shown(state[key])})`;
+	};
+	const providerOptionsProblem = () => at("providerOptions") ? `${field("providerOptions")} must map each provider to an object of JSON values, e.g. { google: { thinkingConfig: { thinkingBudget: 0 } } } (got ${shown(value.providerOptions)})` : void 0;
+	const artifactsProblem = () => at("artifacts") ? `c.judge() artifacts ${badJudgeArtifacts(value.artifacts)}` : void 0;
+	if (site.at === "config") order.push(modelProblem, keyProblem, stateProblem, providerOptionsProblem, thresholdProblem);
+	else if (site.at === "call") order.push(thresholdProblem, maxConcurrentProblem, keyProblem, stateProblem, providerOptionsProblem, artifactsProblem);
+	else order.push(maxConcurrentProblem, thresholdProblem, keyProblem, stateProblem, providerOptionsProblem);
+	for (const check of order) {
+		const problem = check();
+		if (problem) return problem;
+	}
+	return `judge options${loc} aren't valid: ${issues[0]?.message ?? "unknown problem"}`;
+}
+//#endregion
+//#region src/harness/effort.ts
+/**
+* The reasoning effort a harness asked for. Same user-facing name on every factory; the CLI's
+* native flag may say thinking. Undefined when the factory left it unset. Fast is not this.
+*/
+function harnessEffort(h) {
+	return "effort" in h && typeof h.effort === "string" ? h.effort : void 0;
+}
+/**
+* The reasoning effort a served-model label recorded, e.g. `gpt-5.6-luna[effort=medium]` →
+* `medium`. Fast (`[fast=true]`) is not effort and is ignored.
+*/
+function effortFromModelLabel(model) {
+	const inner = model?.match(/\[([^\]]+)]/)?.[1];
+	if (!inner) return void 0;
+	for (const part of inner.split(",")) {
+		const eq = part.indexOf("=");
+		if (eq < 0) continue;
+		const key = part.slice(0, eq).trim();
+		if (key === "fast") continue;
+		if (/^(?:reasoning_)?effort$|^thinking$/.test(key)) {
+			const value = part.slice(eq + 1).trim();
+			if (value) return value;
+		}
+	}
+}
+/** The first reasoning-effort suffix among served-model labels; undefined when none name one. */
+function servedEffortOf(models) {
+	if (!models) return void 0;
+	for (const model of models) {
+		const effort = effortFromModelLabel(model);
+		if (effort) return effort;
+	}
+}
+/** Unique reasoning-effort suffixes among served-model labels, in first-seen order. */
+function resolvedEffortOf(models) {
+	if (!models) return [];
+	const seen = /* @__PURE__ */ new Set();
+	const out = [];
+	for (const model of models) {
+		const effort = effortFromModelLabel(model);
+		if (effort && !seen.has(effort)) {
+			seen.add(effort);
+			out.push(effort);
+		}
+	}
+	return out;
+}
+//#endregion
+//#region src/judge/state-options.ts
+const DEFAULT_STATE = {
+	task: true,
+	output: true,
+	files: true,
+	diff: true,
+	contents: [],
+	transcript: false,
+	turns: "last",
+	checks: false
+};
+/** Every key set, in `STATE_KEYS` order, so equal selections serialise the same. */
+function resolveStateOptions(state) {
+	const out = {};
+	for (const key of STATE_KEYS) out[key] = state?.[key] ?? DEFAULT_STATE[key];
+	return out;
+}
+/** `inner` over `outer`, field by field, and `state` key by key. */
+function mergeJudge(outer, inner) {
+	if (!inner) return outer;
+	const out = {
+		...outer,
+		...inner
+	};
+	if (outer.state || inner.state) out.state = {
+		...outer.state,
+		...inner.state
+	};
+	return out;
+}
+//#endregion
+//#region src/options.ts
+const KNOWN = [
+	"harness",
+	"sandbox",
+	"environment",
+	"compare",
+	"runs",
+	"gate",
+	"env",
+	"egress",
+	"judge",
+	"capture",
+	"question",
+	"timeout",
+	"tags",
+	"pin"
+];
+const SUGGEST = {
+	harnesses: "harness",
+	sandboxes: "sandbox",
+	environments: "environment",
+	baseline: "environment",
+	variants: "compare",
+	repeat: "runs",
+	repeats: "runs",
+	iterations: "runs",
+	trials: "runs",
+	secrets: "env",
+	vars: "env",
+	network: "egress",
+	networkPolicy: "egress",
+	tag: "tags",
+	pins: "pin",
+	passRate: "gate",
+	threshold: "gate"
+};
+/** Throws unless `value` is `'all'` or `{ passRate }` with a rate in (0, 1]. */
+function checkGate(value, where) {
+	const problem = gateProblem(value, where);
+	if (problem) throw new UsageError(problem);
+}
+const PINNABLE = [
+	"harness",
+	"runs",
+	"compare"
+];
+const ID = /^[A-Za-z0-9._:@+\-=[\]]+$/;
+const sanitize = (text) => (text.split("/").at(-1) ?? text).replace(/[^A-Za-z0-9._:@-]+/g, "-");
+function harnessId(h) {
+	if (h.id) return h.id;
+	const effort = harnessEffort(h);
+	const suffix = effort ? `[effort=${sanitize(effort)}]` : "";
+	if (h.kind === "fake") return `fake${suffix}`;
+	return `${h.model ? `${h.kind}:${sanitize(h.model)}` : h.kind}${suffix}`;
+}
+function installId(install) {
+	if (!install?.length) return "";
+	const key = JSON.stringify(install.map((step) => typeof step === "string" ? step : {
+		run: step.run,
+		timeout: step.timeout
+	}));
+	return `+${createHash("sha256").update(key).digest("hex").slice(0, 8)}`;
+}
+/** The provider a sandbox runs on: its kind, or `podman` for a Docker sandbox on Podman. */
+const sandboxProvider = (s) => s.kind === "docker" && s.engine === "podman" ? "podman" : s.kind;
+function sandboxId(s) {
+	if (s.id) return s.id;
+	if (s.kind === "fake") return "fake";
+	const install = s.kind === "docker" ? installId(s.install) : "";
+	const engine = s.engine ?? "docker";
+	if (s.dockerfile) {
+		const [file = "", dir = ""] = s.dockerfile.split(/[\\/]/).reverse();
+		return `${engine}:${sanitize(file === "Dockerfile" ? dir || file : file)}${install}`;
+	}
+	const image = s.image ?? s.from;
+	return image ? `${engine}:${sanitize(image)}${install}` : `${engine}${install}`;
+}
+const cellKey = (harness, environment, sandbox) => `${harness}/${environment}/${sandbox}`;
+const isHarness = (v) => isObjectLike(v) && (v.kind === "fake" || isAgentKind(v.kind));
+function checkMaxConcurrent(value, where) {
+	if (value === void 0 || typeof value === "number" && Number.isInteger(value) && value >= 1) return;
+	throw new UsageError(`${where} must be a whole number of at least 1 (got ${JSON.stringify(value)})`);
+}
+const describe = (value) => {
+	if (Array.isArray(value)) return `a list of ${value.length}`;
+	if (typeof value === "object" && value !== null) {
+		const keys = Object.keys(value);
+		return keys.length ? `an object with ${keys.map((k) => `"${k}"`).join(", ")}` : "an object";
+	}
+	return JSON.stringify(value) ?? String(value);
+};
+/** `compare: false` and `environment` beside `compare.baseline` are never legal options. */
+function refuseIllegalCompare(options, where) {
+	if (options.compare === false) throw new UsageError(`compare: false can't be set in ${where}. A profile turns compare off with limit.compare: false, which runs the baseline and strips the variants. On a group or eval, omit compare or pass compare: { baseline, variants }.`);
+	const baseline = options.compare !== null && typeof options.compare === "object" && "baseline" in options.compare ? options.compare.baseline : void 0;
+	if (options.environment !== void 0 && baseline !== void 0) throw new UsageError(`environment and compare.baseline can't both be set in ${where}. compare: { baseline, variants } names every arm, so drop environment, or drop baseline to make environment the control.`);
+}
+/** Throws on keys that aren't options, and on values of the wrong shape, suggesting a fix. */
+function checkOptions(options, where, allowed = KNOWN) {
+	for (const key of Object.keys(options)) {
+		if (allowed.includes(key)) continue;
+		const near = SUGGEST[key] ?? closest(key, allowed);
+		const hint = key === "pool" ? " pool is an option of evaluate.each, beside the cases: evaluate.each(cases, { pool: false })." : near ? ` Did you mean "${near}"?` : "";
+		throw new UsageError(`Unknown option "${key}" in ${where}.${hint}`);
+	}
+	const o = options;
+	if (o.runs !== void 0 && !(Number.isInteger(o.runs) && o.runs >= 1)) throw new UsageError(`runs in ${where} must be a whole number of at least 1 (got ${JSON.stringify(o.runs)})`);
+	if (o.gate !== void 0) checkGate(o.gate, where);
+	if (o.tags !== void 0 && (!Array.isArray(o.tags) || o.tags.some((t) => typeof t !== "string"))) throw new UsageError(`tags in ${where} must be a list of strings, e.g. tags: ['smoke'] (got ${JSON.stringify(o.tags)})`);
+	if (o.env !== void 0) checkEnv(o.env, where, UsageError);
+	if (o.egress !== void 0) checkEgress(o.egress, where, UsageError);
+	const judgeIssue = o.judge !== void 0 && judgeProblem(o.judge, {
+		at: "options",
+		where
+	});
+	if (judgeIssue) throw new UsageError(judgeIssue);
+	if (o.sandbox !== void 0) {
+		const kind = o.sandbox.kind;
+		if (kind !== "docker" && kind !== "fake") throw new UsageError(`sandbox in ${where} must come from docker(), podman() or fakeSandbox() (got ${describe(o.sandbox)})`);
+		const max = o.sandbox.maxConcurrent;
+		if (max !== void 0 && !(Number.isInteger(max) && max >= 1)) throw new UsageError(`sandbox.maxConcurrent in ${where} must be a whole number of at least 1 (got ${String(max)})`);
+		const problem = sandboxProblem(o.sandbox);
+		if (problem) throw new UsageError(`sandbox in ${where}: ${problem}`);
+	}
+	if (o.environment !== void 0) {
+		if (Array.isArray(o.environment)) throw new UsageError(`environment takes one environment, not a list (${where}). To compare environments, use compare: { baseline, variants: [a, b] }. baseline is the control; variants are not layered on it.`);
+		if (!isEnvironment(o.environment)) throw new UsageError(`environment in ${where} must come from defineEnvironment(…) (got ${describe(o.environment)})`);
+		const problem = environmentDataProblem(o.environment, "environment", where);
+		if (problem) throw new UsageError(problem);
+	}
+	refuseIllegalCompare(options, where);
+	if (o.compare !== void 0) {
+		const problem = compareProblem(o.compare, where, z.custom(isEnvironment));
+		if (problem) throw new UsageError(problem);
+		const { baseline, variants } = o.compare;
+		for (const [label, env] of [["compare.baseline", baseline], ...variants.map((v, i) => [`compare.variants[${i}]`, v])]) {
+			const bad = env && environmentDataProblem(env, label, where);
+			if (bad) throw new UsageError(bad);
+		}
+	}
+	if (o.pin !== void 0) {
+		const bad = Array.isArray(o.pin) ? o.pin.filter((k) => !PINNABLE.includes(k)) : [o.pin];
+		if (bad.length) throw new UsageError(`pin in ${where} lists harness, runs and/or compare (got ${bad.map((k) => JSON.stringify(k)).join(", ")})`);
+	}
+	if (o.capture !== void 0) checkCapture(o.capture, where);
+	const questionIssue = o.question !== void 0 && questionOptionsProblem(o.question, where);
+	if (questionIssue) throw new UsageError(questionIssue);
+}
+const CAPTURE_KEYS = [
+	"viewport",
+	"timeout",
+	"browser",
+	"model"
+];
+const CAPTURE_SHAPE = "{ viewport?, timeout?, browser?, model? }";
+const CAPTURE_CALL_ONLY = /* @__PURE__ */ new Set([
+	"instructions",
+	"wait",
+	"name",
+	"path"
+]);
+/** Throws unless `capture` is a shallow-merge object with a viewport, timeout, browser and/or model. */
+function checkCapture(value, where) {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`capture in ${where} is ${CAPTURE_SHAPE} (got ${JSON.stringify(value)})`);
+	const o = value;
+	if (o.instructions !== void 0) throw new UsageError(`capture.instructions in ${where} can't be set: instructions go on the call, c.capture(port, { instructions }).`);
+	const unknown = Object.keys(o).filter((k) => !CAPTURE_KEYS.includes(k));
+	if (unknown.length) {
+		const key = unknown[0] ?? "";
+		if (key === "use" || key === "fake") throw new UsageError(`Unknown capture key "${key}" in ${where}. Set capture.model; tests pass fakeInstructions from reeval/testing as instructions.`);
+		if (CAPTURE_CALL_ONLY.has(key)) throw new UsageError(`capture.${key} in ${where} can't be set: ${key} goes on the call, c.capture(port, { ${key} }). Use viewport, timeout, browser, model.`);
+		throw new UsageError(`Unknown capture key "${key}" in ${where}. Use viewport, timeout, browser, model.`);
+	}
+	if (o.viewport !== void 0) checkViewport(o.viewport, where);
+	if (o.timeout !== void 0) parseDuration(o.timeout);
+	if (o.browser !== void 0 && typeof o.browser !== "boolean") throw new UsageError(`capture.browser in ${where} is true or false (got ${JSON.stringify(o.browser)})`);
+	if (o.model !== void 0) checkCaptureModel(o.model, where);
+}
+function checkViewport(value, where) {
+	if (value === "full-page") return;
+	if (!isObjectLike(value) || Array.isArray(value)) throw new UsageError(`capture.viewport in ${where} is { width, height } or 'full-page' (got ${JSON.stringify(value)})`);
+	const keys = Object.keys(value);
+	if (keys.includes("fullPage")) throw new UsageError(`capture.viewport in ${where} is { width, height } or 'full-page', not { height, fullPage }. Use viewport: 'full-page' for a tall shot.`);
+	const isSize = (n) => typeof n === "number" && Number.isInteger(n) && n > 0;
+	if (keys.some((k) => k !== "width" && k !== "height") || !isSize(value.width) || !isSize(value.height)) throw new UsageError(`capture.viewport in ${where} is { width, height } or 'full-page' (got ${JSON.stringify(value)})`);
+}
+function checkCaptureModel(value, where) {
+	if (typeof value !== "string" || !value.includes("/")) throw new UsageError(`capture.model in ${where} is a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(value)})`);
+	if (isEvaluationModel(value)) throw new UsageError(`capture.model "${value}" in ${where} is an evaluation model: it scores yes/no questions and can't drive a capture. Use a language model that accepts images, e.g. 'google/gemini-3.8-flash'.`);
+}
+function applyHarness(inherited, value, where) {
+	if (value === void 0) return inherited;
+	const next = typeof value === "function" ? value(inherited) : isHarnessEdit(value) ? editHarness(inherited, value, where) : value;
+	const list = Array.isArray(next) ? next : [next];
+	if (list.length === 0) throw new UsageError(`harness is an empty list in ${where}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
+	checkHarnesses(list, where);
+	return list;
+}
+const isHarnessEdit = (v) => typeof v === "object" && v !== null && !Array.isArray(v) && !("kind" in v) && ("add" in v || "only" in v);
+/** `{ add }` appends to the inherited harnesses; `{ only }` keeps those with these ids. */
+function editHarness(inherited, edit, where) {
+	if ("add" in edit && "only" in edit) throw new UsageError(`harness in ${where} takes { add } or { only }, not both. Use add to append to the inherited harnesses, only to keep some of them.`);
+	if ("add" in edit) return [...inherited, ...Array.isArray(edit.add) ? edit.add : [edit.add]];
+	const ids = Array.isArray(edit.only) ? edit.only : [edit.only];
+	const have = inherited.map(harnessId);
+	const missing = ids.filter((id) => typeof id !== "string" || !have.includes(id));
+	if (missing.length) throw new UsageError(`harness { only: ${missing.map((id) => JSON.stringify(id)).join(", ")} } in ${where} names no inherited harness (inherited: ${have.join(", ")})`);
+	return inherited.filter((h) => ids.includes(harnessId(h)));
+}
+/** Each harness is one, with a valid `maxConcurrent`. */
+function checkHarnesses(list, where) {
+	for (const item of list) {
+		if (!isHarness(item)) throw new UsageError(`harness in ${where} must be like ${HARNESS_EXAMPLE} (got ${describe(item)})`);
+		if (item.kind === "opencode") {
+			const why = opencodeModelError(item.model);
+			if (why) throw new UsageError(`opencode() in ${where}: ${why}`);
+		}
+		checkMaxConcurrent(item.maxConcurrent, `harness.maxConcurrent in ${where}`);
+		const { startTimeout } = item;
+		if (startTimeout !== void 0) try {
+			parseDuration(startTimeout);
+		} catch (error) {
+			throw new UsageError(`harness.startTimeout in ${where}: ${error.message}`);
+		}
+		const problem = harnessProblem(item);
+		if (problem) throw new UsageError(`harness in ${where}: ${problem}`);
+	}
+}
+function checkUnique(list, where) {
+	const seen = /* @__PURE__ */ new Set();
+	for (const item of list) {
+		const key = harnessId(item);
+		if (!ID.test(key)) throw new UsageError(`The harness id "${key}" in ${where} may only use letters, digits and . _ : @ + - [ ] =`);
+		if (seen.has(key)) throw new UsageError(`Two harnesses in ${where} resolve to the id "${key}". Give one an id, e.g. cursor({ model: '…', id: '…' })`);
+		seen.add(key);
+	}
+}
+/** A question's own `across`, else the one `compare.across` names when it names exactly one. */
+const plannedAcross = (own, compare) => {
+	if (own) return own;
+	const list = compare === void 0 ? void 0 : Array.isArray(compare) ? compare : [compare];
+	return list?.length === 1 ? list[0] : void 0;
+};
+const plannedQuestions = (value, compareAcross) => (value ?? []).map((q) => {
+	if (typeof q === "string") {
+		const across = plannedAcross(void 0, compareAcross);
+		return across ? {
+			question: q,
+			across
+		} : { question: q };
+	}
+	const across = plannedAcross(q.across, compareAcross);
+	return {
+		question: q.question,
+		...q.model !== void 0 ? { model: q.model } : {},
+		...across ? { across } : {},
+		...q.state ? { state: q.state } : {},
+		...q.artifacts !== void 0 ? { artifacts: q.artifacts } : {}
+	};
+});
+function normalizeCompare(value) {
+	if (value === void 0) return void 0;
+	return {
+		...value.baseline ? { baseline: value.baseline } : {
+			baseline: emptyEnvironment(EMPTY_BASELINE),
+			from: "empty"
+		},
+		variants: value.variants,
+		gate: value.gate ?? "variants",
+		...value.minLift !== void 0 ? { minLift: value.minLift } : {},
+		...value.harness ? { harness: value.harness.map((h) => typeof h === "string" ? h : harnessId(h)) } : {},
+		of: value.variants.length,
+		questions: plannedQuestions(value.questions, value.across)
+	};
+}
+function applyLevel(out, options, where, egress = "own") {
+	for (const [key, value] of Object.entries(options)) if (value !== void 0) out.sources = {
+		...out.sources,
+		[key]: where
+	};
+	if (options.egress !== void 0) out.egress[egress] = options.egress;
+	out.harness = applyHarness(out.harness, options.harness, where);
+	if (options.sandbox !== void 0) out.sandbox = options.sandbox;
+	if (options.environment !== void 0) out.environment = options.environment;
+	if (options.compare !== void 0) out.compare = normalizeCompare(options.compare);
+	if (options.runs !== void 0) out.runs = options.runs;
+	if (options.gate !== void 0) out.gate = options.gate;
+	if (options.timeout !== void 0) out.timeout = options.timeout;
+	Object.assign(out.env, options.env);
+	out.judge = mergeJudge(out.judge, options.judge);
+	if (options.capture !== void 0) {
+		const { viewport, ...rest } = options.capture;
+		out.capture = {
+			...out.capture,
+			...rest
+		};
+		if (viewport !== void 0) out.capture.viewport = viewport;
+	}
+	out.question = mergeQuestion(out.question, options.question);
+	for (const tag of options.tags ?? []) if (!out.tags.includes(tag)) out.tags.push(tag);
+	for (const key of options.pin ?? []) if (!out.pin.includes(key)) out.pin.push(key);
+}
+/** What a profile sets over the config's judge: its `judge`, then its `defaults.judge`. */
+const profileJudge = (profile) => mergeJudge(profile.judge, profile.defaults.judge);
+/**
+* Config, then the profile's defaults, then each enclosing group, then the eval; then the
+* profile's limit (narrowing only, except pinned keys); then `--runs`.
+*/
+function cascade(base, levels, where, options = {}) {
+	const out = {
+		...base,
+		env: { ...base.env },
+		egress: { ...base.egress },
+		judge: { ...base.judge },
+		capture: { ...base.capture },
+		question: { ...base.question },
+		tags: [...base.tags],
+		pin: [...base.pin]
+	};
+	const { profile } = options;
+	const notes = [];
+	if (profile) applyLevel(out, {
+		...profile.defaults,
+		judge: profileJudge(profile)
+	}, `profiles.${profile.name}.defaults`, "inherited");
+	for (const { options: own, where: at } of levels) applyLevel(out, own, at);
+	if (out.compare?.from && out.environment) out.compare = {
+		...out.compare,
+		baseline: out.environment,
+		from: "environment"
+	};
+	if (out.compare?.questions.length) {
+		const questions = out.compare.questions.map((q) => withQuestionDefaults(q, out.question));
+		const blank = questions.find(showsNothing);
+		if (blank) throw new UsageError(`compare.questions "${blank.question}" in ${where} shows the model nothing: every state key is off and it sends no artifacts. Turn on a state key, such as state: { diff: true }, or send artifacts.`);
+		out.compare = {
+			...out.compare,
+			questions
+		};
+	}
+	checkUnique(out.harness, where);
+	if (profile?.defaults.harness !== void 0 && levels.some((l) => l.options.harness)) notes.push("harness set by the eval");
+	const ids = out.harness.map(harnessId);
+	const missing = (out.compare?.harness)?.filter((id) => !ids.includes(id)) ?? [];
+	if (missing.length) throw new UsageError(`compare.harness lists ${missing.join(", ")}, which ${where} doesn't run (it runs ${ids.join(", ")}).`);
+	for (const v of out.compare?.variants ?? []) if (v.name === out.compare?.baseline.name) throw new UsageError(out.compare.from === "empty" ? `compare in ${where} lists a variant named ${v.name}, the name of the empty control a compare without a baseline or environment runs. Rename the variant, or set compare.baseline.` : out.compare.from === "environment" ? `compare in ${where} lists ${v.name}, which is the control: compare has no baseline, so the eval's environment is. List only the other environments.` : `compare in ${where} lists ${v.name}, which is the baseline. List only the other environments.`);
+	const names = out.compare?.variants.map((v) => v.name) ?? [];
+	const dupe = names.find((n, i) => names.indexOf(n) !== i);
+	if (dupe) throw new UsageError(`compare in ${where} lists two variants named ${dupe}. Give each variant its own name.`);
+	let skip;
+	const limit = profile?.limit;
+	if (profile && limit) {
+		const tag = `profile ${profile.name}`;
+		const pinned = (key) => out.pin.includes(key);
+		if (limit.runs !== void 0 && options.runs === void 0 && !pinned("runs") && out.runs > limit.runs) {
+			notes.push(`runs ${out.runs} → ${limit.runs} (${tag})`);
+			out.runs = limit.runs;
+		}
+		if (limit.harness && !pinned("harness")) {
+			const kept = out.harness.filter((h) => limit.harness?.includes(harnessId(h)));
+			if (kept.length === 0) skip = `no harness left: ${tag} allows ${limit.harness.join(", ") || "none"}`;
+			else if (kept.length < out.harness.length) {
+				notes.push(`harnesses ${out.harness.length} → ${kept.length} (${tag})`);
+				out.harness = kept;
+			}
+		}
+		if (out.compare && limit.compare !== void 0 && !pinned("compare")) {
+			if (limit.compare === false) {
+				notes.push(`compare off (${tag})`);
+				out.compare = {
+					...out.compare,
+					variants: []
+				};
+			} else if (out.compare.variants.length > limit.compare.maxVariants) {
+				notes.push(`variants ${out.compare.variants.length} → ${limit.compare.maxVariants} (${tag})`);
+				out.compare = {
+					...out.compare,
+					variants: out.compare.variants.slice(0, limit.compare.maxVariants)
+				};
+			}
+			if (out.compare.variants.length === 0 && out.compare.questions.length) {
+				notes.push(`questions off (${tag})`);
+				out.compare = {
+					...out.compare,
+					questions: []
+				};
+			}
+		}
+		const pins = out.pin.filter((k) => k === "compare" ? limit.compare !== void 0 && out.compare : limit[k] !== void 0);
+		if (pins.length) notes.push(`pinned: ${pins.join(", ")}`);
+	}
+	if (options.runs !== void 0) out.runs = options.runs;
+	return {
+		resolved: out,
+		notes,
+		...skip ? { skip } : {}
+	};
+}
+/**
+* The sandbox an attempt runs in: the resolved environment's (including an `.extend()`
+* parent's), else the first variant's when this is an omitted-baseline empty control, else
+* the cascaded option (`defineConfig` / profile defaults / group / eval), else implicit Docker.
+* A named environment is a total override: the config environment's sandbox is not inherited.
+*/
+const attemptSandbox = (resolved, env) => {
+	const own = env.layers.findLast((l) => l.spec.sandbox)?.spec.sandbox;
+	if (own) return own;
+	const variant = resolved.compare?.from === "empty" && env.name === "baseline" ? resolved.compare.variants[0] : void 0;
+	return sandboxOf(variant?.layers ?? [], resolved.sandbox);
+};
+/**
+* Which level set this attempt's sandbox: the cascade source of the environment that named one
+* (an `.extend()` child still names that environment), else the first variant when this is an
+* omitted-baseline empty control, else the cascaded `sandbox` option, else the config (implicit
+* Docker).
+*/
+function sandboxSetBy(planned, resolved, configFile) {
+	if (!planned.environment.layers.some((l) => l.spec.sandbox)) {
+		if (resolved.compare?.from === "empty" && planned.cell.role === "control" && resolved.compare.variants[0]?.layers.some((l) => l.spec.sandbox)) return resolved.sources?.compare ?? configFile;
+		return resolved.sources?.sandbox ?? configFile;
+	}
+	if (resolved.compare) {
+		if (planned.cell.role === "control") {
+			if (resolved.compare.from === "environment") return resolved.sources?.environment ?? configFile;
+			if (resolved.compare.from === "empty") return configFile;
+			return resolved.sources?.compare ?? configFile;
+		}
+		return resolved.sources?.compare ?? configFile;
+	}
+	return resolved.sources?.environment ?? configFile;
+}
+/** Per harness: the control, then each variant it compares on; then run numbers. */
+function expand(resolved) {
+	const { compare } = resolved;
+	const control = applied(compare ? compare.baseline : resolved.environment);
+	const policy = resolved.gate ?? "all";
+	const gateIf = (gated) => gated ? policy : "none";
+	const attempts = [];
+	for (const harness of resolved.harness) {
+		const hid = harnessId(harness);
+		const cells = [(() => {
+			const sandbox = attemptSandbox(resolved, control);
+			const sid = sandboxId(sandbox);
+			return {
+				env: control,
+				sandbox,
+				cell: {
+					key: cellKey(hid, control.name, sid),
+					harness: hid,
+					environment: control.name,
+					sandbox: sid,
+					role: "control"
+				},
+				gate: gateIf(!compare || compare.variants.length === 0 || compare.gate === "all")
+			};
+		})()];
+		if (compare && (!compare.harness || compare.harness.includes(hid))) for (const v of compare.variants) {
+			const env = applied(v);
+			const sandbox = attemptSandbox(resolved, env);
+			const sid = sandboxId(sandbox);
+			cells.push({
+				env,
+				sandbox,
+				cell: {
+					key: cellKey(hid, env.name, sid),
+					harness: hid,
+					environment: env.name,
+					sandbox: sid,
+					role: "variant",
+					variant: v.name
+				},
+				gate: gateIf(compare.gate !== "none")
+			});
+		}
+		for (const { env, sandbox, cell, gate } of cells) {
+			const axes = [resolved.harness.length > 1 ? hid : "", compare ? cell.role === "control" ? "control" : env.name : ""].filter(Boolean);
+			for (let n = 1; n <= resolved.runs; n++) {
+				const label = [axes.join(" · "), resolved.runs > 1 ? `#${n}` : ""].filter(Boolean).join(" ");
+				attempts.push({
+					cell,
+					harness,
+					sandbox,
+					environment: env,
+					egress: plannedEgress(resolved.egress, harness, resolved.env, env.layers),
+					gate,
+					...cell.role === "variant" && compare?.minLift !== void 0 ? { minLift: compare.minLift } : {},
+					n,
+					label
+				});
+			}
+		}
+	}
+	return attempts;
+}
+/** Warnings about what an eval's environments pull in, e.g. skills at a moving git ref. */
+function environmentWarnings(attempts) {
+	const out = /* @__PURE__ */ new Set();
+	for (const a of attempts) {
+		const workspace = workspaceOf(a.environment.layers);
+		if (workspace && isMovingRef(workspace.ref)) {
+			const ref = workspace.ref ? `moving ref "${workspace.ref}"` : "no ref (the default branch)";
+			out.add(`workspace git(${workspace.repo}) uses ${ref}: results aren't comparable over time. Pin a tag or SHA (the resolved commit is recorded either way).`);
+		}
+		for (const layer of a.environment.layers) for (const s of layer.spec.skills ?? []) {
+			if (s.source === "git" && isMovingRef(s.ref)) {
+				const ref = s.ref ? `moving ref "${s.ref}"` : "no ref (the default branch)";
+				out.add(`skill.git(${s.repo}) uses ${ref}: results aren't comparable over time. Pin a tag or SHA (the resolved commit is recorded either way).`);
+			}
+			if (s.source === "registry" && !s.version) out.add(`skill.registry(${s.id}) has no version: results aren't comparable over time. Pin one (the resolved commit is recorded either way).`);
+		}
+	}
+	return [...out];
+}
+/** Secrets a Docker attempt would get as real values, because they have no `hosts`. */
+function unbrokered(env, attempts) {
+	const out = /* @__PURE__ */ new Map();
+	for (const a of attempts) {
+		if (a.sandbox.kind !== "docker") continue;
+		for (const s of unbrokeredSecrets(env, a.environment.layers)) out.set(s.label, s);
+	}
+	return [...out.values()];
+}
+/** An optional secret whose host variable isn't set never reaches the sandbox, so it gets no warning. */
+const reachesSandbox = (s) => !s.optional || Boolean(process.env[s.from]);
+/** `GEMINI_API_KEY → generativelanguage.googleapis.com` for each secret with `hosts`. */
+const withHosts = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).flatMap((s) => s.hosts ? [`${s.label} → ${s.hosts.join(", ")}`] : []))];
+/** The secrets an attempt brokers: only Docker's egress gateway swaps a placeholder for its value. */
+const brokeredIn = (env, attempt) => attempt.sandbox.kind === "docker" ? withHosts(env, attempt) : [];
+/** The plan's note for secrets with `hosts` on the fake sandbox, which only ever has their placeholder. */
+function unbrokeredOnFake(env, attempts) {
+	const names = [...new Set(attempts.flatMap((a) => a.sandbox.kind === "fake" ? withHosts(env, a) : []))];
+	return names.length ? [`not brokered (fake sandbox, placeholder only): ${names.join("; ")}`] : [];
+}
+/** The host variables an attempt's secrets read, brokered or not. */
+const secretVarsIn = (env, attempt) => [...new Set(plannedSecrets(env, attempt.environment.layers).map((s) => s.from))];
+const brokerHint = (s) => /^mcp\.[^.]+\.headers\./.test(s.label) ? "give the MCP server an https:// URL (reeval brokers its header secrets to its own host, over HTTPS only)" : `add the hosts it's sent to, e.g. secret.fromEnv('${s.from}', { hosts: ['api.example.com'] })`;
+const unbrokeredWarning = (s) => s.checksOnly ? `${s.label} reaches the checks (c.exec, setup steps) as its real value; the agent never gets it. To give the checks only a placeholder, ${brokerHint(s)}.` : `${s.label} is in the sandbox as its real value. To give the agent only a placeholder, ${brokerHint(s)}.`;
+const unbrokeredError = (s, where, configFile) => `${s.label} in ${where} isn't brokered, and secrets.unbrokered is 'error' in ${configFile}. To broker it, ${brokerHint(s)}.`;
+const levelOf = (where) => where.startsWith("evaluate(") ? "eval" : where.startsWith("group(") ? "group" : where.startsWith("profiles.") ? "profile" : "config";
+/**
+* A warning per privilege an attempt's Docker sandbox adds, naming the level that set the
+* environment which owns that sandbox: the config when no profile, group or eval did.
+*/
+function sandboxWarnings(attempts, resolved, origin) {
+	const out = [];
+	const seen = /* @__PURE__ */ new Set();
+	for (const a of attempts) {
+		if (a.sandbox.kind !== "docker") continue;
+		const where = sandboxSetBy(a, resolved, origin.configFile);
+		const level = levelOf(where);
+		const file = level === "eval" || level === "group" ? origin.file : null;
+		if (a.sandbox.root) {
+			const message = `running as root in the sandbox: ${where} (docker({ root: true }))`;
+			const key = `${file}\0${message}`;
+			if (!seen.has(key)) {
+				seen.add(key);
+				out.push({
+					kind: "root",
+					level,
+					where,
+					file,
+					message
+				});
+			}
+		}
+		if (a.sandbox.capabilities?.length) {
+			const message = `extra capabilities in the sandbox: ${where} (docker({ capabilities: [${a.sandbox.capabilities.map((c) => `'${c}'`).join(", ")}] }))`;
+			const key = `${file}\0${message}`;
+			if (!seen.has(key)) {
+				seen.add(key);
+				out.push({
+					kind: "capabilities",
+					level,
+					where,
+					file,
+					capabilities: [...a.sandbox.capabilities],
+					message
+				});
+			}
+		}
+	}
+	return out;
+}
+/** The plan row for an eval. */
+function evalPlan(r, attempts, origin) {
+	const { resolved, skip } = r;
+	const capture = captureNote(resolved.capture);
+	const notes = [
+		...r.notes,
+		...capture ? [capture] : [],
+		...unbrokeredOnFake(resolved.env, attempts)
+	];
+	return {
+		harnesses: skip ? 0 : resolved.harness.length,
+		variants: resolved.compare ? {
+			run: resolved.compare.variants.length,
+			of: resolved.compare.of
+		} : null,
+		...resolved.compare ? { compare: {
+			baseline: resolved.compare.baseline.name,
+			...resolved.compare.from ? { from: resolved.compare.from } : {}
+		} } : {},
+		...!skip && resolved.compare?.questions.length ? { questions: resolved.compare.questions } : {},
+		runs: resolved.runs,
+		...resolved.gate !== void 0 ? { gate: resolved.gate } : {},
+		attempts: skip ? 0 : attempts.length,
+		notes,
+		...skip ? { skip } : {},
+		warnings: [
+			...environmentWarnings(attempts),
+			...skip ? [] : questionWarnings(resolved),
+			...unbrokered(r.resolved.env, attempts).filter(reachesSandbox).map(unbrokeredWarning)
+		],
+		sandboxWarnings: skip ? [] : sandboxWarnings(attempts, resolved, origin),
+		brokered: [...new Set(attempts.flatMap((a) => brokeredIn(r.resolved.env, a)))],
+		egress: [...new Set(attempts.map((a) => egressLabel(a.egress)).filter(Boolean))],
+		workspaces: [...new Set(attempts.flatMap((a) => {
+			const w = workspaceOf(a.environment.layers);
+			return w ? [workspaceLabel(w)] : [];
+		}))],
+		...capture ? { capture: resolved.capture } : {}
+	};
+}
+/** A question that shows each side's transcript, asked of two harnesses: tool names unblind it. */
+function questionWarnings(resolved) {
+	const { compare } = resolved;
+	if (!compare || resolved.harness.length < 2) return [];
+	return compare.questions.filter((q) => q.across !== "environment" && q.state?.transcript).map((q) => `compare question "${q.question}" shows state.transcript, which names each harness's tools, so its harness pairs aren't blind. Ask it with across: 'environment', or leave the transcript out.`);
+}
+function captureNote(c) {
+	const parts = [];
+	if (c.viewport === "full-page") parts.push("full-page");
+	else if (c.viewport) parts.push(`${c.viewport.width}×${c.viewport.height}`);
+	if (c.timeout !== void 0) parts.push(typeof c.timeout === "number" ? `${c.timeout}ms` : c.timeout);
+	if (c.browser === false) parts.push("browser false");
+	if (c.model) parts.push(`model ${c.model}`);
+	return parts.length ? `capture: ${parts.join(" · ")}` : void 0;
+}
+/** `--tag a --tag !b`: every entry must hold. */
+function matchesTags(tags, filters) {
+	return filters.every((f) => f.startsWith("!") ? !tags.includes(f.slice(1)) : tags.includes(f));
+}
+//#endregion
+export { questionOptionsSchema as A, configJudgeDataSchema as C, decideState as D, badJudgeArtifacts as E, filePartsOf as O, servedEffortOf as S, artifactRefsOf as T, unbrokeredError as _, evalPlan as a, harnessEffort as b, levelOf as c, refuseIllegalCompare as d, sandboxId as f, unbrokered as g, secretVarsIn as h, checkOptions as i, resolveJudgeArtifacts as k, matchesTags as l, sandboxSetBy as m, cascade as n, expand as o, sandboxProvider as p, checkHarnesses as r, harnessId as s, brokeredIn as t, profileJudge as u, mergeJudge as v, judgeProblem as w, resolvedEffortOf as x, resolveStateOptions as y };
+
+//# sourceMappingURL=options-C0d0AzeR.js.map