reeval 0.9.0 → 0.10.0

package/dist/config-DEgnUiU-.js

@@ -0,0 +1,756 @@
+import { t as isObjectLike } from "./object-CLf2-tNq.js";
+import { n as UsageError } from "./errors-B6BSMs03.js";
+import { t as parseDuration } from "./duration-CEOQww97.js";
+import { a as fieldPath, f as valueAt, i as durationSchema, o as genericProblem, s as got, t as closest, u as oneOf } from "./shared-DYpIb1BI.js";
+import { p as harnessDescriptorSchema } from "./harness-B8QC9u0O.js";
+import { A as questionOptionsSchema, C as configJudgeDataSchema, i as checkOptions, r as checkHarnesses, u as profileJudge, w as judgeProblem } from "./options-Bh2W63dK.js";
+import { E as sandboxPaths, F as envVarsSchema, I as gatePolicySchema, M as environmentDataSchema, N as sandboxDescriptorSchema, P as egressPolicySchema, _ as isEnvironment, b as isSecret, d as environmentFrom, f as environmentPaths, g as helpersFor } from "./environment-DNjRk0nU.js";
+import { a as fromCwd } from "./ids-Bp6ut7ZD.js";
+import { i as checkModelString, o as judgeCredentials } from "./judge-DW_GDOmB.js";
+import { t as inCI } from "./env-BQqj93gu.js";
+import { u as ownFileURL } from "./root-DICDHCtp.js";
+import { c as DEFAULT_HARNESS, g as isAgentKind, l as HARNESS_EXAMPLE, m as credentialOf, s as AGENTS } from "./egress-C4PZTSSe.js";
+import { a as reevalJiti, n as defaultIncludeGlobs } from "./adapter-DWlpOtRn.js";
+import { t as docker } from "./sandbox-D3A8oUb7.js";
+import { existsSync, readFileSync } from "node:fs";
+import { availableParallelism, totalmem } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { parseDocument } from "yaml";
+import { z } from "zod";
+//#region src/contract/config.ts
+const count = () => z.number().int().min(1);
+const captureWindowSchema = z.strictObject({
+	width: count().describe("In CSS pixels."),
+	height: count().describe("In CSS pixels.")
+}).describe("Live tab size for `c.capture()`. Default 1280×720 when the call and cascade omit `viewport`.");
+const captureOptionsSchema = z.strictObject({
+	viewport: z.union([captureWindowSchema, z.literal("full-page")]).optional().describe("The window and saved PNG size, or `'full-page'`. Default 1280×720."),
+	timeout: durationSchema.optional().describe("How long to poll before failing. Default `'60s'`."),
+	browser: z.boolean().optional().describe("`false` refuses the host `chrome-headless-shell` binary. `c.capture()` then throws."),
+	model: z.string().includes("/").optional().describe("Vision model for `instructions`. Cascades like `question`. Default is the analysis model when that model accepts images; never Jev.")
+}).describe("Camera defaults for `c.capture()`. Cascades config → profile defaults → group → eval, shallow-merged. `viewport` on a level replaces the whole value. `port` is not an option: pass it as the first argument to `c.capture`.");
+const harnessList = z.union([harnessDescriptorSchema, z.array(harnessDescriptorSchema).min(1)]);
+const budgetSchema = z.strictObject({
+	attempts: count().optional().describe("The most attempts the plan may have."),
+	duration: durationSchema.optional().describe("How long the run may go on, from the first attempt's start.")
+}).describe("Run-level caps. `attempts` is checked against the plan; `duration` while running.");
+const historyConfigSchema = z.strictObject({
+	dir: z.string().optional().describe("Committed history folder (default: `.history` in the directory the evals live in)."),
+	artifacts: z.enum([
+		"all",
+		"failures",
+		"none"
+	]).optional().describe("Artifacts recorded next to each run: every eval's, failing evals' only, or none.")
+});
+const recordMode = z.enum([
+	"manual",
+	"ci",
+	"always"
+]);
+const profileDefaultsDataSchema = z.strictObject({
+	harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them."),
+	sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`."),
+	environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment."),
+	runs: count().optional().describe("Attempts per cell."),
+	gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome."),
+	env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name over the config's."),
+	egress: egressPolicySchema.optional().describe("Where the sandbox may connect; replaces the config's policy."),
+	judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field over the config's."),
+	timeout: durationSchema.optional().describe("Per attempt."),
+	capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`, shallow-merged over the config's."),
+	question: questionOptionsSchema.optional().describe("Defaults for every compare question, merged over the config's.")
+}).describe("`ProfileDefaults` as data: the environment without `extend`, and the judge as data.");
+const profileLimitSchema = z.strictObject({
+	runs: count().optional().describe("The most runs per cell."),
+	compare: z.union([z.literal(false), z.strictObject({ maxVariants: z.number().int().min(0) })]).optional().describe("`false` runs the control only; `{ maxVariants }` keeps the first n variants."),
+	harness: z.array(z.string()).optional().describe("Harness ids to allow; evals left with none are skipped.")
+}).describe("Over every eval, and it can only narrow; `pin` on a group or eval skips it.");
+const profileDataSchema = z.strictObject({
+	defaults: profileDefaultsDataSchema.optional().describe("What evals get when they don't say otherwise: under groups and evals."),
+	limit: profileLimitSchema.optional().describe("Caps over every eval, which only narrow; `pin` on a group or eval skips them."),
+	judge: configJudgeDataSchema.optional().describe("Judge settings for this profile, the same as `defaults.judge`, which wins field by field."),
+	budget: budgetSchema.optional().describe("Merged over the config's `budget`."),
+	record: recordMode.optional().describe("When this profile's runs are copied into the history folder."),
+	tags: z.array(z.string()).optional().describe("Tag filters for this profile, like `--tag` (`'!slow'` excludes).")
+}).describe("`Profile` as data: its defaults and judge as data.");
+const fakeVote = z.enum([
+	"A",
+	"B",
+	"tie"
+]);
+const failure = z.strictObject({ error: z.string() });
+const fakeAnalysisDataSchema = z.strictObject({
+	kind: z.literal("fake-analysis"),
+	output: z.unknown().optional().describe("The model's structured answer."),
+	note: z.union([z.string(), failure]).optional().describe("`analysis.agent`'s note on each eval and harness, or an error to throw."),
+	vote: z.union([
+		fakeVote,
+		z.strictObject({
+			winner: fakeVote,
+			reason: z.string().optional()
+		}),
+		failure
+	]).optional().describe("How it answers compare questions: a fixed pick, or `{ error }` to fail every call.")
+}).describe("`fakeAnalysis()` as data: canned answers, without the functions of the digest or pair.");
+const analysisConfigDataSchema = z.strictObject({
+	model: z.union([z.string(), fakeAnalysisDataSchema]).optional().describe("`'provider/model'`, e.g. `'google/gemini-3.8-flash'`; default: the judge's, if it's one."),
+	agent: z.boolean().optional().describe("`reeval run` writes a note on each eval and harness once its attempts finish, with the analysis model (default false: one call per eval and harness, every run)."),
+	instructions: z.string().optional().describe("Added to the prompt, e.g. \"Fewer tool calls count as a win. The skill under test is reeval-good.\""),
+	maxInputTokens: z.number().int().min(1e3).optional().describe("The digest's cap; evidence is dropped, lowest priority first, to fit (default 20000)."),
+	maxOutputTokens: z.number().int().min(500).optional().describe("The most tokens the model may answer with (default 8000). reeval has no prices, so tokens are the cap.")
+}).describe("`AnalysisConfig` as data: the model by name, or a fake analysis.");
+const reevalConfigDataSchema = z.strictObject({
+	include: z.array(z.string()).optional().describe("Globs of eval files, relative to the config's directory; default every `.eval.ts` under `evals`."),
+	harness: harnessList.optional().describe("The agents every eval runs on, unless a group or eval replaces them; default Cursor."),
+	sandbox: sandboxDescriptorSchema.optional().describe("The default machine when the resolved environment omits `sandbox`. Not an environment: it does not invent skills, files or a name. Implicit `docker()` when nobody sets one."),
+	environment: environmentDataSchema.optional().describe("What every eval runs in when it names no environment, and a baseline-less compare's control."),
+	runs: count().optional().describe("Attempts per cell; default 1."),
+	gate: gatePolicySchema.optional().describe("How each cell's attempts decide its outcome; default `'all'`."),
+	env: envVarsSchema.optional().describe("Variables and secrets for the sandbox, merged by name down the cascade."),
+	egress: egressPolicySchema.optional().describe("Where the sandbox may connect; default `'allow-all'`."),
+	judge: configJudgeDataSchema.optional().describe("Defaults for `c.judge()`, merged field by field down the cascade."),
+	timeout: durationSchema.optional().describe("Per attempt; default `'15m'`."),
+	capture: captureOptionsSchema.optional().describe("Screenshot defaults for `c.capture()`. `REEVAL_CAPTURE_BROWSER=0` refuses the host binary when `c.capture()` runs; it does not change this object."),
+	question: questionOptionsSchema.optional().describe("Defaults for every compare question: the model that answers it and what each side shows. Merged down the cascade."),
+	concurrency: count().optional().describe("How many attempts run at once. `--concurrency` and `REEVAL_CONCURRENCY` override it."),
+	record: recordMode.optional().describe("When runs are copied into the history folder: `reeval record` only, in CI, or on every run."),
+	history: historyConfigSchema.optional().describe("The history folder, and which attempts keep their artifacts there."),
+	budget: budgetSchema.optional().describe("Caps on a run; a profile's `budget` is merged over it."),
+	profiles: z.record(z.string(), profileDataSchema).optional().describe("Named presets of `defaults`, `limit`, `judge`, `budget`, `record` and `tags`."),
+	defaultProfile: z.strictObject({
+		ci: z.string().optional(),
+		local: z.string().optional()
+	}).optional().describe("The profile when neither `--profile` nor `REEVAL_PROFILE` names one: `ci` when `CI` is set."),
+	secrets: z.strictObject({ unbrokered: z.enum(["warn", "error"]).optional() }).optional().describe("`unbrokered: 'error'` refuses Docker secrets without `hosts` instead of warning (default `'warn'`)."),
+	debug: z.strictObject({
+		keep: count().optional(),
+		traceBufferMb: z.number().positive().optional()
+	}).optional().describe("Local debug data in `.reeval/debug`: how many runs to keep (default 20), and how many MB of raw trace each attempt buffers for trace.ndjson (default 8; the oldest entries go first)."),
+	analysis: analysisConfigDataSchema.optional().describe("`reeval analyze`, compare questions without a model, and `agent`'s notes during a run.")
+}).describe("`ReevalConfig` in its data forms, as `reeval.yaml` or another language carries it: the environment without `extend`, judge and analysis models by name or as fakes with fixed answers, and profiles the same way. AI SDK model objects and fake score functions are TypeScript only.");
+/** The object schema an issue's path leads to through `schema`, for "Did you mean". */
+function keysAt(schema, path, value) {
+	const def = schema.def;
+	if (def.innerType) return keysAt(def.innerType, path, value);
+	if (def.getter) return keysAt(def.getter(), path, value);
+	if (def.options) return [...new Set(def.options.flatMap((option) => keysAt(option, path, value)))];
+	const [key, ...rest] = path;
+	if (key === void 0) return def.shape ? Object.keys(def.shape) : [];
+	const next = valueAt(value, [key]);
+	if (def.shape) {
+		const field = def.shape[String(key)];
+		return field ? keysAt(field, rest, next) : [];
+	}
+	const inner = def.valueType ?? def.element;
+	return inner ? keysAt(inner, rest, next) : [];
+}
+/**
+* The issue under a union that went furthest into the value: the branch the value was most
+* likely meant to be. A union where every branch failed at its own path is left as it is.
+*/
+function deepest(issue) {
+	if (issue.code !== "invalid_union" || !("errors" in issue) || !issue.errors.length) return issue;
+	let best;
+	for (const branch of issue.errors) {
+		const inner = branch[0];
+		if (!inner) continue;
+		const found = deepest({
+			...inner,
+			path: [...issue.path, ...inner.path]
+		});
+		if (!best || found.path.length > best.path.length) best = found;
+	}
+	return best && best.path.length > issue.path.length ? best : issue;
+}
+/**
+* reeval's sentence for what the config contract refuses in `value` (the parsed `reeval.yaml`,
+* or a config another language sent), naming the field by its path, or undefined.
+*/
+function configDataProblem(value, file) {
+	const result = reevalConfigDataSchema.safeParse(value);
+	if (result.success) return void 0;
+	const { issues } = result.error;
+	const issue = deepest(issues.find((i) => i.code === "unrecognized_keys") ?? issues[0]);
+	const field = fieldPath(issue.path);
+	const at = field ? `${field} in ${file}` : file;
+	const v = valueAt(value, issue.path);
+	if (issue.code === "unrecognized_keys") {
+		const key = issue.keys[0];
+		const near = closest(key, keysAt(reevalConfigDataSchema, issue.path, value));
+		return `Unknown option "${key}" in ${at}.${near ? ` Did you mean "${near}"?` : ""}`;
+	}
+	if (issue.code === "invalid_union") {
+		const values = "options" in issue ? issue.options : void 0;
+		return values ? `${at} is ${oneOf(values)} (got ${got(v)})` : `${at} matches none of its forms (got ${got(v)})`;
+	}
+	if (issue.code === "invalid_type" && v === void 0) return `${at} is missing`;
+	if ((issue.code === "too_small" || issue.code === "too_big") && issue.origin === "number") {
+		const small = issue.code === "too_small";
+		const bound = small ? issue.minimum : issue.maximum;
+		const strict = small ? "above" : "below";
+		return `${at} must be ${issue.inclusive === false ? strict : small ? "at least" : "at most"} ${String(bound)} (got ${got(v)})`;
+	}
+	return `${at}${genericProblem("", {
+		...issue,
+		path: []
+	}, v)}`;
+}
+//#endregion
+//#region src/config.ts
+const CONFIG_KEYS = [
+	"include",
+	"harness",
+	"sandbox",
+	"environment",
+	"runs",
+	"gate",
+	"env",
+	"egress",
+	"judge",
+	"capture",
+	"question",
+	"timeout",
+	"concurrency",
+	"record",
+	"history",
+	"budget",
+	"profiles",
+	"defaultProfile",
+	"secrets",
+	"debug",
+	"analysis"
+];
+const PROFILE_KEYS = [
+	"defaults",
+	"limit",
+	"judge",
+	"budget",
+	"record",
+	"tags"
+];
+const DEFAULTS_KEYS = [
+	"harness",
+	"sandbox",
+	"environment",
+	"runs",
+	"gate",
+	"env",
+	"egress",
+	"judge",
+	"capture",
+	"question",
+	"timeout"
+];
+const LIMIT_KEYS = [
+	"runs",
+	"compare",
+	"harness"
+];
+/**
+* The config: a plain object, or a callback that destructures the helpers it needs, e.g.
+* `defineConfig(({ secret }) => ({ env: { KEY: secret.fromEnv('KEY') } }))`.
+*/
+const defineConfig = (config) => config;
+/** The script configs `reeval` looks for in each directory, first match first, as Vitest orders its own. */
+const SCRIPT_FILES = [
+	"ts",
+	"mts",
+	"cts",
+	"js",
+	"mjs",
+	"cjs"
+].map((ext) => `reeval.config.${ext}`);
+/** The data configs, for projects without TypeScript: one of them, and no script config beside it. */
+const YAML_FILES = ["reeval.yaml", "reeval.yml"];
+/** Every file name `reeval` looks for in each directory. */
+const CONFIG_FILES = [...SCRIPT_FILES, ...YAML_FILES];
+/** True for a `reeval.yaml`-style config, which is data rather than a module. */
+const isYamlConfig = (path) => /\.ya?ml$/i.test(path);
+const DEFAULT_CONFIG_FILE = "reeval.config.ts";
+/** The names `findConfig` looks for, as doctor and the missing-config error list them. */
+const configFilesPhrase = () => `${CONFIG_FILES[0]} (or ${CONFIG_FILES.slice(1).join(", ")})`;
+/** Usage error when `findConfig()` finds nothing: same names as doctor, plus `reeval init`. */
+const missingConfigMessage = () => `No ${configFilesPhrase()} here or above. Run reeval init.`;
+/** A found config path, or the missing-config usage error. */
+function requireConfig(path) {
+	if (path === void 0) throw new UsageError(missingConfigMessage());
+	return path;
+}
+/**
+* The nearest config file in `from` or a directory above it, up to the filesystem root, the way
+* Vitest and ESLint find theirs. A git or workspace root doesn't stop the search: a nested
+* project's own config is nearer, so it wins.
+*/
+function findConfig(from = process.cwd()) {
+	for (let dir = resolve(from);; dir = dirname(dir)) {
+		const script = SCRIPT_FILES.find((name) => existsSync(join(dir, name)));
+		const yaml = YAML_FILES.filter((name) => existsSync(join(dir, name)));
+		const found = [...script ? [script] : [], ...yaml];
+		if (found.length > 1) {
+			const where = fromCwd(dir) || ".";
+			throw new UsageError(`${found.join(" and ")} are both in ${where}: reeval reads one config per directory. Keep one, or pass --config.`);
+		}
+		if (found[0]) return join(dir, found[0]);
+		if (dirname(dir) === dir) return void 0;
+	}
+}
+/** The config a command reads: `--config` (from the current directory), else the nearest one. */
+const configPathFor = (flag) => flag === void 0 ? findConfig() : resolve(flag);
+/** The config file as the user would type it from the current directory. */
+const configLabel = (path) => fromCwd(path) || path;
+/**
+* Loads the config at `path` (see `configPathFor`); its directory is the project root. Without a
+* path, every key takes its default and the current directory is the root. `reeval run` and
+* `reeval image build` refuse a missing file before this.
+*/
+async function loadConfig(path, command = "run", profile = void 0) {
+	if (path === void 0) return resolveConfig({}, process.cwd(), DEFAULT_CONFIG_FILE);
+	const root = dirname(path);
+	const configPath = configLabel(path);
+	if (!existsSync(path)) throw new UsageError(`Config file not found: ${configPath}`);
+	if (isYamlConfig(path)) return resolveConfigData(parseYamlConfig(readFileSync(path, "utf8"), configPath), root, configPath);
+	const module = await (await reevalJiti(ownFileURL("dist/cli.js").href, { moduleCache: false })).import(path);
+	if (!module || typeof module !== "object" || !("default" in module)) {
+		const names = module && typeof module === "object" ? Object.keys(module) : [];
+		const exportsNote = names.length ? ` (it exports ${names.join(", ")})` : "";
+		throw new UsageError(`${configPath} has no default export${exportsNote}. Write export default defineConfig({…}).`);
+	}
+	let exported = module.default;
+	if (typeof exported === "function") {
+		const hostEnv = process.env;
+		exported = exported({
+			...helpersFor(root),
+			ci: inCI(hostEnv),
+			command,
+			hostEnv,
+			profile
+		});
+		if (exported instanceof Promise) throw new UsageError(`The defineConfig callback in ${configPath} returned a Promise. It runs once when the config loads and must return the config synchronously.`);
+	}
+	if (!exported || typeof exported !== "object" || exported instanceof Promise) throw new UsageError(`${configPath}'s default export must be defineConfig({…}) (got ${exported instanceof Promise ? "a Promise" : JSON.stringify(exported)}). Write export default defineConfig({…}).`);
+	return resolveConfig(exported, root, configPath);
+}
+/** A `reeval.yaml`'s text as data: an empty file is an empty config. */
+function parseYamlConfig(text, file) {
+	const doc = parseDocument(text, { prettyErrors: true });
+	const [error] = doc.errors;
+	if (error) throw new UsageError(`${file} isn't valid YAML: ${error.message.trimEnd()}`);
+	return doc.toJS() ?? {};
+}
+const isRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
+/** A config's Dockerfile that isn't there, named by its field. */
+const missingIn = (file) => (field, full) => {
+	throw new UsageError(`${field} in ${file}: ${full} doesn't exist. Paths in ${file} are relative to its directory.`);
+};
+/**
+* The config with `sandbox` and each profile's `defaults.sandbox`, when written as data, given
+* absolute host paths against `root`. With `environments`, an environment written as data (a
+* `reeval.yaml`'s) gets its layers' paths resolved too; a script config's come from
+* `defineEnvironment`, which resolves them against its own file.
+*/
+function withHostPaths(config, root, file, { environments = false } = {}) {
+	const missing = missingIn(file);
+	const options = (value, prefix) => {
+		if (!isRecord(value)) return value;
+		const out = { ...value };
+		if ("sandbox" in value) out.sandbox = sandboxPaths(value.sandbox, root, `${prefix}sandbox`, missing);
+		if (environments && "environment" in value) out.environment = environmentPaths(value.environment, root, `${prefix}environment`, missing);
+		return out;
+	};
+	const out = options(config, "");
+	const { profiles } = out;
+	if (!isRecord(profiles)) return out;
+	const each = Object.entries(profiles).map(([name, profile]) => isRecord(profile) && "defaults" in profile ? [name, {
+		...profile,
+		defaults: options(profile.defaults, `profiles.${name}.defaults.`)
+	}] : [name, profile]);
+	return {
+		...out,
+		profiles: Object.fromEntries(each)
+	};
+}
+/** The config from its data form: each environment's data becomes an `Environment` again. */
+function configFromData(data) {
+	const withEnvironment = (options) => {
+		if (!isObjectLike(options)) return options;
+		const { environment } = options;
+		return isEnvironment(environment) ? {
+			...options,
+			environment: environmentFrom(environment)
+		} : options;
+	};
+	const config = withEnvironment(data);
+	const { profiles } = config;
+	if (typeof profiles !== "object" || profiles === null || Array.isArray(profiles)) return config;
+	const each = Object.entries(profiles).map(([name, profile]) => {
+		if (!isObjectLike(profile)) return [name, profile];
+		const { defaults } = profile;
+		return [name, defaults === void 0 ? profile : {
+			...profile,
+			defaults: withEnvironment(defaults)
+		}];
+	});
+	return {
+		...config,
+		profiles: Object.fromEntries(each)
+	};
+}
+/**
+* Resolves a config in its data form (a parsed `reeval.yaml`, or `ReevalConfigData` from another
+* language) whose relative host paths resolve against `root`, its directory: `resolveConfig`'s
+* checks name a mistake in their own words first, and the config contract refuses anything they
+* let through, such as a `null` or a misspelt nested key.
+*/
+function resolveConfigData(input, root, file) {
+	if (!isRecord(input)) throw new UsageError(`${file} must be a mapping of config keys, like harness: and judge: (got ${JSON.stringify(input)})`);
+	const data = withHostPaths(input, root, file, { environments: true });
+	const { environment, profiles } = data;
+	if ([environment, ...Object.values(typeof profiles === "object" && profiles ? profiles : {}).map((p) => p?.defaults?.environment)].some((e) => e !== void 0 && !isEnvironment(e))) {
+		const problem = configDataProblem(data, file);
+		if (problem) throw new UsageError(problem);
+	}
+	let resolved;
+	try {
+		resolved = resolveConfig(configFromData(data), root, file);
+	} catch (error) {
+		if (error instanceof UsageError) throw error;
+		const problem = configDataProblem(data, file);
+		if (problem) throw new UsageError(problem);
+		throw error;
+	}
+	const problem = configDataProblem(data, file);
+	if (problem) throw new UsageError(problem);
+	return resolved;
+}
+const GLOB = /[*?[\]{}!]/;
+/** The directory every include pattern starts in: `evals/**\/*.eval.ts` → `evals`. */
+function evalRoot(include) {
+	const dirs = include.map((pattern) => {
+		const parts = pattern.split("/");
+		const stop = parts.findIndex((part) => GLOB.test(part));
+		return (stop < 0 ? parts.slice(0, -1) : parts.slice(0, stop)).filter((p) => p && p !== ".");
+	});
+	const common = [];
+	for (let i = 0; dirs.every((d) => d[i] !== void 0 && d[i] === dirs[0]?.[i]); i++) common.push(dirs[0]?.[i]);
+	return common.join("/");
+}
+const ARTIFACTS = [
+	"all",
+	"failures",
+	"none"
+];
+const RECORD = [
+	"manual",
+	"ci",
+	"always"
+];
+const unknownKey = (value, allowed, where, file) => {
+	const key = Object.keys(value).find((k) => !allowed.includes(k));
+	if (key === void 0) return;
+	const hints = {
+		reuse: "Reusing control results from .history isn’t built yet; every run runs its controls.",
+		compare: "compare only goes on a group or eval; a profile can narrow it with limit.compare."
+	};
+	const near = closest(key, allowed);
+	const hint = hints[key] ?? (near ? `Did you mean "${near}"?` : `Use ${allowed.join(", ")}.`);
+	throw new UsageError(`Unknown option "${key}" in ${where} in ${file}. ${hint}`);
+};
+function checkBudget(budget, where, file) {
+	if (budget === void 0) return {};
+	if (typeof budget !== "object" || budget === null) throw new UsageError(`${where} in ${file} must be an object like { attempts: 20, duration: '1h' } (got ${JSON.stringify(budget)})`);
+	if ("usd" in budget || "tokens" in budget) throw new UsageError(`${where}.${"usd" in budget ? "usd" : "tokens"} in ${file} isn't supported yet: reeval records the tokens harnesses report, but prices none and enforces no token budget. Cap the run with attempts or duration.`);
+	unknownKey(budget, ["attempts", "duration"], where, file);
+	const { attempts, duration } = budget;
+	if (attempts !== void 0 && !(Number.isInteger(attempts) && attempts >= 1)) throw new UsageError(`${where}.attempts in ${file} must be a whole number of at least 1 (got ${JSON.stringify(attempts)})`);
+	if (duration !== void 0) parseDuration(duration);
+	return {
+		...attempts ? { attempts } : {},
+		...duration ? { duration } : {}
+	};
+}
+const describeValue = (v) => {
+	if (!isObjectLike(v)) return JSON.stringify(v);
+	const { kind } = v;
+	if (kind === "fake") return "fakeHarness({ … })";
+	return isAgentKind(kind) ? `${AGENTS[kind].factory}({ … })` : "an object";
+};
+function checkProfile(name, profile, file) {
+	const where = `profiles.${name}`;
+	if (typeof profile !== "object" || profile === null) throw new UsageError(`${where} in ${file} must be an object of profile settings, e.g. { limit: { runs: 1 } } (got ${JSON.stringify(profile)})`);
+	unknownKey(profile, PROFILE_KEYS, where, file);
+	const p = profile;
+	if (p.defaults !== void 0) {
+		unknownKey(p.defaults, DEFAULTS_KEYS, `${where}.defaults`, file);
+		checkOptions(p.defaults, `${where}.defaults in ${file}`);
+	}
+	if (p.limit !== void 0) {
+		unknownKey(p.limit, LIMIT_KEYS, `${where}.limit`, file);
+		const { runs, compare, harness } = p.limit;
+		if (runs !== void 0 && !(Number.isInteger(runs) && runs >= 1)) throw new UsageError(`${where}.limit.runs in ${file} must be a whole number of at least 1 (got ${JSON.stringify(runs)})`);
+		if (compare !== void 0 && compare !== false && !(Number.isInteger(compare?.maxVariants) && compare.maxVariants >= 0)) throw new UsageError(`${where}.limit.compare in ${file} must be false or { maxVariants: n }, e.g. { maxVariants: 1 } (got ${JSON.stringify(compare)})`);
+		if (harness !== void 0) {
+			const bad = Array.isArray(harness) ? harness.find((h) => typeof h !== "string") : harness;
+			if (bad !== void 0) throw new UsageError(`${where}.limit.harness in ${file} can only list harness ids to allow (got ${describeValue(bad)}). To add harnesses, use ${where}.defaults.harness.`);
+		}
+	}
+	if (p.tags !== void 0 && (!Array.isArray(p.tags) || p.tags.some((t) => typeof t !== "string"))) throw new UsageError(`${where}.tags in ${file} must be a list of tag filters, e.g. tags: ['smoke'] (got ${JSON.stringify(p.tags)})`);
+	if (p.record !== void 0 && !RECORD.includes(p.record)) throw new UsageError(`${where}.record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(p.record)})`);
+	if (p.judge !== void 0) resolveJudge(p.judge, file, `${where}.judge`);
+	checkBudget(p.budget, `${where}.budget`, file);
+	return p;
+}
+/** `--profile`, then `REEVAL_PROFILE`, then `defaultProfile.ci` in CI (or `.local`), then none. */
+function selectProfile(config, flag, env = process.env) {
+	const pick = () => {
+		if (flag) return [flag, "--profile"];
+		if (env.REEVAL_PROFILE) return [env.REEVAL_PROFILE, "REEVAL_PROFILE"];
+		const ci = inCI(env);
+		if (ci && config.defaultProfile.ci) return [config.defaultProfile.ci, "auto: CI is set"];
+		if (!ci && config.defaultProfile.local) return [config.defaultProfile.local, "default"];
+		return null;
+	};
+	const picked = pick();
+	if (!picked) return null;
+	const [name, reason] = picked;
+	const profile = config.profiles[name];
+	if (!profile) {
+		const names = Object.keys(config.profiles);
+		throw new UsageError(`Unknown profile "${name}". ${names.length ? `Profiles in ${config.file}: ${names.join(", ")}.` : `${config.file} defines no profiles.`}`);
+	}
+	return {
+		name,
+		reason,
+		defaults: profile.defaults ?? {},
+		limit: profile.limit ?? {},
+		judge: profile.judge ?? {},
+		tags: profile.tags ?? [],
+		budget: {
+			...config.budget,
+			...profile.budget
+		},
+		...profile.record ? { record: profile.record } : {}
+	};
+}
+/**
+* The config with its defaults filled in and checked; `root` is its directory, which relative
+* host paths written as data resolve against.
+*/
+function resolveConfig(input, root, file = DEFAULT_CONFIG_FILE) {
+	const user = withHostPaths(input, root, file);
+	if ("compare" in user) throw new UsageError(`compare can't be set in ${file}: it would multiply every eval. Put it on the group or eval that asks the question.`);
+	checkOptions(user, file, CONFIG_KEYS);
+	const include = user.include ?? defaultIncludeGlobs();
+	if (!Array.isArray(include) || include.some((p) => typeof p !== "string")) throw new UsageError(`include in ${file} must be a list of glob patterns, e.g. include: ['evals/**/*.eval.ts'] (got ${JSON.stringify(include)})`);
+	const record = user.record ?? "ci";
+	if (!RECORD.includes(record)) throw new UsageError(`record in ${file} is ${oneOf(RECORD)} (got ${JSON.stringify(record)})`);
+	const artifacts = user.history?.artifacts ?? "all";
+	if (!ARTIFACTS.includes(artifacts)) throw new UsageError(`history.artifacts in ${file} is ${oneOf(ARTIFACTS)} (got ${JSON.stringify(artifacts)})`);
+	const historyDir = resolve(root, user.history?.dir ?? join(evalRoot(include), ".history"));
+	const harness = user.harness === void 0 ? [DEFAULT_HARNESS] : [user.harness].flat();
+	if (harness.length === 0) throw new UsageError(`harness is an empty list in ${file}. Give at least one, e.g. harness: ${HARNESS_EXAMPLE}`);
+	checkHarnesses(harness, file);
+	const profiles = {};
+	if (user.profiles !== void 0) {
+		if (typeof user.profiles !== "object" || user.profiles === null) throw new UsageError(`profiles in ${file} must be an object of named profiles, e.g. profiles: { pr: { limit: { runs: 1 } } } (got ${JSON.stringify(user.profiles)})`);
+		for (const [name, profile] of Object.entries(user.profiles)) profiles[name] = checkProfile(name, profile, file);
+	}
+	const defaultProfile = user.defaultProfile ?? {};
+	unknownKey(defaultProfile, ["ci", "local"], "defaultProfile", file);
+	for (const key of ["ci", "local"]) {
+		const name = defaultProfile[key];
+		if (name !== void 0 && !profiles[name]) {
+			const names = Object.keys(profiles);
+			throw new UsageError(`defaultProfile.${key} in ${file} names "${name}", which isn't a profile (${names.length ? `profiles: ${names.join(", ")}` : "it defines none"}). Add it under profiles, or name one that's there.`);
+		}
+	}
+	return {
+		root,
+		file,
+		optionKeys: Object.keys(user),
+		include,
+		harness,
+		sandbox: user.sandbox ?? docker(),
+		environment: user.environment,
+		runs: user.runs ?? 1,
+		gate: user.gate,
+		env: user.env ?? {},
+		...user.egress !== void 0 ? { egress: user.egress } : {},
+		budget: checkBudget(user.budget, "budget", file),
+		profiles,
+		defaultProfile,
+		judge: resolveJudge(user.judge, file),
+		capture: user.capture ?? {},
+		question: user.question ?? {},
+		timeoutMs: parseDuration(user.timeout ?? "15m"),
+		concurrency: user.concurrency === void 0 ? void 0 : checkConcurrency(user.concurrency, `concurrency in ${file}`),
+		record,
+		history: {
+			dir: historyDir,
+			artifacts
+		},
+		secrets: checkSecrets(user.secrets, file),
+		debug: checkDebug(user.debug, file),
+		analysis: checkAnalysis(user.analysis, file)
+	};
+}
+const ANALYSIS_DEFAULTS = {
+	maxInputTokens: 2e4,
+	maxOutputTokens: 8e3
+};
+function checkAnalysis(analysis, file) {
+	if (analysis === void 0) return { ...ANALYSIS_DEFAULTS };
+	if (typeof analysis !== "object" || analysis === null) throw new UsageError(`analysis in ${file} must be an object like { model: 'google/gemini-3.8-flash' } (got ${JSON.stringify(analysis)})`);
+	unknownKey(analysis, [
+		"model",
+		"agent",
+		"instructions",
+		"maxInputTokens",
+		"maxOutputTokens"
+	], "analysis", file);
+	const { model, agent, instructions, maxInputTokens = ANALYSIS_DEFAULTS.maxInputTokens, maxOutputTokens = ANALYSIS_DEFAULTS.maxOutputTokens } = analysis;
+	if (typeof model === "string") {
+		const problem = checkModelString(model, `analysis.model in ${file}`, [`google/${model}`]);
+		if (problem) throw new UsageError(problem);
+	} else if (model !== void 0 && !(isObjectLike(model) && model.kind === "fake-analysis")) throw new UsageError(`analysis.model in ${file} must be a 'provider/model' string, e.g. 'google/gemini-3.8-flash' (got ${JSON.stringify(model)})`);
+	if (agent !== void 0 && typeof agent !== "boolean") throw new UsageError(`analysis.agent in ${file} must be true or false (got ${JSON.stringify(agent)})`);
+	if (instructions !== void 0 && typeof instructions !== "string") throw new UsageError(`analysis.instructions in ${file} must be a string (got ${JSON.stringify(instructions)})`);
+	if (!(Number.isInteger(maxInputTokens) && maxInputTokens >= 1e3)) throw new UsageError(`analysis.maxInputTokens in ${file} must be a whole number of at least 1000 (got ${maxInputTokens})`);
+	if (!(Number.isInteger(maxOutputTokens) && maxOutputTokens >= 500)) throw new UsageError(`analysis.maxOutputTokens in ${file} must be a whole number of at least 500 (got ${maxOutputTokens})`);
+	return {
+		...analysis,
+		maxInputTokens,
+		maxOutputTokens
+	};
+}
+/** Runs whose local debug data `.reeval/debug` keeps, and the trace buffer per attempt. */
+const DEBUG_DEFAULTS = {
+	keep: 20,
+	traceBufferMb: 8
+};
+function checkDebug(debug, file) {
+	if (debug === void 0) return {
+		keep: DEBUG_DEFAULTS.keep,
+		traceBufferBytes: DEBUG_DEFAULTS.traceBufferMb * 2 ** 20
+	};
+	if (typeof debug !== "object" || debug === null) throw new UsageError(`debug in ${file} must be an object like { keep: 20, traceBufferMb: 8 } (got ${JSON.stringify(debug)})`);
+	unknownKey(debug, ["keep", "traceBufferMb"], "debug", file);
+	const { keep = DEBUG_DEFAULTS.keep, traceBufferMb = DEBUG_DEFAULTS.traceBufferMb } = debug;
+	if (!(Number.isInteger(keep) && keep >= 1)) throw new UsageError(`debug.keep in ${file} must be a whole number of at least 1 (got ${JSON.stringify(keep)})`);
+	if (!(typeof traceBufferMb === "number" && traceBufferMb > 0)) throw new UsageError(`debug.traceBufferMb in ${file} must be a number of megabytes above 0 (got ${JSON.stringify(traceBufferMb)})`);
+	return {
+		keep,
+		traceBufferBytes: Math.round(traceBufferMb * 2 ** 20)
+	};
+}
+function checkSecrets(secrets, file) {
+	if (secrets === void 0) return { unbrokered: "warn" };
+	if (typeof secrets !== "object" || secrets === null) throw new UsageError(`secrets in ${file} must be an object like { unbrokered: 'error' } (got ${JSON.stringify(secrets)})`);
+	unknownKey(secrets, ["unbrokered"], "secrets", file);
+	const unbrokered = secrets.unbrokered ?? "warn";
+	if (unbrokered !== "warn" && unbrokered !== "error") throw new UsageError(`secrets.unbrokered in ${file} is 'warn' or 'error' (got ${JSON.stringify(unbrokered)})`);
+	return { unbrokered };
+}
+const hostCapacity = () => ({
+	cpus: availableParallelism(),
+	memoryBytes: totalmem()
+});
+/**
+* Attempts mostly wait on a model, so the default is 8, but no more than one per CPU or per GB
+* of memory where the sandboxes run (the Docker VM, under Docker Desktop): an idle agent
+* container takes about 300 MB, and the rest is headroom for what the agent runs.
+*/
+function defaultConcurrency({ cpus, memoryBytes }) {
+	return Math.max(1, Math.min(8, cpus, Math.floor(memoryBytes / 1e9)));
+}
+function checkConcurrency(value, where) {
+	const n = typeof value === "string" && value.trim() !== "" ? Number(value) : value;
+	if (typeof n !== "number" || !Number.isInteger(n) || n < 1) throw new UsageError(`${where} must be a whole number of at least 1 (got ${String(value)})`);
+	return n;
+}
+/** `--concurrency`, then `REEVAL_CONCURRENCY`, then `concurrency` in the config, then the default. */
+async function resolveConcurrency(config, flag, capacity, env = process.env) {
+	if (flag !== void 0) return checkConcurrency(flag, "--concurrency");
+	if (env.REEVAL_CONCURRENCY) return checkConcurrency(env.REEVAL_CONCURRENCY, "REEVAL_CONCURRENCY");
+	return config.concurrency ?? defaultConcurrency(await capacity());
+}
+function resolveJudge(judge, file, path = "judge") {
+	judge ??= {};
+	const problem = judgeProblem(judge, {
+		at: "config",
+		path,
+		file
+	});
+	if (problem) throw new UsageError(problem);
+	const threshold = judge.threshold ?? .8;
+	return {
+		...judge,
+		threshold
+	};
+}
+/** Host variables that required `env` secrets read, as `[sandbox name, host name]`. */
+function secretSources(env) {
+	return Object.entries(env).flatMap(([name, value]) => isSecret(value) && !value.optional ? [[name, value.from ?? name]] : []);
+}
+/** The provider model `analysis.agent` calls during a run, if it's on: `analysis.model`, else the judge's. */
+function agentModel(config) {
+	if (!config.analysis.agent) return null;
+	const model = config.analysis.model ?? config.judge.model;
+	return typeof model === "string" && !model.startsWith("typesafe-ai/") ? model : null;
+}
+/** The run's judge model: the profile's when it sets one, else the config's. */
+function runJudgeModel(config, profile) {
+	return profile ? profileJudge(profile).model ?? config.judge.model : config.judge.model;
+}
+/**
+* Names the environment variables the run's providers need but don't have: the harnesses it
+* plans (default: the config's), the run's judge, the models that answer compare questions,
+* the analysis model with `analysis.agent`, and required config and profile secrets.
+*/
+function missingCredentials(config, env = process.env, options = {}) {
+	const missing = [];
+	const keys = new Set((options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []));
+	for (const key of keys) if (!env[key]) missing.push(key);
+	const judge = runJudgeModel(config, options.profile);
+	const agent = agentModel(config);
+	const models = [
+		...judge !== void 0 ? [judge] : [],
+		...options.questionModels ?? [],
+		...agent ? [agent] : []
+	];
+	for (const model of models) {
+		const modelKeys = judgeCredentials(model);
+		const alternatives = modelKeys.join(" or ");
+		if (modelKeys.length && !modelKeys.some((k) => env[k]) && !missing.includes(alternatives)) missing.push(alternatives);
+	}
+	const vars = {
+		...config.env,
+		...options.profile?.defaults.env
+	};
+	for (const [name, from] of secretSources(vars)) if (!env[from]) missing.push(from === name ? name : `${from} (for ${name})`);
+	return missing;
+}
+/** Where a value has its first whitespace or control character, from 1; 0 when it has none. */
+const badCharacterAt = (value) => [...value].findIndex((c) => /[\s\p{Cc}]/u.test(c)) + 1;
+/**
+* Keys the run sends in request headers (the agents' credentials, the run's judge's and the
+* compare questions' models') whose value has
+* whitespace or a control character in it, with where the first one is: a key pasted from a
+* wrapped terminal line picks up a line break, and the agent then hangs on a request it can't send
+* rather than failing to authenticate.
+*/
+function malformedCredentials(config, env = process.env, options = {}) {
+	const names = (options.harnesses ?? config.harness).flatMap((h) => credentialOf(h) ?? []);
+	const judge = runJudgeModel(config, options.profile);
+	if (judge !== void 0) names.push(...judgeCredentials(judge));
+	for (const model of options.questionModels ?? []) names.push(...judgeCredentials(model));
+	const analyst = agentModel(config);
+	if (analyst) names.push(...judgeCredentials(analyst));
+	return [...new Set(names)].flatMap((name) => {
+		const at = badCharacterAt(env[name] ?? "");
+		return at ? [`${name} (character ${at})`] : [];
+	});
+}
+//#endregion
+export { configPathFor as a, findConfig as c, malformedCredentials as d, missingCredentials as f, selectProfile as h, configLabel as i, hostCapacity as l, resolveConcurrency as m, badCharacterAt as n, defaultConcurrency as o, requireConfig as p, configFilesPhrase as r, defineConfig as s, CONFIG_FILES as t, loadConfig as u };
+
+//# sourceMappingURL=config-DEgnUiU-.js.map