toolroll 0.9.81 → 0.9.83

package/dist/eval-oracle.js

@@ -0,0 +1,217 @@
+/**
+ * The eval set's trusted oracle (docs/plans/evals-nightly.md, part C, "Grading: a trusted oracle outside the fence").
+ *
+ * The plane's runner starts the oracle as a separate process from the installed release (`runOracle`, which spawns
+ * eval-oracle-main.js), only after the attempt's whole process tree is gone, and hands it two folders: the installed
+ * case (`evals/cases/<id>/`, from the plane's own package) and the graded files the runner extracted from the built
+ * commit's verified Git objects. The oracle owns the grading (`gradeCase`):
+ *
+ *   1. It copies only the case's named files from the graded folder into a fresh grading folder, reading each with
+ *      O_NOFOLLOW and refusing links and anything but a regular file; then the case's own `probe.mjs` from the
+ *      installed case (an edited launcher in the build is never the one run).
+ *   2. It starts the probe with the installed node and a fixed argv inside the native fence (the release check's:
+ *      Seatbelt or bubblewrap via the agent fence), writing only the grading folder, with no network and the installed
+ *      eval set and the plane's state unreadable, under a time limit.
+ *   3. It sends a fresh nonce and inputs it chose from that nonce, reads at most 64 KiB of output, and passes the
+ *      attempt only when the output names that nonce with the values it computed itself (`oracle.mjs` `expected`).
+ *      An exit code alone never passes: an early `process.exit(0)` leaves no answer.
+ *
+ * The build's code never runs in the oracle's own process, and nothing the build wrote (a result file, a verdict) is
+ * ever read as one: the verdict travels from the oracle to the runner over the oracle's own pipe.
+ */
+import { spawn } from "node:child_process";
+import { createHash, randomBytes } from "node:crypto";
+import { closeSync, constants, lstatSync, mkdirSync, mkdtempSync, openSync, realpathSync, rmSync, writeSync } from "node:fs";
+import { dirname, isAbsolute, join, relative } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { checkLaunchRoot, fenceFor, linuxFenceAvailable, macosFenceAvailable, runFolders } from "./agent-fence.js";
+import { startEgressProxy, proxyEnv } from "./egress.js";
+import { run, OVERFLOW_CODE } from "./exec.js";
+import { readNoFollow } from "./eval-gate.js";
+/** The most a graded program may print, and the most of each graded file the oracle copies. */
+export const ORACLE_OUTPUT_BYTES = 64 * 1024;
+const GRADED_FILE_BYTES = 256 * 1024;
+const EVALS_ROOT = fileURLToPath(new URL("../evals/", import.meta.url));
+/** A random source seeded by the nonce (mulberry32 over its digest): the inputs are fresh for every grading. */
+export function seededRandom(nonce) {
+    let state = createHash("sha256").update(nonce).digest().readUInt32LE(0);
+    return () => {
+        state = (state + 0x6d2b79f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/** One value in a canonical form (object keys sorted), so two answers compare by content. */
+export function canonical(value) {
+    return JSON.stringify(value, (_key, inner) => inner !== null && typeof inner === "object" && !Array.isArray(inner)
+        ? Object.fromEntries(Object.keys(inner).sort().map(key => [key, inner[key]]))
+        : inner);
+}
+/** The default: the installed node, `probe.mjs`, inside the release check's fence — writes only the grading folder, no
+ * network (a proxy of its own that allows no host), the installed eval set and whatever else `deny` names unreadable. No
+ * fence on this machine: nothing is graded (the attempt is an error, never a pass). */
+export const fencedGradeRunner = async ({ cwd, stdin, timeoutMs, deny }) => {
+    if (!macosFenceAvailable() && !linuxFenceAvailable())
+        throw new Error("this machine can't fence the graded program");
+    const egress = await startEgressProxy({ hosts: [] });
+    try {
+        const network = { mode: "allowlist", proxyPort: egress.port, brokerPorts: [], loopback: "proxy-only", socketRoots: [cwd], token: egress.secret };
+        const launch = fenceFor("check", { databaseFile: null, worktree: cwd, boundary: null, binding: null, runFolders: runFolders(null), poolRoots: [], extra: [...deny],
+            writes: { writes: { subpaths: [cwd], prefixes: [] }, readOnly: [] }, network });
+        const result = await run(process.execPath, ["probe.mjs"], { cwd, stdin, timeoutMs, maxBuffer: ORACLE_OUTPUT_BYTES, processGroup: true, envAllowlist: ["PATH"],
+            env: { ...proxyEnv(egress, "proxy-only"), HOME: cwd, TMPDIR: cwd }, fence: launch.fence, fenceGrants: launch.fenceGrants, confinement: "required" });
+        return { code: result.code, stdout: result.stdout, timedOut: result.timedOut };
+    }
+    finally {
+        await egress.close();
+    }
+};
+/** Copy a file the oracle trusts nothing about: a regular file, never through a link, bounded; into a path that must
+ * not exist yet. */
+function copyIn(from, to, maxBytes) {
+    const bytes = readNoFollow(from, maxBytes);
+    if (bytes === null) {
+        try {
+            return lstatSync(from).isSymbolicLink() ? "is a link, not a file" : "isn't a regular file it can read";
+        }
+        catch {
+            return "is missing";
+        }
+    }
+    mkdirSync(dirname(to), { recursive: true });
+    const fd = openSync(to, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
+    try {
+        writeSync(fd, bytes);
+    }
+    finally {
+        closeSync(fd);
+    }
+    return null;
+}
+const inside = (root, path) => { const rel = relative(root, path); return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel); };
+/** Grade one attempt, in the oracle's own process. `caseDir` must be a case of the installed eval set (`evalsRoot`). */
+export async function gradeCase(input) {
+    const evalsRoot = realpathSync(input.evalsRoot ?? EVALS_ROOT);
+    let caseDir, inputDir;
+    try {
+        caseDir = realpathSync(input.caseDir);
+        inputDir = realpathSync(input.inputDir);
+        if (!inside(evalsRoot, caseDir) || lstatSync(input.caseDir).isSymbolicLink())
+            return { outcome: "error", why: "the case isn't one of the installed eval set" };
+        if (lstatSync(input.inputDir).isSymbolicLink() || !lstatSync(inputDir).isDirectory())
+            return { outcome: "error", why: "the graded folder isn't a folder" };
+    }
+    catch {
+        return { outcome: "error", why: "the case or the graded folder can't be read" };
+    }
+    for (const name of ["oracle.mjs", "probe.mjs"]) {
+        try {
+            if (!lstatSync(join(caseDir, name)).isFile())
+                return { outcome: "error", why: `the case's ${name} isn't a file` };
+        }
+        catch {
+            return { outcome: "error", why: `the case has no ${name}` };
+        }
+    }
+    let oracle;
+    try {
+        oracle = await import(pathToFileURL(join(caseDir, "oracle.mjs")).href);
+    }
+    catch {
+        return { outcome: "error", why: "the case's oracle can't be loaded" };
+    }
+    if (!Array.isArray(oracle.files) || typeof oracle.inputs !== "function" || typeof oracle.expected !== "function")
+        return { outcome: "error", why: "the case's oracle is malformed" };
+    const grading = realpathSync(mkdtempSync(join(checkLaunchRoot(), "so-eval-grade-")));
+    try {
+        for (const file of oracle.files) {
+            if (typeof file !== "string" || file === "probe.mjs" || file.split("/").some(part => part === "" || part === "." || part === "..") || isAbsolute(file))
+                return { outcome: "error", why: "the case's oracle names a file it can't grade" };
+            const problem = copyIn(join(inputDir, ...file.split("/")), join(grading, ...file.split("/")), GRADED_FILE_BYTES);
+            if (problem !== null)
+                return { outcome: "fail", why: `the build's ${file} ${problem}` };
+        }
+        const probeProblem = copyIn(join(caseDir, "probe.mjs"), join(grading, "probe.mjs"), GRADED_FILE_BYTES);
+        if (probeProblem !== null)
+            return { outcome: "error", why: `the case's probe ${probeProblem}` };
+        const nonce = randomBytes(16).toString("hex");
+        let inputs, expected;
+        try {
+            inputs = oracle.inputs(seededRandom(nonce));
+            expected = inputs.map(one => canonical(oracle.expected(structuredClone(one))));
+        }
+        catch {
+            return { outcome: "error", why: "the case's oracle failed to work out its answers" };
+        }
+        let ran;
+        try {
+            ran = await (input.runner ?? fencedGradeRunner)({ cwd: grading, stdin: `${JSON.stringify({ nonce, inputs })}\n`, timeoutMs: input.timeoutMs,
+                deny: [evalsRoot, inputDir, ...(input.deny ?? [])] });
+        }
+        catch (error) {
+            return { outcome: "error", why: `the graded program couldn't be started: ${error instanceof Error ? error.message.slice(0, 160) : String(error)}` };
+        }
+        if (ran.timedOut)
+            return { outcome: "fail", why: "the graded program timed out" };
+        if (ran.code === OVERFLOW_CODE)
+            return { outcome: "fail", why: `the graded program printed more than ${ORACLE_OUTPUT_BYTES} bytes` };
+        // The answer: a line naming this grading's nonce, with one output per input. The last such line counts.
+        let outputs = null;
+        for (const line of ran.stdout.slice(0, ORACLE_OUTPUT_BYTES).split("\n")) {
+            try {
+                const parsed = JSON.parse(line);
+                if (parsed !== null && parsed["nonce"] === nonce && Array.isArray(parsed["outputs"]))
+                    outputs = parsed["outputs"];
+            }
+            catch { /* not the answer */ }
+        }
+        if (outputs === null)
+            return { outcome: "fail", why: ran.code === 0 ? "the graded program exited without an answer (an exit code alone never passes)" : "the graded program crashed without an answer" };
+        // An answer and then a crash or a nonzero exit is still a crash (Codex round 1 #11): it must answer AND finish cleanly.
+        if (ran.code !== 0)
+            return { outcome: "fail", why: `the graded program exited with ${ran.code} after answering` };
+        if (outputs.length !== expected.length)
+            return { outcome: "fail", why: "the graded program answered a different number of inputs" };
+        const wrong = expected.findIndex((one, index) => canonical(outputs[index]) !== one);
+        return wrong === -1 ? { outcome: "pass", why: null } : { outcome: "fail", why: `wrong answer for input ${wrong + 1} of ${expected.length}` };
+    }
+    finally {
+        rmSync(grading, { recursive: true, force: true });
+    }
+}
+/** The oracle process's entry file, from the installed release (the sources under a test). */
+export function oracleEntry() {
+    const source = import.meta.url.endsWith(".ts");
+    const entry = fileURLToPath(new URL(source ? "./eval-oracle-main.ts" : "./eval-oracle-main.js", import.meta.url));
+    return { file: process.execPath, args: [...(source ? ["--import", import.meta.resolve("tsx")] : []), entry] };
+}
+/**
+ * Grade in a separate process from the installed release (the plane's runner calls this once the attempt's process
+ * tree is gone). Its verdict comes back over its own stdout; anything else — it crashed, timed out, said nothing
+ * readable — is an error the plane recorded, never a pass.
+ */
+export async function runOracle(input, entry = oracleEntry()) {
+    const child = spawn(entry.file, entry.args, { stdio: ["pipe", "pipe", "ignore"], env: { PATH: process.env["PATH"] ?? "/usr/bin:/bin", HOME: process.env["HOME"] ?? "/" }, detached: true });
+    let stdout = "";
+    child.stdout.on("data", (chunk) => { if (stdout.length < 16 * 1024)
+        stdout += chunk.toString("utf8"); });
+    child.stdin.end(`${JSON.stringify({ caseDir: input.caseDir, inputDir: input.inputDir, timeoutMs: input.timeoutMs, deny: input.deny ?? [] })}\n`);
+    const code = await new Promise(done => {
+        const timer = setTimeout(() => { try {
+            process.kill(-(child.pid ?? 0), "SIGKILL");
+        }
+        catch { /* gone */ } }, input.timeoutMs + 60_000);
+        child.once("error", () => { clearTimeout(timer); done(null); });
+        child.once("close", exit => { clearTimeout(timer); done(exit); });
+    });
+    try {
+        const parsed = JSON.parse(stdout.trim().split("\n").pop() ?? "");
+        const outcome = parsed["outcome"];
+        if (code === 0 && (outcome === "pass" || outcome === "fail" || outcome === "error"))
+            return { outcome, why: typeof parsed["why"] === "string" ? parsed["why"].slice(0, 300) : null };
+    }
+    catch { /* below */ }
+    return { outcome: "error", why: "the oracle process failed" };
+}

package/dist/eval-world.d.ts

@@ -0,0 +1,17 @@
+import { type EvalCase } from "./evals.js";
+/** The installed CLI, as the world's commands run it (the sources under a test). */
+export declare function evalCliEntry(): {
+    file: string;
+    args: string[];
+};
+export type WorldReport = {
+    evalStage: "world-start" | "done";
+    why: string | null;
+};
+/** Make the world and run its build to the end. Never throws: a failure is the report. */
+export declare function runEvalWorld(one: EvalCase, input: {
+    worldDir: string;
+    databaseFile: string;
+    deny: readonly string[];
+    timeoutMs: number;
+}): Promise<WorldReport>;

package/dist/eval-world.js

@@ -0,0 +1,97 @@
+/**
+ * One eval attempt's world (docs/plans/evals-nightly.md, part C, "Where it runs"): what `toolroll evals attempt` does
+ * in its own child process of the installed binary. Everything is throwaway and the world's own — its database, its
+ * repository made from the case's fixture, its worktrees, its approver and worker — driven through the installed CLI
+ * exactly as a person's project is (the real-model pilot's choreography, scripts/pilot.mjs). The world never learns
+ * the plane's database: every command names the world's own (`--db`).
+ *
+ * Its builds run under the project's own fence, with the installed eval set (the hidden checks) and anything else the
+ * runner names unreadable (`project sandbox --read-deny`), and every provider dispatch takes a slot of the runner's
+ * gate (eval-gate.ts) through TOOLROLL_EVAL_GATE, which the runner set and this process never changes.
+ *
+ * The last line on stdout says how far it got: `world-start` (the world failed before the agent could start: the
+ * runner records an error only if the world's database agrees nothing reached a provider), or `done`.
+ */
+import { closeSync, constants, mkdirSync, openSync, writeSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import { run } from "./exec.js";
+import { EVALS_ROOT, worldSecret } from "./evals.js";
+/** The installed CLI, as the world's commands run it (the sources under a test). */
+export function evalCliEntry() {
+    const source = import.meta.url.endsWith(".ts");
+    const entry = fileURLToPath(new URL(source ? "./bin.ts" : "./bin.js", import.meta.url));
+    return { file: process.execPath, args: [...(source ? ["--import", import.meta.resolve("tsx")] : []), entry] };
+}
+/** Git for the world's own fixture repository, before any agent exists: no system or global config, no hooks. */
+const GIT_ENV = { GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" };
+const GIT_ARGS = ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"];
+function writeNew(path, body, mode = 0o644) {
+    mkdirSync(dirname(path), { recursive: true });
+    const fd = openSync(path, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, mode);
+    try {
+        writeSync(fd, body);
+    }
+    finally {
+        closeSync(fd);
+    }
+}
+/** Make the world and run its build to the end. Never throws: a failure is the report. */
+export async function runEvalWorld(one, input) {
+    const { worldDir, databaseFile } = input;
+    const repo = join(worldDir, "repo"), pool = join(worldDir, "pool");
+    const cli = evalCliEntry();
+    const say = async (argv, timeoutMs = 120_000) => {
+        const result = await run(cli.file, [...cli.args, ...argv, "--db", databaseFile, "--json"], { cwd: worldDir, timeoutMs, maxBuffer: 4 * 1024 * 1024 });
+        let body = {};
+        try {
+            body = JSON.parse(result.stdout);
+        }
+        catch { /* below */ }
+        if (result.code !== 0 || body["ok"] === false)
+            throw new Error(`${argv.slice(0, 2).join(" ")}: ${String(body["message"] ?? result.stderr).slice(0, 300)}`);
+        return body;
+    };
+    try {
+        mkdirSync(repo, { recursive: true });
+        for (const [path, body] of Object.entries(one.fixture))
+            writeNew(join(repo, ...path.split("/")), body);
+        // The worker later commits the agent's changes through checkoutGit. Keep the world's synthetic identity in its
+        // repository, where that helper carries it, instead of supplying it only to this first fixture commit.
+        for (const argv of [["init", "-q", "-b", "main"], ["config", "user.name", "Toolroll eval"], ["config", "user.email", "eval@toolroll.invalid"],
+            ["config", "user.useConfigOnly", "true"], ["add", "-A"], ["commit", "-q", "-m", `Eval fixture ${one.id}`]]) {
+            const git = await run("git", [...GIT_ARGS, ...argv], { cwd: repo, env: GIT_ENV, timeoutMs: 30_000 });
+            if (git.code !== 0)
+                throw new Error(`git ${argv[0]}: ${git.stderr.slice(0, 200)}`);
+        }
+        const password = worldSecret();
+        const auth = ["--as", "eval", "--token", password];
+        await say(["approver", "add", "eval", "--password", password]);
+        const registered = await say(["runner", "register", "worker", "--repo", repo, ...auth]);
+        if (typeof registered["token"] !== "string")
+            throw new Error("the world's worker got no token");
+        for (const phase of ["plan", "build", "repair"])
+            await say(["config", "set", phase, "--provider", one.pin.provider, "--model", one.pin.model, ...auth]);
+        await say(["mode", "set", "--repo", repo, "--name", "standard", "--days", "1", ...auth]);
+        await say(["task", "add", one.title, "--id", "work", "--repo", repo]);
+        // The hidden checks and the plane's own state stay out of the agent's reach (sealed into the scope filed next).
+        await say(["project", "sandbox", "--repo", repo, "--read-deny", [EVALS_ROOT.replace(/\/$/, ""), ...input.deny].join(","), ...auth]);
+        await say(["verify", "set", "--repo", repo, "--command", "node --test", "--timeout-seconds", "120", "--yes", ...auth]);
+        await say(["task", "scope", "work", "--goal", one.goal, "--touches", one.touches.join(","), "--acceptance", `${one.goal.replace(/[|;]/g, ",").slice(0, 600)}|check`,
+            "--not", "Changing package or dependency files, or anything the goal doesn't ask for."]);
+        const shown = await say(["task", "show", "work"]);
+        const digest = shown["scope"]?.["digest"];
+        if (typeof digest !== "string")
+            throw new Error("the world's task has no scope to approve");
+        await say(["task", "approve", "work", "--yes", "--digest", digest, ...auth]);
+        writeNew(join(dirname(databaseFile), "worker-token"), String(registered["token"]), 0o600);
+    }
+    catch (error) {
+        return { evalStage: "world-start", why: error instanceof Error ? error.message : String(error) };
+    }
+    // The agent's part: the worker builds (or parks with its question) until there is nothing left to do. Its outcome is
+    // read from the world's records by the runner, never from here.
+    await run(cli.file, [...cli.args, "watch", "--runner", "worker", "--token-file", join(dirname(databaseFile), "worker-token"), "--repo", repo, "--pool", pool,
+        "--for", "1000", "--tick-every", "200", "--reconcile-every", "1000", "--bridge-every", "3600000", "--db", databaseFile, "--json"], { cwd: worldDir, timeoutMs: input.timeoutMs, maxBuffer: 4 * 1024 * 1024 });
+    return { evalStage: "done", why: null };
+}

package/dist/evals.d.ts

@@ -0,0 +1,299 @@
+import { type OracleVerdict } from "./eval-oracle.js";
+import { type ReportedAmount } from "./reported-usage.js";
+import { Store } from "./store.js";
+export declare const EVALS_ROOT: string;
+export declare const EVAL_K = 3;
+/** An errored attempt gets one replacement when the plane confirms the error; at most this many a campaign. */
+export declare const EVAL_MAX_REPLACEMENTS = 3;
+/** Attempts at once within one campaign. */
+export declare const EVAL_CONCURRENCY = 2;
+export type EvalCaseKind = "build" | "ask";
+export type EvalCase = {
+    id: string;
+    version: string;
+    dir: string;
+    kind: EvalCaseKind;
+    title: string;
+    /** What the task asks, as the agent's brief gets it. */
+    goal: string;
+    touches: string[];
+    pin: {
+        provider: "claude";
+        model: string;
+    };
+    maxDispatches: number;
+    timeoutMinutes: number;
+    /** build: the files the oracle copies from the built commit. ask: the options the question must offer (patterns). */
+    grade: {
+        files: string[];
+    } | {
+        options: string[];
+    };
+    /** The person who validated the expected outcome, or null: a draft, counted in no canonical campaign. */
+    validatedBy: string | null;
+    fixture: Record<string, string>;
+};
+export type EvalSet = {
+    ok: true;
+    root: string;
+    manifestDigest: string;
+    cases: EvalCase[];
+} | {
+    ok: false;
+    problem: string;
+};
+/** Why a pin isn't an exact model id, or null. An alias, an empty or odd value, or a non-Claude provider is no pin. */
+export declare function exactModelProblem(pin: {
+    provider?: unknown;
+    model?: unknown;
+} | null | undefined): string | null;
+/** The installed eval set: the manifest's cases, each read whole and versioned. Any case that can't be read, and any
+ * folder the manifest doesn't list, refuses the whole set. */
+export declare function loadEvalSet(root?: string): EvalSet;
+export type CampaignAsk = {
+    cases?: string[];
+    k?: number;
+    diagnostic?: boolean;
+};
+export type CampaignPlan = {
+    canonical: boolean;
+    k: number;
+    cases: EvalCase[];
+};
+/** What a campaign runs. Canonical (nothing asked): every case, k = 3, refused while any case is a draft. Diagnostic
+ * (anything asked): a report only. Either way every case runs under an exact model pin. */
+export declare function planCampaign(set: Extract<EvalSet, {
+    ok: true;
+}>, ask: CampaignAsk): {
+    ok: true;
+    plan: CampaignPlan;
+} | {
+    ok: false;
+    problem: string;
+};
+export type Admission = {
+    ok: true;
+    id: string;
+} | {
+    ok: false;
+    reason: "person-only" | "release-busy" | "already-running";
+    message: string;
+};
+/**
+ * Admit a campaign (eval or reviewer): a person proved now, nothing releasing or deploying, and no other canonical
+ * campaign running (canonical campaigns settle in the order they were admitted). One transaction: the proof, the checks
+ * and the row, with the runner that owns it (this process, on this computer) for recovery.
+ */
+export declare function admitEvalCampaign(store: Store, input: {
+    kind: "eval" | "reviewer";
+    canonical: boolean;
+    manifestDigest: string;
+    buildDigest: string;
+    version: string;
+    k: number;
+    sealed: unknown;
+    as: string;
+    token: string;
+    now: Date;
+    id?: string;
+    runner?: {
+        host: string;
+        pid: number;
+        started?: string | null;
+    };
+}): Admission;
+/** Record one attempt's start (its identity is unique). Refused when the campaign isn't running. */
+export declare function startEvalAttempt(store: Store, input: {
+    campaign: string;
+    caseId: string;
+    caseVersion: string;
+    n: number;
+    replaces: number;
+    modelPinned: string;
+    now: Date;
+}): {
+    ok: true;
+    attempt: number;
+} | {
+    ok: false;
+    why: string;
+};
+export type EvalOutcome = "pass" | "fail" | "error" | "unevaluated";
+/** One provider dispatch an attempt's world made: its record (`run:<id>`, `chat_turn:<id>`) or the gate slot no record
+ * accounts for (`slot:<n>`), what it was billed as, and the model it reported. */
+export type EvalDispatchRecord = {
+    kind: string;
+    worldRef: string;
+    recorded: boolean;
+    amount: ReportedAmount;
+    modelReported: string | null;
+};
+export type AttemptResult = {
+    outcome: EvalOutcome;
+    why: string | null;
+    modelReported: string | null;
+    dispatches: number | null;
+    apiMicrousd: number | null;
+    subscriptionMicrousd: number | null;
+    usageUnknown: boolean;
+    turns: number | null;
+    tokensIn: number | null;
+    cacheRead: number | null;
+    cacheWrite: number | null;
+    tokensOut: number | null;
+    toolCalls: number | null;
+    wallMs: number;
+    invocations: EvalDispatchRecord[];
+};
+/** Record a finished attempt, once: its outcome, each dispatch as it was billed, and its totals. */
+export declare function recordEvalAttempt(store: Store, attempt: number, result: AttemptResult, now: Date): void;
+/** End a campaign, once: complete or incomplete. */
+export declare function finishEvalCampaign(store: Store, id: string, state: "complete" | "incomplete", why: string | null, now: Date): void;
+/** What a campaign spent, from its attempts as recorded: API dollars (null when any dispatch's money is unknown),
+ * subscription usage (null when any is unknown or unreported), and how many attempts had unknown spending. Never one
+ * total of the three. */
+export declare function campaignSpending(store: Store, id: string): {
+    apiMicrousd: number | null;
+    subscriptionMicrousd: number | null;
+    unknownAttempts: number;
+    attempts: number;
+};
+export type WorldDispatches = {
+    invocations: EvalDispatchRecord[];
+    modelReported: string | null;
+    dispatches: number | null;
+    /** Asked past the case's limit (the gate refused it), or more dispatches than the limit recorded. */
+    overLimit: boolean;
+    /** A dispatch the gate and the records don't agree on: a record with no slot (it went around the gate), a slot with no
+     * record, or slots that can't be read. Its spending is unknown and the attempt is an error. */
+    unaccounted: boolean;
+    apiMicrousd: number | null;
+    subscriptionMicrousd: number | null;
+    usageUnknown: boolean;
+    turns: number | null;
+    tokensIn: number | null;
+    cacheRead: number | null;
+    cacheWrite: number | null;
+    tokensOut: number | null;
+    toolCalls: number | null;
+};
+/** Every provider dispatch a world made, from its own records (its database, opened read only; null when there is none
+ * or it can't be read) and its gate's slots: each with what it was billed as and the model it reported. A slot no
+ * record accounts for, and a record no slot does, is a dispatch with unknown money and an unknown model. */
+export declare function collectWorld(world: Store | null, gateDir: string, maxDispatches: number, evidenceRoot: string, now: Date): WorldDispatches;
+/** The installed build's identity: its version, a digest of every file of the code that is running (dist when
+ * installed, the sources in a checkout), and the service definitions that say which runtime the plane runs (a deploy
+ * rewrites them to another runtime's folder). A change between attempts, or before the end, stops the campaign as
+ * incomplete. */
+export declare function installedBuildDigest(version: string, home?: string): string;
+/** How an attempt's world is started: the child command (the installed binary's `evals attempt`, or a test's world). */
+export type AttemptCommand = (input: {
+    campaign: string;
+    caseId: string;
+    n: number;
+    worldDir: string;
+    databaseFile: string;
+    gateDir: string;
+}) => {
+    file: string;
+    args: string[];
+    env: Record<string, string>;
+};
+export type AttemptDeps = {
+    command: AttemptCommand;
+    /** The oracle (eval-oracle.ts runOracle by default: a separate process from the installed release). */
+    oracle?: (input: {
+        caseDir: string;
+        inputDir: string;
+        timeoutMs: number;
+        deny?: readonly string[];
+    }) => Promise<OracleVerdict>;
+    /** Where worlds are made (a folder the plane owns). */
+    workRoot: string;
+    /** How long the world's processes get to end after its child does, before the attempt is an error. */
+    settleMs?: number;
+    /** Called the moment the whole process tree is proven gone, before anything is read or graded (tests). */
+    onTreeGone?: () => void;
+    keepWorlds?: boolean;
+    now?: () => Date;
+};
+/**
+ * One attempt: record its start, make the world, run its child to the end of its whole process tree, read what it
+ * did, grade it, record it. What the world dispatched is collected and recorded on every road out, the runner's own
+ * failure included, before the world is removed. Outcomes (plan, "Grading"): an error only where the plane itself
+ * recorded one (the world failed before the agent started, the provider refused or was down, the oracle failed, the
+ * world's processes wouldn't end, a dispatch the gate and the records don't agree on); the agent timing out, crashing
+ * or going past its dispatch limit is a fail; a reported model other than the pin is unevaluated.
+ */
+export declare function runEvalAttempt(store: Store, input: {
+    campaign: string;
+    one: EvalCase;
+    n: number;
+    replaces: number;
+}, deps: AttemptDeps): Promise<{
+    attempt: number;
+    result: AttemptResult;
+    stopCampaign: string | null;
+} | {
+    attempt: null;
+    why: string;
+}>;
+export type CampaignDeps = AttemptDeps & {
+    /** The installed build's identity now (installedBuildDigest): a change stops the campaign as incomplete. */
+    buildDigest: () => string;
+    /** The installed eval set now (loadEvalSet): a change stops the campaign as incomplete too. */
+    evalSet?: () => EvalSet;
+    concurrency?: number;
+};
+/** Run an admitted campaign to its end: every sealed attempt (EVAL_CONCURRENCY at once), one replacement for an
+ * attempt the plane recorded an error for (at most EVAL_MAX_REPLACEMENTS a campaign), then complete — or incomplete
+ * when the installed build or eval set changed (checked before every attempt and again at the end), a deploy is
+ * draining, or the runner itself failed. */
+export declare function runEvalCampaign(store: Store, campaign: string, cases: readonly EvalCase[], deps: CampaignDeps): Promise<{
+    state: "complete" | "incomplete";
+    why: string | null;
+}>;
+/**
+ * Recover campaigns whose runner stopped (Codex round 1 #12): a running campaign whose runner was this computer's and
+ * whose process is gone. Each unfinished attempt is recorded as an error with whatever its world recorded (its spending
+ * unknown where the records can't say), once its world's process group is proven gone; then the campaign ends
+ * incomplete. A campaign whose runner is alive, ran on another computer, or still has a world running stays as it is.
+ */
+export declare function recoverEvalCampaigns(store: Store, now: Date, options?: {
+    host?: string;
+    alive?: (pid: number, started: string | null) => boolean;
+}): {
+    id: string;
+    recovered: boolean;
+    why: string;
+}[];
+/** A process's start time as the system reports it (`ps -o lstart`, whole seconds, in UTC and the C locale), or null
+ * when it can't be read: the identity a saved pid is compared by, so a later process that reuses the number is never
+ * taken for the one recorded. */
+export declare function processStartOf(pid: number): string | null;
+/** Whether a campaign's runner may still be running (PR #263 review): its pid is held, and by the same process — its
+ * start time, where one was recorded, matches. A pid now held by a process that started at another time is someone
+ * else's. A start that can't be read now counts as alive (a person ends such a campaign: `evals recover --abandon`). */
+export declare function runnerMayBeAlive(pid: number, started: string | null): boolean;
+/**
+ * End a stuck campaign by a person's word (PR #263 review): a campaign recovery can't settle (its runner can't be
+ * proven gone, or a world's processes can't). A person proved now with their password (a lead token or remembered
+ * login is no proof); in one transaction every unfinished attempt is recorded as an error with unknown spending, the
+ * campaign's spending is marked unknown, and it ends incomplete. No process is signalled and no world is read or
+ * removed: whatever is still running is left to the person.
+ */
+export declare function abandonEvalCampaign(store: Store, input: {
+    id: string;
+    as: string;
+    token: string;
+    now: Date;
+}): {
+    ok: true;
+    attempts: number;
+} | {
+    ok: false;
+    reason: "person-only" | "not-running";
+    message: string;
+};
+/** A random hex id, for the world's throwaway approver password. */
+export declare const worldSecret: () => string;