toolroll 0.9.104 → 0.9.105

package/dist/evals.js

@@ -1,27 +1,3 @@
-/**
- * The eval set (docs/plans/evals-nightly.md, part C1): versioned cases run by the installed plane, each attempt in a
- * throwaway world of its own, graded by a trusted oracle outside the fence, every provider dispatch through a gate
- * and every dispatch's spending recorded as it was reported.
- *
- * - **Cases** live in `evals/` (shipped in the package): `manifest.json` lists them; each `cases/<id>/` holds
- *   `case.json`, `fixture/` (the project the agent starts from, plain files), `oracle.mjs` + `probe.mjs` (the hidden
- *   check) and `source.md` (where it came from). A case's version is the sha256 of its folder; a link or anything but a
- *   regular file in it refuses the whole set. A case is a draft until a person validates its expected outcome
- *   (`validatedBy` in case.json, changed only in a release a person approves: `evals/` is a policy path).
- * - **Campaigns**: canonical (no flags) seals every case, its version, its pin, k = 3 and the installed build; it is
- *   refused while any case is a draft or any pin isn't an exact model id. Anything else is diagnostic. Admission is one
- *   transaction with a person's proof: no release operation reserved, no deploy draining, one canonical campaign at a
- *   time (settlement order). The runner that owns a campaign is recorded, so an interrupted one can be recovered.
- * - **Attempts** run as `toolroll evals attempt …`, a child process of the installed binary in its own process group,
- *   with its own world (database, repository, worktrees, dispatch gate). The runner waits for the whole process tree to
- *   be gone, reads the world's database (read only), extracts the built files from verified Git objects, and only then
- *   asks the oracle (eval-oracle.ts).
- * - **Money**: no limit (the owner, 2026-10-30). What every dispatch spent is recorded on every exit path: API dollars,
- *   subscription usage and unknown kept apart and never added together; a dispatch the world's records don't account
- *   for, or one that went around the gate, makes the attempt an error with unknown spending.
- *
- * Result-writing authority is this module's alone: a world never gets a handle on the plane's database.
- */
 import { execFileSync, spawn } from "node:child_process";
 import { createHash, randomBytes } from "node:crypto";
 import { closeSync, constants, lstatSync, mkdirSync, mkdtempSync, openSync, readdirSync, realpathSync, rmSync, writeSync } from "node:fs";
@@ -41,23 +17,17 @@ import { combineReportedModels } from "./reported-usage.js";
 import { authenticateApprover } from "./scope.js";
 import { deriveRunEfficiency } from "./run-efficiency.js";
 import { chatTurnAccounting, runAccounting, Store } from "./store.js";
-// Loaded on first use, like the store: a static import prints Node's SQLite warning and can't be bundled.
 const sqlite = () => createRequire(import.meta.url)("node:sqlite");
 export const EVALS_ROOT = fileURLToPath(new URL("../evals/", import.meta.url));
 export const EVAL_K = 3;
-/** An errored attempt gets one replacement when the plane confirms the error; at most this many a campaign. */
 export const EVAL_MAX_REPLACEMENTS = 3;
-/** Attempts at once within one campaign. */
 export const EVAL_CONCURRENCY = 2;
 const CASE_ID = /^[a-z0-9][a-z0-9-]{0,47}$/;
 const FIXTURE_PATH = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,63}(\/[A-Za-z0-9_][A-Za-z0-9_.-]{0,63}){0,3}$/;
-/** Names that pick a model for you: never a pin. */
 const MODEL_ALIASES = new Set(["sonnet", "opus", "haiku", "default", "best", "opusplan", "sonnet[1m]", "opus[1m]"]);
 const MAX_CASE_FILES = 64;
 const MAX_CASE_FILE_BYTES = 256 * 1024;
 const sha256 = (value) => createHash("sha256").update(value).digest("hex");
-/** Every regular file under `dir` (relative path → bytes), never through a link; null with why when anything else is
- * there or a limit is passed. */
 function folderFiles(dir, limit = MAX_CASE_FILES) {
     const files = new Map();
     const walk = (at, depth) => {
@@ -93,14 +63,12 @@ function folderFiles(dir, limit = MAX_CASE_FILES) {
     const problem = walk(dir, 0);
     return problem === null ? { ok: true, files } : { ok: false, problem };
 }
-/** A folder's digest: every file's path and bytes, in path order. */
 function digestOf(files) {
     const hash = createHash("sha256");
     for (const [path, bytes] of [...files].sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0))
         hash.update(path).update("\0").update(sha256(bytes)).update("\0");
     return hash.digest("hex");
 }
-/** Why a pin isn't an exact model id, or null. An alias, an empty or odd value, or a non-Claude provider is no pin. */
 export function exactModelProblem(pin) {
     if (pin === null || pin === undefined || pin.provider !== "claude")
         return "its pin names no provider an eval can cap and read the exact model of (claude)";
@@ -171,8 +139,6 @@ function caseOf(id, dir, files) {
     return { ok: true, value: { id, version: digestOf(files), dir, kind, title, goal, touches, pin: { provider: "claude", model: pin.model },
             maxDispatches, timeoutMinutes, grade: graded, validatedBy, fixture } };
 }
-/** The installed eval set: the manifest's cases, each read whole and versioned. Any case that can't be read, and any
- * folder the manifest doesn't list, refuses the whole set. */
 export function loadEvalSet(root = EVALS_ROOT) {
     const manifestBytes = readNoFollow(join(root, "manifest.json"), 64 * 1024);
     if (manifestBytes === null)
@@ -217,8 +183,6 @@ export function loadEvalSet(root = EVALS_ROOT) {
     }
     return { ok: true, root, manifestDigest: sha256(JSON.stringify({ manifest: sha256(manifestBytes), cases: cases.map(one => [one.id, one.version]) })), cases };
 }
-/** What a campaign runs. Canonical (nothing asked): every case, k = 3, refused while any case is a draft. Diagnostic
- * (anything asked): a report only. Either way every case runs under an exact model pin. */
 export function planCampaign(set, ask) {
     const canonical = ask.cases === undefined && ask.k === undefined && ask.diagnostic !== true;
     const k = ask.k ?? EVAL_K;
@@ -242,14 +206,8 @@ export function planCampaign(set, ask) {
     }
     return { ok: true, plan: { canonical, k, cases } };
 }
-/**
- * Admit a campaign (eval or reviewer): a person proved now, nothing releasing or deploying, and no other canonical
- * campaign running (canonical campaigns settle in the order they were admitted). One transaction: the proof, the checks
- * and the row, with the runner that owns it (this process, on this computer) for recovery.
- */
 export function admitEvalCampaign(store, input) {
     const id = input.id ?? `ev-${input.now.toISOString().slice(0, 10).replace(/-/g, "")}-${randomBytes(4).toString("hex")}`;
-    // The runner's process start time too: recovery never takes a later process that reuses its pid for the runner.
     const runner = input.runner ?? { host: hostname(), pid: process.pid, started: processStartOf(process.pid) };
     try {
         return store.transact(() => {
@@ -273,13 +231,11 @@ export function admitEvalCampaign(store, input) {
         });
     }
     catch (error) {
-        // The unique running index is the last word on "one canonical at a time" when two admissions race.
         if (/UNIQUE constraint failed: eval_campaign/.test(String(error.message)))
             return { ok: false, reason: "already-running", message: "Another canonical campaign started at the same moment; they run one at a time." };
         throw error;
     }
 }
-/** Record one attempt's start (its identity is unique). Refused when the campaign isn't running. */
 export function startEvalAttempt(store, input) {
     return store.transact(() => {
         const campaign = store.handle.prepare("SELECT state FROM eval_campaign WHERE id = ?").get(input.campaign);
@@ -290,15 +246,12 @@ export function startEvalAttempt(store, input) {
         return { ok: true, attempt: Number(made.lastInsertRowid) };
     });
 }
-/** Where an attempt's world is, and its process group once spawned: what recovery reads if the runner stops. */
 function markAttemptWorld(store, attempt, worldDir, pgid) {
     store.handle.prepare("UPDATE eval_attempt SET world_dir = ?, world_pgid = COALESCE(?, world_pgid) WHERE id = ? AND outcome IS NULL").run(worldDir, pgid, attempt);
 }
-/** Saved outside the world: recovery must check detached groups as well as the world's original group. */
 function recordWorldProcesses(store, attempt, processes) {
     store.handle.prepare("UPDATE eval_attempt SET world_processes_json = ? WHERE id = ? AND outcome IS NULL").run(JSON.stringify(processes), attempt);
 }
-/** Record a finished attempt, once: its outcome, each dispatch as it was billed, and its totals. */
 export function recordEvalAttempt(store, attempt, result, now) {
     store.transact(() => {
         const row = store.handle.prepare("SELECT outcome FROM eval_attempt WHERE id = ?").get(attempt);
@@ -318,25 +271,17 @@ export function recordEvalAttempt(store, attempt, result, now) {
             .run(result.outcome, result.why === null ? null : result.why.slice(0, 500), result.modelReported, result.dispatches, result.apiMicrousd, result.subscriptionMicrousd, result.usageUnknown ? 1 : 0, result.turns, result.tokensIn, result.cacheRead, result.cacheWrite, result.tokensOut, result.toolCalls, result.wallMs, now.toISOString(), attempt);
     });
 }
-/** End a campaign, once: complete or incomplete. */
 export function finishEvalCampaign(store, id, state, why, now) {
     store.handle.prepare("UPDATE eval_campaign SET state = ?, why = ?, finished_at = ? WHERE id = ? AND state = 'running'").run(state, why, now.toISOString(), id);
 }
-/** What a campaign spent, from its attempts as recorded: API dollars (null when any dispatch's money is unknown),
- * subscription usage (null when any is unknown or unreported), and how many attempts had unknown spending. Never one
- * total of the three. */
 export function campaignSpending(store, id) {
     const rows = store.handle.prepare("SELECT api_microusd, subscription_microusd, usage_unknown FROM eval_attempt WHERE campaign = ? AND outcome IS NOT NULL").all(id);
-    // An abandoned campaign's spending is unknown whatever its finished attempts recorded: its runner may have gone on.
     const abandoned = Number(store.handle.prepare("SELECT spending_unknown FROM eval_campaign WHERE id = ?").get(id)?.["spending_unknown"] ?? 0) !== 0;
     const total = (column) => abandoned || rows.some(row => row[column] === null) ? null : rows.reduce((sum, row) => sum + Number(row[column]), 0);
     return { apiMicrousd: total("api_microusd"), subscriptionMicrousd: total("subscription_microusd"), unknownAttempts: rows.filter(row => Number(row["usage_unknown"]) === 1).length, attempts: rows.length };
 }
 const sum = (values) => values.some(one => one === null) ? null : values.reduce((total, one) => total + one, 0);
 const UNKNOWN = { kind: "unknown" };
-/** Every provider dispatch a world made, from its own records (its database, opened read only; null when there is none
- * or it can't be read) and its gate's slots: each with what it was billed as and the model it reported. A slot no
- * record accounts for, and a record no slot does, is a dispatch with unknown money and an unknown model. */
 export function collectWorld(world, gateDir, maxDispatches, evidenceRoot, now) {
     const slots = evalGateSlots(gateDir);
     const invocations = [];
@@ -379,13 +324,10 @@ export function collectWorld(world, gateDir, maxDispatches, evidenceRoot, now) {
                 cacheRead.push(read?.cacheReadTokens ?? null);
                 cacheWrite.push(read?.cacheWriteTokens ?? null);
                 tokensOut.push(read?.tokensOut ?? null);
-                // A chat request's own tool calls aren't step-recorded: unknown.
                 toolCalls.push(null);
             }
             if (chats.length > chatSlots.length)
                 unaccounted = true;
-            // A run slot whose run never reached its provider spent nothing; a run slot with no run at all, or a chat slot past
-            // the recorded chat requests, may have been sent.
             for (const slot of slots ?? []) {
                 if (!slot.kind.startsWith("run:"))
                     continue;
@@ -401,8 +343,6 @@ export function collectWorld(world, gateDir, maxDispatches, evidenceRoot, now) {
             }
         }
         catch {
-            // Keep every charge already read. A damaged or older world must not prevent settlement or turn missing
-            // dispatches into zero spending. The synthetic slot is explicitly unknown, never a measured call.
             unrecorded({ n: 0, kind: "unreadable" });
             unaccounted = true;
         }
@@ -423,8 +363,6 @@ export function collectWorld(world, gateDir, maxDispatches, evidenceRoot, now) {
         usageUnknown, turns: unaccounted ? null : sum(turns), tokensIn: unaccounted ? null : sum(tokensIn), cacheRead: unaccounted ? null : sum(cacheRead), cacheWrite: unaccounted ? null : sum(cacheWrite), tokensOut: unaccounted ? null : sum(tokensOut), toolCalls: unaccounted ? null : sum(toolCalls),
     };
 }
-/** A run's tool calls from its step records (run-efficiency.ts stepsOf), read before the world is removed; null when
- * the records are absent, expired or incomplete. */
 function toolCallsOf(world, runId, evidenceRoot, now) {
     const run = world.getRun(runId);
     if (run === null)
@@ -438,7 +376,6 @@ function toolCallsOf(world, runId, evidenceRoot, now) {
         return null;
     }
 }
-/** A world's records, read only; null when there is no database or it can't be read. */
 function openWorld(databaseFile) {
     try {
         lstatSync(databaseFile);
@@ -453,11 +390,6 @@ function openWorld(databaseFile) {
         return null;
     }
 }
-// ------------------------------------------------------------------ one attempt
-/** The installed build's identity: its version, a digest of every file of the code that is running (dist when
- * installed, the sources in a checkout), and the service definitions that say which runtime the plane runs (a deploy
- * rewrites them to another runtime's folder). A change between attempts, or before the end, stops the campaign as
- * incomplete. */
 export function installedBuildDigest(version, home = homedir()) {
     const here = dirname(fileURLToPath(import.meta.url));
     const units = [];
@@ -492,29 +424,23 @@ function folderFilesDeep(dir) {
     walk(dir);
     return hash.digest("hex");
 }
-/** Whether every process witness is gone: the child's own group, and each descendant the observer saw, checked as what
- * it was seen as (a process, or a group whose leader it was). */
 function treeGone(pgid, seen) {
     return pgid > 0 && !processMayBeAlive(pgid, true) && [...seen].every(([pid, group]) => !processMayBeAlive(pid, group));
 }
-/** Stop what is left of a world: the child's group, and each witness as what it was seen as. Its own processes only. */
 function stopWorld(pgid, seen) {
     if (pgid > 0) {
         try {
             process.kill(-pgid, "SIGKILL");
         }
-        catch { /* gone */ }
+        catch { }
     }
     for (const [pid, group] of seen) {
         try {
             process.kill(group ? -pid : pid, "SIGKILL");
         }
-        catch { /* gone */ }
+        catch { }
     }
 }
-/** Run the child and wait for its whole process tree: the child's exit, its process group gone, and every descendant
- * (and every group one led) the process-tree observer saw gone. A tree that won't end, or one whose ancestry couldn't
- * be observed, is stopped and reported (`lingering`), never graded. */
 async function runWorldProcess(spec, cwd, timeoutMs, settleMs, onSpawn, persist) {
     persist({ state: "starting", witnesses: [] });
     const child = spawn(spec.file, spec.args, { cwd, env: spec.env, detached: true, stdio: ["ignore", "pipe", "pipe"] });
@@ -522,7 +448,6 @@ async function runWorldProcess(spec, cwd, timeoutMs, settleMs, onSpawn, persist)
     let unknownAncestry = false;
     const pgid = child.pid ?? -1;
     const save = () => persist({ state: unknownAncestry ? "unknown" : "watching", witnesses: [...(pgid > 0 ? [[pgid, true]] : []), ...seen] });
-    // A persistence failure after spawn must still stop the owned child and await its exit.
     try {
         if (pgid > 0)
             onSpawn(pgid);
@@ -553,7 +478,7 @@ async function runWorldProcess(spec, cwd, timeoutMs, settleMs, onSpawn, persist)
                 try {
                     process.kill(-pgid, "SIGKILL");
                 }
-                catch { /* gone */ }
+                catch { }
             }
         }, timeoutMs);
         child.once("error", () => { clearTimeout(timer); done(null); });
@@ -573,7 +498,6 @@ async function runWorldProcess(spec, cwd, timeoutMs, settleMs, onSpawn, persist)
         while (pgid > 0 && !treeGone(pgid, seen) && Date.now() < last)
             await new Promise(done => setTimeout(done, 50));
     }
-    // Its report: the whole output as one JSON document (the CLI's envelope), or its last line. None: the agent's stage.
     let report = null;
     for (const candidate of [stdout.trim(), stdout.trim().split("\n").pop() ?? ""]) {
         try {
@@ -584,14 +508,12 @@ async function runWorldProcess(spec, cwd, timeoutMs, settleMs, onSpawn, persist)
                 break;
             }
         }
-        catch { /* not this one */ }
+        catch { }
     }
     const stopped = pgid < 0 || (!unknownAncestry && treeGone(pgid, seen));
     persist({ state: stopped ? "stopped" : "unknown", witnesses: [...(pgid > 0 ? [[pgid, true]] : []), ...seen] });
     return { timedOut, report, lingering, stopped, why, code };
 }
-/** Write each graded file of the built commit, from verified Git objects (never the checkout, never a link), into a
- * fresh folder for the oracle. */
 function extractGraded(repo, head, files, into) {
     let gitDir;
     try {
@@ -611,7 +533,7 @@ function extractGraded(repo, head, files, into) {
     for (const file of files) {
         const blob = blobs.value.get(file);
         if (blob === undefined)
-            continue; // missing: the oracle fails it
+            continue;
         mkdirSync(dirname(join(into, file)), { recursive: true });
         const fd = openSync(join(into, file), constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
         try {
@@ -623,21 +545,12 @@ function extractGraded(repo, head, files, into) {
     }
     return { ok: true };
 }
-/** An attempt's result from what its world recorded, with the outcome and why. */
 const resultOf = (outcome, why, collected, wallMs) => ({
     outcome, why, modelReported: collected.modelReported, dispatches: collected.dispatches,
     apiMicrousd: collected.apiMicrousd, subscriptionMicrousd: collected.subscriptionMicrousd, usageUnknown: collected.usageUnknown,
     turns: collected.turns, tokensIn: collected.tokensIn, cacheRead: collected.cacheRead, cacheWrite: collected.cacheWrite, tokensOut: collected.tokensOut, toolCalls: collected.toolCalls,
     wallMs, invocations: collected.invocations
 });
-/**
- * One attempt: record its start, make the world, run its child to the end of its whole process tree, read what it
- * did, grade it, record it. What the world dispatched is collected and recorded on every road out, the runner's own
- * failure included, before the world is removed. Outcomes (plan, "Grading"): an error only where the plane itself
- * recorded one (the world failed before the agent started, the provider refused or was down, the oracle failed, the
- * world's processes wouldn't end, a dispatch the gate and the records don't agree on); the agent timing out, crashing
- * or going past its dispatch limit is a fail; a reported model other than the pin is unevaluated.
- */
 export async function runEvalAttempt(store, input, deps) {
     const now = deps.now ?? (() => new Date());
     const { one } = input;
@@ -650,8 +563,6 @@ export async function runEvalAttempt(store, input, deps) {
     let ran = null;
     let failure = null;
     let spawned = false;
-    // The world's state folder is named as Toolroll's own, so every agent fence in the world denies it whole: the
-    // database, the worker's token and the dispatch gate (agent-fence.ts agentFence, a dedicated state folder).
     const paths = (dir) => ({ stateDir: join(dir, "toolroll"), gateDir: join(dir, "toolroll", "eval-gate"), databaseFile: join(dir, "toolroll", "orders.db") });
     try {
         mkdirSync(deps.workRoot, { recursive: true, mode: 0o700 });
@@ -670,9 +581,7 @@ export async function runEvalAttempt(store, input, deps) {
     catch (error) {
         failure = `the eval runner failed: ${error instanceof Error ? error.message.slice(0, 200) : String(error)}`;
     }
-    // Whatever happened, what the world recorded is read before anything is decided or removed.
     const where = worldDir === null ? null : paths(worldDir);
-    // A lingering tree has been stopped by now: its records are read all the same, so known spending is kept.
     const world = where === null ? null : openWorld(where.databaseFile);
     let result;
     const collected = where === null
@@ -696,8 +605,6 @@ export async function runEvalAttempt(store, input, deps) {
             const startedRuns = Number(world.handle.prepare("SELECT COUNT(*) AS n FROM run WHERE provider_started_at IS NOT NULL").get()?.["n"] ?? 0);
             const refused = Number(world.handle.prepare("SELECT COUNT(*) AS n FROM run WHERE outcome = 'refused' AND provider_started_at IS NULL").get()?.["n"] ?? 0);
             if (collected.invocations.length === 0 && startedRuns === 0) {
-                // Nothing reached a provider: the world failed before the agent started (its child said so), or the provider
-                // refused (a plane-side record); anything else is the agent's (a fail).
                 const why = ran.report?.stage === "world-start" ? `the world failed before the agent started: ${ran.report.why ?? "no reason given"}`
                     : refused > 0 ? "the provider refused or was unavailable before the agent started" : null;
                 result = why !== null ? resultOf("error", why, collected, wall()) : resultOf("fail", ran.timedOut ? "timed out before the agent started" : "the agent never started", collected, wall());
@@ -732,9 +639,6 @@ export async function runEvalAttempt(store, input, deps) {
     finally {
         world?.close();
     }
-    // PR #263 review: a world whose process tree isn't proven stopped (its ancestry unknown, a process that outlived
-    // SIGKILL, or a runner failure after the spawn) may still be dispatching. What it spent is unknown, never the part
-    // its records show, and the campaign ends incomplete.
     let stopCampaign = null;
     if ((ran !== null && !ran.stopped) || (ran === null && spawned)) {
         result = { ...result, apiMicrousd: null, subscriptionMicrousd: null, usageUnknown: true,
@@ -742,13 +646,10 @@ export async function runEvalAttempt(store, input, deps) {
         stopCampaign = `${one.id} #${input.n}'s world couldn't be proven stopped, so what it spent is unknown`;
     }
     recordEvalAttempt(store, attempt, result, now());
-    // Only once it is recorded is the world removed.
     if (worldDir !== null && ran?.stopped === true && deps.keepWorlds !== true)
         rmSync(worldDir, { recursive: true, force: true });
     return { attempt, result, stopCampaign };
 }
-/** Grade from what the world's installed code recorded: the built commit's files through the oracle (build), or the
- * question a run parked with (ask). The agent's own words are never read as a verdict. */
 async function grade(world, worldDir, one, deps, deny) {
     if ("options" in one.grade) {
         const asked = world.handle.prepare("SELECT d.options FROM decision d JOIN run r ON r.id = d.run WHERE r.role = 'builder' ORDER BY d.id LIMIT 1").get();
@@ -771,10 +672,6 @@ async function grade(world, worldDir, one, deps, deny) {
     const verdict = await (deps.oracle ?? runOracle)({ caseDir: one.dir, inputDir, timeoutMs: 60_000, deny });
     return { outcome: verdict.outcome, why: verdict.why };
 }
-/** Run an admitted campaign to its end: every sealed attempt (EVAL_CONCURRENCY at once), one replacement for an
- * attempt the plane recorded an error for (at most EVAL_MAX_REPLACEMENTS a campaign), then complete — or incomplete
- * when the installed build or eval set changed (checked before every attempt and again at the end), a deploy is
- * draining, or the runner itself failed. */
 export async function runEvalCampaign(store, campaign, cases, deps) {
     const now = deps.now ?? (() => new Date());
     const row = store.handle.prepare("SELECT k, build_digest, manifest_digest, state FROM eval_campaign WHERE id = ?").get(campaign);
@@ -819,7 +716,6 @@ export async function runEvalCampaign(store, campaign, cases, deps) {
                 stop ??= `the eval runner failed: ${error instanceof Error ? error.message.slice(0, 200) : String(error)}`;
             }
         }))));
-        // The identity again at the end: a change during the last attempts leaves the campaign incomplete too.
         stop ??= changed();
     }
     catch (error) {
@@ -829,12 +725,6 @@ export async function runEvalCampaign(store, campaign, cases, deps) {
     finishEvalCampaign(store, campaign, state, stop, now());
     return { state, why: stop };
 }
-/**
- * Recover campaigns whose runner stopped (Codex round 1 #12): a running campaign whose runner was this computer's and
- * whose process is gone. Each unfinished attempt is recorded as an error with whatever its world recorded (its spending
- * unknown where the records can't say), once its world's process group is proven gone; then the campaign ends
- * incomplete. A campaign whose runner is alive, ran on another computer, or still has a world running stays as it is.
- */
 export function recoverEvalCampaigns(store, now, options = {}) {
     const host = options.host ?? hostname();
     const alive = options.alive ?? runnerMayBeAlive;
@@ -857,7 +747,7 @@ export function recoverEvalCampaigns(store, now, options = {}) {
         }
         const unproven = open.find(one => {
             if (one["world_dir"] === null)
-                return false; // No world was ever made.
+                return false;
             try {
                 const saved = JSON.parse(String(one["world_processes_json"]));
                 if (saved === null || !Array.isArray(saved.witnesses))
@@ -882,7 +772,6 @@ export function recoverEvalCampaigns(store, now, options = {}) {
             const dir = one["world_dir"] === null ? null : String(one["world_dir"]);
             const world = dir === null ? null : openWorld(join(dir, "toolroll", "orders.db"));
             try {
-                // No world folder recorded: the attempt never got as far as making one (its gate reads as unreadable: unknown).
                 const collected = collectWorld(world, dir === null ? "" : join(dir, "toolroll", "eval-gate"), Number.MAX_SAFE_INTEGER, dir === null ? "" : join(dir, "toolroll", "evidence"), now);
                 recordEvalAttempt(store, Number(one["id"]), resultOf("error", "interrupted: its runner stopped before the attempt finished", collected, 0), now);
             }
@@ -895,9 +784,6 @@ export function recoverEvalCampaigns(store, now, options = {}) {
     }
     return out;
 }
-/** A process's start time as the system reports it (`ps -o lstart`, whole seconds, in UTC and the C locale), or null
- * when it can't be read: the identity a saved pid is compared by, so a later process that reuses the number is never
- * taken for the one recorded. */
 export function processStartOf(pid) {
     if (process.platform === "win32" || !Number.isSafeInteger(pid) || pid <= 0)
         return null;
@@ -911,20 +797,14 @@ export function processStartOf(pid) {
         return null;
     }
 }
-/** Whether a campaign's runner may still be running (PR #263 review): its pid is held, and by the same process — its
- * start time, where one was recorded, matches. A pid now held by a process that started at another time is someone
- * else's. A start that can't be read now counts as alive (a person ends such a campaign: `evals recover --abandon`). */
 export function runnerMayBeAlive(pid, started) {
     if (!processMayBeAlive(pid, false))
         return false;
     const was = started === null ? null : startMs(started);
     const now = processStartOf(pid);
     const is = now === null ? null : startMs(now);
-    // Within two seconds is the same process: a system that derives the start from its boot time can shift it by a
-    // second, and a pid can't be reused while the process holding it lives.
     return was === null || is === null || Math.abs(is - was) <= 2_000;
 }
-/** A `ps -o lstart` reading (UTC, C locale) as epoch milliseconds, or null when it isn't one exactly. */
 function startMs(raw) {
     const match = /^(Sun|Mon|Tue|Wed|Thu|Fri|Sat) (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) (\d{1,2}) (\d{2}:\d{2}:\d{2}) (\d{4})$/.exec(raw);
     if (match === null)
@@ -933,13 +813,6 @@ function startMs(raw) {
     const ms = Date.parse(utc);
     return Number.isFinite(ms) && new Date(ms).toUTCString() === utc ? ms : null;
 }
-/**
- * End a stuck campaign by a person's word (PR #263 review): a campaign recovery can't settle (its runner can't be
- * proven gone, or a world's processes can't). A person proved now with their password (a lead token or remembered
- * login is no proof); in one transaction every unfinished attempt is recorded as an error with unknown spending, the
- * campaign's spending is marked unknown, and it ends incomplete. No process is signalled and no world is read or
- * removed: whatever is still running is left to the person.
- */
 export function abandonEvalCampaign(store, input) {
     return store.transact(() => {
         const proved = authenticateApprover(store, input.as, input.token, null);
@@ -961,5 +834,4 @@ export function abandonEvalCampaign(store, input) {
         return { ok: true, attempts: open.length };
     });
 }
-/** A random hex id, for the world's throwaway approver password. */
 export const worldSecret = () => randomBytes(24).toString("hex");

package/dist/evidence-pack.js

@@ -1,15 +1,3 @@
-/**
- * v103: a change's evidence pack. It is everything an auditor asks about one
- * piece of work, in one place: who asked for it, the exact terms and who
- * approved them (under which rules), which agents ran and what they cost,
- * what changed, the checks, who marked it complete, where it was published,
- * and every ledger entry about it with its seal. JSON for tools; the same
- * facts as a printable page for people.
- *
- * Each ledger entry carries its seal (`prev`, `hash`), so anyone can
- * recompute it; the chain report and the latest checkpoint tie those
- * entries to the whole ledger.
- */
 import { createHash } from "node:crypto";
 import { rulesWords } from "./approval-policy.js";
 import { assignmentOf } from "./assignment.js";
@@ -18,12 +6,9 @@ import { changedFilesOf } from "./result-completion.js";
 import { approvalProofOf, personProof } from "./scope.js";
 import { html, htmlString, joinHtml, styleElement } from "./html.js";
 export const EVIDENCE_PACK_FORMAT = "standing-orders/evidence-pack/v1";
-/** How to recheck one entry's seal, for whoever reads the JSON. */
 export const SEAL_RECIPE = 'hash = sha256(prev + "\\n" + JSON.stringify([id, at, actor, repo, taskId, runId, action, outcome, source, detail]))';
-/** A pack carries up to this many of the task's own entries, and the newest this many requests made about it. */
 const MAX_ENTRIES = 5000;
 const MAX_REQUESTS = 500;
-/** The pack for one task (the whole family: the task and every revision), as `who` may see it. null when they can't. */
 export function evidencePack(store, taskId, access, who, now, evidenceRoot, checked) {
     const family = store.taskFamilyOf(taskId, access.repos, access.principal === "operator" && access.includeUnplaced === true);
     if (family === null)
@@ -102,7 +87,6 @@ export function evidencePack(store, taskId, access, who, now, evidenceRoot, chec
 export function packDigest(body) {
     return createHash("sha256").update(JSON.stringify(body), "utf8").digest("hex");
 }
-/** A day as the export takes it (YYYY-MM-DD, UTC), or null. */
 export function exportDay(value) {
     if (value === null || !/^\d{4}-\d{2}-\d{2}$/.test(value))
         return null;
@@ -110,14 +94,9 @@ export function exportDay(value) {
     return Number.isNaN(day.getTime()) || day.toISOString().slice(0, 10) !== value ? null : day.toISOString();
 }
 const EXPORT_ENTRIES = 100_000;
-/** Entries per piece of an export: small enough that a slow, steady reader finishes each well inside the server's minute. */
 const EXPORT_PAGE = 250;
 const EXPORT_PACKS = 200;
-/** Every ledger entry in [from, to) that `who` may read, sealed, with a pack for each task they name: one JSON
- * document (a `LedgerExport`), a piece at a time (a page of entries, or one pack), so an export never holds
- * the whole range at once. */
 export function* ledgerExportChunks(store, range, scope, access, who, now, evidenceRoot) {
-    // An export is the record someone takes away: the whole chain is walked for it (or was, within the last minute).
     const chain = store.ledgerChain({ full: true, fresh: 60_000 });
     const latest = store.ledgerCheckpoints(1)[0] ?? null;
     const head = {
@@ -174,11 +153,9 @@ export function* ledgerExportChunks(store, range, scope, access, who, now, evide
     }
     yield `],"truncated":${JSON.stringify({ entries: moreEntries, packs: morePacks })}}`;
 }
-// ---- The printable page -----------------------------------------------------
 const when = (at) => at === null ? html `—` : html `${at.slice(0, 16).replace("T", " ")} UTC`;
 const short = (hash) => hash === null ? html `—` : html `<code title="${hash}">${hash.slice(0, 12)}</code>`;
 const usd = (value) => value === null ? "—" : `$${value.toFixed(value < 1 ? 4 : 2)}`;
-/** Only a web address links; anything else (a stored value that isn't one) is shown as text. */
 const link = (url) => /^https:\/\/[^\s"'<>]+$/.test(url) ? html `<a href="${url}">${url}</a>` : html `${url}`;
 const filer = (who) => who === null || who.name === null ? html `automation` : who.kind === "person" ? html `${who.name}` : html `${who.name} (${who.kind})`;
 export const EVIDENCE_PACK_CSS = `.evidence-pack{max-width:960px;min-width:0}.evidence-pack h1{margin:0}.evidence-pack h2{font-size:.9375rem;margin:24px 0 8px}.evidence-pack h3{font-size:.875rem;margin:16px 0 6px}` +
@@ -190,7 +167,6 @@ export const EVIDENCE_PACK_CSS = `.evidence-pack{max-width:960px;min-width:0}.ev
     `@media(max-width:600px){.evidence-pack dl{grid-template-columns:minmax(0,1fr)}.evidence-pack dd{margin-bottom:6px}}` +
     `@media print{.side,.tabbar,.mobile-top,.scope-bar,.banner,.evidence-actions{display:none!important}body,.app,.content,.detail,main{background:#fff!important;color:#000!important;margin:0!important;padding:0!important;box-shadow:none!important;border:0!important}` +
     `.evidence-pack{max-width:none}.evidence-pack section{break-inside:avoid-page}.evidence-pack a{color:#000;text-decoration:none}}`;
-/** The pack as a page: the same facts, in the order an auditor reads them. No script. */
 export function evidencePackHtml(pack) {
     const task = pack.task;
     const chain = pack.ledger.chain.ok
@@ -232,7 +208,6 @@ export function evidencePackHtml(pack) {
     const ledger = html `<section><h2>Ledger</h2>${decisions.length === 0 ? html `<p class="meta">No entries.</p>` :
         html `<div class="evidence-table"><table><thead><tr><th>#</th><th>Time</th><th>Who</th><th>What</th><th>Outcome</th><th>Seal</th></tr></thead><tbody>${decisions.map(one => html `<tr data-ledger-id="${one.id}"><td>${one.id}</td><td>${when(one.at)}</td><td>${one.actor}</td><td>${one.action}${one.detail ? html `<br><span class="meta">${one.detail}</span>` : ""}</td><td>${one.outcome}</td><td>${short(one.seal?.hash ?? null)}</td></tr>`)}</tbody></table></div>`}${requests > 0 ? html `<p class="meta">Plus ${askedWords}, listed in the JSON.</p>` : ""}${pack.ledger.truncated ? html `<p class="meta">This pack holds ${decisions.length} of the task's entries and the newest ${requests} requests; the ledger export has the rest.</p>` : ""}</section>`;
     const cost = pack.totals.runs === 0 ? "None ran" : `${pack.totals.runs} ${pack.totals.runs === 1 ? "run" : "runs"} · ${usd(pack.totals.costUsd)}`;
-    // The rules as they stand, and each change to them (the approval above was under whichever applied then).
     const rules = pack.rules === null ? html `None` : html `${pack.rules.now}${pack.rules.changes.length === 0 ? "" :
         html `<ul class="criteria">${pack.rules.changes.slice(-5).map(one => html `<li class="meta">${when(one.at)} · ${one.by}: ${one.change}</li>`)}</ul>`}`;
     return joinHtml([html `<article class="evidence-pack" data-evidence-pack="${task.id}">`,
@@ -244,16 +219,13 @@ export function evidencePackHtml(pack) {
         versions, outcome, ledger,
         html `<p class="evidence-foot">Pack digest <code>${pack.digest}</code>. Each ledger entry's seal can be rechecked from the JSON: ${pack.ledger.recipe}.</p></article>`]);
 }
-/** The page as a file on its own (the CLI's --html): the same facts with just enough style to print. */
 export function standaloneEvidenceHtml(pack) {
     const css = `:root{--muted-foreground:#5c5c5c;--border:#dedede;--success:#1a7f37;--font-mono:ui-monospace,SFMono-Regular,Menlo,monospace}` +
         `body{font:14px/1.5 system-ui,-apple-system,sans-serif;margin:32px auto;padding:0 16px;max-width:960px;color:#171717;background:#fff}.meta{color:var(--muted-foreground);font-size:.8125rem}.evidence-actions{display:none}` +
         EVIDENCE_PACK_CSS;
-    // The file is the sink: its bytes.
     return htmlString(html `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1"><title>Evidence pack · ${pack.task.id}</title>${styleElement(css)}</head><body>${evidencePackHtml(pack)}</body></html>
 `);
 }
-/** Who approved, in the pack's words (approval policy §1.5): `person` only when a person's proof stands behind it. */
 function approvalBasisWord(store, taskId, basis, by, digest) {
     if (basis === "mode")
         return "mode";
@@ -261,7 +233,6 @@ function approvalBasisWord(store, taskId, basis, by, digest) {
         return "flow";
     if (basis === "schedule")
         return "schedule";
-    // A later release commit carried under a person's release approval (release-once.md): never a person's own yes.
     if (basis === "carried")
         return "carried";
     if (basis === "lead")

package/dist/evidence-pair.js

@@ -1,20 +1,9 @@
-/**
- * Before-and-after evidence (docs/plans/fleet-follow-through.md, part 4), and a bug fix's optional failing test first.
- *
- * The before side is saved from a sealed failed check: its run, commit, receipt digest and command digest. The after
- * side is never stored. It is read each time from the family's newest result, so a newer commit reads as out of date
- * without anything having to notice. Rerunning uses the follow-up check road (result-follow-ups.ts): the same approved
- * command, in a fresh checkout, on the latest commit, with no agent.
- *
- * A failing-test-first report is the agent's own words, shown as that. It is not a check Toolroll ran.
- */
 import { readVerificationView } from "./contracts/verification-view.js";
 import { followUpChecksOf, requestFollowUpChecks } from "./result-follow-ups.js";
 import { readingFleet } from "./fleet-schema.js";
 import { cleanFindingText, familyRootOf, NOT_ON_TASK, runOnTask } from "./findings.js";
 import { runCheckLevel } from "./check-levels.js";
 import { failedVerificationEvidence, verificationEvidence } from "./verification-evidence.js";
-/** The family's builder runs that finished with a commit, newest first. */
 function familyRuns(store, taskId) {
     const family = store.taskFamilyOf(taskId, null, true);
     const ids = family === null || family.problem !== null ? [taskId] : family.versions.map(one => one.id);
@@ -24,7 +13,6 @@ function familyRuns(store, taskId) {
     }).sort((a, b) => b.id - a.id);
 }
 const screenshotsOf = (store, runId) => store.artifactsFor(runId).filter(one => one.kind === "screenshot").map(one => one.id);
-/** The command a run's sealed receipt ran, and whether it passed. Null when it has no readable receipt. */
 function sealedCheck(store, root, runId) {
     const gate = verificationEvidence(store, root, runId);
     if (!gate.ok || gate.bytes === null)
@@ -35,7 +23,6 @@ function sealedCheck(store, root, runId) {
     const result = view.value.result;
     return { digest: view.value.command.digest, passed: result.configured && result.ran === true && result.exitCode === 0 };
 }
-/** The newest result of the family that ran `digest`, by the build's own check or a follow-up check on its commit. */
 function afterSide(store, root, runs, beforeRun, digest, now) {
     let otherCommand = false;
     for (const run of runs) {
@@ -80,12 +67,10 @@ function pairOf(store, root, row, taskId, now) {
 export function evidencePairsOf(store, root, taskId, now) {
     return readingFleet(() => store.handle.prepare("SELECT * FROM evidence_pair WHERE family_root = ? ORDER BY id DESC").all(familyRootOf(store, taskId)).map(row => pairOf(store, root, row, taskId, now)), []);
 }
-/** Save a run's failed check as the before side. Only a sealed failure can be saved: there's nothing else to compare. */
 export function saveEvidencePair(store, root, input, now) {
     const ref = store.lookupRef(input.taskId);
     if (ref === null)
         return { ok: false, reason: "unknown", message: `no task \`${input.taskId}\`` };
-    // A run named by a caller is checked first, and said plainly: every road (the CLI's --run, the console's form) comes here.
     if (input.runId !== undefined && !runOnTask(store, input.runId, input.taskId))
         return { ok: false, reason: "not-on-task", message: NOT_ON_TASK };
     const runId = input.runId ?? familyRuns(store, input.taskId)[0]?.id;
@@ -104,7 +89,6 @@ export function saveEvidencePair(store, root, input, now) {
     const row = store.handle.prepare("SELECT id FROM evidence_pair WHERE family_root = ? AND before_run = ?").get(root1, run.id);
     return { ok: true, id: Number(row?.["id"]), existing: true };
 }
-/** Ask for the same check on the latest commit. Its level is the one the saved failure ran at. The caller proves the person. */
 export function rerunEvidence(store, taskId, actor, now) {
     const latest = familyRuns(store, taskId)[0];
     if (latest === undefined)
@@ -114,7 +98,6 @@ export function rerunEvidence(store, taskId, actor, now) {
     return asked.ok ? { ok: true, run: latest.id, head: latest.headRevision, request: asked.request, existing: asked.existing } : { ok: false, message: asked.message };
 }
 const short = (head) => head === null ? "no commit" : head.slice(0, 7);
-/** How a pair reads, in one sentence. */
 export function evidenceWords(pair) {
     const before = `Failed on ${short(pair.before.head)} (build #${pair.before.run})`;
     switch (pair.reading) {