toolroll 0.9.104 → 0.9.105

package/dist/eval-oracle.js

@@ -1,24 +1,3 @@
-/**
- * The eval set's trusted oracle (docs/plans/evals-nightly.md, part C, "Grading: a trusted oracle outside the fence").
- *
- * The plane's runner starts the oracle as a separate process from the installed release (`runOracle`, which spawns
- * eval-oracle-main.js), only after the attempt's whole process tree is gone, and hands it two folders: the installed
- * case (`evals/cases/<id>/`, from the plane's own package) and the graded files the runner extracted from the built
- * commit's verified Git objects. The oracle owns the grading (`gradeCase`):
- *
- *   1. It copies only the case's named files from the graded folder into a fresh grading folder, reading each with
- *      O_NOFOLLOW and refusing links and anything but a regular file; then the case's own `probe.mjs` from the
- *      installed case (an edited launcher in the build is never the one run).
- *   2. It starts the probe with the installed node and a fixed argv inside the native fence (the release check's:
- *      Seatbelt or bubblewrap via the agent fence), writing only the grading folder, with no network and the installed
- *      eval set and the plane's state unreadable, under a time limit.
- *   3. It sends a fresh nonce and inputs it chose from that nonce, reads at most 64 KiB of output, and passes the
- *      attempt only when the output names that nonce with the values it computed itself (`oracle.mjs` `expected`).
- *      An exit code alone never passes: an early `process.exit(0)` leaves no answer.
- *
- * The build's code never runs in the oracle's own process, and nothing the build wrote (a result file, a verdict) is
- * ever read as one: the verdict travels from the oracle to the runner over the oracle's own pipe.
- */
 import { prepareProtectedStorage } from "./protected-storage.js";
 import { spawn } from "node:child_process";
 import { createHash, randomBytes } from "node:crypto";
@@ -29,11 +8,9 @@ import { checkLaunchRoot, fenceFor, linuxFenceAvailable, macosFenceAvailable, ru
 import { startEgressProxy, proxyEnv } from "./egress.js";
 import { run, OVERFLOW_CODE } from "./exec.js";
 import { readNoFollow } from "./eval-gate.js";
-/** The most a graded program may print, and the most of each graded file the oracle copies. */
 export const ORACLE_OUTPUT_BYTES = 64 * 1024;
 const GRADED_FILE_BYTES = 256 * 1024;
 const EVALS_ROOT = fileURLToPath(new URL("../evals/", import.meta.url));
-/** A random source seeded by the nonce (mulberry32 over its digest): the inputs are fresh for every grading. */
 export function seededRandom(nonce) {
     let state = createHash("sha256").update(nonce).digest().readUInt32LE(0);
     return () => {
@@ -44,19 +21,14 @@ export function seededRandom(nonce) {
         return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
     };
 }
-/** One value in a canonical form (object keys sorted), so two answers compare by content. */
 export function canonical(value) {
     return JSON.stringify(value, (_key, inner) => inner !== null && typeof inner === "object" && !Array.isArray(inner)
         ? Object.fromEntries(Object.keys(inner).sort().map(key => [key, inner[key]]))
         : inner);
 }
-/** The default: the installed node, `probe.mjs`, inside the release check's fence — writes only the grading folder, no
- * network (a proxy of its own that allows no host), the installed eval set and whatever else `deny` names unreadable. No
- * fence on this machine: nothing is graded (the attempt is an error, never a pass). */
 export const fencedGradeRunner = async ({ cwd, stdin, timeoutMs, deny }) => {
     if (!macosFenceAvailable() && !linuxFenceAvailable())
         throw new Error("this machine can't fence the graded program");
-    // The storage check, awaited off this thread (protected-storage.ts); a refusal keeps its words.
     const storage = await prepareProtectedStorage();
     const egress = await startEgressProxy({ hosts: [] });
     try {
@@ -71,8 +43,6 @@ export const fencedGradeRunner = async ({ cwd, stdin, timeoutMs, deny }) => {
         await egress.close();
     }
 };
-/** Copy a file the oracle trusts nothing about: a regular file, never through a link, bounded; into a path that must
- * not exist yet. */
 function copyIn(from, to, maxBytes) {
     const bytes = readNoFollow(from, maxBytes);
     if (bytes === null) {
@@ -94,7 +64,6 @@ function copyIn(from, to, maxBytes) {
     return null;
 }
 const inside = (root, path) => { const rel = relative(root, path); return rel !== "" && !rel.startsWith("..") && !isAbsolute(rel); };
-/** Grade one attempt, in the oracle's own process. `caseDir` must be a case of the installed eval set (`evalsRoot`). */
 export async function gradeCase(input) {
     const evalsRoot = realpathSync(input.evalsRoot ?? EVALS_ROOT);
     let caseDir, inputDir;
@@ -160,7 +129,6 @@ export async function gradeCase(input) {
             return { outcome: "fail", why: "the graded program timed out" };
         if (ran.code === OVERFLOW_CODE)
             return { outcome: "fail", why: `the graded program printed more than ${ORACLE_OUTPUT_BYTES} bytes` };
-        // The answer: a line naming this grading's nonce, with one output per input. The last such line counts.
         let outputs = null;
         for (const line of ran.stdout.slice(0, ORACLE_OUTPUT_BYTES).split("\n")) {
             try {
@@ -168,11 +136,10 @@ export async function gradeCase(input) {
                 if (parsed !== null && parsed["nonce"] === nonce && Array.isArray(parsed["outputs"]))
                     outputs = parsed["outputs"];
             }
-            catch { /* not the answer */ }
+            catch { }
         }
         if (outputs === null)
             return { outcome: "fail", why: ran.code === 0 ? "the graded program exited without an answer (an exit code alone never passes)" : "the graded program crashed without an answer" };
-        // An answer and then a crash or a nonzero exit is still a crash (Codex round 1 #11): it must answer AND finish cleanly.
         if (ran.code !== 0)
             return { outcome: "fail", why: `the graded program exited with ${ran.code} after answering` };
         if (outputs.length !== expected.length)
@@ -184,17 +151,11 @@ export async function gradeCase(input) {
         rmSync(grading, { recursive: true, force: true });
     }
 }
-/** The oracle process's entry file, from the installed release (the sources under a test). */
 export function oracleEntry() {
     const source = import.meta.url.endsWith(".ts");
     const entry = fileURLToPath(new URL(source ? "./eval-oracle-main.ts" : "./eval-oracle-main.js", import.meta.url));
     return { file: process.execPath, args: [...(source ? ["--import", import.meta.resolve("tsx")] : []), entry] };
 }
-/**
- * Grade in a separate process from the installed release (the plane's runner calls this once the attempt's process
- * tree is gone). Its verdict comes back over its own stdout; anything else — it crashed, timed out, said nothing
- * readable — is an error the plane recorded, never a pass.
- */
 export async function runOracle(input, entry = oracleEntry()) {
     const child = spawn(entry.file, entry.args, { stdio: ["pipe", "pipe", "ignore"], env: { PATH: process.env["PATH"] ?? "/usr/bin:/bin", HOME: process.env["HOME"] ?? "/" }, detached: true });
     let stdout = "";
@@ -205,7 +166,7 @@ export async function runOracle(input, entry = oracleEntry()) {
         const timer = setTimeout(() => { try {
             process.kill(-(child.pid ?? 0), "SIGKILL");
         }
-        catch { /* gone */ } }, input.timeoutMs + 60_000);
+        catch { } }, input.timeoutMs + 60_000);
         child.once("error", () => { clearTimeout(timer); done(null); });
         child.once("close", exit => { clearTimeout(timer); done(exit); });
     });
@@ -215,6 +176,6 @@ export async function runOracle(input, entry = oracleEntry()) {
         if (code === 0 && (outcome === "pass" || outcome === "fail" || outcome === "error"))
             return { outcome, why: typeof parsed["why"] === "string" ? parsed["why"].slice(0, 300) : null };
     }
-    catch { /* below */ }
+    catch { }
     return { outcome: "error", why: "the oracle process failed" };
 }

package/dist/eval-world.js

@@ -1,29 +1,13 @@
-/**
- * One eval attempt's world (docs/plans/evals-nightly.md, part C, "Where it runs"): what `toolroll evals attempt` does
- * in its own child process of the installed binary. Everything is throwaway and the world's own — its database, its
- * repository made from the case's fixture, its worktrees, its approver and worker — driven through the installed CLI
- * exactly as a person's project is (the real-model pilot's choreography, scripts/pilot.mjs). The world never learns
- * the plane's database: every command names the world's own (`--db`).
- *
- * Its builds run under the project's own fence, with the installed eval set (the hidden checks) and anything else the
- * runner names unreadable (`project sandbox --read-deny`), and every provider dispatch takes a slot of the runner's
- * gate (eval-gate.ts) through TOOLROLL_EVAL_GATE, which the runner set and this process never changes.
- *
- * The last line on stdout says how far it got: `world-start` (the world failed before the agent could start: the
- * runner records an error only if the world's database agrees nothing reached a provider), or `done`.
- */
 import { closeSync, constants, mkdirSync, openSync, writeSync } from "node:fs";
 import { dirname, join } from "node:path";
 import { fileURLToPath } from "node:url";
 import { run } from "./exec.js";
 import { EVALS_ROOT, worldSecret } from "./evals.js";
-/** The installed CLI, as the world's commands run it (the sources under a test). */
 export function evalCliEntry() {
     const source = import.meta.url.endsWith(".ts");
     const entry = fileURLToPath(new URL(source ? "./bin.ts" : "./bin.js", import.meta.url));
     return { file: process.execPath, args: [...(source ? ["--import", import.meta.resolve("tsx")] : []), entry] };
 }
-/** Git for the world's own fixture repository, before any agent exists: no system or global config, no hooks. */
 const GIT_ENV = { GIT_CONFIG_NOSYSTEM: "1", GIT_CONFIG_GLOBAL: "/dev/null", GIT_TERMINAL_PROMPT: "0" };
 const GIT_ARGS = ["-c", "core.hooksPath=/dev/null", "-c", "core.fsmonitor=false"];
 function writeNew(path, body, mode = 0o644) {
@@ -36,7 +20,6 @@ function writeNew(path, body, mode = 0o644) {
         closeSync(fd);
     }
 }
-/** Make the world and run its build to the end. Never throws: a failure is the report. */
 export async function runEvalWorld(one, input) {
     const { worldDir, databaseFile } = input;
     const repo = join(worldDir, "repo"), pool = join(worldDir, "pool");
@@ -47,7 +30,7 @@ export async function runEvalWorld(one, input) {
         try {
             body = JSON.parse(result.stdout);
         }
-        catch { /* below */ }
+        catch { }
         if (result.code !== 0 || body["ok"] === false)
             throw new Error(`${argv.slice(0, 2).join(" ")}: ${String(body["message"] ?? result.stderr).slice(0, 300)}`);
         return body;
@@ -56,8 +39,6 @@ export async function runEvalWorld(one, input) {
         mkdirSync(repo, { recursive: true });
         for (const [path, body] of Object.entries(one.fixture))
             writeNew(join(repo, ...path.split("/")), body);
-        // The worker later commits the agent's changes through checkoutGit. Keep the world's synthetic identity in its
-        // repository, where that helper carries it, instead of supplying it only to this first fixture commit.
         for (const argv of [["init", "-q", "-b", "main"], ["config", "user.name", "Toolroll eval"], ["config", "user.email", "eval@toolroll.invalid"],
             ["config", "user.useConfigOnly", "true"], ["add", "-A"], ["commit", "-q", "-m", `Eval fixture ${one.id}`]]) {
             const git = await run("git", [...GIT_ARGS, ...argv], { cwd: repo, env: GIT_ENV, timeoutMs: 30_000 });
@@ -74,7 +55,6 @@ export async function runEvalWorld(one, input) {
             await say(["config", "set", phase, "--provider", one.pin.provider, "--model", one.pin.model, ...auth]);
         await say(["mode", "set", "--repo", repo, "--name", "standard", "--days", "1", ...auth]);
         await say(["task", "add", one.title, "--id", "work", "--repo", repo]);
-        // The hidden checks and the plane's own state stay out of the agent's reach (sealed into the scope filed next).
         await say(["project", "sandbox", "--repo", repo, "--read-deny", [EVALS_ROOT.replace(/\/$/, ""), ...input.deny].join(","), ...auth]);
         await say(["verify", "set", "--repo", repo, "--command", "node --test", "--timeout-seconds", "120", "--yes", ...auth]);
         await say(["task", "scope", "work", "--goal", one.goal, "--touches", one.touches.join(","), "--acceptance", `${one.goal.replace(/[|;]/g, ",").slice(0, 600)}|check`,
@@ -89,8 +69,6 @@ export async function runEvalWorld(one, input) {
     catch (error) {
         return { evalStage: "world-start", why: error instanceof Error ? error.message : String(error) };
     }
-    // The agent's part: the worker builds (or parks with its question) until there is nothing left to do. Its outcome is
-    // read from the world's records by the runner, never from here.
     await run(cli.file, [...cli.args, "watch", "--runner", "worker", "--token-file", join(dirname(databaseFile), "worker-token"), "--repo", repo, "--pool", pool,
         "--for", "1000", "--tick-every", "200", "--reconcile-every", "1000", "--bridge-every", "3600000", "--db", databaseFile, "--json"], { cwd: worldDir, timeoutMs: input.timeoutMs, maxBuffer: 4 * 1024 * 1024 });
     return { evalStage: "done", why: null };