sakura-agent 0.1.0 → 0.1.1

package/dist/packages/env/src/connect-ssh.js

@@ -0,0 +1,132 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, rm } from "node:fs/promises";
+import { createRemoteEnvironment, RemoteConnectionError } from "./remote.js";
+/** Output kept from one command, as for a local one. */
+const MAX_OUTPUT_CHARS = 20_000_000;
+const quote = (text) => `'${text.replace(/'/g, `'\\''`)}'`;
+/** Runs ssh on this machine and collects what it prints. */
+function runSsh(program, args, options) {
+    return new Promise((resolve, reject) => {
+        const child = spawn(program, args, { stdio: ["ignore", "pipe", "pipe"] });
+        let output = "";
+        let stderr = "";
+        let timedOut = false;
+        const keep = (chunk, target) => {
+            const text = chunk.toString("utf8");
+            output += text;
+            if (output.length > MAX_OUTPUT_CHARS)
+                output = output.slice(output.length - MAX_OUTPUT_CHARS);
+            if (target === "stderr")
+                stderr += text;
+        };
+        child.stdout.on("data", (chunk) => keep(chunk, "both"));
+        child.stderr.on("data", (chunk) => keep(chunk, "stderr"));
+        const stop = () => {
+            child.kill("SIGTERM");
+            setTimeout(() => child.kill("SIGKILL"), 1000).unref();
+        };
+        const timer = options.timeoutMs !== undefined
+            ? setTimeout(() => {
+                timedOut = true;
+                stop();
+            }, options.timeoutMs)
+            : undefined;
+        if (options.signal?.aborted)
+            stop();
+        options.signal?.addEventListener("abort", stop, { once: true });
+        child.on("error", (error) => {
+            clearTimeout(timer);
+            reject(new RemoteConnectionError(`could not run ${program}: ${error.message}`));
+        });
+        const finish = (code) => {
+            clearTimeout(timer);
+            options.signal?.removeEventListener("abort", stop);
+            resolve({ exitCode: code, output, timedOut, stderr });
+        };
+        child.on("close", finish);
+        // An ssh that makes the connection leaves a copy of itself behind in the background (the first one does, and so does one that has to make it again), and
+        // that copy may keep our pipes open for as long as it lives. So the answer is the ssh we started exiting, with a moment for what it wrote to arrive.
+        child.on("exit", (code) => setTimeout(() => {
+            child.stdout.destroy();
+            child.stderr.destroy();
+            finish(code);
+        }, 100));
+    });
+}
+/**
+ * Opens one connection to the machine and runs every command over it. The first ssh logs in (once) and stays in the background; each later ssh finds it
+ * through a socket in a directory of ours and only starts a new process on the other side, so a command costs milliseconds and not a login. If the
+ * connection dies, the next command makes a new one by itself. Nothing is promised about the connection: when it cannot be reached, the command's
+ * result is a RemoteConnectionError and the caller says so.
+ */
+export async function connectSsh(options) {
+    const program = options.ssh ?? "ssh";
+    // sockets have a short length limit (104 characters on macOS), so the directory is under /tmp and the name is a hash
+    const directory = await mkdtemp("/tmp/sk-ssh-");
+    const control = `${directory}/%C`;
+    const common = [
+        "-o",
+        `ControlPath=${control}`,
+        "-o",
+        "BatchMode=yes",
+        "-o",
+        `ConnectTimeout=${options.connectTimeoutSec ?? 15}`,
+        "-o",
+        "ServerAliveInterval=15",
+        "-o",
+        "ServerAliveCountMax=3",
+        ...(options.port !== undefined ? ["-p", String(options.port)] : []),
+        ...(options.identityFile !== undefined ? ["-i", options.identityFile] : []),
+        ...Object.entries(options.options ?? {}).flatMap(([name, value]) => ["-o", `${name}=${value}`]),
+    ];
+    let closed = false;
+    const cleanup = () => rm(directory, { recursive: true, force: true });
+    try {
+        // log in now, so a wrong host or key shows up here and not in the middle of a task
+        const opened = await runSsh(program, [...common, "-o", "ControlMaster=yes", "-o", "ControlPersist=10m", "-N", options.host], {});
+        if (opened.exitCode !== 0)
+            throw new RemoteConnectionError(`could not connect to ${options.host}: ${opened.stderr.trim() || `ssh exited with ${opened.exitCode}`}`);
+    }
+    catch (error) {
+        await cleanup();
+        throw error;
+    }
+    const run = async (command, runOptions) => {
+        if (closed)
+            throw new RemoteConnectionError("the connection was closed");
+        const remote = `${runOptions.cwd !== undefined ? `cd ${quote(runOptions.cwd)} && ` : ""}bash -c ${quote(command)}`;
+        const result = await runSsh(program, [...common, "-o", "ControlMaster=auto", "-o", "ControlPersist=10m", options.host, remote], { timeoutMs: runOptions.timeoutMs, signal: runOptions.signal });
+        // ssh itself exits with 255 when it could not reach the machine; a command that exits with 255 on its own is taken for that too
+        if (result.exitCode === 255 && !result.timedOut && !runOptions.signal?.aborted)
+            throw new RemoteConnectionError(result.stderr.trim() || `could not reach ${options.host}`);
+        return { exitCode: result.exitCode, output: result.output, timedOut: result.timedOut };
+    };
+    return {
+        run,
+        async close() {
+            if (closed)
+                return;
+            closed = true;
+            await runSsh(program, [...common, "-O", "exit", options.host], {}).catch(() => undefined);
+            await cleanup();
+        },
+    };
+}
+/** What the model is told about where its tools run, over ssh. */
+export function describeSsh(host, cwd) {
+    return (`a remote machine (${host}), not the computer that runs you. Every command is sent over an SSH connection and starts fresh in ${cwd}: cd and export do not carry over to the next command. ` +
+        "The connection can be slow or drop, and a command can fail for reasons that have nothing to do with it, so results are not guaranteed to be repeatable. " +
+        "If a tool says the connection failed, the command may or may not have run: look at the current state (list the files, read the output file) before you repeat anything that changes something. " +
+        "Files are on the remote machine.");
+}
+/** An environment whose tools run on another machine over ssh. Close it (or dispose of the agent) to hang up. */
+export async function createSshEnvironment(options) {
+    const connection = await connectSsh(options);
+    return createRemoteEnvironment({
+        cwd: options.cwd,
+        run: connection.run,
+        close: connection.close,
+        description: describeSsh(options.host, options.cwd),
+        ...(options.workDir !== undefined ? { workDir: options.workDir } : {}),
+    });
+}

package/dist/packages/env/src/index.d.ts

@@ -1,3 +1,7 @@
 export { createLocalEnvironment } from "./local.js";
-export { createRemoteEnvironment } from "./remote.js";
+export type { LocalOptions } from "./local.js";
+export { OutsideDirectoryError } from "./confine.js";
+export { createRemoteEnvironment, RemoteConnectionError } from "./remote.js";
 export type { RemoteRunner } from "./remote.js";
+export { connectSsh, createSshEnvironment } from "./connect-ssh.js";
+export type { SshConnection, SshOptions } from "./connect-ssh.js";

package/dist/packages/env/src/index.js

@@ -1,2 +1,4 @@
 export { createLocalEnvironment } from "./local.js";
-export { createRemoteEnvironment } from "./remote.js";
+export { OutsideDirectoryError } from "./confine.js";
+export { createRemoteEnvironment, RemoteConnectionError } from "./remote.js";
+export { connectSsh, createSshEnvironment } from "./connect-ssh.js";

package/dist/packages/env/src/local.d.ts

@@ -1,3 +1,12 @@
 import type { Environment } from "../../kernel/src/index.js";
-/** The environment tools run in when there is no sandbox: the local machine. */
-export declare function createLocalEnvironment(cwd: string): Environment;
+export interface LocalOptions {
+    /**
+     * Keep every file operation inside the working directory: reading, writing, listing and the directory of a command are refused (OutsideDirectoryError)
+     * for a path outside it, however it is written (absolute, with .., through a link). This is a boundary for the file tools. It is not a sandbox: a shell
+     * command can still reach anything this process can, so with a shell tool use it together with the review, or without a shell tool at all.
+     * It guards against honest mistakes and a model that wanders, not against a program that races the check.
+     */
+    confine?: boolean;
+}
+/** The environment tools run in by default: this machine, with the permissions of this process. It confines nothing unless `confine` is set. */
+export declare function createLocalEnvironment(cwd: string, options?: LocalOptions): Environment;

package/dist/packages/env/src/local.js

@@ -1,8 +1,9 @@
 import { spawn as spawnProcess } from "node:child_process";
 import { createWriteStream } from "node:fs";
-import { mkdir, mkdtemp, readdir, readFile, stat, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readdir, readFile, realpath, stat, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { dirname, isAbsolute, join, resolve } from "node:path";
+import { isInside, OutsideDirectoryError, realLocation } from "./confine.js";
 const MAX_OUTPUT_CHARS = 20_000_000;
 const KILL_GRACE_MS = 1500;
 function killGroup(pid, signal) {
@@ -15,7 +16,9 @@ function killGroup(pid, signal) {
         // the group is already gone
     }
 }
-// Commands run in their own process group, so they would outlive a host process that is stopped without closing the environment
+// Commands run in their own process group, so that stopping one stops everything it started. (A command that starts a session of its own, with
+// setsid for example, leaves the group and is not reached by that; to stop everything, run the commands in a container.)
+// They would also outlive a host process that is stopped without closing the environment
 // (Ctrl-C, process.exit, a script that ends while a background command still runs). Every group still running is stopped when the host exits.
 const liveGroups = new Set();
 let exitHooked = false;
@@ -39,15 +42,43 @@ function hookExit() {
         process.on(signal, onSignal);
     }
 }
-/** The environment tools run in when there is no sandbox: the local machine. */
-export function createLocalEnvironment(cwd) {
+/** The environment tools run in by default: this machine, with the permissions of this process. It confines nothing unless `confine` is set. */
+export function createLocalEnvironment(cwd, options = {}) {
     const root = resolve(cwd);
-    const resolvePath = (path) => (isAbsolute(path) ? resolve(path) : resolve(root, path));
+    const confine = options.confine === true;
+    /** Where the output files of the commands of this environment are: outside the working directory, and the model is told to read them. */
+    let outputRoot;
+    const inOutputRoot = (absolute) => outputRoot !== undefined && isInside(outputRoot, absolute);
+    const resolvePath = (path) => {
+        const absolute = isAbsolute(path) ? resolve(path) : resolve(root, path);
+        if (confine && !isInside(root, absolute) && !inOutputRoot(absolute))
+            throw new OutsideDirectoryError(absolute, root);
+        return absolute;
+    };
+    let realRoot;
+    /** For file operations: the text check of resolvePath, and then where the path really is, because a link inside the folder can lead out of it. */
+    const checked = async (path, forWriting = false) => {
+        const absolute = resolvePath(path);
+        if (confine && inOutputRoot(absolute)) {
+            if (forWriting)
+                throw new OutsideDirectoryError(absolute, root); // the output files may be read, not written
+            return absolute;
+        }
+        if (confine) {
+            realRoot ??= realpath(root);
+            if (!isInside(await realRoot, await realLocation(absolute, root)))
+                throw new OutsideDirectoryError(absolute, root);
+        }
+        return absolute;
+    };
     const running = new Set();
     let outputDir;
     let counter = 0;
     const nextOutputFile = async () => {
-        outputDir ??= mkdtemp(join(tmpdir(), "sakura-env-"));
+        outputDir ??= mkdtemp(join(tmpdir(), "sakura-env-")).then((directory) => {
+            outputRoot = resolve(directory);
+            return directory;
+        });
         return join(await outputDir, `command-${++counter}.log`);
     };
     function start(command, options) {
@@ -129,16 +160,17 @@ export function createLocalEnvironment(cwd) {
     }
     return {
         cwd: root,
+        ...(confine ? { description: `this machine (${process.platform}). The file tools work only inside ${root}; a path outside it is refused.` } : {}),
         resolve: resolvePath,
-        readFile: async (path) => new Uint8Array(await readFile(resolvePath(path))),
+        readFile: async (path) => new Uint8Array(await readFile(await checked(path))),
         async writeFile(path, data) {
-            const target = resolvePath(path);
+            const target = await checked(path, true);
             await mkdir(dirname(target), { recursive: true });
             await writeFile(target, data);
         },
         async stat(path) {
             try {
-                const info = await stat(resolvePath(path));
+                const info = await stat(await checked(path));
                 return { isFile: info.isFile(), isDirectory: info.isDirectory(), size: info.size };
             }
             catch (error) {
@@ -148,7 +180,7 @@ export function createLocalEnvironment(cwd) {
             }
         },
         async list(path) {
-            return (await readdir(resolvePath(path))).sort();
+            return (await readdir(await checked(path))).sort();
         },
         spawn: start,
         async exec(command, options) {

package/dist/packages/env/src/remote.d.ts

@@ -1,5 +1,15 @@
 import type { Environment, ExecResult } from "../../kernel/src/index.js";
-/** The one thing a remote environment needs: a way to run a shell command somewhere else and get its output back. */
+/**
+ * Thrown by a runner when it could not reach the other side, so nothing is known about whether the command ran. A command that did run and failed
+ * is not this: it is a result with its own exit code. Whoever writes a runner translates its transport's failures (ssh exits with 255, for example) into this.
+ */
+export declare class RemoteConnectionError extends Error {
+    constructor(message: string);
+}
+/**
+ * The one thing a remote environment needs: a way to run a shell command somewhere else and get its output back.
+ * It resolves with the result of a command that ran, whatever its exit code, and rejects with a RemoteConnectionError when it could not reach the other side.
+ */
 export type RemoteRunner = (command: string, options: {
     cwd?: string;
     timeoutMs?: number;
@@ -12,4 +22,10 @@ export type RemoteRunner = (command: string, options: {
 export declare function createRemoteEnvironment(options: {
     cwd: string;
     run: RemoteRunner;
+    /** Called once when the environment is disposed, after its commands were stopped: close the connection the runner uses. */
+    close?: () => Promise<void>;
+    /** What the model is told about where its tools run (see Environment.description). */
+    description?: string;
+    /** Where the scripts and logs of started commands are kept on the other machine, and where a background command's output file is. Default /tmp/.sakura-remote, so the working directory is left as the task has it. */
+    workDir?: string;
 }): Environment;

package/dist/packages/env/src/remote.js

@@ -1,5 +1,15 @@
 import { randomBytes } from "node:crypto";
 import { posix } from "node:path";
+/**
+ * Thrown by a runner when it could not reach the other side, so nothing is known about whether the command ran. A command that did run and failed
+ * is not this: it is a result with its own exit code. Whoever writes a runner translates its transport's failures (ssh exits with 255, for example) into this.
+ */
+export class RemoteConnectionError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "RemoteConnectionError";
+    }
+}
 const quote = (text) => `'${text.replace(/'/g, `'\\''`)}'`;
 const CHUNK = 48_000;
 /**
@@ -8,7 +18,7 @@ const CHUNK = 48_000;
  */
 export function createRemoteEnvironment(options) {
     const root = posix.resolve("/", options.cwd);
-    const workDir = posix.join(root, ".sakura", "remote");
+    const workDir = posix.resolve("/", options.workDir ?? "/tmp/.sakura-remote");
     const { run } = options;
     const resolvePath = (path) => (path.startsWith("/") ? posix.normalize(path) : posix.resolve(root, path));
     const running = new Set();
@@ -20,6 +30,7 @@ export function createRemoteEnvironment(options) {
             throw new Error(result.output.trim() || `command failed with exit code ${result.exitCode}`);
         return result.output;
     }
+    const killing = new Map();
     function spawn(command, spawnOptions) {
         const id = `${prefix}-${++counter}`;
         const log = posix.join(workDir, `${id}.log`);
@@ -28,16 +39,34 @@ export function createRemoteEnvironment(options) {
         let pid = 0;
         let finished = false;
         let result;
+        let startFailure;
         const script = `cd ${quote(spawnOptions.cwd ? resolvePath(spawnOptions.cwd) : root)} || exit 1\n${command}\n`;
         const started = run(`mkdir -p ${quote(workDir)} && echo ${Buffer.from(script).toString("base64")} | base64 -d > ${quote(scriptFile)} || exit $?; ` +
             `setsid bash -c ${quote(`bash ${quote(scriptFile)} > ${quote(log)} 2>&1; echo $? > ${quote(codeFile)}`)} > /dev/null 2>&1 & echo $!`, { cwd: root }).then((outcome) => {
             if (outcome.exitCode === 0)
                 pid = Number.parseInt(outcome.output.trim().split("\n").at(-1) ?? "", 10) || 0;
+        }, (error) => {
+            if (!(error instanceof RemoteConnectionError))
+                throw error;
+            startFailure = error;
         });
+        // Stopping is done once, however often it is asked for, and can be waited for (dispose does).
         const kill = () => {
-            void started.then(() => (pid > 0 ? run(`kill -TERM -- -${pid} 2>/dev/null; sleep 1; kill -KILL -- -${pid} 2>/dev/null; true`, { cwd: root }) : undefined)).catch(() => undefined);
+            if (killing.has(handle))
+                return;
+            killing.set(handle, started
+                .then(() => (pid > 0 ? run(`kill -TERM -- -${pid} 2>/dev/null; sleep 1; kill -KILL -- -${pid} 2>/dev/null; true`, { cwd: root }) : undefined))
+                .then(() => undefined, () => undefined));
         };
         spawnOptions.signal?.addEventListener("abort", kill, { once: true });
+        /** The connection failed and nothing can be said about how the command ended: say that, and where its output is. */
+        const lost = (error, what) => {
+            finished = true;
+            running.delete(handle);
+            return error instanceof RemoteConnectionError
+                ? new RemoteConnectionError(`${what} (${error.message}). The command may still be running; its output is being written to ${log} on the remote machine.`)
+                : error;
+        };
         let done;
         const handle = {
             get pid() {
@@ -52,6 +81,11 @@ export function createRemoteEnvironment(options) {
         };
         done = (async () => {
             await started;
+            if (startFailure) {
+                finished = true;
+                running.delete(handle);
+                throw new RemoteConnectionError(`The connection to the remote machine failed while the command was being started, so it may or may not be running (${startFailure.message}). If it is, its output is being written to ${log} on the remote machine.`);
+            }
             if (pid === 0) {
                 finished = true;
                 running.delete(handle);
@@ -60,10 +94,14 @@ export function createRemoteEnvironment(options) {
             for (let attempt = 0;; attempt++) {
                 const state = await run(`if [ -f ${quote(codeFile)} ]; then cat ${quote(codeFile)}; elif kill -0 ${pid} 2>/dev/null; then echo running; else echo gone; fi`, {
                     cwd: root,
+                }).catch((error) => {
+                    throw lost(error, "The connection to the remote machine was lost while waiting for the command");
                 });
                 const text = state.output.trim();
                 if (text !== "running") {
-                    const output = await run(`cat ${quote(log)} 2>/dev/null`, { cwd: root });
+                    const output = await run(`cat ${quote(log)} 2>/dev/null`, { cwd: root }).catch((error) => {
+                        throw lost(error, "The connection to the remote machine was lost while reading the output of the command");
+                    });
                     const exitCode = /^\d+$/.test(text) ? Number(text) : null;
                     finished = true;
                     running.delete(handle);
@@ -73,11 +111,14 @@ export function createRemoteEnvironment(options) {
                 await new Promise((resolve) => setTimeout(resolve, attempt < 8 ? 250 : 1000));
             }
         })();
+        done.catch(() => undefined); // a caller that never waits must not cause an unhandled rejection
         running.add(handle);
         return handle;
     }
+    let disposed = false;
     return {
         cwd: root,
+        ...(options.description !== undefined ? { description: options.description } : {}),
         resolve: resolvePath,
         async readFile(path) {
             const output = await must(`set -o pipefail; base64 < ${quote(resolvePath(path))} | tr -d '\\n'`);
@@ -113,8 +154,17 @@ export function createRemoteEnvironment(options) {
             ...(execOptions.signal ? { signal: execOptions.signal } : {}),
         }),
         async dispose() {
-            for (const handle of [...running])
+            if (disposed)
+                return;
+            disposed = true;
+            const handles = [...running];
+            for (const handle of handles)
                 handle.kill();
+            await Promise.all(handles.map((handle) => killing.get(handle)));
+            // the scripts, logs and exit codes of this session are of no use to anyone afterwards
+            if (counter > 0)
+                await run(`rm -f ${quote(posix.join(workDir, `${prefix}-`))}*`, { cwd: root }).catch(() => undefined);
+            await options.close?.();
         },
     };
 }

package/dist/packages/evals/src/bfcl.d.ts

@@ -0,0 +1,24 @@
+export interface BfclFunction {
+    name: string;
+    description: string;
+    parameters: Record<string, unknown>;
+}
+export interface BfclItem {
+    id: string;
+    question: Array<Array<{
+        role: string;
+        content: string;
+    }>>;
+    function: BfclFunction[];
+}
+export type GroundTruth = Array<Record<string, Record<string, unknown[]>>>;
+/** The wire tool name: BFCL names may contain dots, which tool name rules do not allow. */
+export declare const wireName: (name: string) => string;
+/** BFCL schemas use "dict", "float", "tuple" and similar; map them to JSON Schema types. */
+export declare function toJsonSchema(value: unknown): unknown;
+export interface ActualCall {
+    name: string;
+    args: Record<string, unknown>;
+}
+/** Every expected call must be matched by a different actual call (order does not matter). */
+export declare function scoreCalls(calls: ActualCall[], truth: GroundTruth): boolean;

package/dist/packages/evals/src/bfcl.js

@@ -0,0 +1,71 @@
+/*
+ * Scoring for the Berkeley Function Calling Leaderboard (Gorilla, Apache-2.0) "simple", "multiple" and
+ * "parallel" sets, in a simplified form: function names and argument values are compared; types are
+ * compared loosely. Data files live in external/bfcl (downloaded, not committed).
+ */
+/** The wire tool name: BFCL names may contain dots, which tool name rules do not allow. */
+export const wireName = (name) => name.replace(/[^A-Za-z0-9_-]/g, "_");
+/** BFCL schemas use "dict", "float", "tuple" and similar; map them to JSON Schema types. */
+export function toJsonSchema(value) {
+    if (Array.isArray(value))
+        return value.map(toJsonSchema);
+    if (typeof value !== "object" || value === null)
+        return value;
+    const out = {};
+    for (const [key, inner] of Object.entries(value)) {
+        if (key === "type" && typeof inner === "string") {
+            out.type = { dict: "object", float: "number", tuple: "array", any: "string", integer: "integer" }[inner] ?? inner;
+        }
+        else
+            out[key] = toJsonSchema(inner);
+    }
+    return out;
+}
+const normalizeString = (value) => value.toLowerCase().replace(/[\s,.\-_/:'"`()]/g, "");
+function valuesEqual(actual, allowed) {
+    if (typeof allowed === "number" && typeof actual === "number")
+        return Math.abs(allowed - actual) < 1e-9 * Math.max(1, Math.abs(allowed));
+    if (typeof allowed === "number" && typeof actual === "string")
+        return Number(actual) === allowed;
+    if (typeof allowed === "string" && typeof actual === "string")
+        return normalizeString(allowed) === normalizeString(actual);
+    if (typeof allowed === "string" && typeof actual === "number")
+        return normalizeString(allowed) === normalizeString(String(actual));
+    if (Array.isArray(allowed) && Array.isArray(actual))
+        return allowed.length === actual.length && allowed.every((item, index) => valuesEqual(actual[index], item));
+    if (typeof allowed === "object" && allowed && typeof actual === "object" && actual) {
+        const keys = Object.keys(allowed);
+        return keys.length === Object.keys(actual).length && keys.every((key) => valuesEqual(actual[key], allowed[key]));
+    }
+    return actual === allowed;
+}
+/** One call matches one ground-truth entry when the name matches and every argument is an allowed value. */
+function callMatches(call, truth) {
+    const [name, allowedArgs] = Object.entries(truth)[0] ?? [];
+    if (!name || !allowedArgs || wireName(name) !== call.name)
+        return false;
+    for (const [parameter, allowed] of Object.entries(allowedArgs)) {
+        const given = call.args[parameter];
+        if (given === undefined) {
+            if (allowed.includes(""))
+                continue; // optional parameter
+            return false;
+        }
+        if (!allowed.some((candidate) => valuesEqual(given, candidate)))
+            return false;
+    }
+    return Object.keys(call.args).every((parameter) => parameter in allowedArgs);
+}
+/** Every expected call must be matched by a different actual call (order does not matter). */
+export function scoreCalls(calls, truth) {
+    if (calls.length !== truth.length)
+        return false;
+    const used = new Set();
+    return truth.every((expected) => {
+        const index = calls.findIndex((call, position) => !used.has(position) && callMatches(call, expected));
+        if (index === -1)
+            return false;
+        used.add(index);
+        return true;
+    });
+}

package/dist/packages/evals/src/index.d.ts

@@ -0,0 +1,3 @@
+export { liveProviders, apiKeyFor } from "./providers.js";
+export type { LiveProvider } from "./providers.js";
+export { solidPng } from "./png.js";

package/dist/packages/evals/src/index.js

@@ -0,0 +1,2 @@
+export { liveProviders, apiKeyFor } from "./providers.js";
+export { solidPng } from "./png.js";

package/dist/packages/evals/src/png.d.ts

@@ -0,0 +1,2 @@
+/** A solid-colour PNG, used to check that a model can really see an image. */
+export declare function solidPng(width: number, height: number, [red, green, blue]: [number, number, number]): Buffer;

package/dist/packages/evals/src/png.js

@@ -0,0 +1,30 @@
+import { deflateSync } from "node:zlib";
+function crc32(bytes) {
+    let crc = 0xffffffff;
+    for (const byte of bytes) {
+        crc ^= byte;
+        for (let bit = 0; bit < 8; bit++)
+            crc = crc & 1 ? (crc >>> 1) ^ 0xedb88320 : crc >>> 1;
+    }
+    return (crc ^ 0xffffffff) >>> 0;
+}
+function chunk(type, data) {
+    const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
+    const out = Buffer.alloc(8 + data.length + 4);
+    out.writeUInt32BE(data.length, 0);
+    body.copy(out, 4);
+    out.writeUInt32BE(crc32(body), 8 + data.length);
+    return out;
+}
+/** A solid-colour PNG, used to check that a model can really see an image. */
+export function solidPng(width, height, [red, green, blue]) {
+    const header = Buffer.alloc(13);
+    header.writeUInt32BE(width, 0);
+    header.writeUInt32BE(height, 4);
+    header.set([8, 2, 0, 0, 0], 8);
+    const row = Buffer.alloc(1 + width * 3);
+    for (let x = 0; x < width; x++)
+        row.set([red, green, blue], 1 + x * 3);
+    const raw = Buffer.concat(Array.from({ length: height }, () => row));
+    return Buffer.concat([Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]), chunk("IHDR", header), chunk("IDAT", deflateSync(raw)), chunk("IEND", Buffer.alloc(0))]);
+}

package/dist/packages/evals/src/pool.d.ts

@@ -0,0 +1,3 @@
+/** Runs async jobs with a fixed number at a time and returns their results in order. */
+export declare function mapPool<T, R>(items: readonly T[], limit: number, work: (item: T, index: number) => Promise<R>): Promise<R[]>;
+export declare function parseArgs(argv: string[]): Record<string, string>;

package/dist/packages/evals/src/pool.js

@@ -0,0 +1,22 @@
+/** Runs async jobs with a fixed number at a time and returns their results in order. */
+export async function mapPool(items, limit, work) {
+    const results = new Array(items.length);
+    let next = 0;
+    async function worker() {
+        while (next < items.length) {
+            const index = next++;
+            results[index] = await work(items[index], index);
+        }
+    }
+    await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
+    return results;
+}
+export function parseArgs(argv) {
+    const args = {};
+    for (let index = 0; index < argv.length; index++) {
+        const key = argv[index];
+        if (key?.startsWith("--"))
+            args[key.slice(2)] = argv[index + 1] ?? "true";
+    }
+    return args;
+}

package/dist/packages/evals/src/providers.d.ts

@@ -0,0 +1,13 @@
+import type { ChatCompat } from "../../llm/src/index.js";
+/** The real endpoints used by live tests. Keys are read from the environment and never printed. */
+export interface LiveProvider {
+    name: string;
+    baseUrl: string;
+    keyEnv: string[];
+    model: string;
+    contextWindow: number;
+    compat: Partial<ChatCompat>;
+    headers?: Record<string, string>;
+}
+export declare const liveProviders: LiveProvider[];
+export declare function apiKeyFor(provider: LiveProvider): string | undefined;

package/dist/packages/evals/src/providers.js

@@ -0,0 +1,23 @@
+import { randomUUID } from "node:crypto";
+import { compatPresets } from "../../llm/src/index.js";
+export const liveProviders = [
+    { name: "deepseek", baseUrl: "https://api.deepseek.com", keyEnv: ["DEEPSEEK_API_KEY"], model: "deepseek-flash", contextWindow: 128_000, compat: compatPresets.deepseek },
+    { name: "kimi", baseUrl: "https://api.moonshot.cn/v1", keyEnv: ["MOONSHOT_API_KEY", "KIMI_API_KEY"], model: "kimi-k3", contextWindow: 128_000, compat: compatPresets.moonshot },
+    {
+        name: "opencode",
+        baseUrl: "https://opencode.ai/zen/go/v1",
+        keyEnv: ["OPENCODE_API_KEY"],
+        model: "deepseek-v4.1-flash",
+        contextWindow: 128_000,
+        compat: compatPresets.deepseek,
+        headers: { "x-opencode-session": randomUUID() },
+    },
+];
+export function apiKeyFor(provider) {
+    for (const name of provider.keyEnv) {
+        const value = process.env[name];
+        if (value)
+            return value;
+    }
+    return undefined;
+}