toolnexus 0.21.0 → 0.23.0

package/dist/builtin.js

@@ -14,11 +14,12 @@ import { compareCodePoints, sortEntriesByName, toPosixPath } from "./order.js";
 import path from "node:path";
 import { pending } from "./types.js";
 import { mediaTypeFor } from "./content.js";
+import { bashOutput, killShell, runWebSearch, editNotebook, goJsonIndent, } from "./builtin_extras.js";
 /**
- * How long a job gets between "please stop" and "stop". Fixed, and identical in
- * every port, so a timeout means the same thing everywhere.
+ * How long a job gets between "please stop" and "stop" when `killGraceMs` is
+ * not set. Identical in every port, so a timeout means the same thing everywhere.
  */
-const KILL_GRACE_MS = 2000;
+const DEFAULT_KILL_GRACE_MS = 2000;
 const WIN_RESERVED = new Set([
     "CON", "PRN", "AUX", "NUL",
     "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9",
@@ -68,8 +69,16 @@ class BuiltinEnv {
     shellError;
     baseDir;
     confine;
+    killGraceMs;
+    background;
+    webSearch;
+    notebookEdit;
     constructor(cfg) {
         const obj = typeof cfg === "object" && cfg !== null ? cfg : {};
+        this.background = obj.background;
+        this.webSearch = obj.webSearch;
+        this.notebookEdit = obj.notebookEdit === true;
+        this.killGraceMs = typeof obj.killGraceMs === "number" && obj.killGraceMs > 0 ? obj.killGraceMs : DEFAULT_KILL_GRACE_MS;
         this.baseDir = obj.baseDir ?? "";
         this.confine = obj.confineToBaseDir === true;
         if (obj.shell && obj.shell.length > 0) {
@@ -276,14 +285,21 @@ function walkFiles(root) {
 // individual tools
 // ---------------------------------------------------------------------------
 function bashTool(env) {
+    const properties = {
+        command: { type: "string", description: "The shell command to run" },
+        workdir: { type: "string", description: "Working directory (default: process cwd)" },
+        timeout: { type: "number", description: "Timeout in milliseconds (default 60000)" },
+        description: { type: "string", description: "Human-readable description of the command" },
+    };
+    if (env.background) {
+        properties.run_in_background = {
+            type: "boolean",
+            description: "Start the command and return at once with a background shell id; read it with bash_output, stop it with kill_shell. No timeout applies.",
+        };
+    }
     return builtin("bash", "Run a shell command and return its combined stdout+stderr. Non-zero exit is an error.", {
         type: "object",
-        properties: {
-            command: { type: "string", description: "The shell command to run" },
-            workdir: { type: "string", description: "Working directory (default: process cwd)" },
-            timeout: { type: "number", description: "Timeout in milliseconds (default 60000)" },
-            description: { type: "string", description: "Human-readable description of the command" },
-        },
+        properties,
         required: ["command"],
         additionalProperties: false,
     }, (args, ctx) => new Promise((resolve) => {
@@ -301,6 +317,10 @@ function bashTool(env) {
         }
         const timeout = typeof args.timeout === "number" ? args.timeout : 60_000;
         const meta = { shell: env.shellLabel };
+        if (env.background && args.run_in_background === true) {
+            const j = env.background.start(argv, command, workdir, env.killGraceMs);
+            return resolve(ok(`Started background shell ${j.id}. Read its output with bash_output, stop it with kill_shell.`, { ...meta, backgroundId: j.id }));
+        }
         // NOT `shell: true`: on Windows that silently means cmd.exe while every
         // other port fails loudly, and it hides WHICH interpreter ran. `detached`
         // puts the child in its own process group so a kill reaches the command
@@ -362,7 +382,7 @@ function bashTool(env) {
             graceTimer = setTimeout(() => {
                 if (killJob(false))
                     killedTree = true;
-            }, KILL_GRACE_MS);
+            }, env.killGraceMs);
         };
         let graceTimer;
         const timer = setTimeout(() => stop("timeout"), timeout);
@@ -459,7 +479,16 @@ function readTool(env) {
         const offset = typeof args.offset === "number" ? Math.max(1, Math.trunc(args.offset)) : 1;
         const start = offset - 1;
         const limit = typeof args.limit === "number" ? Math.max(0, Math.trunc(args.limit)) : lines.length - start;
-        return ok(lines.slice(start, start + limit).join("\n"));
+        const end = Math.min(Math.max(start, start + limit), lines.length);
+        const out = lines.slice(start, start + limit).join("\n");
+        // A window that stops before the end of the file says so in metadata, with the offset to
+        // continue from (SPEC §4C). The text stays byte-identical. A trailing newline is not a line.
+        let total = lines.length;
+        if (total > 0 && lines[total - 1] === "")
+            total--;
+        if (end < total)
+            return ok(out, { truncated: true, totalLines: total, nextOffset: end + 1 });
+        return ok(out);
     });
 }
 function writeTool(env) {
@@ -483,10 +512,11 @@ function writeTool(env) {
         catch (e) {
             return err(`write: ${e instanceof Error ? e.message : String(e)}`);
         }
+        const before = await readIfExists(full);
         await fs.mkdir(path.dirname(path.resolve(full)), { recursive: true });
         await fs.writeFile(full, content, "utf8");
         const bytes = Buffer.byteLength(content, "utf8");
-        return ok(`Wrote ${bytes} bytes to ${p}`, { bytes });
+        return ok(`Wrote ${bytes} bytes to ${p}`, { bytes, ...editMetadata(env, [{ path: full, before, after: content }]) });
     });
 }
 function editTool(env) {
@@ -530,9 +560,10 @@ function editTool(env) {
                 return err(`edit: oldString is not unique in ${p} (${count} occurrences); use replaceAll`);
             next = content.replace(oldString, newString);
         }
-        await fs.writeFile(p, next, "utf8");
+        await fs.writeFile(full, next, "utf8");
         return ok(`Edited ${p} (${args.replaceAll === true ? count : 1} replacement${(args.replaceAll === true ? count : 1) === 1 ? "" : "s"})`, {
             replacements: args.replaceAll === true ? count : 1,
+            ...editMetadata(env, [{ path: full, before: content, after: next }]),
         });
     });
 }
@@ -544,6 +575,9 @@ function grepTool(env) {
             path: { type: "string", description: "Directory to search (default: process cwd)" },
             include: { type: "string", description: "Glob filter for file names" },
             limit: { type: "number", description: "Maximum number of matches (default 100)" },
+            context: { type: "number", description: "Lines of context before and after each match" },
+            before: { type: "number", description: "Lines of context before each match (overrides context)" },
+            after: { type: "number", description: "Lines of context after each match (overrides context)" },
         },
         required: ["pattern"],
         additionalProperties: false,
@@ -571,6 +605,14 @@ function grepTool(env) {
         // CONTENT and the order of the results were whatever the walk happened to reach first.
         // Now: collect every match, order by relative path (code point) then line number ascending,
         // and truncate last. The emitted path and the sort key are the same string (A28).
+        let before = 0, after = 0;
+        if (typeof args.context === "number" && args.context > 0)
+            before = after = Math.trunc(args.context);
+        if (typeof args.before === "number" && args.before >= 0)
+            before = Math.trunc(args.before);
+        if (typeof args.after === "number" && args.after >= 0)
+            after = Math.trunc(args.after);
+        const fileLines = new Map(); // only filled when context is asked for
         const hits = [];
         for (const file of walkFiles(root)) {
             const rel = toPosixPath(path.relative(root, file), path.sep);
@@ -584,14 +626,58 @@ function grepTool(env) {
                 continue;
             }
             const lines = text.split("\n");
+            if (before > 0 || after > 0)
+                fileLines.set(rel, lines);
             for (let i = 0; i < lines.length; i++) {
                 if (re.test(lines[i]))
                     hits.push({ rel, line: i + 1, text: lines[i] });
             }
         }
         hits.sort((a, b) => compareCodePoints(a.rel, b.rel) || a.line - b.line);
-        const matches = hits.slice(0, limit).map((h) => `${h.rel}:${h.line}:${h.text}`);
-        return ok(matches.join("\n"), { count: matches.length });
+        const total = hits.length;
+        const kept = hits.slice(0, limit);
+        let outLines;
+        if (before === 0 && after === 0) {
+            outLines = kept.map((h) => `${h.rel}:${h.line}:${h.text}`);
+        }
+        else {
+            // ripgrep-style: match lines path:N:text, context lines path-N-text, "--" between
+            // groups that do not touch.
+            outLines = [];
+            const matchAt = new Map();
+            for (const h of kept) {
+                if (!matchAt.has(h.rel))
+                    matchAt.set(h.rel, new Set());
+                matchAt.get(h.rel).add(h.line);
+            }
+            let lastRel = "", lastLine = 0;
+            for (const h of kept) {
+                const lines = fileLines.get(h.rel) ?? [];
+                let n = lines.length;
+                if (n > 0 && lines[n - 1] === "")
+                    n--; // a trailing newline is not a line
+                let from = Math.max(1, h.line - before);
+                const to = Math.min(n, h.line + after);
+                if (h.rel === lastRel && from <= lastLine + 1)
+                    from = lastLine + 1;
+                else if (lastRel !== "")
+                    outLines.push("--");
+                for (let k = from; k <= to; k++) {
+                    const sep = matchAt.get(h.rel).has(k) ? ":" : "-";
+                    outLines.push(`${h.rel}${sep}${k}${sep}${lines[k - 1]}`);
+                }
+                if (to > lastLine || h.rel !== lastRel)
+                    lastLine = to;
+                lastRel = h.rel;
+            }
+        }
+        const meta = { count: kept.length };
+        // Say the list was cut, and by how much, in metadata (SPEC §4C). The text is unchanged.
+        if (total > kept.length) {
+            meta.truncated = true;
+            meta.total = total;
+        }
+        return ok(outLines.join("\n"), meta);
     });
 }
 function globTool(env) {
@@ -872,21 +958,26 @@ function applyPatchTool(env) {
         // Stage every write/delete first; only touch the filesystem once all hunks apply.
         const writes = [];
         const deletes = [];
+        const changes = [];
         try {
             for (const op of ops) {
                 if (op.type === "add") {
                     if (existsSync(op.path))
                         throw new Error(`file already exists: ${op.path}`);
                     writes.push({ path: op.path, content: op.content });
+                    changes.push({ path: op.path, before: undefined, after: op.content });
                 }
                 else if (op.type === "delete") {
                     if (!existsSync(op.path))
                         throw new Error(`file not found: ${op.path}`);
                     deletes.push(op.path);
+                    changes.push({ path: op.path, before: await fs.readFile(op.path, "utf8"), after: undefined });
                 }
                 else {
                     const content = await fs.readFile(op.path, "utf8");
-                    writes.push({ path: op.path, content: applyUpdate(content, op.body) });
+                    const next = applyUpdate(content, op.body);
+                    writes.push({ path: op.path, content: next });
+                    changes.push({ path: op.path, before: content, after: next });
                 }
             }
         }
@@ -904,9 +995,119 @@ function applyPatchTool(env) {
             added: ops.filter((o) => o.type === "add").length,
             updated: ops.filter((o) => o.type === "update").length,
             deleted: ops.filter((o) => o.type === "delete").length,
+            ...editMetadata(env, changes),
         });
     });
 }
+async function readIfExists(p) {
+    try {
+        return await fs.readFile(p, "utf8");
+    }
+    catch {
+        return undefined;
+    }
+}
+/** Lines of a file's text; a trailing newline does not make an extra empty line. */
+function diffLines(text) {
+    if (!text)
+        return [];
+    const lines = text.split("\n");
+    if (lines[lines.length - 1] === "")
+        lines.pop();
+    return lines;
+}
+/** Line diff by longest common subsequence — minimal, so the counts agree across ports. */
+function lineDiff(a0, b0) {
+    // Common prefix/suffix first: an edit touches a few lines, so the LCS table stays small.
+    let pre = 0;
+    while (pre < a0.length && pre < b0.length && a0[pre] === b0[pre])
+        pre++;
+    let suf = 0;
+    while (suf < a0.length - pre && suf < b0.length - pre && a0[a0.length - 1 - suf] === b0[b0.length - 1 - suf])
+        suf++;
+    const head = a0.slice(0, pre).map((line) => ({ kind: " ", line }));
+    const tail = a0.slice(a0.length - suf).map((line) => ({ kind: " ", line }));
+    const a = a0.slice(pre, a0.length - suf), b = b0.slice(pre, b0.length - suf);
+    const n = a.length, m = b.length;
+    const lcs = Array.from({ length: n + 1 }, () => new Array(m + 1).fill(0));
+    for (let i = n - 1; i >= 0; i--) {
+        for (let j = m - 1; j >= 0; j--) {
+            lcs[i][j] = a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
+        }
+    }
+    const ops = [];
+    let i = 0, j = 0;
+    while (i < n && j < m) {
+        if (a[i] === b[j]) {
+            ops.push({ kind: " ", line: a[i] });
+            i++;
+            j++;
+        }
+        else if (lcs[i + 1][j] >= lcs[i][j + 1])
+            ops.push({ kind: "-", line: a[i++] });
+        else
+            ops.push({ kind: "+", line: b[j++] });
+    }
+    while (i < n)
+        ops.push({ kind: "-", line: a[i++] });
+    while (j < m)
+        ops.push({ kind: "+", line: b[j++] });
+    return [...head, ...ops, ...tail];
+}
+/** Unified hunks with 3 lines of context. */
+function unifiedHunks(ops) {
+    const CTX = 3;
+    const out = [];
+    const changed = ops.map((o, k) => (o.kind === " " ? -1 : k)).filter((k) => k >= 0);
+    if (!changed.length)
+        return out;
+    // group change indices whose gap of context is <= 2*CTX
+    const groups = [];
+    for (const k of changed) {
+        const g = groups[groups.length - 1];
+        if (g && k - g[1] <= 2 * CTX)
+            g[1] = k;
+        else
+            groups.push([k, k]);
+    }
+    // old/new line number before each op
+    const oldAt = [], newAt = [];
+    let o = 1, nn = 1;
+    for (const op of ops) {
+        oldAt.push(o);
+        newAt.push(nn);
+        if (op.kind !== "+")
+            o++;
+        if (op.kind !== "-")
+            nn++;
+    }
+    for (const [s0, e0] of groups) {
+        const s = Math.max(0, s0 - CTX), e = Math.min(ops.length - 1, e0 + CTX);
+        const slice = ops.slice(s, e + 1);
+        const oldLen = slice.filter((x) => x.kind !== "+").length;
+        const newLen = slice.filter((x) => x.kind !== "-").length;
+        const oldStart = oldLen ? oldAt[s] : oldAt[s] - 1;
+        const newStart = newLen ? newAt[s] : newAt[s] - 1;
+        out.push(`@@ -${oldStart},${oldLen} +${newStart},${newLen} @@`);
+        for (const x of slice)
+            out.push(x.kind + x.line);
+    }
+    return out;
+}
+function editMetadata(env, changes) {
+    const files = [];
+    const diff = [];
+    for (const c of changes) {
+        const rel = toPosixPath(env.baseDir ? path.relative(path.resolve(env.baseDir), path.resolve(c.path)) : c.path, path.sep);
+        const ops = lineDiff(diffLines(c.before), diffLines(c.after));
+        const additions = ops.filter((x) => x.kind === "+").length;
+        const deletions = ops.filter((x) => x.kind === "-").length;
+        const status = c.before === undefined ? "added" : c.after === undefined ? "deleted" : "modified";
+        files.push({ path: rel, status, additions, deletions });
+        diff.push(`--- ${c.before === undefined ? "/dev/null" : `a/${rel}`}`, `+++ ${c.after === undefined ? "/dev/null" : `b/${rel}`}`, ...unifiedHunks(ops));
+    }
+    return { files, diff: diff.length ? diff.join("\n") + "\n" : "" };
+}
 /**
  * Build the ten built-in tools (each source:"builtin"). The order is fixed
  * for parity: bash, read, write, edit, grep, glob, webfetch, question,
@@ -914,7 +1115,7 @@ function applyPatchTool(env) {
  */
 export function createBuiltinTools(cfg) {
     const env = new BuiltinEnv(cfg);
-    return [
+    const tools = [
         bashTool(env),
         readTool(env),
         writeTool(env),
@@ -926,6 +1127,103 @@ export function createBuiltinTools(cfg) {
         applyPatchTool(env),
         todowriteTool(),
     ];
+    if (env.background)
+        tools.push(bashOutputTool(env.background), killShellTool(env.background));
+    if (env.webSearch)
+        tools.push(webSearchTool(env.webSearch));
+    if (env.notebookEdit)
+        tools.push(notebookEditTool(env));
+    for (const t of tools)
+        t.hints = { ...BUILTIN_HINTS[t.name] };
+    return tools;
+}
+const hint = (readOnly, destructive, openWorld) => ({ readOnly, destructive, openWorld });
+/**
+ * Hints on every builtin, for host permission gates (SPEC §4C). bash can do anything, so it is
+ * mutating, destructive and open-world. todowrite and question only touch the agent's own state
+ * or ask the human, so they count as read-only for the host's files and systems.
+ */
+const BUILTIN_HINTS = {
+    bash: hint(false, true, true),
+    read: hint(true, false, false),
+    write: hint(false, true, false),
+    edit: hint(false, true, false),
+    grep: hint(true, false, false),
+    glob: hint(true, false, false),
+    webfetch: hint(true, false, true),
+    question: hint(true, false, false),
+    apply_patch: hint(false, true, false),
+    todowrite: hint(true, false, false),
+    bash_output: hint(true, false, false),
+    kill_shell: hint(false, true, false),
+    websearch: hint(true, false, true),
+    notebook_edit: hint(false, true, false),
+};
+const idSchema = {
+    type: "object",
+    properties: { id: { type: "string", description: "The background shell id bash returned" } },
+    required: ["id"],
+    additionalProperties: false,
+};
+function bashOutputTool(bg) {
+    return builtin("bash_output", "Read the output a background bash command (started with run_in_background) produced since the last read, and whether it is still running.", idSchema, async (args) => bashOutput(bg, typeof args.id === "string" ? args.id : ""));
+}
+function killShellTool(bg) {
+    return builtin("kill_shell", "Stop a background bash command and everything it started.", idSchema, async (args) => killShell(bg, typeof args.id === "string" ? args.id : ""));
+}
+function webSearchTool(search) {
+    return builtin("websearch", "Search the web. Returns numbered results with title, URL and snippet.", {
+        type: "object",
+        properties: {
+            query: { type: "string", description: "The search query" },
+            limit: { type: "number", description: "Maximum number of results (default 5)" },
+        },
+        required: ["query"],
+        additionalProperties: false,
+    }, async (args, ctx) => runWebSearch(search, args, ctx?.signal));
+}
+function notebookEditTool(env) {
+    return builtin("notebook_edit", "Edit a Jupyter notebook (.ipynb) cell: replace its source, insert a new cell after it, or delete it. Replacing a code cell clears its outputs.", {
+        type: "object",
+        properties: {
+            path: { type: "string", description: "Path to the .ipynb file" },
+            cell_id: { type: "string", description: "The cell's id (or use cell_index)" },
+            cell_index: { type: "number", description: "0-based cell position (used when cell_id is absent)" },
+            new_source: { type: "string", description: "The new cell source (replace/insert)" },
+            cell_type: { type: "string", enum: ["code", "markdown"], description: "Cell type for insert (default code) or to change on replace" },
+            edit_mode: { type: "string", enum: ["replace", "insert", "delete"], description: "Default replace. insert adds the new cell AFTER the given cell, or first when none is given" },
+        },
+        required: ["path"],
+        additionalProperties: false,
+    }, async (args) => {
+        const p = typeof args.path === "string" ? args.path : "";
+        if (!p)
+            return err("notebook_edit: path is required");
+        let full;
+        let raw;
+        try {
+            full = env.resolvePath(p);
+            raw = await fs.readFile(full, "utf8");
+        }
+        catch (e) {
+            return err(`notebook_edit: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        let nb;
+        try {
+            nb = JSON.parse(raw);
+            if (!nb || typeof nb !== "object" || Array.isArray(nb))
+                throw new Error("not a JSON object");
+        }
+        catch (e) {
+            return err(`notebook_edit: ${p} is not a notebook: ${e instanceof Error ? e.message : String(e)}`);
+        }
+        const r = editNotebook(nb, args, p);
+        if (!("summary" in r))
+            return r;
+        const out = goJsonIndent(nb) + "\n";
+        await fs.writeFile(full, out, "utf8");
+        return ok(r.summary, { cells: nb.cells.length, ...editMetadata(env, [{ path: full, before: raw, after: out }]) });
+    });
 }
 /**
  * The interpreter the builtins resolved for `bash` — what a host prints, and

package/dist/builtin_extras.d.ts

@@ -0,0 +1,80 @@
+/**
+ * Opt-in builtins for coding-agent hosts (SPEC §4C): background shells (`bash`
+ * run_in_background + `bash_output` + `kill_shell`), `websearch` over a host-supplied provider,
+ * and `notebook_edit`. None is on by default: a host that sets nothing sees the same ten builtins.
+ */
+import { type ChildProcess } from "node:child_process";
+import type { ToolResult } from "./types.js";
+/** One background command, for a host UI. */
+export interface BackgroundShellInfo {
+    id: string;
+    command: string;
+    started: Date;
+    running: boolean;
+    /** Valid when not running; -1 when killed by a signal. */
+    exitCode: number;
+    killed: boolean;
+}
+interface BgJob {
+    id: string;
+    command: string;
+    started: Date;
+    child: ChildProcess;
+    pgid?: number;
+    grace: number;
+    out: string;
+    read: number;
+    running: boolean;
+    exitCode: number;
+    killed: boolean;
+    done: Promise<void>;
+}
+/**
+ * Owns the commands `bash` started with `run_in_background`. Pass one as
+ * `builtins.background`, and call `closeAll()` when the host shuts down — a background command
+ * is NOT tied to the turn that started it, so nothing else stops it.
+ */
+export declare class BackgroundShells {
+    private next;
+    private readonly jobs;
+    /** @internal Start `command` under `argv` in its own process group. */
+    start(argv: string[], command: string, workdir: string, grace: number): BgJob;
+    /** @internal */
+    get(id: string): BgJob | undefined;
+    private signal;
+    /** @internal Stop the job's whole process group: SIGTERM, the grace window, SIGKILL.
+     * Resolves whether it was still running. */
+    kill(j: BgJob): Promise<boolean>;
+    static info(j: BgJob): BackgroundShellInfo;
+    /** Every background command, oldest first. */
+    list(): BackgroundShellInfo[];
+    /** Stop one background command's process group. Unknown id ⇒ throws. */
+    killShell(id: string): Promise<void>;
+    /** Stop every background command still running, concurrently. */
+    closeAll(): Promise<void>;
+}
+export declare function bgStatus(i: BackgroundShellInfo): string;
+/** `bash_output(id)`. */
+export declare function bashOutput(bg: BackgroundShells, id: string): ToolResult;
+/** `kill_shell(id)`. */
+export declare function killShell(bg: BackgroundShells, id: string): Promise<ToolResult>;
+/** One hit from a host search provider. */
+export interface WebSearchResult {
+    title: string;
+    url: string;
+    snippet?: string;
+}
+/** The host's search provider. The library ships none. */
+export type WebSearchFunc = (query: string, limit: number, ctx: {
+    signal?: AbortSignal;
+}) => Promise<WebSearchResult[]>;
+export declare function runWebSearch(search: WebSearchFunc, args: Record<string, unknown>, signal?: AbortSignal): Promise<ToolResult>;
+/** Split text into nbformat's list of lines, each keeping its "\n". */
+export declare function notebookSource(s: string): string[];
+/** Go's json.MarshalIndent(v, "", " ") with sorted keys, plus Go's HTML escaping. */
+export declare function goJsonIndent(v: unknown): string;
+/** Apply one notebook_edit to the parsed notebook, in place. Returns the summary, or an error result. */
+export declare function editNotebook(nb: Record<string, any>, args: Record<string, unknown>, p: string): {
+    summary: string;
+} | ToolResult;
+export {};