toolnexus 0.16.0 → 0.18.0

package/README.md

@@ -235,6 +235,52 @@ For `httpTool`: URL `{placeholders}` are filled from args; `query` arg names go
 querystring (all args on GET); the rest become the JSON body (`body: "json" | "form" | "raw"`).
 Non-2xx → `{ isError: true, output: "HTTP <status>: <body>" }`.
 
+## Images, PDFs and audio (content parts)
+
+Parts go in the **first** argument, next to your text, because the order of text and image is
+semantic to a model:
+
+```ts
+import { attach, text } from "toolnexus"
+
+const { text: answer } = await agent.run(
+  ["What is broken in this screenshot?", await attach("./shot.png")],
+  { toolkit: tk },
+)
+```
+
+`attach(source, opts?)` takes what you already hold: a filesystem path, a `data:` URL, an
+`https:` URL, native bytes (`Uint8Array`, `Buffer`, `ArrayBuffer`, any `ArrayBufferView` —
+`mimeType` required, bytes carry no extension), or a `Blob`/`File` straight off an upload.
+`fromPath`, `fromBytes`, `fromBlob`, `fromDataUrl`, `fromUrl` are the same thing spelled
+explicitly. Mime types come from a fixed extension table (`png jpg jpeg gif webp pdf mp3 wav`) —
+never sniffed, never read from the machine's mime database, so every port agrees. A plain string
+prompt is untouched and byte-identical to before.
+
+A tool can answer with media too — `parts` alongside the still-required `output`, which is what
+the transcript, compaction and any text-only provider keep seeing:
+
+```ts
+defineTool({
+  name: "screenshot",
+  description: "Grab the screen",
+  inputSchema: { type: "object", properties: {} },
+  run: async () => ({
+    output: "screenshot, 1280x720 png",
+    parts: [await attach("./shot.png")],
+  }),
+})
+```
+
+**MCP tools' images now survive.** Non-text content from an MCP server — image, audio, embedded
+resource, `resource_link` — used to be filtered out silently; it is preserved as `parts` and
+emitted per provider style (Anthropic inside `tool_result`; OpenAI, whose `tool` message rejects
+an image, in a synthetic user message that never enters your transcript).
+
+Whatever you pass in, a part stores **bytes plus a `mimeType`** — never a handle, never a path —
+so a persisted transcript replays without the original file. A `Blob` is read **eagerly** at
+construction, and a handle you supply is read, not closed.
+
 ## Built-in tools
 
 A fifth source ships **10 built-in tools** — `bash`, `read`, `write`, `edit`, `grep`, `glob`,

package/dist/agents/agent.d.ts

@@ -1,3 +1,13 @@
+/**
+ * Level-1 subagent surface — SPEC.md §7D.
+ *
+ * One new noun: `agent(name, spec)`. An Agent is (identity/system prompt × a
+ * filtered toolkit view × the shipped client loop), and — the §7A/§7B axiom
+ * completed locally — an Agent IS a Tool: `asTool()` bridges it into the classic
+ * API's `extraTools`; `run()` executes one-shot against a private runtime.
+ * Everything compiles down to the six host verbs in runtime.ts.
+ */
+import type { PromptInput } from "../content.js";
 import type { Answer, Request, Tool } from "../types.js";
 import { AgentRuntime, type AgentDef, type Budget, type RuntimeOptions, type TaskResult } from "./runtime.js";
 import { Loop, type Completion, type Guardrail } from "./loop.js";
@@ -71,7 +81,7 @@ export declare class Agent {
      * the tree down — unless the run suspends durably (`status:"pending"`), in which
      * case the tree stays parked and `result.runtime.resume(answer)` continues it.
      */
-    run(prompt: string, opts?: AgentRunOptions): Promise<AgentRunResult>;
+    run(prompt: PromptInput, opts?: AgentRunOptions): Promise<AgentRunResult>;
     /**
      * The bridge: an Agent IS a Tool. Drop the result into the classic API's
      * `extraTools`; the agent's own loop runs in isolation and the classic run

package/dist/agents/agent.js

@@ -1,12 +1,3 @@
-/**
- * Level-1 subagent surface — SPEC.md §7D.
- *
- * One new noun: `agent(name, spec)`. An Agent is (identity/system prompt × a
- * filtered toolkit view × the shipped client loop), and — the §7A/§7B axiom
- * completed locally — an Agent IS a Tool: `asTool()` bridges it into the classic
- * API's `extraTools`; `run()` executes one-shot against a private runtime.
- * Everything compiles down to the six host verbs in runtime.ts.
- */
 import fs from "node:fs";
 import { defineTool } from "../native.js";
 import { AgentRuntime, isVerbError } from "./runtime.js";

package/dist/agents/compaction.d.ts

@@ -1,28 +1,3 @@
-/**
- * Context compaction (SPEC.md §7F) — keep a long-lived agent under budget.
- *
- * A long-running agent grows its transcript until it overflows the model's window.
- * `compactor(opts)` returns a **`beforeLLM` hook** (§8) that summarizes the older
- * transcript and keeps a recent tail. It rides the existing seam: the client loop
- * applies a `beforeLLM` message rewrite by *replacing* the working transcript, and
- * that array flows on into `RunResult.messages` and the `ConversationStore`. So
- * compaction is a pure `messages → messages` helper and adds **no loop behavior** —
- * it is the canonical use of the §8 `beforeLLM` hook.
- *
- * Invariants (SPEC §7F):
- *   1. At/below `maxTokens` the hook is a **no-op** — it returns nothing and the run
- *      is byte-identical to a run with no compactor.
- *   2. **Tool-pair safety** — the retained tail always begins at a `user` turn, so no
- *      `tool` message is ever orphaned from the `assistant` carrying its `tool_call_id`.
- *   3. The **leading system prompt** (identity / soul / skills) is preserved verbatim;
- *      only the body between it and the tail is summarized.
- *
- *   import { agents } from "toolnexus"
- *   const client = createClient({
- *     …,
- *     hooks: { beforeLLM: agents.compactor({ maxTokens: 120_000, summarize }) },
- *   })
- */
 /**
  * A chat message in the working transcript. Structural and provider-neutral — only the
  * fields compaction reasons about are named; anything else on the message is carried

package/dist/agents/compaction.js

@@ -23,6 +23,7 @@
  *     hooks: { beforeLLM: agents.compactor({ maxTokens: 120_000, summarize }) },
  *   })
  */
+import { partTokens } from "../content.js";
 /**
  * Cheap, deterministic token estimate: `ceil(chars/4)` over each message's JSON
  * serialization, summed. This is an *estimator*, not a tokenizer — exactness is the
@@ -30,10 +31,48 @@
  */
 export function estimateTokens(messages) {
     let total = 0;
-    for (const m of messages)
-        total += Math.ceil(JSON.stringify(m).length / 4);
+    for (const m of messages) {
+        const { text, partBytes } = stripPartPayloads(m);
+        total += Math.ceil(text.length / 4);
+        // §1B: a media part is charged from its BYTE LENGTH. Scoring it by its `mimeType`
+        // string would price a 5 MB image at ~3 tokens, so the compactor would evict text and
+        // keep every image forever — the exact opposite of what a budget is for. Charging the
+        // base64 verbatim is the other failure: it prices an 82-byte PNG like 30 lines of prose.
+        for (const bytes of partBytes)
+            total += partTokens(bytes);
+    }
     return total;
 }
+/** Keys whose string value is a media payload rather than prose. */
+const PAYLOAD_KEYS = new Set(["data", "file_data", "blob"]);
+/**
+ * Serialize a message for estimation with its media payloads replaced by "", returning the
+ * decoded byte length of each one so the caller can charge it properly. Recognises both the
+ * canonical `ContentPart` (`data`) and every provider block shape it encodes to
+ * (`image_url.url` as a data: URL, `file.file_data`, `input_audio.data`).
+ */
+function stripPartPayloads(m) {
+    const partBytes = [];
+    const walk = (v, key) => {
+        if (typeof v === "string") {
+            const b64 = PAYLOAD_KEYS.has(key ?? "") ? v : v.startsWith("data:") ? v.slice(v.indexOf(",") + 1) : undefined;
+            if (b64 === undefined)
+                return v;
+            partBytes.push(Math.max(0, Math.floor((b64.length * 3) / 4)));
+            return "";
+        }
+        if (Array.isArray(v))
+            return v.map((x) => walk(x));
+        if (v && typeof v === "object") {
+            const out = {};
+            for (const [k, val] of Object.entries(v))
+                out[k] = walk(val, k);
+            return out;
+        }
+        return v;
+    };
+    return { text: JSON.stringify(walk(m)) ?? "", partBytes };
+}
 /** The system reminder injected when `flushToMemory` is set. */
 const FLUSH_TO_MEMORY_REMINDER = "Before continuing: if anything from earlier is worth keeping, save it with the memory tool now — the earlier transcript is about to be summarized.";
 /** Prefix on the summary system message; hosts/tests key off this to detect a compaction. */

package/dist/agents/loop.d.ts

@@ -12,6 +12,7 @@
  *
  * So Loop takes no options: it is read, not configured.
  */
+import type { PromptInput } from "../content.js";
 import { type ClientOptions, type Hooks, type RunResult } from "../client.js";
 import type { Toolkit } from "../toolkit.js";
 import type { Agent } from "./agent.js";
@@ -77,7 +78,7 @@ export declare function allTodosDone(result: RunResult): Verdict;
  * keeps `pending` and `incomplete` distinct — the caller can always tell whether
  * it owes an Answer or a fix.
  */
-export declare function runGated(ask: (prompt: string) => Promise<RunResult>, prompt: string, completion: Completion | undefined): Promise<RunResult>;
+export declare function runGated(ask: (prompt: PromptInput) => Promise<RunResult>, prompt: PromptInput, completion: Completion | undefined): Promise<RunResult>;
 /** A live execution of an Agent. Its only verbs are `run` and reading state. */
 export declare class Loop {
     #private;
@@ -94,5 +95,5 @@ export declare class Loop {
     get status(): string;
     /** Model round trips this loop has spent. */
     get turns(): number;
-    run(prompt: string, opts?: LoopRunOptions): Promise<Outcome>;
+    run(prompt: PromptInput, opts?: LoopRunOptions): Promise<Outcome>;
 }

package/dist/agents/loop.js

@@ -1,17 +1,3 @@
-/**
- * Loop — a live execution of an Agent, and the completion gate that stops it
- * claiming `done` too early. A layer over the shipped §8 client: nothing here
- * changes existing behaviour.
- *
- * The placement law this encodes:
- *
- *   AgentSpec (the harness) answers "MAY it?"    — capability, ceilings. Per problem.
- *   RunOptions             answers "with WHAT?"  — model for this call.  Per call.
- *   Loop                   answers "DID it?"     — status, turns.        Observed.
- *   none of them           answers "is it RIGHT?"— a tool, skill or agent.
- *
- * So Loop takes no options: it is read, not configured.
- */
 import { createClient } from "../client.js";
 /**
  * Compiles a spec's guardrails into one `beforeTool` with FIRST-DENY-WINS,
@@ -27,6 +13,16 @@ export function guardedHooks(guardrails, hooks) {
         beforeTool: async (ev) => {
             for (const g of guardrails) {
                 const verdict = g(ev);
+                // A guardrail is SYNCHRONOUS by contract. An async one returns a Promise,
+                // which is truthy and !== "allow", so it would silently DENY every call
+                // with `denied: [object Promise]`. Fail loudly instead, and say where
+                // asynchronous work belongs.
+                if (verdict != null && typeof verdict !== "string") {
+                    const what = typeof verdict.then === "function" ? "a Promise" : typeof verdict;
+                    throw new TypeError(`guardrail returned ${what}: a guardrail must return a string synchronously ` +
+                        `("" or "allow" to permit, any other string to deny). ` +
+                        `Do asynchronous work in hooks.beforeTool, which is awaited.`);
+                }
                 if (verdict && verdict !== "allow") {
                     return { result: { output: `denied: ${verdict}`, isError: true } };
                 }

package/dist/agents/runtime.d.ts

@@ -14,6 +14,7 @@
  * the ConversationStore. SPEC pins transitions, never scheduling — conformance is
  * identical per-handle transition traces against `examples/subagent-*` fixtures.
  */
+import type { PromptInput } from "../content.js";
 import { type ClientStyle, type ConversationStore, type Hooks, type MetricEvent } from "../client.js";
 import { type Completion } from "./loop.js";
 import type { Answer, Request, Tool } from "../types.js";
@@ -132,11 +133,11 @@ interface Pools {
  */
 interface Checkpoint {
     /** The exact turn input (prompt + rendered inbox drain) to replay. */
-    input: string;
+    input: PromptInput;
 }
 interface QueuedWake {
     h: Handle;
-    prompt?: string;
+    prompt?: PromptInput;
 }
 /**
  * A live agent. Public surface is the observable state (§7D pins transitions, not
@@ -266,7 +267,7 @@ export declare class AgentRuntime {
      * the wake queues FIFO and a completing sibling transfers its slot. Waking a
      * suspended handle is a no-op (items buffer; only the Answer wakes it).
      */
-    wake(h: Handle, prompt?: string): {
+    wake(h: Handle, prompt?: PromptInput): {
         ok: true;
     } | ({
         ok: false;

package/dist/agents/runtime.js

@@ -1,19 +1,3 @@
-/**
- * Agent runtime substrate — SPEC.md §7D.
- *
- * A Handle is a live agent: state machine (`idle → running → idle|suspended|closed`,
- * `suspended → running` only via the Answer to its pending Request), an inbox held as
- * agent state, hierarchical budgets, and a deterministic parent-scoped id
- * (`root/coordinator.1/explore.2`). The runtime exposes exactly six host verbs
- * (spawn/post/wake/wait/interrupt/close) plus read-only list/inspect views, and owns
- * the cross-cutting infrastructure: ONE ConversationStore for all handles
- * (conversation id = handle id, so transcripts genuinely survive turns), an
- * injectable clock (fixtures run virtual), and the handle table.
- *
- * Everything rides shipped machinery: the §8 client loop, §10 pending/waitFor, and
- * the ConversationStore. SPEC pins transitions, never scheduling — conformance is
- * identical per-handle transition traces against `examples/subagent-*` fixtures.
- */
 import { createClient, InMemoryConversationStore } from "../client.js";
 import { runGated } from "./loop.js";
 import { createToolkit } from "../toolkit.js";
@@ -563,7 +547,14 @@ export class AgentRuntime {
      * snapshot and the input is checkpointed — resume replays the whole turn.
      */
     async executeTurn(h, prompt, oneShotWaitFor) {
-        const input = (prompt ?? "") + this.drainInbox(h);
+        // A parts prompt keeps its ordering; the drained inbox joins it as a trailing text part,
+        // which is the same append the string path does.
+        const drained = this.drainInbox(h);
+        const input = typeof prompt === "string" || prompt === undefined
+            ? (prompt ?? "") + drained
+            : drained
+                ? [...prompt, { type: "text", text: drained }]
+                : prompt;
         let result;
         try {
             await h.ready;

package/dist/builtin.js

@@ -12,6 +12,7 @@ import fs from "node:fs/promises";
 import { existsSync, readdirSync } from "node:fs";
 import path from "node:path";
 import { pending } from "./types.js";
+import { mediaTypeFor } from "./content.js";
 /**
  * Whether the builtin source is on. Default ON. Same precedence as MCP:
  * `disabled:true` wins, else `enabled:false` disables, otherwise enabled.
@@ -171,7 +172,7 @@ function bashTool() {
     }));
 }
 function readTool() {
-    return builtin("read", "Read a UTF-8 text file. With offset/limit, return only that line window.", {
+    return builtin("read", "Read a file. Recognised media (png/jpg/jpeg/gif/webp/pdf/mp3/wav) comes back as a content part; anything else is read as UTF-8 text, with offset/limit returning that line window.", {
         type: "object",
         properties: {
             path: { type: "string", description: "Path to the file to read" },
@@ -184,13 +185,33 @@ function readTool() {
         const p = String(args.path ?? "");
         if (!p)
             return err("read: path is required");
-        let content;
+        let bytes;
         try {
-            content = await fs.readFile(p, "utf8");
+            bytes = await fs.readFile(p);
         }
         catch (e) {
             return err(`read: ${e instanceof Error ? e.message : String(e)}`);
         }
+        // §6: a recognised media extension comes back as a part. The table is fixed — no magic-byte
+        // sniffing and no platform mime database, whose contents vary per machine.
+        const media = mediaTypeFor(p);
+        if (media) {
+            return {
+                output: `${p} (${media.mimeType}, ${bytes.length} bytes)`,
+                isError: false,
+                parts: [{ type: media.type, mimeType: media.mimeType, data: bytes.toString("base64"), ...(media.type === "file" ? { name: path.basename(p) } : {}) }],
+            };
+        }
+        let content;
+        try {
+            // `fatal` so undecodable bytes are an isError RESULT, not U+FFFD soup — and never a
+            // raised exception escaping execute() into the loop. `ignoreBOM` keeps a leading BOM,
+            // as readFile(…, "utf8") does, so text reads stay byte-identical.
+            content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
+        }
+        catch {
+            return err(`read: ${p} is not valid UTF-8 text and has no recognised media extension`);
+        }
         if (args.offset === undefined && args.limit === undefined)
             return ok(content);
         const lines = content.split("\n");