toolnexus 0.16.0 → 0.18.0

package/dist/translate.js

@@ -12,6 +12,7 @@
  * OpenAI function-calling posture). When toolnexus owns the conversation, use the agent
  * loop with relay tools + `runWithAnswer` (§10) instead.
  */
+import { encodeParts, inboundPart } from "./content.js";
 /**
  * Maps a provider stop reason onto an OpenAI finish reason. Tool calls win: a turn that
  * emitted any tool call is always `"tool_calls"` to a conforming client.
@@ -134,17 +135,44 @@ export function openAIMessagesToAnthropic(messages) {
             }
             default: {
                 flush();
+                // A `content` array has its text parts concatenated and its NON-text parts translated
+                // into Anthropic's block shape (§11). Six ports used to pass a text-empty array through
+                // raw and undocumented; one specified mapping replaces that, so nothing is flattened
+                // away and nothing is dropped.
+                const blocks = contentPartBlocks(m.content);
+                if (blocks) {
+                    out.push({ role: "user", content: blocks });
+                    break;
+                }
                 const s = contentText(m.content);
                 if (s)
                     out.push({ role: "user", content: s });
-                else if (Array.isArray(m.content) && m.content.length)
-                    out.push({ role: "user", content: m.content });
             }
         }
     }
     flush();
     return { messages: out, system: systemParts.join("\n\n") };
 }
+/**
+ * Translate an OpenAI-shaped `content` array into Anthropic blocks — but ONLY when it
+ * actually carries something non-text; an all-text array still concatenates, so the
+ * common case is byte-identical. Both spellings are accepted: a literal `ContentPart`
+ * and the provider-native block that same part encodes to.
+ */
+function contentPartBlocks(content) {
+    if (!Array.isArray(content) || content.length === 0)
+        return undefined;
+    const parts = [];
+    for (const raw of content) {
+        const p = inboundPart(raw);
+        if (!p)
+            return undefined; // an array we do not understand is left to contentText, as before
+        parts.push(p);
+    }
+    if (parts.every((p) => p.type === "text"))
+        return undefined;
+    return encodeParts(parts, { style: "anthropic", provenance: "attached" });
+}
 /**
  * Converts an OpenAI `tools` array into Anthropic tool declarations. Entries that are
  * already provider-native pass through; anything unrecognized is skipped.

package/dist/types.d.ts

@@ -8,10 +8,18 @@ export type JSONSchema = {
     additionalProperties?: boolean;
     [k: string]: unknown;
 };
+import type { ContentPart } from "./content.js";
 export type ToolSource = "mcp" | "skill" | "builtin" | "native" | "http" | "a2a" | "custom";
 export interface ToolResult {
     output: string;
     isError: boolean;
+    /**
+     * §1B non-text output. `output` stays required and remains what the transcript,
+     * compaction, token estimation and any text-only provider see; `parts` is the extra
+     * channel a screenshot / PDF / audio clip travels on. Absent ⇒ byte-identical to a
+     * pre-0.17 result, which is why it is optional rather than a fourth required field.
+     */
+    parts?: ContentPart[];
     metadata?: Record<string, unknown>;
 }
 export interface ToolContext {

package/dist/wire.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Canonical transcript → provider wire messages (SPEC.md §8A).
+ *
+ * The loop's `messages` array is the **canonical transcript**: a user turn carrying parts
+ * holds `ContentPart[]`, and a tool turn carrying parts holds them under a `parts` key.
+ * Neither shape is a wire shape. This module is the single place that turns the canonical
+ * transcript into what each style actually accepts, so the four loop sites (run/stream ×
+ * openai/anthropic) only ever have to *record* parts, never encode them.
+ *
+ * It is also where the **tool-result relocation rule** lives. Anthropic defines blocks
+ * inside `tool_result.content`, so its parts are emitted natively, keyed to their
+ * `tool_use_id`. OpenAI's `tool` message rejects an image outright (a hard 400 — "Image URLs
+ * are only allowed for messages with role 'user'"), so there every non-text part from every
+ * tool result answering one assistant turn is relocated, in tool-call order, into ONE
+ * synthetic `user` message emitted immediately after the last tool message. That synthetic
+ * message is an adapter artifact: it is built here, on the way out, and never written back
+ * to the transcript, the `ConversationStore` or `translate` output — which is what keeps a
+ * mid-conversation provider switch free of OpenAI-shaped residue.
+ */
+import type { UnsupportedPartMode } from "./content.js";
+/** Shared shaping options for a wire build. */
+export interface WireOptions {
+    onUnsupportedPart?: UnsupportedPartMode;
+    maxPartBytes?: number;
+}
+/** The text preceding each relocated part, so the model can attribute it to its call. */
+export declare const relocationHeader: (name: string, id: string) => string;
+/**
+ * Build the OpenAI-style wire messages. Tool messages carry `output` plus text parts only;
+ * their non-text parts are collected and flushed as one synthetic user message the moment
+ * the run of tool messages ends.
+ */
+export declare function toOpenAIWire(messages: any[], opts?: WireOptions): any[];
+/**
+ * Build the Anthropic-style wire messages. Nothing is relocated: a `tool_result` block that
+ * carries parts gets them natively inside its own `content`, keyed to its `tool_use_id`.
+ */
+export declare function toAnthropicWire(messages: any[], opts?: WireOptions): any[];
+/** Validate every attached part before assembly, so `maxPartBytes` cannot be bypassed. */
+export declare function checkPromptParts(prompt: unknown, opts: WireOptions): void;

package/dist/wire.js

@@ -0,0 +1,104 @@
+import { encodeParts, isContentPart, validatePart } from "./content.js";
+/** The text preceding each relocated part, so the model can attribute it to its call. */
+export const relocationHeader = (name, id) => `Output of tool ${name} (${id}):`;
+/** Split parts into the text ones (which ride along in place) and the rest. */
+function split(parts) {
+    const texts = [];
+    const others = [];
+    for (const p of parts)
+        (p.type === "text" ? texts : others).push(p);
+    return { texts, others };
+}
+function partsOf(m) {
+    const p = m?.parts;
+    return Array.isArray(p) && p.length ? p : undefined;
+}
+/** True when a message's `content` is a canonical `ContentPart[]` rather than a wire array. */
+function isPartArray(content) {
+    return Array.isArray(content) && content.length > 0 && content.every(isContentPart);
+}
+/**
+ * Build the OpenAI-style wire messages. Tool messages carry `output` plus text parts only;
+ * their non-text parts are collected and flushed as one synthetic user message the moment
+ * the run of tool messages ends.
+ */
+export function toOpenAIWire(messages, opts = {}) {
+    const out = [];
+    let relocated = [];
+    const flush = () => {
+        if (!relocated.length)
+            return;
+        const blocks = [];
+        for (const r of relocated) {
+            blocks.push({ type: "text", text: relocationHeader(r.name, r.id) });
+            blocks.push(...encodeParts(r.parts, { style: "openai", provenance: "derived", ...opts }));
+        }
+        relocated = [];
+        out.push({ role: "user", content: blocks });
+    };
+    for (const m of messages ?? []) {
+        if (m?.role === "tool") {
+            const parts = partsOf(m);
+            const { parts: _drop, ...rest } = m ?? {};
+            if (!parts) {
+                out.push(m);
+                continue;
+            }
+            const { texts, others } = split(parts);
+            const msg = { ...rest };
+            if (texts.length) {
+                msg.content = [{ type: "text", text: String(rest.content ?? "") }, ...encodeParts(texts, { style: "openai", provenance: "derived", ...opts })];
+            }
+            out.push(msg);
+            if (others.length)
+                relocated.push({ name: String(rest.name ?? ""), id: String(rest.tool_call_id ?? ""), parts: others });
+            continue;
+        }
+        flush();
+        if (isPartArray(m?.content)) {
+            out.push({ ...m, content: encodeParts(m.content, { style: "openai", provenance: "attached", ...opts }) });
+            continue;
+        }
+        out.push(m);
+    }
+    flush();
+    return out;
+}
+/**
+ * Build the Anthropic-style wire messages. Nothing is relocated: a `tool_result` block that
+ * carries parts gets them natively inside its own `content`, keyed to its `tool_use_id`.
+ */
+export function toAnthropicWire(messages, opts = {}) {
+    return (messages ?? []).map((m) => {
+        if (isPartArray(m?.content)) {
+            return { ...m, content: encodeParts(m.content, { style: "anthropic", provenance: "attached", ...opts }) };
+        }
+        if (!Array.isArray(m?.content))
+            return m;
+        let touched = false;
+        const content = m.content.map((block) => {
+            if (block?.type !== "tool_result")
+                return block;
+            const parts = partsOf(block);
+            if (!parts)
+                return block;
+            touched = true;
+            const { parts: _drop, ...rest } = block;
+            return {
+                ...rest,
+                content: [
+                    { type: "text", text: String(rest.content ?? "") },
+                    ...encodeParts(parts, { style: "anthropic", provenance: "derived", ...opts }),
+                ],
+            };
+        });
+        return touched ? { ...m, content } : m;
+    });
+}
+/** Validate every attached part before assembly, so `maxPartBytes` cannot be bypassed. */
+export function checkPromptParts(prompt, opts) {
+    if (!Array.isArray(prompt))
+        return;
+    for (const p of prompt)
+        validatePart(p, opts.maxPartBytes);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "toolnexus",
-  "version": "0.16.0",
+  "version": "0.18.0",
   "description": "Dynamic MCP servers + agent skills as uniform tools for any LLM (opencode-style).",
   "type": "module",
   "main": "./dist/index.js",
@@ -17,8 +17,10 @@
   ],
   "scripts": {
     "build": "tsc -p tsconfig.json",
-    "test": "npm run build && node --experimental-strip-types --test test/unit.test.ts test/agents.test.ts test/agent-hooks.test.ts test/home.test.ts test/compaction.test.ts test/loop.test.ts test/in-process.test.ts",
-    "example": "node --experimental-strip-types examples/basic.ts"
+    "test": "npm run build && node --experimental-strip-types --test test/unit.test.ts test/agents.test.ts test/agent-hooks.test.ts test/home.test.ts test/compaction.test.ts test/loop.test.ts test/in-process.test.ts test/translate.test.ts test/multimodal.test.ts test/classifier.test.ts",
+    "example": "node --experimental-strip-types examples/basic.ts",
+    "example:judge": "node --experimental-strip-types examples/judge.ts",
+    "example:multimodal": "node --experimental-strip-types examples/multimodal.ts"
   },
   "keywords": [
     "mcp",

package/src/agents/agent.ts

@@ -7,6 +7,7 @@
  * API's `extraTools`; `run()` executes one-shot against a private runtime.
  * Everything compiles down to the six host verbs in runtime.ts.
  */
+import type { PromptInput } from "../content.js"
 import fs from "node:fs"
 import { defineTool } from "../native.js"
 import type { Answer, Request, Tool } from "../types.js"
@@ -113,7 +114,7 @@ export class Agent {
    * the tree down — unless the run suspends durably (`status:"pending"`), in which
    * case the tree stays parked and `result.runtime.resume(answer)` continues it.
    */
-  async run(prompt: string, opts: AgentRunOptions = {}): Promise<AgentRunResult> {
+  async run(prompt: PromptInput, opts: AgentRunOptions = {}): Promise<AgentRunResult> {
     const runtime = this.createRuntime(opts)
     const h = runtime.spawn(runtime.root, this.name)
     if (isVerbError(h)) {

package/src/agents/compaction.ts

@@ -23,6 +23,7 @@
  *     hooks: { beforeLLM: agents.compactor({ maxTokens: 120_000, summarize }) },
  *   })
  */
+import { partTokens } from "../content.js"
 
 /**
  * A chat message in the working transcript. Structural and provider-neutral — only the
@@ -44,10 +45,47 @@ export interface Message {
  */
 export function estimateTokens(messages: Message[]): number {
   let total = 0
-  for (const m of messages) total += Math.ceil(JSON.stringify(m).length / 4)
+  for (const m of messages) {
+    const { text, partBytes } = stripPartPayloads(m)
+    total += Math.ceil(text.length / 4)
+    // §1B: a media part is charged from its BYTE LENGTH. Scoring it by its `mimeType`
+    // string would price a 5 MB image at ~3 tokens, so the compactor would evict text and
+    // keep every image forever — the exact opposite of what a budget is for. Charging the
+    // base64 verbatim is the other failure: it prices an 82-byte PNG like 30 lines of prose.
+    for (const bytes of partBytes) total += partTokens(bytes)
+  }
   return total
 }
 
+/** Keys whose string value is a media payload rather than prose. */
+const PAYLOAD_KEYS = new Set(["data", "file_data", "blob"])
+
+/**
+ * Serialize a message for estimation with its media payloads replaced by "", returning the
+ * decoded byte length of each one so the caller can charge it properly. Recognises both the
+ * canonical `ContentPart` (`data`) and every provider block shape it encodes to
+ * (`image_url.url` as a data: URL, `file.file_data`, `input_audio.data`).
+ */
+function stripPartPayloads(m: Message): { text: string; partBytes: number[] } {
+  const partBytes: number[] = []
+  const walk = (v: unknown, key?: string): unknown => {
+    if (typeof v === "string") {
+      const b64 = PAYLOAD_KEYS.has(key ?? "") ? v : v.startsWith("data:") ? v.slice(v.indexOf(",") + 1) : undefined
+      if (b64 === undefined) return v
+      partBytes.push(Math.max(0, Math.floor((b64.length * 3) / 4)))
+      return ""
+    }
+    if (Array.isArray(v)) return v.map((x) => walk(x))
+    if (v && typeof v === "object") {
+      const out: Record<string, unknown> = {}
+      for (const [k, val] of Object.entries(v as Record<string, unknown>)) out[k] = walk(val, k)
+      return out
+    }
+    return v
+  }
+  return { text: JSON.stringify(walk(m)) ?? "", partBytes }
+}
+
 /** Options for {@link compactor}. */
 export interface CompactorOptions {
   /** Compact only when the transcript estimate exceeds this; at/below ⇒ no-op. */

package/src/agents/loop.ts

@@ -12,6 +12,7 @@
  *
  * So Loop takes no options: it is read, not configured.
  */
+import type { PromptInput } from "../content.js"
 import { createClient, type ClientOptions, type Hooks, type RunResult } from "../client.js"
 import type { Toolkit } from "../toolkit.js"
 import type { Agent } from "./agent.js"
@@ -71,6 +72,18 @@ export function guardedHooks(guardrails: Guardrail[] | undefined, hooks: Hooks |
     beforeTool: async (ev) => {
       for (const g of guardrails) {
         const verdict = g(ev)
+        // A guardrail is SYNCHRONOUS by contract. An async one returns a Promise,
+        // which is truthy and !== "allow", so it would silently DENY every call
+        // with `denied: [object Promise]`. Fail loudly instead, and say where
+        // asynchronous work belongs.
+        if (verdict != null && typeof verdict !== "string") {
+          const what = typeof (verdict as { then?: unknown }).then === "function" ? "a Promise" : typeof verdict
+          throw new TypeError(
+            `guardrail returned ${what}: a guardrail must return a string synchronously ` +
+              `("" or "allow" to permit, any other string to deny). ` +
+              `Do asynchronous work in hooks.beforeTool, which is awaited.`,
+          )
+        }
         if (verdict && verdict !== "allow") {
           return { result: { output: `denied: ${verdict}`, isError: true } }
         }
@@ -114,8 +127,8 @@ export function allTodosDone(result: RunResult): Verdict {
  * it owes an Answer or a fix.
  */
 export async function runGated(
-  ask: (prompt: string) => Promise<RunResult>,
-  prompt: string,
+  ask: (prompt: PromptInput) => Promise<RunResult>,
+  prompt: PromptInput,
   completion: Completion | undefined,
 ): Promise<RunResult> {
   if (!completion) return ask(prompt)
@@ -187,14 +200,14 @@ export class Loop {
     return this.#turns
   }
 
-  async run(prompt: string, opts: LoopRunOptions = {}): Promise<Outcome> {
+  async run(prompt: PromptInput, opts: LoopRunOptions = {}): Promise<Outcome> {
     const completion = this.agent.spec.completion
     const client = createClient(this.#clientOptions(opts))
     this.#status = "running"
 
     let attempts = 0
     let history: any[] = []
-    const ask = async (p: string): Promise<RunResult> => {
+    const ask = async (p: PromptInput): Promise<RunResult> => {
       attempts++
       const r = await client.run(p, { toolkit: this.toolkit, history })
       this.#turns += r.turns

package/src/agents/runtime.ts

@@ -14,6 +14,7 @@
  * the ConversationStore. SPEC pins transitions, never scheduling — conformance is
  * identical per-handle transition traces against `examples/subagent-*` fixtures.
  */
+import type { PromptInput } from "../content.js"
 import { createClient, InMemoryConversationStore, type ClientStyle, type ConversationStore, type Hooks, type MetricEvent } from "../client.js"
 import { runGated, type Completion } from "./loop.js"
 import { createToolkit, type Toolkit } from "../toolkit.js"
@@ -176,12 +177,12 @@ interface Pools {
  */
 interface Checkpoint {
   /** The exact turn input (prompt + rendered inbox drain) to replay. */
-  input: string
+  input: PromptInput
 }
 
 interface QueuedWake {
   h: Handle
-  prompt?: string
+  prompt?: PromptInput
 }
 
 /**
@@ -387,7 +388,7 @@ export class AgentRuntime {
    * the wake queues FIFO and a completing sibling transfers its slot. Waking a
    * suspended handle is a no-op (items buffer; only the Answer wakes it).
    */
-  wake(h: Handle, prompt?: string): { ok: true } | ({ ok: false } & VerbError) {
+  wake(h: Handle, prompt?: PromptInput): { ok: true } | ({ ok: false } & VerbError) {
     if (h.state === "closed") return { ok: false, error: `closed: ${h.id}` }
     if (h.state === "suspended") return { ok: true } // buffers; suspended→running ONLY via the Answer
     if (h.state === "running") return { ok: true } // already awake; inbox drains next turn
@@ -750,8 +751,15 @@ export class AgentRuntime {
    * When the Run halts pending, the stored transcript is REWOUND to the pre-turn
    * snapshot and the input is checkpointed — resume replays the whole turn.
    */
-  private async executeTurn(h: Handle, prompt?: string, oneShotWaitFor?: (req: Request) => Promise<Answer>): Promise<TaskResult> {
-    const input = (prompt ?? "") + this.drainInbox(h)
+  private async executeTurn(h: Handle, prompt?: PromptInput, oneShotWaitFor?: (req: Request) => Promise<Answer>): Promise<TaskResult> {
+    // A parts prompt keeps its ordering; the drained inbox joins it as a trailing text part,
+    // which is the same append the string path does.
+    const drained = this.drainInbox(h)
+    const input: PromptInput = typeof prompt === "string" || prompt === undefined
+      ? (prompt ?? "") + drained
+      : drained
+        ? [...prompt, { type: "text" as const, text: drained }]
+        : prompt
     let result: TaskResult
     try {
       await h.ready

package/src/builtin.ts

@@ -13,6 +13,7 @@ import { existsSync, readdirSync } from "node:fs"
 import path from "node:path"
 import type { JSONSchema, Tool, ToolContext, ToolResult } from "./types.js"
 import { pending } from "./types.js"
+import { mediaTypeFor } from "./content.js"
 
 /** Config for the single global builtin toggle (mirrors MCP isEnabled precedence). */
 export type BuiltinsConfig =
@@ -184,7 +185,7 @@ function bashTool(): Tool {
 function readTool(): Tool {
   return builtin(
     "read",
-    "Read a UTF-8 text file. With offset/limit, return only that line window.",
+    "Read a file. Recognised media (png/jpg/jpeg/gif/webp/pdf/mp3/wav) comes back as a content part; anything else is read as UTF-8 text, with offset/limit returning that line window.",
     {
       type: "object",
       properties: {
@@ -198,12 +199,31 @@ function readTool(): Tool {
     async (args) => {
       const p = String(args.path ?? "")
       if (!p) return err("read: path is required")
-      let content: string
+      let bytes: Buffer
       try {
-        content = await fs.readFile(p, "utf8")
+        bytes = await fs.readFile(p)
       } catch (e) {
         return err(`read: ${e instanceof Error ? e.message : String(e)}`)
       }
+      // §6: a recognised media extension comes back as a part. The table is fixed — no magic-byte
+      // sniffing and no platform mime database, whose contents vary per machine.
+      const media = mediaTypeFor(p)
+      if (media) {
+        return {
+          output: `${p} (${media.mimeType}, ${bytes.length} bytes)`,
+          isError: false,
+          parts: [{ type: media.type, mimeType: media.mimeType, data: bytes.toString("base64"), ...(media.type === "file" ? { name: path.basename(p) } : {}) } as any],
+        }
+      }
+      let content: string
+      try {
+        // `fatal` so undecodable bytes are an isError RESULT, not U+FFFD soup — and never a
+        // raised exception escaping execute() into the loop. `ignoreBOM` keeps a leading BOM,
+        // as readFile(…, "utf8") does, so text reads stay byte-identical.
+        content = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes)
+      } catch {
+        return err(`read: ${p} is not valid UTF-8 text and has no recognised media extension`)
+      }
       if (args.offset === undefined && args.limit === undefined) return ok(content)
       const lines = content.split("\n")
       const offset = typeof args.offset === "number" ? Math.max(1, Math.trunc(args.offset)) : 1