toolnexus 0.16.0 → 0.17.0

package/src/mcp.ts

@@ -11,6 +11,7 @@ import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"
 import type { Tool as McpToolDef, ElicitRequest, ElicitResult } from "@modelcontextprotocol/sdk/types.js"
 import { ElicitRequestSchema } from "@modelcontextprotocol/sdk/types.js"
 import { type JSONSchema, type Tool, type ToolContext, type ToolResult, type McpStatus, type Request, type Answer, sanitize } from "./types.js"
+import { type ContentPart, summarizePart } from "./content.js"
 
 // ---------------------------------------------------------------------------
 // MCP elicitation bridge (§10). A server can ask US for input mid-`tools/call`
@@ -108,6 +109,43 @@ export function expandEnvHeaders(headers?: Record<string, string>): Record<strin
   return out
 }
 
+/**
+ * §0.4 / §2. Map every non-text `content[]` entry to a `ContentPart`, so nothing an MCP
+ * server returns is dropped silently: image ⇒ image, audio ⇒ audio, `resource_link` ⇒
+ * `file{url}`, an embedded `resource` carrying a blob ⇒ `file{data}`. An embedded resource
+ * carrying *text* is not a part — it belongs in `output`, and {@link joinTextContent}
+ * takes it there.
+ */
+export function mcpContentToParts(content: unknown): ContentPart[] | undefined {
+  if (!Array.isArray(content)) return undefined
+  const parts: ContentPart[] = []
+  for (const item of content) {
+    const it = item as any
+    if (!it || typeof it !== "object") continue
+    // `data` / `blob` are base64 STRINGS in the SDK, never bytes — passed through verbatim
+    // so no encoder of ours can drift them away from what the server sent.
+    if (it.type === "image" && typeof it.data === "string") {
+      parts.push({ type: "image", mimeType: String(it.mimeType ?? "application/octet-stream"), data: it.data })
+    } else if (it.type === "audio" && typeof it.data === "string") {
+      parts.push({ type: "audio", mimeType: String(it.mimeType ?? "application/octet-stream"), data: it.data })
+    } else if (it.type === "resource_link" && it.uri) {
+      const part: ContentPart = { type: "file", mimeType: String(it.mimeType ?? "application/octet-stream"), url: String(it.uri) }
+      if (it.name) (part as any).name = String(it.name)
+      parts.push(part)
+    } else if (it.type === "resource" && typeof it.resource?.blob === "string") {
+      const part: ContentPart = { type: "file", mimeType: String(it.resource.mimeType ?? "application/octet-stream"), data: it.resource.blob }
+      if (it.resource.uri) (part as any).name = String(it.resource.uri)
+      parts.push(part)
+    }
+  }
+  return parts.length ? parts : undefined
+}
+
+/** An image-only result must not answer the model with "". Name what came back instead. */
+function describeMcpParts(parts: ContentPart[] | undefined): string {
+  return parts ? parts.map(summarizePart).join("\n") : ""
+}
+
 function formatToolErrorContent(content: unknown): string {
   if (!Array.isArray(content)) return "MCP tool returned an error"
   return (
@@ -127,10 +165,17 @@ function isTextContent(value: unknown): value is { type: "text"; text: string }
   )
 }
 
+/** An embedded resource carrying text is text: it joins `output` in place, keeping the
+ *  server's ordering, rather than being discarded with the other non-text entries. */
+function isResourceText(value: unknown): value is { resource: { text: string } } {
+  const v = value as any
+  return v?.type === "resource" && typeof v?.resource?.text === "string"
+}
+
 function joinTextContent(content: unknown): string {
   if (!Array.isArray(content)) return ""
   return content
-    .flatMap((item) => (isTextContent(item) ? [item.text] : []))
+    .flatMap((item) => (isTextContent(item) ? [item.text] : isResourceText(item) ? [item.resource.text] : []))
     .join("\n")
 }
 
@@ -202,13 +247,19 @@ function convertTool(server: string, def: McpToolDef, client: Client, timeout: n
           undefined,
           { resetTimeoutOnProgress: true, signal: ctx?.signal, timeout: ctx?.timeout ?? timeout },
         )
+        // Parts are collected BEFORE any branch returns: the isError and structuredContent
+        // short-circuits both read `content[]` on the way past, so neither reintroduces the
+        // silent drop this mapping exists to remove (§2).
+        const parts = mcpContentToParts(result.content)
+        const withParts = (r: ToolResult): ToolResult => (parts ? { ...r, parts } : r)
         if (result.isError) {
-          return { output: formatToolErrorContent(result.content), isError: true, metadata: { server } }
+          return withParts({ output: formatToolErrorContent(result.content), isError: true, metadata: { server } })
         }
         if (result.structuredContent !== undefined && result.structuredContent !== null) {
-          return { output: JSON.stringify(result.structuredContent), isError: false, metadata: { server } }
+          return withParts({ output: JSON.stringify(result.structuredContent), isError: false, metadata: { server } })
         }
-        return { output: joinTextContent(result.content), isError: false, metadata: { server } }
+        const text = joinTextContent(result.content)
+        return withParts({ output: text || describeMcpParts(parts), isError: false, metadata: { server } })
       } catch (e) {
         return { output: e instanceof Error ? e.message : String(e), isError: true, metadata: { server } }
       }

package/src/mcpserve.ts

@@ -14,7 +14,8 @@
  */
 import { Server } from "@modelcontextprotocol/sdk/server/index.js"
 import { ListToolsRequestSchema, CallToolRequestSchema, McpError, ErrorCode } from "@modelcontextprotocol/sdk/types.js"
-import type { Tool, ToolSource } from "./types.js"
+import type { Tool, ToolSource, ToolResult } from "./types.js"
+import type { ContentPart } from "./content.js"
 
 const DEFAULT_NAME = "toolnexus"
 const DEFAULT_VERSION = "0.1.0"
@@ -75,7 +76,7 @@ export function buildMcpServer(tools: Tool[], cfg?: MCPServeConfig, onCall?: OnC
     // The call is a single synchronous tool invocation honoring the MCP
     // request's cancellation. An execute() throw becomes an isError result —
     // never a crash.
-    let result
+    let result: ToolResult
     try {
       result = await tool.execute(req.params.arguments ?? {}, { signal: extra.signal })
     } catch (e) {
@@ -88,8 +89,30 @@ export function buildMcpServer(tools: Tool[], cfg?: MCPServeConfig, onCall?: OnC
         // Host callback errors are isolated.
       }
     }
-    return { content: [{ type: "text", text: result.output }], isError: result.isError }
+    return { content: [{ type: "text", text: result.output }, ...partBlocks(result.parts)], isError: result.isError }
   })
 
   return server
 }
+
+/**
+ * §7C. A served tool's `ToolResult.parts` become MCP content blocks appended after the
+ * text part, in order — image ⇒ image content, audio ⇒ audio content, file ⇒ an embedded
+ * resource (a url-backed file keeps its uri, a byte-backed one carries the blob).
+ */
+function partBlocks(parts: ContentPart[] | undefined): any[] {
+  if (!parts?.length) return []
+  const out: any[] = []
+  for (const p of parts) {
+    if (p.type === "text") out.push({ type: "text", text: p.text })
+    else if (p.type === "image" && p.data) out.push({ type: "image", data: p.data, mimeType: p.mimeType })
+    else if (p.type === "audio" && p.data) out.push({ type: "audio", data: p.data, mimeType: p.mimeType })
+    else {
+      const uri = p.url ?? `file:///${(p as any).name ?? "resource"}`
+      const resource: any = { uri, mimeType: p.mimeType }
+      if (p.data) resource.blob = p.data
+      out.push({ type: "resource", resource })
+    }
+  }
+  return out
+}

package/src/translate.ts

@@ -14,6 +14,7 @@
  */
 
 import type { Toolkit } from "./toolkit.js"
+import { encodeParts, inboundPart, type ContentPart } from "./content.js"
 import type { Usage } from "./client.js"
 
 /** An OpenAI-shaped chat request, handed over verbatim (§11). */
@@ -187,9 +188,14 @@ export function openAIMessagesToAnthropic(messages: any[]): { messages: any[]; s
       }
       default: {
         flush()
+        // A `content` array has its text parts concatenated and its NON-text parts translated
+        // into Anthropic's block shape (§11). Six ports used to pass a text-empty array through
+        // raw and undocumented; one specified mapping replaces that, so nothing is flattened
+        // away and nothing is dropped.
+        const blocks = contentPartBlocks(m.content)
+        if (blocks) { out.push({ role: "user", content: blocks }); break }
         const s = contentText(m.content)
         if (s) out.push({ role: "user", content: s })
-        else if (Array.isArray(m.content) && m.content.length) out.push({ role: "user", content: m.content })
       }
     }
   }
@@ -197,6 +203,24 @@ export function openAIMessagesToAnthropic(messages: any[]): { messages: any[]; s
   return { messages: out, system: systemParts.join("\n\n") }
 }
 
+/**
+ * Translate an OpenAI-shaped `content` array into Anthropic blocks — but ONLY when it
+ * actually carries something non-text; an all-text array still concatenates, so the
+ * common case is byte-identical. Both spellings are accepted: a literal `ContentPart`
+ * and the provider-native block that same part encodes to.
+ */
+function contentPartBlocks(content: unknown): any[] | undefined {
+  if (!Array.isArray(content) || content.length === 0) return undefined
+  const parts: ContentPart[] = []
+  for (const raw of content) {
+    const p = inboundPart(raw)
+    if (!p) return undefined // an array we do not understand is left to contentText, as before
+    parts.push(p)
+  }
+  if (parts.every((p) => p.type === "text")) return undefined
+  return encodeParts(parts, { style: "anthropic", provenance: "attached" })
+}
+
 /**
  * Converts an OpenAI `tools` array into Anthropic tool declarations. Entries that are
  * already provider-native pass through; anything unrecognized is skipped.

package/src/types.ts

@@ -10,11 +10,20 @@ export type JSONSchema = {
   [k: string]: unknown
 }
 
+import type { ContentPart } from "./content.js"
+
 export type ToolSource = "mcp" | "skill" | "builtin" | "native" | "http" | "a2a" | "custom"
 
 export interface ToolResult {
   output: string
   isError: boolean
+  /**
+   * §1B non-text output. `output` stays required and remains what the transcript,
+   * compaction, token estimation and any text-only provider see; `parts` is the extra
+   * channel a screenshot / PDF / audio clip travels on. Absent ⇒ byte-identical to a
+   * pre-0.17 result, which is why it is optional rather than a fourth required field.
+   */
+  parts?: ContentPart[]
   metadata?: Record<string, unknown>
 }
 

package/src/wire.ts

@@ -0,0 +1,131 @@
+/**
+ * Canonical transcript → provider wire messages (SPEC.md §8A).
+ *
+ * The loop's `messages` array is the **canonical transcript**: a user turn carrying parts
+ * holds `ContentPart[]`, and a tool turn carrying parts holds them under a `parts` key.
+ * Neither shape is a wire shape. This module is the single place that turns the canonical
+ * transcript into what each style actually accepts, so the four loop sites (run/stream ×
+ * openai/anthropic) only ever have to *record* parts, never encode them.
+ *
+ * It is also where the **tool-result relocation rule** lives. Anthropic defines blocks
+ * inside `tool_result.content`, so its parts are emitted natively, keyed to their
+ * `tool_use_id`. OpenAI's `tool` message rejects an image outright (a hard 400 — "Image URLs
+ * are only allowed for messages with role 'user'"), so there every non-text part from every
+ * tool result answering one assistant turn is relocated, in tool-call order, into ONE
+ * synthetic `user` message emitted immediately after the last tool message. That synthetic
+ * message is an adapter artifact: it is built here, on the way out, and never written back
+ * to the transcript, the `ConversationStore` or `translate` output — which is what keeps a
+ * mid-conversation provider switch free of OpenAI-shaped residue.
+ */
+import type { ContentPart, PartStyle, UnsupportedPartMode } from "./content.js"
+import { encodeParts, isContentPart, validatePart } from "./content.js"
+
+/** Shared shaping options for a wire build. */
+export interface WireOptions {
+  onUnsupportedPart?: UnsupportedPartMode
+  maxPartBytes?: number
+}
+
+/** The text preceding each relocated part, so the model can attribute it to its call. */
+export const relocationHeader = (name: string, id: string) => `Output of tool ${name} (${id}):`
+
+/** Split parts into the text ones (which ride along in place) and the rest. */
+function split(parts: readonly ContentPart[]): { texts: ContentPart[]; others: ContentPart[] } {
+  const texts: ContentPart[] = []
+  const others: ContentPart[] = []
+  for (const p of parts) (p.type === "text" ? texts : others).push(p)
+  return { texts, others }
+}
+
+function partsOf(m: any): ContentPart[] | undefined {
+  const p = m?.parts
+  return Array.isArray(p) && p.length ? (p as ContentPart[]) : undefined
+}
+
+/** True when a message's `content` is a canonical `ContentPart[]` rather than a wire array. */
+function isPartArray(content: unknown): content is ContentPart[] {
+  return Array.isArray(content) && content.length > 0 && content.every(isContentPart)
+}
+
+/**
+ * Build the OpenAI-style wire messages. Tool messages carry `output` plus text parts only;
+ * their non-text parts are collected and flushed as one synthetic user message the moment
+ * the run of tool messages ends.
+ */
+export function toOpenAIWire(messages: any[], opts: WireOptions = {}): any[] {
+  const out: any[] = []
+  let relocated: { name: string; id: string; parts: ContentPart[] }[] = []
+
+  const flush = () => {
+    if (!relocated.length) return
+    const blocks: any[] = []
+    for (const r of relocated) {
+      blocks.push({ type: "text", text: relocationHeader(r.name, r.id) })
+      blocks.push(...encodeParts(r.parts, { style: "openai", provenance: "derived", ...opts }))
+    }
+    relocated = []
+    out.push({ role: "user", content: blocks })
+  }
+
+  for (const m of messages ?? []) {
+    if (m?.role === "tool") {
+      const parts = partsOf(m)
+      const { parts: _drop, ...rest } = m ?? {}
+      if (!parts) {
+        out.push(m)
+        continue
+      }
+      const { texts, others } = split(parts)
+      const msg: any = { ...rest }
+      if (texts.length) {
+        msg.content = [{ type: "text", text: String(rest.content ?? "") }, ...encodeParts(texts, { style: "openai", provenance: "derived", ...opts })]
+      }
+      out.push(msg)
+      if (others.length) relocated.push({ name: String(rest.name ?? ""), id: String(rest.tool_call_id ?? ""), parts: others })
+      continue
+    }
+    flush()
+    if (isPartArray(m?.content)) {
+      out.push({ ...m, content: encodeParts(m.content, { style: "openai", provenance: "attached", ...opts }) })
+      continue
+    }
+    out.push(m)
+  }
+  flush()
+  return out
+}
+
+/**
+ * Build the Anthropic-style wire messages. Nothing is relocated: a `tool_result` block that
+ * carries parts gets them natively inside its own `content`, keyed to its `tool_use_id`.
+ */
+export function toAnthropicWire(messages: any[], opts: WireOptions = {}): any[] {
+  return (messages ?? []).map((m) => {
+    if (isPartArray(m?.content)) {
+      return { ...m, content: encodeParts(m.content, { style: "anthropic", provenance: "attached", ...opts }) }
+    }
+    if (!Array.isArray(m?.content)) return m
+    let touched = false
+    const content = m.content.map((block: any) => {
+      if (block?.type !== "tool_result") return block
+      const parts = partsOf(block)
+      if (!parts) return block
+      touched = true
+      const { parts: _drop, ...rest } = block
+      return {
+        ...rest,
+        content: [
+          { type: "text", text: String(rest.content ?? "") },
+          ...encodeParts(parts, { style: "anthropic", provenance: "derived", ...opts }),
+        ],
+      }
+    })
+    return touched ? { ...m, content } : m
+  })
+}
+
+/** Validate every attached part before assembly, so `maxPartBytes` cannot be bypassed. */
+export function checkPromptParts(prompt: unknown, opts: WireOptions): void {
+  if (!Array.isArray(prompt)) return
+  for (const p of prompt) validatePart(p as ContentPart, opts.maxPartBytes)
+}