toolnexus 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/agents/agent.d.ts +11 -1
- package/dist/agents/agent.js +0 -9
- package/dist/agents/compaction.d.ts +0 -25
- package/dist/agents/compaction.js +41 -2
- package/dist/agents/loop.d.ts +3 -2
- package/dist/agents/loop.js +0 -14
- package/dist/agents/runtime.d.ts +4 -3
- package/dist/agents/runtime.js +8 -17
- package/dist/builtin.js +24 -3
- package/dist/client.d.ts +14 -4
- package/dist/client.js +35 -8
- package/dist/content.d.ts +158 -0
- package/dist/content.js +379 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.js +2 -0
- package/dist/mcp.d.ts +9 -0
- package/dist/mcp.js +59 -4
- package/dist/mcpserve.js +27 -1
- package/dist/translate.js +30 -2
- package/dist/types.d.ts +8 -0
- package/dist/wire.d.ts +40 -0
- package/dist/wire.js +104 -0
- package/package.json +2 -2
- package/src/agents/agent.ts +2 -1
- package/src/agents/compaction.ts +39 -1
- package/src/agents/loop.ts +5 -4
- package/src/agents/runtime.ts +13 -5
- package/src/builtin.ts +23 -3
- package/src/client.ts +53 -16
- package/src/content.ts +448 -0
- package/src/index.ts +2 -0
- package/src/mcp.ts +55 -4
- package/src/mcpserve.ts +26 -3
- package/src/translate.ts +25 -1
- package/src/types.ts +9 -0
- package/src/wire.ts +131 -0
package/src/client.ts
CHANGED
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
import type { Toolkit } from "./toolkit.js"
|
|
7
7
|
import type { ToolResult, Request, Answer } from "./types.js"
|
|
8
8
|
import { pendingOf } from "./types.js"
|
|
9
|
+
import type { ContentPart, PromptInput, UnsupportedPartMode } from "./content.js"
|
|
10
|
+
import { toOpenAIWire, toAnthropicWire, checkPromptParts, type WireOptions } from "./wire.js"
|
|
9
11
|
import type { TranslateRequest, TranslateResult, TranslatedToolCall } from "./translate.js"
|
|
10
12
|
import {
|
|
11
13
|
finishReasonFor,
|
|
@@ -62,6 +64,13 @@ export interface ClientOptions {
|
|
|
62
64
|
* today. A "retry" is always bounded by `retries`; the classifier cannot loop unbounded.
|
|
63
65
|
* There is no "suspend" tier — suspension (§10) stays a user-action pause, not an error path. */
|
|
64
66
|
onError?: (info: ErrorInfo) => ErrorTier
|
|
67
|
+
/** §1B. Reject a content part carrying more than this many **decoded** bytes (never the
|
|
68
|
+
* +33% base64 string). Unset ⇒ no limit, as today. */
|
|
69
|
+
maxPartBytes?: number
|
|
70
|
+
/** §8A. Overrides the provenance rule uniformly: `"error"` fails on any part the style
|
|
71
|
+
* cannot represent (attached or tool-derived), `"text"` degrades every one of them to a
|
|
72
|
+
* text placeholder. Unset ⇒ attached parts error, derived parts degrade. */
|
|
73
|
+
onUnsupportedPart?: UnsupportedPartMode
|
|
65
74
|
}
|
|
66
75
|
|
|
67
76
|
/** §8 Resilience. What to do with a failed LLM call. */
|
|
@@ -384,7 +393,7 @@ export class Client {
|
|
|
384
393
|
/** §8 Gap 5. OpenAI-style body; omits tools/tool_choice when the tool list is empty. Key order
|
|
385
394
|
* matches the pre-change body so a no-options call is byte-identical. */
|
|
386
395
|
private openaiBody(messages: any[], tools: any[], stream: boolean): Record<string, any> {
|
|
387
|
-
const b: Record<string, any> = { model: this.opts.model, messages }
|
|
396
|
+
const b: Record<string, any> = { model: this.opts.model, messages: toOpenAIWire(messages, this.wireOpts()) }
|
|
388
397
|
if (tools.length) {
|
|
389
398
|
b.tools = tools
|
|
390
399
|
b.tool_choice = "auto"
|
|
@@ -398,12 +407,17 @@ export class Client {
|
|
|
398
407
|
|
|
399
408
|
/** §8 Gap 5. Anthropic-style body; omits tools when the tool list is empty. */
|
|
400
409
|
private anthropicBody(system: string, messages: any[], tools: any[], stream: boolean): Record<string, any> {
|
|
401
|
-
const b: Record<string, any> = { model: this.opts.model, max_tokens: 4096, system, messages }
|
|
410
|
+
const b: Record<string, any> = { model: this.opts.model, max_tokens: 4096, system, messages: toAnthropicWire(messages, this.wireOpts()) }
|
|
402
411
|
if (tools.length) b.tools = tools
|
|
403
412
|
if (stream) b.stream = true
|
|
404
413
|
return this.finalizeBody(b)
|
|
405
414
|
}
|
|
406
415
|
|
|
416
|
+
/** The §8A/§1B shaping options every wire build shares. */
|
|
417
|
+
private wireOpts(): WireOptions {
|
|
418
|
+
return { onUnsupportedPart: this.opts.onUnsupportedPart, maxPartBytes: this.opts.maxPartBytes }
|
|
419
|
+
}
|
|
420
|
+
|
|
407
421
|
/** Prometheus text exposition of cumulative metrics (§8). Always valid, empty-but-valid before activity. */
|
|
408
422
|
metrics(): string {
|
|
409
423
|
return this.registry.render()
|
|
@@ -427,7 +441,8 @@ export class Client {
|
|
|
427
441
|
this.emit({ event: "run", model: this.opts.model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart, error: e instanceof Error ? e.message : String(e) })
|
|
428
442
|
}
|
|
429
443
|
|
|
430
|
-
async run(prompt:
|
|
444
|
+
async run(prompt: PromptInput, ctx: { toolkit: Toolkit; signal?: AbortSignal; history?: any[] }): Promise<RunResult> {
|
|
445
|
+
checkPromptParts(prompt, this.wireOpts())
|
|
431
446
|
return this.opts.style === "anthropic"
|
|
432
447
|
? this.runAnthropic(prompt, ctx.toolkit, ctx.signal, ctx.history)
|
|
433
448
|
: this.runOpenAI(prompt, ctx.toolkit, ctx.signal, ctx.history)
|
|
@@ -592,7 +607,7 @@ export class Client {
|
|
|
592
607
|
* the answer — so the next `ask` with the same `id` continues it. Without an
|
|
593
608
|
* `id` it is a stateless one-shot (identical to `run`).
|
|
594
609
|
*/
|
|
595
|
-
async ask(prompt:
|
|
610
|
+
async ask(prompt: PromptInput, ctx: { toolkit: Toolkit; id?: string; on_text?: (delta: string) => void; signal?: AbortSignal }): Promise<RunResult> {
|
|
596
611
|
// Block-style streaming: run the streaming loop, forward text deltas, still return the
|
|
597
612
|
// final RunResult. Memory (id load/save) is handled by stream() itself, so no duplication.
|
|
598
613
|
if (ctx.on_text) {
|
|
@@ -620,7 +635,8 @@ export class Client {
|
|
|
620
635
|
* With an `id`, it is stateful (like `ask`): the thread's transcript is loaded as history before
|
|
621
636
|
* streaming, and saved back to the ConversationStore on the terminal `done` event. No `id` ⇒ stateless.
|
|
622
637
|
*/
|
|
623
|
-
async *stream(prompt:
|
|
638
|
+
async *stream(prompt: PromptInput, ctx: { toolkit: Toolkit; id?: string; signal?: AbortSignal }): AsyncGenerator<StreamEvent, void, unknown> {
|
|
639
|
+
checkPromptParts(prompt, this.wireOpts())
|
|
624
640
|
const history = ctx.id ? (await this.store.get(ctx.id)) ?? [] : undefined
|
|
625
641
|
const gen = this.opts.style === "anthropic"
|
|
626
642
|
? this.streamAnthropic(prompt, ctx.toolkit, ctx.signal, history)
|
|
@@ -730,7 +746,7 @@ export class Client {
|
|
|
730
746
|
}
|
|
731
747
|
|
|
732
748
|
// ---- OpenAI-style: POST {baseUrl}/chat/completions ----
|
|
733
|
-
private async runOpenAI(prompt:
|
|
749
|
+
private async runOpenAI(prompt: PromptInput, toolkit: Toolkit, external?: AbortSignal, history?: any[]): Promise<RunResult> {
|
|
734
750
|
const key = resolveKey(this.opts)
|
|
735
751
|
const signal = this.makeSignal(external)
|
|
736
752
|
const runStart = Date.now()
|
|
@@ -783,7 +799,7 @@ export class Client {
|
|
|
783
799
|
// on resume). Mirrors the streaming path.
|
|
784
800
|
for (const s of settled) {
|
|
785
801
|
toolCalls.push({ name: s.call.function.name, args: s.args, output: s.result.output, isError: s.result.isError, metadata: s.result.metadata })
|
|
786
|
-
messages.push(
|
|
802
|
+
messages.push(toolMessage(s.call.id, s.call.function.name, s.result))
|
|
787
803
|
if (s.halted) return this.pendingRun(runStart, s.halted, messages, toolCalls, turns, usage)
|
|
788
804
|
}
|
|
789
805
|
}
|
|
@@ -830,7 +846,7 @@ export class Client {
|
|
|
830
846
|
}
|
|
831
847
|
|
|
832
848
|
// ---- Anthropic-style: POST {baseUrl}/messages ----
|
|
833
|
-
private async runAnthropic(prompt:
|
|
849
|
+
private async runAnthropic(prompt: PromptInput, toolkit: Toolkit, external?: AbortSignal, history?: any[]): Promise<RunResult> {
|
|
834
850
|
const key = resolveKey(this.opts)
|
|
835
851
|
const signal = this.makeSignal(external)
|
|
836
852
|
const base = this.opts.baseUrl.replace(/\/$/, "")
|
|
@@ -890,7 +906,7 @@ export class Client {
|
|
|
890
906
|
let halted: Request | undefined
|
|
891
907
|
for (const s of settled) {
|
|
892
908
|
toolCalls.push({ name: s.use.name, args: s.args, output: s.result.output, isError: s.result.isError, metadata: s.result.metadata })
|
|
893
|
-
content.push(
|
|
909
|
+
content.push(toolResultBlock(s.use.id, s.result))
|
|
894
910
|
if (s.halted) { halted = s.halted; break }
|
|
895
911
|
}
|
|
896
912
|
messages.push({ role: "user", content })
|
|
@@ -904,7 +920,7 @@ export class Client {
|
|
|
904
920
|
}
|
|
905
921
|
|
|
906
922
|
// ---- Streaming: OpenAI-style ----
|
|
907
|
-
private async *streamOpenAI(prompt:
|
|
923
|
+
private async *streamOpenAI(prompt: PromptInput, toolkit: Toolkit, external?: AbortSignal, history?: any[]): AsyncGenerator<StreamEvent, void, unknown> {
|
|
908
924
|
const key = resolveKey(this.opts)
|
|
909
925
|
const signal = this.makeSignal(external)
|
|
910
926
|
const runStart = Date.now()
|
|
@@ -982,13 +998,13 @@ export class Client {
|
|
|
982
998
|
result = r.result
|
|
983
999
|
if (r.halted) {
|
|
984
1000
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
985
|
-
messages.push(
|
|
1001
|
+
messages.push(toolMessage(c.id, c.name, result))
|
|
986
1002
|
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) }
|
|
987
1003
|
return
|
|
988
1004
|
}
|
|
989
1005
|
}
|
|
990
1006
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
991
|
-
messages.push(
|
|
1007
|
+
messages.push(toolMessage(c.id, c.name, result))
|
|
992
1008
|
yield { type: "tool_result", id: c.id, name: c.name, output: result.output, isError: result.isError }
|
|
993
1009
|
}
|
|
994
1010
|
}
|
|
@@ -1000,7 +1016,7 @@ export class Client {
|
|
|
1000
1016
|
}
|
|
1001
1017
|
|
|
1002
1018
|
// ---- Streaming: Anthropic-style ----
|
|
1003
|
-
private async *streamAnthropic(prompt:
|
|
1019
|
+
private async *streamAnthropic(prompt: PromptInput, toolkit: Toolkit, external?: AbortSignal, history?: any[]): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1004
1020
|
const key = resolveKey(this.opts)
|
|
1005
1021
|
const signal = this.makeSignal(external)
|
|
1006
1022
|
const runStart = Date.now()
|
|
@@ -1073,14 +1089,14 @@ export class Client {
|
|
|
1073
1089
|
result = r.result
|
|
1074
1090
|
if (r.halted) {
|
|
1075
1091
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
1076
|
-
results.push(
|
|
1092
|
+
results.push(toolResultBlock(u.id, result))
|
|
1077
1093
|
messages.push({ role: "user", content: results })
|
|
1078
1094
|
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) }
|
|
1079
1095
|
return
|
|
1080
1096
|
}
|
|
1081
1097
|
}
|
|
1082
1098
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
1083
|
-
results.push(
|
|
1099
|
+
results.push(toolResultBlock(u.id, result))
|
|
1084
1100
|
yield { type: "tool_result", id: u.id, name: u.name, output: result.output, isError: result.isError }
|
|
1085
1101
|
}
|
|
1086
1102
|
messages.push({ role: "user", content: results })
|
|
@@ -1233,7 +1249,7 @@ export class Conversation {
|
|
|
1233
1249
|
constructor(private readonly client: Client, private readonly toolkit: Toolkit, private readonly signal?: AbortSignal) {}
|
|
1234
1250
|
|
|
1235
1251
|
/** Send the next user turn; prior history is retained automatically. */
|
|
1236
|
-
async send(prompt:
|
|
1252
|
+
async send(prompt: PromptInput): Promise<RunResult> {
|
|
1237
1253
|
const result = await this.client.run(prompt, { toolkit: this.toolkit, signal: this.signal, history: this.messages })
|
|
1238
1254
|
this.messages = result.messages
|
|
1239
1255
|
return result
|
|
@@ -1279,6 +1295,27 @@ function safeJson(s: string): Record<string, unknown> {
|
|
|
1279
1295
|
}
|
|
1280
1296
|
}
|
|
1281
1297
|
|
|
1298
|
+
/**
|
|
1299
|
+
* A canonical `tool` turn. With no parts this is byte-identical to the pre-0.17 message;
|
|
1300
|
+
* with parts it also carries them (and the tool `name` the §8A relocation header needs) —
|
|
1301
|
+
* both consumed and stripped by the wire build, so neither reaches OpenAI.
|
|
1302
|
+
*/
|
|
1303
|
+
function toolMessage(id: string | undefined, name: string, result: ToolResult): any {
|
|
1304
|
+
const m: any = { role: "tool", tool_call_id: id, content: result.output }
|
|
1305
|
+
if (result.parts?.length) {
|
|
1306
|
+
m.name = name
|
|
1307
|
+
m.parts = result.parts
|
|
1308
|
+
}
|
|
1309
|
+
return m
|
|
1310
|
+
}
|
|
1311
|
+
|
|
1312
|
+
/** A canonical Anthropic `tool_result` block; parts ride along and are emitted natively. */
|
|
1313
|
+
function toolResultBlock(id: string | undefined, result: ToolResult): any {
|
|
1314
|
+
const b: any = { type: "tool_result", tool_use_id: id, content: result.output, is_error: result.isError }
|
|
1315
|
+
if (result.parts?.length) b.parts = result.parts
|
|
1316
|
+
return b
|
|
1317
|
+
}
|
|
1318
|
+
|
|
1282
1319
|
function lastText(messages: any[]): string {
|
|
1283
1320
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1284
1321
|
if (messages[i].role === "assistant" && typeof messages[i].content === "string") return messages[i].content
|
package/src/content.ts
ADDED
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multimodal content parts (SPEC.md §1B, §8A).
|
|
3
|
+
*
|
|
4
|
+
* One `ContentPart` union — `text | image | file | audio` — is how non-text data enters
|
|
5
|
+
* the loop (a prompt), comes back from a tool (`ToolResult.parts`), and survives MCP.
|
|
6
|
+
* A part is **bytes-or-URL, never a path**: a path does not survive a persisted and
|
|
7
|
+
* replayed transcript, nor the MCP / A2A process boundary, so the edge constructors
|
|
8
|
+
* (`attach`, `text`) normalise a path / native bytes / a `data:` URL at construction.
|
|
9
|
+
*
|
|
10
|
+
* Emission lives here too, not in `adapters.ts` (which is tool-*schema* only): each
|
|
11
|
+
* `(style, part.type)` pair has either a defined block shape or an explicit refusal, and
|
|
12
|
+
* the encoded block is checked against a **positive allowlist** before it can reach the
|
|
13
|
+
* wire — because an unknown block type upstream returns HTTP 200 with the content
|
|
14
|
+
* silently discarded, which is the exact bug this module exists to remove.
|
|
15
|
+
*/
|
|
16
|
+
import fs from "node:fs/promises"
|
|
17
|
+
import path from "node:path"
|
|
18
|
+
|
|
19
|
+
export type ContentPart =
|
|
20
|
+
| { type: "text"; text: string }
|
|
21
|
+
| { type: "image"; mimeType: string; data?: string; url?: string }
|
|
22
|
+
| { type: "file"; mimeType: string; data?: string; url?: string; name?: string }
|
|
23
|
+
| { type: "audio"; mimeType: string; data?: string; url?: string }
|
|
24
|
+
|
|
25
|
+
/** A prompt is a string or an ordered list of parts — ordering is semantic to a model. */
|
|
26
|
+
export type PromptInput = string | ContentPart[]
|
|
27
|
+
|
|
28
|
+
/** The two provider styles that have a request path (`toGemini` is declarations only). */
|
|
29
|
+
export type PartStyle = "openai" | "anthropic"
|
|
30
|
+
|
|
31
|
+
/** How a part got here. Drives the unsupported-part rule (§8A): the caller asked for an
|
|
32
|
+
* attached part, so silently changing it is the betrayal; nobody asked for a part an MCP
|
|
33
|
+
* server volunteered, so failing the run over it is a regression. */
|
|
34
|
+
export type PartProvenance = "attached" | "derived"
|
|
35
|
+
|
|
36
|
+
/** What to do with a part the style cannot represent; overrides provenance uniformly. */
|
|
37
|
+
export type UnsupportedPartMode = "error" | "text"
|
|
38
|
+
|
|
39
|
+
/** Why a part could not be constructed or sent (§1B / §8A). `code` is the machine-readable half. */
|
|
40
|
+
export class ContentPartError extends Error {
|
|
41
|
+
constructor(
|
|
42
|
+
message: string,
|
|
43
|
+
readonly code: "source-conflict" | "source-missing" | "unknown-extension" | "too-large" | "unsupported",
|
|
44
|
+
) {
|
|
45
|
+
super(message)
|
|
46
|
+
this.name = "ContentPartError"
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* The media extension table (§6 `read`) — fixed, shared with the edge constructors,
|
|
52
|
+
* identical in every port. No magic-byte sniffing and no platform mime database:
|
|
53
|
+
* `/etc/mime.types` varies per machine and would break cross-port parity.
|
|
54
|
+
*/
|
|
55
|
+
export const MEDIA_TYPES: Record<string, { mimeType: string; type: "image" | "file" | "audio" }> = {
|
|
56
|
+
png: { mimeType: "image/png", type: "image" },
|
|
57
|
+
jpg: { mimeType: "image/jpeg", type: "image" },
|
|
58
|
+
jpeg: { mimeType: "image/jpeg", type: "image" },
|
|
59
|
+
gif: { mimeType: "image/gif", type: "image" },
|
|
60
|
+
webp: { mimeType: "image/webp", type: "image" },
|
|
61
|
+
pdf: { mimeType: "application/pdf", type: "file" },
|
|
62
|
+
mp3: { mimeType: "audio/mpeg", type: "audio" },
|
|
63
|
+
wav: { mimeType: "audio/wav", type: "audio" },
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Lower-cased extension without the dot, or "" when there is none. */
|
|
67
|
+
export function extensionOf(p: string): string {
|
|
68
|
+
const ext = path.extname(p)
|
|
69
|
+
return ext ? ext.slice(1).toLowerCase() : ""
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The table entry for a path/URL, or undefined when the extension is not media. */
|
|
73
|
+
export function mediaTypeFor(p: string): { mimeType: string; type: "image" | "file" | "audio" } | undefined {
|
|
74
|
+
return MEDIA_TYPES[extensionOf(p)]
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Part kind for a mime type: `image/*` ⇒ image, `audio/*` ⇒ audio, everything else ⇒ file. */
|
|
78
|
+
export function partTypeForMime(mimeType: string): "image" | "file" | "audio" {
|
|
79
|
+
if (mimeType.startsWith("image/")) return "image"
|
|
80
|
+
if (mimeType.startsWith("audio/")) return "audio"
|
|
81
|
+
return "file"
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/** Decoded byte length of standard, padded base64 — without decoding it (RFC 4648 §4:
|
|
85
|
+
* `floor(len/4)*3 - padCount`). `<bytes>` in every rendered string is a plain integer, never
|
|
86
|
+
* a fraction, so this floors the group count rather than assuming `len` is a multiple of 4. */
|
|
87
|
+
export function base64Bytes(b64: string): number {
|
|
88
|
+
const n = b64.length
|
|
89
|
+
if (n === 0) return 0
|
|
90
|
+
let pad = 0
|
|
91
|
+
if (b64.endsWith("==")) pad = 2
|
|
92
|
+
else if (b64.endsWith("=")) pad = 1
|
|
93
|
+
return Math.floor(n / 4) * 3 - pad
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Decoded byte length of a part's payload; a url-only part carries no bytes here. */
|
|
97
|
+
export function partBytes(part: ContentPart): number {
|
|
98
|
+
if (part.type === "text") return Buffer.byteLength(part.text, "utf8")
|
|
99
|
+
return part.data ? base64Bytes(part.data) : 0
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** §1B token estimate for a part, from its decoded byte count: `max(85, floor(bytes/750))`.
|
|
103
|
+
* Measured against real providers: image cost is not proportional to bytes, so this is an
|
|
104
|
+
* explicit estimate, not a count — but it is normative and lives with the part logic so a
|
|
105
|
+
* budgeting helper (e.g. `agents.estimateTokens`) cannot silently diverge from it. */
|
|
106
|
+
export const PART_BYTES_PER_TOKEN = 750
|
|
107
|
+
export const MIN_PART_TOKENS = 85
|
|
108
|
+
export function partTokens(bytes: number): number {
|
|
109
|
+
return Math.max(MIN_PART_TOKENS, Math.floor(bytes / PART_BYTES_PER_TOKEN))
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* How a part appears in a log line or a §9 event: `{type, mimeType, bytes}` — never `data`.
|
|
114
|
+
* Same rule as never-log-headers: part payloads are user content.
|
|
115
|
+
*/
|
|
116
|
+
export function describePart(part: ContentPart): Record<string, unknown> {
|
|
117
|
+
if (part.type === "text") return { type: "text", bytes: partBytes(part) }
|
|
118
|
+
return { type: part.type, mimeType: part.mimeType, bytes: partBytes(part) }
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** A one-line, payload-free rendering of a part — for `output` text and placeholders. */
|
|
122
|
+
export function summarizePart(part: ContentPart): string {
|
|
123
|
+
return part.type === "text" ? part.text : `${part.type} (${part.mimeType}, ${partBytes(part)} bytes)`
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Validate the §1B invariant: exactly one of `data` / `url`. */
|
|
127
|
+
export function validatePart(part: ContentPart, maxPartBytes?: number): ContentPart {
|
|
128
|
+
if (part.type === "text") return part
|
|
129
|
+
const hasData = part.data !== undefined && part.data !== null
|
|
130
|
+
const hasUrl = part.url !== undefined && part.url !== null
|
|
131
|
+
if (hasData && hasUrl) {
|
|
132
|
+
throw new ContentPartError(`content part "${part.type}" carries both data and url — supply exactly one`, "source-conflict")
|
|
133
|
+
}
|
|
134
|
+
if (!hasData && !hasUrl) {
|
|
135
|
+
throw new ContentPartError(`content part "${part.type}" carries neither data nor url — supply exactly one`, "source-missing")
|
|
136
|
+
}
|
|
137
|
+
if (maxPartBytes !== undefined && hasData) {
|
|
138
|
+
const bytes = base64Bytes(part.data!)
|
|
139
|
+
if (bytes > maxPartBytes) {
|
|
140
|
+
throw new ContentPartError(`content part "${part.type}" is ${bytes} decoded bytes, over the maxPartBytes limit of ${maxPartBytes}`, "too-large")
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return part
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** True when a value looks like a `ContentPart` (as opposed to a provider-native block). */
|
|
147
|
+
export function isContentPart(v: unknown): v is ContentPart {
|
|
148
|
+
if (!v || typeof v !== "object") return false
|
|
149
|
+
const t = (v as any).type
|
|
150
|
+
if (t === "text") return typeof (v as any).text === "string"
|
|
151
|
+
if (t === "image" || t === "file" || t === "audio") return typeof (v as any).mimeType === "string"
|
|
152
|
+
return false
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
// --------------------------------------------------------------------------- //
|
|
156
|
+
// Edge constructors — a path / bytes / a data: URL never enters the part itself.
|
|
157
|
+
// --------------------------------------------------------------------------- //
|
|
158
|
+
|
|
159
|
+
/** A text part. */
|
|
160
|
+
export function text(value: string): ContentPart {
|
|
161
|
+
return { type: "text", text: value }
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** Options for {@link attach}. */
|
|
165
|
+
export interface AttachOptions {
|
|
166
|
+
/** Overrides the extension table; **required** for raw bytes and for an unlisted extension. */
|
|
167
|
+
mimeType?: string
|
|
168
|
+
/** Filename carried on a `file` part (OpenAI's `file.filename`). Defaults to the basename. */
|
|
169
|
+
name?: string
|
|
170
|
+
/** Reject a payload over this many **decoded** bytes (never the +33% base64 string). */
|
|
171
|
+
maxPartBytes?: number
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Everything the edge constructor accepts (§1B): a path / `data:` / `https:` string, native
|
|
175
|
+
* bytes in any of their JS spellings, or the host's own file object. Whatever comes in, only
|
|
176
|
+
* `mimeType` + base64 `data` (or a `url`) lands in the part — accept broadly, store narrowly. */
|
|
177
|
+
export type AttachSource = string | Uint8Array | ArrayBuffer | ArrayBufferView | Blob
|
|
178
|
+
|
|
179
|
+
/** A `Blob`/`File` — duck-typed, so a Blob from another realm or a polyfill is accepted too. */
|
|
180
|
+
function isBlobLike(v: unknown): v is Blob {
|
|
181
|
+
return !!v && typeof v === "object" && typeof (v as any).arrayBuffer === "function" && !ArrayBuffer.isView(v)
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** Normalise any native byte spelling to a `Uint8Array` view over the same bytes — an
|
|
185
|
+
* `ArrayBufferView` that is not a `Uint8Array` must be re-viewed, not passed to
|
|
186
|
+
* `Buffer.from`, which would read it as an array of numbers. */
|
|
187
|
+
function toBytes(source: Uint8Array | ArrayBuffer | ArrayBufferView): Uint8Array {
|
|
188
|
+
if (source instanceof Uint8Array) return source
|
|
189
|
+
if (ArrayBuffer.isView(source)) return new Uint8Array(source.buffer, source.byteOffset, source.byteLength)
|
|
190
|
+
return new Uint8Array(source)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** The typed §1B refusal for a name/path whose extension is not in the fixed table. */
|
|
194
|
+
function unknownExtension(nameOrPath: string): ContentPartError {
|
|
195
|
+
const ext = extensionOf(nameOrPath)
|
|
196
|
+
return new ContentPartError(
|
|
197
|
+
`no mime type for extension "${ext || "(none)"}" — pass an explicit mimeType (mime is never sniffed)`,
|
|
198
|
+
"unknown-extension",
|
|
199
|
+
)
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* The edge constructor: hand it whatever you actually have and get a path-free part.
|
|
204
|
+
*
|
|
205
|
+
* | given | becomes |
|
|
206
|
+
* |---|---|
|
|
207
|
+
* | a filesystem path | bytes read now, base64d now; mime from the fixed extension table |
|
|
208
|
+
* | native bytes (`Uint8Array`/`Buffer`/`ArrayBuffer`/any `ArrayBufferView`) | base64d now; `mimeType` required |
|
|
209
|
+
* | a `Blob` / `File` | read to bytes **now**; mime from `blob.type`, else the extension table via `File.name` |
|
|
210
|
+
* | `data:<mime>;base64,<b64>` | parsed into `{mimeType, data}` — never stored as a url |
|
|
211
|
+
* | an `http(s):` URL | kept as `url` |
|
|
212
|
+
*/
|
|
213
|
+
export async function attach(source: AttachSource, opts: AttachOptions = {}): Promise<ContentPart> {
|
|
214
|
+
if (typeof source === "string") {
|
|
215
|
+
if (source.startsWith("data:")) return fromDataUrl(source, opts)
|
|
216
|
+
if (/^https?:\/\//i.test(source)) return fromUrl(source, opts)
|
|
217
|
+
return fromPath(source, opts)
|
|
218
|
+
}
|
|
219
|
+
if (isBlobLike(source)) return fromBlob(source, opts)
|
|
220
|
+
if (!opts.mimeType) {
|
|
221
|
+
throw new ContentPartError("attach(bytes) requires an explicit mimeType — bytes carry no extension to read", "unknown-extension")
|
|
222
|
+
}
|
|
223
|
+
return fromBytes(toBytes(source), opts.mimeType, opts)
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* Build a part from a `Blob` or `File`, consuming it **eagerly** — a part holding a half-read
|
|
228
|
+
* stream would not survive the transcript boundary any better than a path does (§1B).
|
|
229
|
+
*
|
|
230
|
+
* Mime comes from `blob.type` when it is non-empty, else from the fixed extension table applied
|
|
231
|
+
* to `File.name`; an explicit `opts.mimeType` beats both. A `File`'s name is carried onto a
|
|
232
|
+
* `file` part unless `opts.name` overrides it.
|
|
233
|
+
*/
|
|
234
|
+
export async function fromBlob(blob: Blob, opts: AttachOptions = {}): Promise<ContentPart> {
|
|
235
|
+
const name = typeof (blob as any).name === "string" ? String((blob as any).name) : undefined
|
|
236
|
+
const mimeType = opts.mimeType || (blob.type || undefined) || (name ? mediaTypeFor(name)?.mimeType : undefined)
|
|
237
|
+
if (!mimeType) throw unknownExtension(name ?? "")
|
|
238
|
+
const bytes = new Uint8Array(await blob.arrayBuffer())
|
|
239
|
+
return finish(partTypeForMime(mimeType), mimeType, { data: Buffer.from(bytes).toString("base64") }, name ? { name, ...opts } : opts)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** Build a part from in-memory bytes. `mimeType` is required — bytes carry no extension. */
|
|
243
|
+
export function fromBytes(bytes: Uint8Array | ArrayBuffer | ArrayBufferView, mimeType: string, opts: AttachOptions = {}): ContentPart {
|
|
244
|
+
const data = Buffer.from(toBytes(bytes)).toString("base64")
|
|
245
|
+
return finish(partTypeForMime(mimeType), mimeType, { data }, opts)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Read a file now and base64 it now, so the part never carries the path (§1B). */
|
|
249
|
+
export async function fromPath(p: string, opts: AttachOptions = {}): Promise<ContentPart> {
|
|
250
|
+
const mimeType = opts.mimeType ?? mediaTypeFor(p)?.mimeType
|
|
251
|
+
if (!mimeType) throw unknownExtension(p)
|
|
252
|
+
const bytes = await fs.readFile(p)
|
|
253
|
+
return finish(partTypeForMime(mimeType), mimeType, { data: bytes.toString("base64") }, { name: path.basename(p), ...opts })
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** Parse `data:<mime>;base64,<b64>` into `{mimeType, data}`, so two spellings of the same
|
|
257
|
+
* bytes cannot diverge downstream. */
|
|
258
|
+
export function fromDataUrl(url: string, opts: AttachOptions = {}): ContentPart {
|
|
259
|
+
const m = /^data:([^;,]*)(;base64)?,(.*)$/s.exec(url)
|
|
260
|
+
if (!m) throw new ContentPartError("malformed data: URL", "source-missing")
|
|
261
|
+
const mimeType = opts.mimeType ?? (m[1] || "application/octet-stream")
|
|
262
|
+
const data = m[2] ? m[3] : Buffer.from(decodeURIComponent(m[3]), "utf8").toString("base64")
|
|
263
|
+
return finish(partTypeForMime(mimeType), mimeType, { data }, opts)
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
/** Keep an `http(s):` URL as a url part; mime comes from the table or an explicit override. */
|
|
267
|
+
export function fromUrl(url: string, opts: AttachOptions = {}): ContentPart {
|
|
268
|
+
const mimeType = opts.mimeType ?? mediaTypeFor(new URL(url).pathname)?.mimeType
|
|
269
|
+
if (!mimeType) throw unknownExtension(new URL(url).pathname)
|
|
270
|
+
return finish(partTypeForMime(mimeType), mimeType, { url }, opts)
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
function finish(
|
|
274
|
+
type: "image" | "file" | "audio",
|
|
275
|
+
mimeType: string,
|
|
276
|
+
source: { data?: string; url?: string },
|
|
277
|
+
opts: AttachOptions,
|
|
278
|
+
): ContentPart {
|
|
279
|
+
const part: any = { type, mimeType, ...source }
|
|
280
|
+
if (type === "file" && opts.name) part.name = opts.name
|
|
281
|
+
return validatePart(part as ContentPart, opts.maxPartBytes)
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// --------------------------------------------------------------------------- //
|
|
285
|
+
// §8A emission — the positive allowlist, and the unsupported-part rule.
|
|
286
|
+
// --------------------------------------------------------------------------- //
|
|
287
|
+
|
|
288
|
+
/** The ONLY block types that may reach each style's wire. A part encoding to anything
|
|
289
|
+
* else never leaves this module (map-and-hope is how content gets silently discarded). */
|
|
290
|
+
const ALLOWLIST: Record<PartStyle, ReadonlySet<string>> = {
|
|
291
|
+
openai: new Set(["text", "image_url", "file", "input_audio"]),
|
|
292
|
+
anthropic: new Set(["text", "image", "document"]),
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
/** OpenAI's `input_audio.format` is a bare format name, not a mime type. */
|
|
296
|
+
const AUDIO_FORMATS: Record<string, string> = { "audio/mpeg": "mp3", "audio/mp3": "mp3", "audio/wav": "wav", "audio/x-wav": "wav" }
|
|
297
|
+
|
|
298
|
+
const dataUrl = (mimeType: string, data: string) => `data:${mimeType};base64,${data}`
|
|
299
|
+
|
|
300
|
+
/**
|
|
301
|
+
* Encode one part into its provider block, or `undefined` when the style defines no shape
|
|
302
|
+
* for it. Refusals are explicit, never a fall-through: `openai × file+url` (Chat Completions
|
|
303
|
+
* has no URL form for a file) and `anthropic × audio` (the provider defines no audio block).
|
|
304
|
+
*/
|
|
305
|
+
export function encodePart(part: ContentPart, style: PartStyle): Record<string, any> | undefined {
|
|
306
|
+
if (part.type === "text") return { type: "text", text: part.text }
|
|
307
|
+
if (style === "openai") {
|
|
308
|
+
switch (part.type) {
|
|
309
|
+
case "image":
|
|
310
|
+
return { type: "image_url", image_url: { url: part.data ? dataUrl(part.mimeType, part.data) : part.url } }
|
|
311
|
+
case "file":
|
|
312
|
+
// `file_data` REQUIRES the `data:<mime>;base64,` prefix — a bare base64 string is a 400.
|
|
313
|
+
if (!part.data) return undefined
|
|
314
|
+
return { type: "file", file: { filename: part.name ?? "file", file_data: dataUrl(part.mimeType, part.data) } }
|
|
315
|
+
case "audio":
|
|
316
|
+
if (!part.data) return undefined
|
|
317
|
+
return { type: "input_audio", input_audio: { data: part.data, format: AUDIO_FORMATS[part.mimeType] ?? part.mimeType.replace(/^audio\//, "") } }
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
switch (part.type) {
|
|
321
|
+
case "image":
|
|
322
|
+
return part.data
|
|
323
|
+
? { type: "image", source: { type: "base64", media_type: part.mimeType, data: part.data } }
|
|
324
|
+
: { type: "image", source: { type: "url", url: part.url } }
|
|
325
|
+
case "file":
|
|
326
|
+
return part.data
|
|
327
|
+
? { type: "document", source: { type: "base64", media_type: part.mimeType, data: part.data } }
|
|
328
|
+
: { type: "document", source: { type: "url", url: part.url } }
|
|
329
|
+
case "audio":
|
|
330
|
+
return undefined // Anthropic defines no audio block — a named refusal, not an oversight.
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
/** Context for {@link encodeParts}: which wire, who asked for the part, and the override. */
|
|
335
|
+
export interface EncodeOptions {
|
|
336
|
+
style: PartStyle
|
|
337
|
+
provenance: PartProvenance
|
|
338
|
+
onUnsupportedPart?: UnsupportedPartMode
|
|
339
|
+
maxPartBytes?: number
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
const warnedUnsupported = new Set<string>()
|
|
343
|
+
|
|
344
|
+
/** Test seam: forget which unsupported (style, type) pairs have already been warned about. */
|
|
345
|
+
export function resetUnsupportedWarnings(): void {
|
|
346
|
+
warnedUnsupported.clear()
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** The text a degraded (non-failing) unsupported part leaves behind — never silence. */
|
|
350
|
+
export function unsupportedPlaceholder(part: ContentPart): string {
|
|
351
|
+
return `[unsupported ${part.type} part (${(part as any).mimeType}, ${partBytes(part)} bytes)]`
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Warn once per (style, type, reason) — the same latch the unsupported-block path uses,
|
|
355
|
+
* keyed separately (by `reason`, not by the message text) so an oversize warning and a
|
|
356
|
+
* no-block warning never suppress each other, and a differently-sized oversize part of the
|
|
357
|
+
* same type doesn't re-trigger the latch. */
|
|
358
|
+
function warnUnsupportedOnce(style: PartStyle, type: string, reason: "unsupported" | "too-large", message: string): void {
|
|
359
|
+
const key = `${style}:${type}:${reason}`
|
|
360
|
+
if (warnedUnsupported.has(key)) return
|
|
361
|
+
warnedUnsupported.add(key)
|
|
362
|
+
console.warn(`[toolnexus] ${message}`)
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
/**
|
|
366
|
+
* Encode a list of parts for one style, applying the §8A rules:
|
|
367
|
+
* an **attached** part the style cannot represent raises before any HTTP call; a **derived**
|
|
368
|
+
* part degrades to a text placeholder and warns once. `onUnsupportedPart` overrides both.
|
|
369
|
+
* Every emitted block is asserted against the style's allowlist.
|
|
370
|
+
*
|
|
371
|
+
* `maxPartBytes` (§1B) is enforced here too, at assembly, over EVERY part regardless of
|
|
372
|
+
* provenance — a part that arrived from an MCP server never passed through an edge
|
|
373
|
+
* constructor, so a limit it can walk around is not a limit. Going over follows the exact
|
|
374
|
+
* same provenance split as an unsupported block: an attached part errors, a derived one
|
|
375
|
+
* degrades to the placeholder so a remote server still cannot fail the run.
|
|
376
|
+
*/
|
|
377
|
+
export function encodeParts(parts: readonly ContentPart[], opts: EncodeOptions): Record<string, any>[] {
|
|
378
|
+
const mode: UnsupportedPartMode = opts.onUnsupportedPart ?? (opts.provenance === "attached" ? "error" : "text")
|
|
379
|
+
const out: Record<string, any>[] = []
|
|
380
|
+
for (const raw of parts) {
|
|
381
|
+
const part = validatePart(raw)
|
|
382
|
+
if (opts.maxPartBytes !== undefined && part.type !== "text") {
|
|
383
|
+
const bytes = partBytes(part)
|
|
384
|
+
if (bytes > opts.maxPartBytes) {
|
|
385
|
+
if (mode === "error") {
|
|
386
|
+
throw new ContentPartError(
|
|
387
|
+
`content part "${part.type}" is ${bytes} decoded bytes, over the maxPartBytes limit of ${opts.maxPartBytes}`,
|
|
388
|
+
"too-large",
|
|
389
|
+
)
|
|
390
|
+
}
|
|
391
|
+
warnUnsupportedOnce(
|
|
392
|
+
opts.style,
|
|
393
|
+
part.type,
|
|
394
|
+
"too-large",
|
|
395
|
+
`content part "${part.type}" is ${bytes} decoded bytes, over the maxPartBytes limit of ${opts.maxPartBytes} — sending a text placeholder`,
|
|
396
|
+
)
|
|
397
|
+
out.push({ type: "text", text: unsupportedPlaceholder(part) })
|
|
398
|
+
continue
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
const block = encodePart(part, opts.style)
|
|
402
|
+
if (block && ALLOWLIST[opts.style].has(String(block.type))) {
|
|
403
|
+
out.push(block)
|
|
404
|
+
continue
|
|
405
|
+
}
|
|
406
|
+
if (mode === "error") {
|
|
407
|
+
throw new ContentPartError(
|
|
408
|
+
`provider style "${opts.style}" defines no block for a "${part.type}" content part` +
|
|
409
|
+
(part.type !== "text" ? ` (${part.mimeType})` : ""),
|
|
410
|
+
"unsupported",
|
|
411
|
+
)
|
|
412
|
+
}
|
|
413
|
+
warnUnsupportedOnce(opts.style, part.type, "unsupported", `provider style "${opts.style}" has no block for a "${part.type}" part — sending a text placeholder`)
|
|
414
|
+
out.push({ type: "text", text: unsupportedPlaceholder(part) })
|
|
415
|
+
}
|
|
416
|
+
return out
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Read an inbound OpenAI-shaped content block back into a `ContentPart` (§11). Accepts both
|
|
421
|
+
* a `ContentPart` written literally and the provider-native block the same part encodes to,
|
|
422
|
+
* so a caller's own OpenAI messages translate as faithfully as ours do.
|
|
423
|
+
*/
|
|
424
|
+
export function inboundPart(block: unknown): ContentPart | undefined {
|
|
425
|
+
if (isContentPart(block)) return block
|
|
426
|
+
if (!block || typeof block !== "object") return undefined
|
|
427
|
+
const b = block as any
|
|
428
|
+
if (b.type === "image_url" && b.image_url?.url) {
|
|
429
|
+
const url = String(b.image_url.url)
|
|
430
|
+
try {
|
|
431
|
+
return url.startsWith("data:") ? fromDataUrl(url) : { type: "image", mimeType: mediaTypeFor(new URL(url).pathname)?.mimeType ?? "image/*", url }
|
|
432
|
+
} catch {
|
|
433
|
+
return undefined
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
if (b.type === "file" && typeof b.file?.file_data === "string") {
|
|
437
|
+
const part = fromDataUrl(String(b.file.file_data)) as any
|
|
438
|
+
part.type = "file"
|
|
439
|
+
if (b.file.filename) part.name = String(b.file.filename)
|
|
440
|
+
return part as ContentPart
|
|
441
|
+
}
|
|
442
|
+
if (b.type === "input_audio" && typeof b.input_audio?.data === "string") {
|
|
443
|
+
const fmt = String(b.input_audio.format ?? "")
|
|
444
|
+
const mimeType = fmt === "mp3" ? "audio/mpeg" : fmt === "wav" ? "audio/wav" : `audio/${fmt || "*"}`
|
|
445
|
+
return { type: "audio", mimeType, data: String(b.input_audio.data) }
|
|
446
|
+
}
|
|
447
|
+
return undefined
|
|
448
|
+
}
|