toolnexus 0.29.0 → 0.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/acp.js +137 -15
- package/dist/client.d.ts +7 -1
- package/dist/client.js +26 -5
- package/dist/media.d.ts +1 -1
- package/dist/media.js +2 -2
- package/package.json +1 -1
- package/src/acp.ts +132 -17
- package/src/client.ts +30 -6
- package/src/media.ts +2 -2
package/dist/acp.js
CHANGED
|
@@ -34,27 +34,79 @@
|
|
|
34
34
|
import { spawn } from "node:child_process";
|
|
35
35
|
import { createInterface } from "node:readline";
|
|
36
36
|
import path from "node:path";
|
|
37
|
+
import { isContentPart, summarizePart } from "./content.js";
|
|
37
38
|
/** The literal marker every rendered prompt ends with, naming the current
|
|
38
39
|
* turn as the one to answer. Kept as a named constant (not just inlined)
|
|
39
40
|
* because the exact wording is shared, by convention, with the other
|
|
40
41
|
* language ports and the spike that proved it fixes the stale-answer bug. */
|
|
41
42
|
const SUPERSEDES_MARKER = "SUPERSEDES-ALL-PRIOR:";
|
|
42
|
-
function renderContentPart(part) {
|
|
43
|
+
function renderContentPart(part, media) {
|
|
43
44
|
if (typeof part === "string")
|
|
44
45
|
return part;
|
|
45
46
|
if (part == null)
|
|
46
47
|
return "";
|
|
47
48
|
if (typeof part === "object" && "text" in part && typeof part.text === "string")
|
|
48
49
|
return part.text;
|
|
50
|
+
// A media part (OpenAI chat wire shape) is named by its §1B one-liner, never its bytes;
|
|
51
|
+
// the part itself travels as a native ACP block (SPEC §8C "ACP content parts").
|
|
52
|
+
const p = acpWirePart(part);
|
|
53
|
+
if (p && p.type !== "text") {
|
|
54
|
+
media?.push(p);
|
|
55
|
+
return summarizePart(p);
|
|
56
|
+
}
|
|
49
57
|
return JSON.stringify(part);
|
|
50
58
|
}
|
|
51
|
-
/**
|
|
52
|
-
function
|
|
59
|
+
/** `data:<mime>;base64,<payload>` (non-empty mime) ⇒ {mimeType, data}; anything else ⇒ undefined (SPEC §8C). */
|
|
60
|
+
function parseBase64DataUrl(u) {
|
|
61
|
+
if (!u.startsWith("data:"))
|
|
62
|
+
return undefined;
|
|
63
|
+
const comma = u.indexOf(",");
|
|
64
|
+
if (comma < 0)
|
|
65
|
+
return undefined;
|
|
66
|
+
const meta = u.slice(5, comma);
|
|
67
|
+
if (!meta.endsWith(";base64"))
|
|
68
|
+
return undefined;
|
|
69
|
+
const mimeType = meta.slice(0, -";base64".length);
|
|
70
|
+
return mimeType ? { mimeType, data: u.slice(comma + 1) } : undefined;
|
|
71
|
+
}
|
|
72
|
+
/** One OpenAI chat media block read back into a §1B part, as SPEC §8C pins it (Go acpWirePart). */
|
|
73
|
+
function acpWirePart(part) {
|
|
74
|
+
switch (part?.type) {
|
|
75
|
+
case "image_url": {
|
|
76
|
+
const url = typeof part.image_url?.url === "string" ? part.image_url.url : "";
|
|
77
|
+
const d = parseBase64DataUrl(url);
|
|
78
|
+
if (d)
|
|
79
|
+
return { type: "image", ...d };
|
|
80
|
+
// A URL-only image part: EMPTY mimeType, never inferred from the URL.
|
|
81
|
+
return url ? { type: "image", mimeType: "", url } : undefined;
|
|
82
|
+
}
|
|
83
|
+
case "input_audio": {
|
|
84
|
+
let mimeType = "audio/" + String(part.input_audio?.format ?? "");
|
|
85
|
+
if (mimeType === "audio/mp3")
|
|
86
|
+
mimeType = "audio/mpeg";
|
|
87
|
+
return { type: "audio", mimeType, data: String(part.input_audio?.data ?? "") };
|
|
88
|
+
}
|
|
89
|
+
case "file": {
|
|
90
|
+
const d = parseBase64DataUrl(String(part.file?.file_data ?? ""));
|
|
91
|
+
if (!d)
|
|
92
|
+
return undefined;
|
|
93
|
+
const type = d.mimeType.startsWith("image/") ? "image" : d.mimeType.startsWith("audio/") ? "audio" : "file";
|
|
94
|
+
const p = { type, ...d };
|
|
95
|
+
if (part.file?.filename)
|
|
96
|
+
p.name = String(part.file.filename);
|
|
97
|
+
return p;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return isContentPart(part) && part.type !== "text" ? part : undefined;
|
|
101
|
+
}
|
|
102
|
+
/** Flattens `content` (a string, or an array of content parts) to plain text; media parts
|
|
103
|
+
* found along the way are appended to `media`. */
|
|
104
|
+
function renderContent(content, media) {
|
|
53
105
|
if (typeof content === "string")
|
|
54
106
|
return content;
|
|
55
107
|
if (Array.isArray(content))
|
|
56
|
-
return content.map(renderContentPart).join("
|
|
57
|
-
return renderContentPart(content);
|
|
108
|
+
return content.map((p) => renderContentPart(p, media)).join("\n");
|
|
109
|
+
return renderContentPart(content, media);
|
|
58
110
|
}
|
|
59
111
|
/** Renders the FULL assembled request as one prompt string — role + content
|
|
60
112
|
* per message — then appends the supersedes marker naming the latest user
|
|
@@ -64,7 +116,8 @@ function renderContent(content) {
|
|
|
64
116
|
* the caller's own conversation state. */
|
|
65
117
|
function renderPrompt(request) {
|
|
66
118
|
const messages = request.messages ?? [];
|
|
67
|
-
const
|
|
119
|
+
const parts = [];
|
|
120
|
+
const lines = messages.map((m) => `${m?.role ?? "user"}: ${renderContent(m?.content, parts)}`);
|
|
68
121
|
let latest = "";
|
|
69
122
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
70
123
|
if (messages[i]?.role === "user") {
|
|
@@ -75,7 +128,25 @@ function renderPrompt(request) {
|
|
|
75
128
|
if (!latest && messages.length > 0)
|
|
76
129
|
latest = renderContent(messages[messages.length - 1]?.content);
|
|
77
130
|
lines.push(`${SUPERSEDES_MARKER} ${latest}`);
|
|
78
|
-
return lines.join("\n");
|
|
131
|
+
return { text: lines.join("\n"), parts };
|
|
132
|
+
}
|
|
133
|
+
/** The session/prompt ContentBlock array: the text block, then one native block per media
|
|
134
|
+
* part the agent advertised support for. A part it cannot take stays named only by its
|
|
135
|
+
* description in the text; base64 never enters the text block. */
|
|
136
|
+
function promptBlocks(text, parts, caps) {
|
|
137
|
+
const blocks = [{ type: "text", text }];
|
|
138
|
+
for (const p of parts) {
|
|
139
|
+
if (p.type === "text" || !p.data)
|
|
140
|
+
continue; // ACP media blocks require inline data
|
|
141
|
+
if (p.type === "image" && caps.image)
|
|
142
|
+
blocks.push({ type: "image", mimeType: p.mimeType, data: p.data });
|
|
143
|
+
else if (p.type === "audio" && caps.audio)
|
|
144
|
+
blocks.push({ type: "audio", mimeType: p.mimeType, data: p.data });
|
|
145
|
+
else if (p.type === "file" && caps.embeddedContext) {
|
|
146
|
+
blocks.push({ type: "resource", resource: { uri: `attachment:///${p.name || "attachment"}`, mimeType: p.mimeType, blob: p.data } });
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
return blocks;
|
|
79
150
|
}
|
|
80
151
|
/**
|
|
81
152
|
* Opens one ACP connection: spawns `command`, performs `initialize` →
|
|
@@ -268,6 +339,8 @@ export async function loadACP(opts) {
|
|
|
268
339
|
}
|
|
269
340
|
};
|
|
270
341
|
const initializeResult = await step("initialize", () => call("initialize", { protocolVersion: opts.protocolVersion ?? 1, clientCapabilities }));
|
|
342
|
+
const pc = initializeResult?.agentCapabilities?.promptCapabilities ?? {};
|
|
343
|
+
const promptCaps = { image: pc.image === true, audio: pc.audio === true, embeddedContext: pc.embeddedContext === true };
|
|
271
344
|
const sessionParams = { cwd, mcpServers: opts.mcpServers ?? [] };
|
|
272
345
|
if (opts.sessionMeta)
|
|
273
346
|
sessionParams._meta = opts.sessionMeta;
|
|
@@ -302,9 +375,11 @@ export async function loadACP(opts) {
|
|
|
302
375
|
async function promptOnce(request, emit, signal) {
|
|
303
376
|
if (signal?.aborted)
|
|
304
377
|
throw acpAbortError();
|
|
305
|
-
const text = renderPrompt(request);
|
|
378
|
+
const { text, parts } = renderPrompt(request);
|
|
306
379
|
const id = `c-${++nextId}`;
|
|
307
380
|
let content = "";
|
|
381
|
+
const files = []; // inbound media, in arrival order
|
|
382
|
+
const messageParts = []; // media the agent sent as message chunks
|
|
308
383
|
const toolNames = new Map();
|
|
309
384
|
const send = (ev) => {
|
|
310
385
|
try {
|
|
@@ -320,6 +395,13 @@ export async function loadACP(opts) {
|
|
|
320
395
|
const u = params?.update;
|
|
321
396
|
switch (u?.sessionUpdate) {
|
|
322
397
|
case "agent_message_chunk": {
|
|
398
|
+
const mp = acpBlockPart(u?.content);
|
|
399
|
+
if (mp) {
|
|
400
|
+
// An image/audio chunk is a part, never text (SPEC §8C ACP content parts).
|
|
401
|
+
messageParts.push(mp);
|
|
402
|
+
files.push(acpRunFile(mp, "acp", ""));
|
|
403
|
+
break;
|
|
404
|
+
}
|
|
323
405
|
const t = u?.content?.text ?? "";
|
|
324
406
|
content += t;
|
|
325
407
|
if (t)
|
|
@@ -352,11 +434,15 @@ export async function loadACP(opts) {
|
|
|
352
434
|
}
|
|
353
435
|
if (u.status !== "completed" && u.status !== "failed")
|
|
354
436
|
break;
|
|
437
|
+
const { output, parts: toolParts } = acpToolContent(u.content);
|
|
438
|
+
for (const p of toolParts)
|
|
439
|
+
files.push(acpRunFile(p, name, callId));
|
|
355
440
|
send({
|
|
356
441
|
type: "tool_result",
|
|
357
442
|
id: callId,
|
|
358
443
|
name,
|
|
359
|
-
output
|
|
444
|
+
output,
|
|
445
|
+
...(toolParts.length ? { parts: toolParts } : {}),
|
|
360
446
|
isError: u.status === "failed",
|
|
361
447
|
via: "acp",
|
|
362
448
|
});
|
|
@@ -367,7 +453,7 @@ export async function loadACP(opts) {
|
|
|
367
453
|
if (exited)
|
|
368
454
|
throw new Error(exitMessage("session/prompt"));
|
|
369
455
|
const replyPromise = new Promise((resolve) => pending.set(id, { method: "session/prompt", cb: resolve }));
|
|
370
|
-
write({ jsonrpc: "2.0", id, method: "session/prompt", params: { sessionId, prompt:
|
|
456
|
+
write({ jsonrpc: "2.0", id, method: "session/prompt", params: { sessionId, prompt: promptBlocks(text, parts, promptCaps) } });
|
|
371
457
|
let timer;
|
|
372
458
|
const timeout = new Promise((_, reject) => {
|
|
373
459
|
timer = setTimeout(() => reject(new Error(`toolnexus: ACP turn timed out after ${permissionTimeoutMs}ms`)), permissionTimeoutMs);
|
|
@@ -400,7 +486,9 @@ export async function loadACP(opts) {
|
|
|
400
486
|
throw new Error(msg.error.message);
|
|
401
487
|
if (msg.error)
|
|
402
488
|
throw new Error(`toolnexus: ACP error ${msg.error.code}: ${msg.error.message}`);
|
|
403
|
-
|
|
489
|
+
// With no message text but media chunks, the answer is their §1B description.
|
|
490
|
+
const answer = content === "" && messageParts.length ? messageParts.map(summarizePart).join("\n") : content;
|
|
491
|
+
return files.length ? { content: answer, files } : { content: answer };
|
|
404
492
|
}
|
|
405
493
|
finally {
|
|
406
494
|
clearTimeout(timer);
|
|
@@ -478,9 +566,43 @@ function killGroup(pid, signal) {
|
|
|
478
566
|
// already gone
|
|
479
567
|
}
|
|
480
568
|
}
|
|
481
|
-
/**
|
|
482
|
-
|
|
569
|
+
/** Reads an ACP tool_call_update's `content` array ([{type:"content", content:ContentBlock}]):
|
|
570
|
+
* text blocks join into the output; image/audio blocks become parts. With no text, the
|
|
571
|
+
* output is the parts' §1B description (as for an image-only MCP result, §0.4). */
|
|
572
|
+
function acpToolContent(content) {
|
|
483
573
|
if (!Array.isArray(content))
|
|
484
|
-
return "";
|
|
485
|
-
|
|
574
|
+
return { output: "", parts: [] };
|
|
575
|
+
const parts = [];
|
|
576
|
+
let output = "";
|
|
577
|
+
for (const item of content) {
|
|
578
|
+
const p = acpBlockPart(item?.content);
|
|
579
|
+
if (p)
|
|
580
|
+
parts.push(p);
|
|
581
|
+
else if (typeof item?.content?.text === "string")
|
|
582
|
+
output += item.content.text;
|
|
583
|
+
}
|
|
584
|
+
if (output === "" && parts.length)
|
|
585
|
+
output = parts.map(summarizePart).join("\n");
|
|
586
|
+
return { output, parts };
|
|
587
|
+
}
|
|
588
|
+
/** An ACP image/audio ContentBlock ({type, mimeType, data, uri?}) as a §1B part: data wins,
|
|
589
|
+
* a uri-only block is a URL part; any other block is not a part. */
|
|
590
|
+
function acpBlockPart(blk) {
|
|
591
|
+
if (!blk || (blk.type !== "image" && blk.type !== "audio"))
|
|
592
|
+
return undefined;
|
|
593
|
+
const mimeType = blk.mimeType ?? "";
|
|
594
|
+
if (typeof blk.data === "string" && blk.data)
|
|
595
|
+
return { type: blk.type, mimeType, data: blk.data };
|
|
596
|
+
if (typeof blk.uri === "string" && blk.uri)
|
|
597
|
+
return { type: blk.type, mimeType, url: blk.uri };
|
|
598
|
+
return undefined;
|
|
599
|
+
}
|
|
600
|
+
/** One inbound part as a RunResult.files entry. */
|
|
601
|
+
function acpRunFile(p, from, callId) {
|
|
602
|
+
const f = { type: p.type, mimeType: p.mimeType, from, callId };
|
|
603
|
+
if (p.url)
|
|
604
|
+
f.url = p.url;
|
|
605
|
+
if (p.data)
|
|
606
|
+
f.data = p.data;
|
|
607
|
+
return f;
|
|
486
608
|
}
|
package/dist/client.d.ts
CHANGED
|
@@ -435,12 +435,15 @@ export type StreamEvent = {
|
|
|
435
435
|
name: string;
|
|
436
436
|
args: Record<string, unknown>;
|
|
437
437
|
via: "acp";
|
|
438
|
-
}
|
|
438
|
+
}
|
|
439
|
+
/** `parts`: the result's non-text output (§1B), e.g. an image an ACP agent's own tool returned. */
|
|
440
|
+
| {
|
|
439
441
|
type: "tool_result";
|
|
440
442
|
id: string;
|
|
441
443
|
name: string;
|
|
442
444
|
output: string;
|
|
443
445
|
isError: boolean;
|
|
446
|
+
parts?: ContentPart[];
|
|
444
447
|
metadata?: Record<string, unknown>;
|
|
445
448
|
via?: "call_tool" | "acp";
|
|
446
449
|
} | {
|
|
@@ -703,6 +706,9 @@ export interface InProcessResponse {
|
|
|
703
706
|
completionTokens?: number;
|
|
704
707
|
totalTokens?: number;
|
|
705
708
|
};
|
|
709
|
+
/** Media the model's own agent-side tools or messages produced (e.g. an ACP agent's
|
|
710
|
+
* images). Appended to RunResult.files after the loop's own tool files. */
|
|
711
|
+
files?: RunFile[];
|
|
706
712
|
}
|
|
707
713
|
/** A streaming in-process model: `emit` each event as it happens — `text` deltas become
|
|
708
714
|
* the streamed answer as they arrive; any other event (thought, an agent's own tool_call /
|
package/dist/client.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* ("openai" | "anthropic"), and it runs the tool-calling agent loop against a
|
|
4
4
|
* Toolkit. See ../../SPEC.md §8.
|
|
5
5
|
*/
|
|
6
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
6
7
|
import { mediaPartPath } from "./media.js";
|
|
7
8
|
import { pendingOf } from "./types.js";
|
|
8
9
|
import { toOpenAI, toAnthropic } from "./adapters.js";
|
|
@@ -414,9 +415,14 @@ export class Client {
|
|
|
414
415
|
*/
|
|
415
416
|
async run(prompt, ctx) {
|
|
416
417
|
checkPromptParts(prompt, this.wireOpts());
|
|
417
|
-
|
|
418
|
+
const agentFiles = [];
|
|
419
|
+
const r = await agentFilesSink.run(agentFiles, () => this.opts.style === "anthropic"
|
|
418
420
|
? this.runAnthropic(prompt, ctx?.toolkit, ctx?.signal, ctx?.history)
|
|
419
|
-
: this.runOpenAI(prompt, ctx?.toolkit, ctx?.signal, ctx?.history);
|
|
421
|
+
: this.runOpenAI(prompt, ctx?.toolkit, ctx?.signal, ctx?.history));
|
|
422
|
+
// Agent-side media (an in-process model's own tools) follow the loop's tool files.
|
|
423
|
+
if (agentFiles.length)
|
|
424
|
+
r.files = [...r.files, ...agentFiles];
|
|
425
|
+
return r;
|
|
420
426
|
}
|
|
421
427
|
/**
|
|
422
428
|
* Single-turn translation (§11): exactly ONE provider call, returned in OpenAI shape.
|
|
@@ -1336,6 +1342,9 @@ export class Client {
|
|
|
1336
1342
|
messages.push({ role: "user", content: prompt });
|
|
1337
1343
|
let tools = toOpenAI(this.requestTools(toolkit));
|
|
1338
1344
|
const toolCalls = [];
|
|
1345
|
+
// Agent-side media from an in-process answer (§8C ACP content parts), after the loop's own.
|
|
1346
|
+
const agentFiles = [];
|
|
1347
|
+
const withAgentFiles = (r) => (agentFiles.length ? { ...r, files: [...r.files, ...agentFiles] } : r);
|
|
1339
1348
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
1340
1349
|
let turns = 0;
|
|
1341
1350
|
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
@@ -1393,6 +1402,10 @@ export class Client {
|
|
|
1393
1402
|
yield j[IN_PROCESS_SIDE_KEY];
|
|
1394
1403
|
continue;
|
|
1395
1404
|
}
|
|
1405
|
+
if (j && j[IN_PROCESS_FILES_KEY] && this.opts.baseUrl === IN_PROCESS_BASE_URL) {
|
|
1406
|
+
agentFiles.push(...j[IN_PROCESS_FILES_KEY]);
|
|
1407
|
+
continue;
|
|
1408
|
+
}
|
|
1396
1409
|
if (j?.usage)
|
|
1397
1410
|
addUsage(usage, j.usage, "openai");
|
|
1398
1411
|
const choice = j?.choices?.[0];
|
|
@@ -1435,7 +1448,7 @@ export class Client {
|
|
|
1435
1448
|
continue;
|
|
1436
1449
|
}
|
|
1437
1450
|
yield { type: "usage", usage };
|
|
1438
|
-
yield { type: "done", result: this.endRun(runStart, lastModel, (rs?.carry ?? "") + content, messages, toolCalls, turns, usage) };
|
|
1451
|
+
yield { type: "done", result: withAgentFiles(this.endRun(runStart, lastModel, (rs?.carry ?? "") + content, messages, toolCalls, turns, usage)) };
|
|
1439
1452
|
return;
|
|
1440
1453
|
}
|
|
1441
1454
|
if (rs)
|
|
@@ -1460,7 +1473,7 @@ export class Client {
|
|
|
1460
1473
|
if (r.halted) {
|
|
1461
1474
|
toolCalls.push(this.toolCallRecord(c.name, args, result, c.id));
|
|
1462
1475
|
messages.push(toolMessage(c.id, c.name, result));
|
|
1463
|
-
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
|
|
1476
|
+
yield { type: "done", result: withAgentFiles(this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage)) };
|
|
1464
1477
|
return;
|
|
1465
1478
|
}
|
|
1466
1479
|
}
|
|
@@ -1470,7 +1483,7 @@ export class Client {
|
|
|
1470
1483
|
}
|
|
1471
1484
|
await this.roundEnd(messages, turn, false, "");
|
|
1472
1485
|
}
|
|
1473
|
-
yield { type: "done", result: this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true) };
|
|
1486
|
+
yield { type: "done", result: withAgentFiles(this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true)) };
|
|
1474
1487
|
}
|
|
1475
1488
|
catch (e) {
|
|
1476
1489
|
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
@@ -1634,6 +1647,10 @@ const IN_PROCESS_BASE_URL = "http://in-process.invalid/v1";
|
|
|
1634
1647
|
/** The in-band SSE key that carries a non-text stream event from an in-process
|
|
1635
1648
|
* streaming generate to the consumer (honoured only on {@link IN_PROCESS_BASE_URL}). */
|
|
1636
1649
|
const IN_PROCESS_SIDE_KEY = "toolnexus_event";
|
|
1650
|
+
/** The in-band SSE key carrying an in-process answer's `files` to `stream()` (same sentinel rule). */
|
|
1651
|
+
const IN_PROCESS_FILES_KEY = "toolnexus_files";
|
|
1652
|
+
/** Per-`run()` sink for an in-process answer's `files` — they are agent-side, not loop tool calls. */
|
|
1653
|
+
const agentFilesSink = new AsyncLocalStorage();
|
|
1637
1654
|
/**
|
|
1638
1655
|
* Build a `fetch`-shaped adapter over a semantic `generate` function (ADR 0030):
|
|
1639
1656
|
* the reusable piece of {@link createInProcessClient} — assembles the wire request,
|
|
@@ -1663,6 +1680,8 @@ export function createInProcessFetch(generate, generateStream) {
|
|
|
1663
1680
|
"complete answer. Use the non-streaming run, or supply a `fetch`/transport that streams.");
|
|
1664
1681
|
}
|
|
1665
1682
|
const answer = generate ? await generate(request) : await generateStream(request, () => { });
|
|
1683
|
+
if (answer.files?.length)
|
|
1684
|
+
agentFilesSink.getStore()?.push(...answer.files);
|
|
1666
1685
|
const message = { role: "assistant" };
|
|
1667
1686
|
if (answer.toolCalls && answer.toolCalls.length > 0) {
|
|
1668
1687
|
message.tool_calls = wireToolCalls(answer.toolCalls, false);
|
|
@@ -1725,6 +1744,8 @@ function inProcessStream(generateStream, request) {
|
|
|
1725
1744
|
}
|
|
1726
1745
|
chunk({ [IN_PROCESS_SIDE_KEY]: ev });
|
|
1727
1746
|
});
|
|
1747
|
+
if (answer.files?.length)
|
|
1748
|
+
chunk({ [IN_PROCESS_FILES_KEY]: answer.files });
|
|
1728
1749
|
let finish = "stop";
|
|
1729
1750
|
if (answer.toolCalls && answer.toolCalls.length > 0) {
|
|
1730
1751
|
chunk({ choices: [{ index: 0, delta: { tool_calls: wireToolCalls(answer.toolCalls, true) } }] });
|
package/dist/media.d.ts
CHANGED
|
@@ -167,7 +167,7 @@ export declare const A2E_STYLE: MediaStyle & {
|
|
|
167
167
|
};
|
|
168
168
|
/**
|
|
169
169
|
* Built-in CLI presets for {@link loadMediaStyle} (ADR 0042): media through a locally installed,
|
|
170
|
-
* logged-in CLI
|
|
170
|
+
* logged-in CLI: the vendor's official CLI in its documented headless mode, signed in by the user; the provider's terms apply. The `*Mock` twins behave the same
|
|
171
171
|
* (name, verbs, args, prompt folding, errors) but never start a process and return a fixed file.
|
|
172
172
|
*/
|
|
173
173
|
export declare const MediaStyle: {
|
package/dist/media.js
CHANGED
|
@@ -361,7 +361,7 @@ export const A2E_STYLE = {
|
|
|
361
361
|
};
|
|
362
362
|
/**
|
|
363
363
|
* Built-in CLI presets for {@link loadMediaStyle} (ADR 0042): media through a locally installed,
|
|
364
|
-
* logged-in CLI
|
|
364
|
+
* logged-in CLI: the vendor's official CLI in its documented headless mode, signed in by the user; the provider's terms apply. The `*Mock` twins behave the same
|
|
365
365
|
* (name, verbs, args, prompt folding, errors) but never start a process and return a fixed file.
|
|
366
366
|
*/
|
|
367
367
|
export const MediaStyle = {
|
|
@@ -1780,7 +1780,7 @@ const GROK_SPEC = {
|
|
|
1780
1780
|
command: "grok",
|
|
1781
1781
|
templates: {
|
|
1782
1782
|
image: "Generate exactly one image with your image generation tool: {prompt}. Do not write code. Reply only with the file path.",
|
|
1783
|
-
video: "Generate exactly one
|
|
1783
|
+
video: "Generate exactly one video with your video generation tool: {prompt}. Do not write code. Reply only with the file path.",
|
|
1784
1784
|
},
|
|
1785
1785
|
argv: (prompt, cwd, model) => [
|
|
1786
1786
|
"-p", prompt, "--cwd", cwd, "--output-format", "streaming-json", "--yolo", "--no-auto-update",
|
package/package.json
CHANGED
package/src/acp.ts
CHANGED
|
@@ -34,7 +34,8 @@
|
|
|
34
34
|
import { spawn } from "node:child_process"
|
|
35
35
|
import { createInterface } from "node:readline"
|
|
36
36
|
import path from "node:path"
|
|
37
|
-
import type { InProcessRequest, InProcessResponse, StreamEvent } from "./client.js"
|
|
37
|
+
import type { InProcessRequest, InProcessResponse, RunFile, StreamEvent } from "./client.js"
|
|
38
|
+
import { isContentPart, summarizePart, type ContentPart } from "./content.js"
|
|
38
39
|
|
|
39
40
|
/** Options for {@link loadACP}. `command`/`args` spawn the ACP agent as a child
|
|
40
41
|
* process; everything else tunes the session handshake. */
|
|
@@ -135,18 +136,64 @@ interface RpcMessage {
|
|
|
135
136
|
error?: { code: number; message: string }
|
|
136
137
|
}
|
|
137
138
|
|
|
138
|
-
function renderContentPart(part: any): string {
|
|
139
|
+
function renderContentPart(part: any, media?: ContentPart[]): string {
|
|
139
140
|
if (typeof part === "string") return part
|
|
140
141
|
if (part == null) return ""
|
|
141
142
|
if (typeof part === "object" && "text" in part && typeof part.text === "string") return part.text
|
|
143
|
+
// A media part (OpenAI chat wire shape) is named by its §1B one-liner, never its bytes;
|
|
144
|
+
// the part itself travels as a native ACP block (SPEC §8C "ACP content parts").
|
|
145
|
+
const p = acpWirePart(part)
|
|
146
|
+
if (p && p.type !== "text") {
|
|
147
|
+
media?.push(p)
|
|
148
|
+
return summarizePart(p)
|
|
149
|
+
}
|
|
142
150
|
return JSON.stringify(part)
|
|
143
151
|
}
|
|
144
152
|
|
|
145
|
-
/**
|
|
146
|
-
function
|
|
153
|
+
/** `data:<mime>;base64,<payload>` (non-empty mime) ⇒ {mimeType, data}; anything else ⇒ undefined (SPEC §8C). */
|
|
154
|
+
function parseBase64DataUrl(u: string): { mimeType: string; data: string } | undefined {
|
|
155
|
+
if (!u.startsWith("data:")) return undefined
|
|
156
|
+
const comma = u.indexOf(",")
|
|
157
|
+
if (comma < 0) return undefined
|
|
158
|
+
const meta = u.slice(5, comma)
|
|
159
|
+
if (!meta.endsWith(";base64")) return undefined
|
|
160
|
+
const mimeType = meta.slice(0, -";base64".length)
|
|
161
|
+
return mimeType ? { mimeType, data: u.slice(comma + 1) } : undefined
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/** One OpenAI chat media block read back into a §1B part, as SPEC §8C pins it (Go acpWirePart). */
|
|
165
|
+
function acpWirePart(part: any): ContentPart | undefined {
|
|
166
|
+
switch (part?.type) {
|
|
167
|
+
case "image_url": {
|
|
168
|
+
const url = typeof part.image_url?.url === "string" ? part.image_url.url : ""
|
|
169
|
+
const d = parseBase64DataUrl(url)
|
|
170
|
+
if (d) return { type: "image", ...d } as ContentPart
|
|
171
|
+
// A URL-only image part: EMPTY mimeType, never inferred from the URL.
|
|
172
|
+
return url ? ({ type: "image", mimeType: "", url } as ContentPart) : undefined
|
|
173
|
+
}
|
|
174
|
+
case "input_audio": {
|
|
175
|
+
let mimeType = "audio/" + String(part.input_audio?.format ?? "")
|
|
176
|
+
if (mimeType === "audio/mp3") mimeType = "audio/mpeg"
|
|
177
|
+
return { type: "audio", mimeType, data: String(part.input_audio?.data ?? "") } as ContentPart
|
|
178
|
+
}
|
|
179
|
+
case "file": {
|
|
180
|
+
const d = parseBase64DataUrl(String(part.file?.file_data ?? ""))
|
|
181
|
+
if (!d) return undefined
|
|
182
|
+
const type = d.mimeType.startsWith("image/") ? "image" : d.mimeType.startsWith("audio/") ? "audio" : "file"
|
|
183
|
+
const p: any = { type, ...d }
|
|
184
|
+
if (part.file?.filename) p.name = String(part.file.filename)
|
|
185
|
+
return p as ContentPart
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
return isContentPart(part) && part.type !== "text" ? part : undefined
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** Flattens `content` (a string, or an array of content parts) to plain text; media parts
|
|
192
|
+
* found along the way are appended to `media`. */
|
|
193
|
+
function renderContent(content: any, media?: ContentPart[]): string {
|
|
147
194
|
if (typeof content === "string") return content
|
|
148
|
-
if (Array.isArray(content)) return content.map(renderContentPart).join("
|
|
149
|
-
return renderContentPart(content)
|
|
195
|
+
if (Array.isArray(content)) return content.map((p) => renderContentPart(p, media)).join("\n")
|
|
196
|
+
return renderContentPart(content, media)
|
|
150
197
|
}
|
|
151
198
|
|
|
152
199
|
/** Renders the FULL assembled request as one prompt string — role + content
|
|
@@ -155,9 +202,10 @@ function renderContent(content: any): string {
|
|
|
155
202
|
* every turn, not a delta, so this stays stateless like every other
|
|
156
203
|
* toolnexus model source and never risks a shadow transcript drifting from
|
|
157
204
|
* the caller's own conversation state. */
|
|
158
|
-
function renderPrompt(request: InProcessRequest): string {
|
|
205
|
+
function renderPrompt(request: InProcessRequest): { text: string; parts: ContentPart[] } {
|
|
159
206
|
const messages = request.messages ?? []
|
|
160
|
-
const
|
|
207
|
+
const parts: ContentPart[] = []
|
|
208
|
+
const lines = messages.map((m) => `${m?.role ?? "user"}: ${renderContent(m?.content, parts)}`)
|
|
161
209
|
|
|
162
210
|
let latest = ""
|
|
163
211
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
@@ -169,7 +217,30 @@ function renderPrompt(request: InProcessRequest): string {
|
|
|
169
217
|
if (!latest && messages.length > 0) latest = renderContent(messages[messages.length - 1]?.content)
|
|
170
218
|
|
|
171
219
|
lines.push(`${SUPERSEDES_MARKER} ${latest}`)
|
|
172
|
-
return lines.join("\n")
|
|
220
|
+
return { text: lines.join("\n"), parts }
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/** The agent's `initialize` `agentCapabilities.promptCapabilities` (absent ⇒ false). */
|
|
224
|
+
interface PromptCaps {
|
|
225
|
+
image: boolean
|
|
226
|
+
audio: boolean
|
|
227
|
+
embeddedContext: boolean
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/** The session/prompt ContentBlock array: the text block, then one native block per media
|
|
231
|
+
* part the agent advertised support for. A part it cannot take stays named only by its
|
|
232
|
+
* description in the text; base64 never enters the text block. */
|
|
233
|
+
function promptBlocks(text: string, parts: ContentPart[], caps: PromptCaps): Record<string, unknown>[] {
|
|
234
|
+
const blocks: Record<string, unknown>[] = [{ type: "text", text }]
|
|
235
|
+
for (const p of parts) {
|
|
236
|
+
if (p.type === "text" || !p.data) continue // ACP media blocks require inline data
|
|
237
|
+
if (p.type === "image" && caps.image) blocks.push({ type: "image", mimeType: p.mimeType, data: p.data })
|
|
238
|
+
else if (p.type === "audio" && caps.audio) blocks.push({ type: "audio", mimeType: p.mimeType, data: p.data })
|
|
239
|
+
else if (p.type === "file" && caps.embeddedContext) {
|
|
240
|
+
blocks.push({ type: "resource", resource: { uri: `attachment:///${p.name || "attachment"}`, mimeType: p.mimeType, blob: p.data } })
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
return blocks
|
|
173
244
|
}
|
|
174
245
|
|
|
175
246
|
/**
|
|
@@ -364,6 +435,8 @@ export async function loadACP(opts: ACPOptions): Promise<ACPClient> {
|
|
|
364
435
|
const initializeResult = await step("initialize", () =>
|
|
365
436
|
call("initialize", { protocolVersion: opts.protocolVersion ?? 1, clientCapabilities }),
|
|
366
437
|
)
|
|
438
|
+
const pc = initializeResult?.agentCapabilities?.promptCapabilities ?? {}
|
|
439
|
+
const promptCaps: PromptCaps = { image: pc.image === true, audio: pc.audio === true, embeddedContext: pc.embeddedContext === true }
|
|
367
440
|
const sessionParams: Record<string, unknown> = { cwd, mcpServers: opts.mcpServers ?? [] }
|
|
368
441
|
if (opts.sessionMeta) sessionParams._meta = opts.sessionMeta
|
|
369
442
|
const sessionResult = await step("session/new", () => call("session/new", sessionParams))
|
|
@@ -401,10 +474,12 @@ export async function loadACP(opts: ACPOptions): Promise<ACPClient> {
|
|
|
401
474
|
signal?: AbortSignal,
|
|
402
475
|
): Promise<InProcessResponse> {
|
|
403
476
|
if (signal?.aborted) throw acpAbortError()
|
|
404
|
-
const text = renderPrompt(request)
|
|
477
|
+
const { text, parts } = renderPrompt(request)
|
|
405
478
|
const id = `c-${++nextId}`
|
|
406
479
|
|
|
407
480
|
let content = ""
|
|
481
|
+
const files: RunFile[] = [] // inbound media, in arrival order
|
|
482
|
+
const messageParts: ContentPart[] = [] // media the agent sent as message chunks
|
|
408
483
|
const toolNames = new Map<string, string>()
|
|
409
484
|
const send = (ev: StreamEvent) => {
|
|
410
485
|
try {
|
|
@@ -419,6 +494,13 @@ export async function loadACP(opts: ACPOptions): Promise<ACPClient> {
|
|
|
419
494
|
const u = params?.update
|
|
420
495
|
switch (u?.sessionUpdate) {
|
|
421
496
|
case "agent_message_chunk": {
|
|
497
|
+
const mp = acpBlockPart(u?.content)
|
|
498
|
+
if (mp) {
|
|
499
|
+
// An image/audio chunk is a part, never text (SPEC §8C ACP content parts).
|
|
500
|
+
messageParts.push(mp)
|
|
501
|
+
files.push(acpRunFile(mp, "acp", ""))
|
|
502
|
+
break
|
|
503
|
+
}
|
|
422
504
|
const t: string = u?.content?.text ?? ""
|
|
423
505
|
content += t
|
|
424
506
|
if (t) send({ type: "text", delta: t })
|
|
@@ -447,11 +529,14 @@ export async function loadACP(opts: ACPOptions): Promise<ACPClient> {
|
|
|
447
529
|
send({ type: "tool_call_args", id: callId, name, args: raw, via: "acp" })
|
|
448
530
|
}
|
|
449
531
|
if (u.status !== "completed" && u.status !== "failed") break
|
|
532
|
+
const { output, parts: toolParts } = acpToolContent(u.content)
|
|
533
|
+
for (const p of toolParts) files.push(acpRunFile(p, name, callId))
|
|
450
534
|
send({
|
|
451
535
|
type: "tool_result",
|
|
452
536
|
id: callId,
|
|
453
537
|
name,
|
|
454
|
-
output
|
|
538
|
+
output,
|
|
539
|
+
...(toolParts.length ? { parts: toolParts } : {}),
|
|
455
540
|
isError: u.status === "failed",
|
|
456
541
|
via: "acp",
|
|
457
542
|
})
|
|
@@ -462,7 +547,7 @@ export async function loadACP(opts: ACPOptions): Promise<ACPClient> {
|
|
|
462
547
|
|
|
463
548
|
if (exited) throw new Error(exitMessage("session/prompt"))
|
|
464
549
|
const replyPromise = new Promise<RpcMessage>((resolve) => pending.set(id, { method: "session/prompt", cb: resolve }))
|
|
465
|
-
write({ jsonrpc: "2.0", id, method: "session/prompt", params: { sessionId, prompt:
|
|
550
|
+
write({ jsonrpc: "2.0", id, method: "session/prompt", params: { sessionId, prompt: promptBlocks(text, parts, promptCaps) } })
|
|
466
551
|
|
|
467
552
|
let timer: NodeJS.Timeout | undefined
|
|
468
553
|
const timeout = new Promise<never>((_, reject) => {
|
|
@@ -498,7 +583,9 @@ export async function loadACP(opts: ACPOptions): Promise<ACPClient> {
|
|
|
498
583
|
}
|
|
499
584
|
if (msg.error && msg.error.code === 0) throw new Error(msg.error.message)
|
|
500
585
|
if (msg.error) throw new Error(`toolnexus: ACP error ${msg.error.code}: ${msg.error.message}`)
|
|
501
|
-
|
|
586
|
+
// With no message text but media chunks, the answer is their §1B description.
|
|
587
|
+
const answer = content === "" && messageParts.length ? messageParts.map(summarizePart).join("\n") : content
|
|
588
|
+
return files.length ? { content: answer, files } : { content: answer }
|
|
502
589
|
} finally {
|
|
503
590
|
clearTimeout(timer)
|
|
504
591
|
if (onAbort) signal!.removeEventListener("abort", onAbort)
|
|
@@ -583,8 +670,36 @@ function killGroup(pid: number, signal: NodeJS.Signals): void {
|
|
|
583
670
|
}
|
|
584
671
|
}
|
|
585
672
|
|
|
586
|
-
/**
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
673
|
+
/** Reads an ACP tool_call_update's `content` array ([{type:"content", content:ContentBlock}]):
|
|
674
|
+
* text blocks join into the output; image/audio blocks become parts. With no text, the
|
|
675
|
+
* output is the parts' §1B description (as for an image-only MCP result, §0.4). */
|
|
676
|
+
function acpToolContent(content: any): { output: string; parts: ContentPart[] } {
|
|
677
|
+
if (!Array.isArray(content)) return { output: "", parts: [] }
|
|
678
|
+
const parts: ContentPart[] = []
|
|
679
|
+
let output = ""
|
|
680
|
+
for (const item of content) {
|
|
681
|
+
const p = acpBlockPart(item?.content)
|
|
682
|
+
if (p) parts.push(p)
|
|
683
|
+
else if (typeof item?.content?.text === "string") output += item.content.text
|
|
684
|
+
}
|
|
685
|
+
if (output === "" && parts.length) output = parts.map(summarizePart).join("\n")
|
|
686
|
+
return { output, parts }
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
/** An ACP image/audio ContentBlock ({type, mimeType, data, uri?}) as a §1B part: data wins,
|
|
690
|
+
* a uri-only block is a URL part; any other block is not a part. */
|
|
691
|
+
function acpBlockPart(blk: any): ContentPart | undefined {
|
|
692
|
+
if (!blk || (blk.type !== "image" && blk.type !== "audio")) return undefined
|
|
693
|
+
const mimeType: string = blk.mimeType ?? ""
|
|
694
|
+
if (typeof blk.data === "string" && blk.data) return { type: blk.type, mimeType, data: blk.data }
|
|
695
|
+
if (typeof blk.uri === "string" && blk.uri) return { type: blk.type, mimeType, url: blk.uri }
|
|
696
|
+
return undefined
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
/** One inbound part as a RunResult.files entry. */
|
|
700
|
+
function acpRunFile(p: ContentPart, from: string, callId: string): RunFile {
|
|
701
|
+
const f: RunFile = { type: p.type as RunFile["type"], mimeType: (p as any).mimeType, from, callId }
|
|
702
|
+
if ((p as any).url) f.url = (p as any).url
|
|
703
|
+
if ((p as any).data) f.data = (p as any).data
|
|
704
|
+
return f
|
|
590
705
|
}
|
package/src/client.ts
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
* ("openai" | "anthropic"), and it runs the tool-calling agent loop against a
|
|
4
4
|
* Toolkit. See ../../SPEC.md §8.
|
|
5
5
|
*/
|
|
6
|
+
import { AsyncLocalStorage } from "node:async_hooks"
|
|
6
7
|
import { mediaPartPath } from "./media.js"
|
|
7
8
|
import type { Toolkit } from "./toolkit.js"
|
|
8
9
|
import type { Tool, ToolHints, ToolResult, Request, Answer } from "./types.js"
|
|
@@ -465,7 +466,8 @@ export type StreamEvent =
|
|
|
465
466
|
/** An ACP agent's `tool_call_update` carrying `rawInput`: the call's arguments, known after
|
|
466
467
|
* the initial `tool_call` (#188). Reported only; never executed by the loop. */
|
|
467
468
|
| { type: "tool_call_args"; id: string; name: string; args: Record<string, unknown>; via: "acp" }
|
|
468
|
-
|
|
469
|
+
/** `parts`: the result's non-text output (§1B), e.g. an image an ACP agent's own tool returned. */
|
|
470
|
+
| { type: "tool_result"; id: string; name: string; output: string; isError: boolean; parts?: ContentPart[]; metadata?: Record<string, unknown>; via?: "call_tool" | "acp" }
|
|
469
471
|
| { type: "usage"; usage: Usage }
|
|
470
472
|
/** Only with `fallbackModels`: switching from model `from` to `to` after overload (§8). */
|
|
471
473
|
| { type: "fallback"; from: string; to: string }
|
|
@@ -744,9 +746,13 @@ export class Client {
|
|
|
744
746
|
*/
|
|
745
747
|
async run(prompt: PromptInput, ctx?: { toolkit?: Toolkit; signal?: AbortSignal; history?: any[] }): Promise<RunResult> {
|
|
746
748
|
checkPromptParts(prompt, this.wireOpts())
|
|
747
|
-
|
|
749
|
+
const agentFiles: RunFile[] = []
|
|
750
|
+
const r = await agentFilesSink.run(agentFiles, () => this.opts.style === "anthropic"
|
|
748
751
|
? this.runAnthropic(prompt, ctx?.toolkit, ctx?.signal, ctx?.history)
|
|
749
|
-
: this.runOpenAI(prompt, ctx?.toolkit, ctx?.signal, ctx?.history)
|
|
752
|
+
: this.runOpenAI(prompt, ctx?.toolkit, ctx?.signal, ctx?.history))
|
|
753
|
+
// Agent-side media (an in-process model's own tools) follow the loop's tool files.
|
|
754
|
+
if (agentFiles.length) r.files = [...r.files, ...agentFiles]
|
|
755
|
+
return r
|
|
750
756
|
}
|
|
751
757
|
|
|
752
758
|
/**
|
|
@@ -1631,6 +1637,9 @@ export class Client {
|
|
|
1631
1637
|
messages.push({ role: "user", content: prompt })
|
|
1632
1638
|
let tools = toOpenAI(this.requestTools(toolkit))
|
|
1633
1639
|
const toolCalls: ToolCallRecord[] = []
|
|
1640
|
+
// Agent-side media from an in-process answer (§8C ACP content parts), after the loop's own.
|
|
1641
|
+
const agentFiles: RunFile[] = []
|
|
1642
|
+
const withAgentFiles = (r: RunResult): RunResult => (agentFiles.length ? { ...r, files: [...r.files, ...agentFiles] } : r)
|
|
1634
1643
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
1635
1644
|
let turns = 0
|
|
1636
1645
|
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
@@ -1682,6 +1691,10 @@ export class Client {
|
|
|
1682
1691
|
yield j[IN_PROCESS_SIDE_KEY] as StreamEvent
|
|
1683
1692
|
continue
|
|
1684
1693
|
}
|
|
1694
|
+
if (j && j[IN_PROCESS_FILES_KEY] && this.opts.baseUrl === IN_PROCESS_BASE_URL) {
|
|
1695
|
+
agentFiles.push(...j[IN_PROCESS_FILES_KEY])
|
|
1696
|
+
continue
|
|
1697
|
+
}
|
|
1685
1698
|
if (j?.usage) addUsage(usage, j.usage, "openai")
|
|
1686
1699
|
const choice = j?.choices?.[0]
|
|
1687
1700
|
const d = choice?.delta
|
|
@@ -1710,7 +1723,7 @@ export class Client {
|
|
|
1710
1723
|
if (calls.length === 0) {
|
|
1711
1724
|
if (await this.roundEnd(messages, turn, true, (rs?.carry ?? "") + content)) { if (rs) rs.carry = ""; continue }
|
|
1712
1725
|
yield { type: "usage", usage }
|
|
1713
|
-
yield { type: "done", result: this.endRun(runStart, lastModel, (rs?.carry ?? "") + content, messages, toolCalls, turns, usage) }
|
|
1726
|
+
yield { type: "done", result: withAgentFiles(this.endRun(runStart, lastModel, (rs?.carry ?? "") + content, messages, toolCalls, turns, usage)) }
|
|
1714
1727
|
return
|
|
1715
1728
|
}
|
|
1716
1729
|
if (rs) rs.carry = ""
|
|
@@ -1732,7 +1745,7 @@ export class Client {
|
|
|
1732
1745
|
if (r.halted) {
|
|
1733
1746
|
toolCalls.push(this.toolCallRecord(c.name, args, result, c.id))
|
|
1734
1747
|
messages.push(toolMessage(c.id, c.name, result))
|
|
1735
|
-
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) }
|
|
1748
|
+
yield { type: "done", result: withAgentFiles(this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage)) }
|
|
1736
1749
|
return
|
|
1737
1750
|
}
|
|
1738
1751
|
}
|
|
@@ -1742,7 +1755,7 @@ export class Client {
|
|
|
1742
1755
|
}
|
|
1743
1756
|
await this.roundEnd(messages, turn, false, "")
|
|
1744
1757
|
}
|
|
1745
|
-
yield { type: "done", result: this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true) }
|
|
1758
|
+
yield { type: "done", result: withAgentFiles(this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true)) }
|
|
1746
1759
|
} catch (e) {
|
|
1747
1760
|
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e)
|
|
1748
1761
|
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done))
|
|
@@ -1905,6 +1918,9 @@ export interface InProcessResponse {
|
|
|
1905
1918
|
content?: string
|
|
1906
1919
|
toolCalls?: InProcessToolCall[]
|
|
1907
1920
|
usage?: { promptTokens?: number; completionTokens?: number; totalTokens?: number }
|
|
1921
|
+
/** Media the model's own agent-side tools or messages produced (e.g. an ACP agent's
|
|
1922
|
+
* images). Appended to RunResult.files after the loop's own tool files. */
|
|
1923
|
+
files?: RunFile[]
|
|
1908
1924
|
}
|
|
1909
1925
|
|
|
1910
1926
|
/** A streaming in-process model: `emit` each event as it happens — `text` deltas become
|
|
@@ -1940,6 +1956,12 @@ const IN_PROCESS_BASE_URL = "http://in-process.invalid/v1"
|
|
|
1940
1956
|
* streaming generate to the consumer (honoured only on {@link IN_PROCESS_BASE_URL}). */
|
|
1941
1957
|
const IN_PROCESS_SIDE_KEY = "toolnexus_event"
|
|
1942
1958
|
|
|
1959
|
+
/** The in-band SSE key carrying an in-process answer's `files` to `stream()` (same sentinel rule). */
|
|
1960
|
+
const IN_PROCESS_FILES_KEY = "toolnexus_files"
|
|
1961
|
+
|
|
1962
|
+
/** Per-`run()` sink for an in-process answer's `files` — they are agent-side, not loop tool calls. */
|
|
1963
|
+
const agentFilesSink = new AsyncLocalStorage<RunFile[]>()
|
|
1964
|
+
|
|
1943
1965
|
/**
|
|
1944
1966
|
* Build a `fetch`-shaped adapter over a semantic `generate` function (ADR 0030):
|
|
1945
1967
|
* the reusable piece of {@link createInProcessClient} — assembles the wire request,
|
|
@@ -1974,6 +1996,7 @@ export function createInProcessFetch(generate?: InProcessGenerate, generateStrea
|
|
|
1974
1996
|
}
|
|
1975
1997
|
|
|
1976
1998
|
const answer = generate ? await generate(request) : await generateStream!(request, () => {})
|
|
1999
|
+
if (answer.files?.length) agentFilesSink.getStore()?.push(...answer.files)
|
|
1977
2000
|
|
|
1978
2001
|
const message: Record<string, any> = { role: "assistant" }
|
|
1979
2002
|
if (answer.toolCalls && answer.toolCalls.length > 0) {
|
|
@@ -2041,6 +2064,7 @@ function inProcessStream(generateStream: InProcessGenerateStream, request: InPro
|
|
|
2041
2064
|
}
|
|
2042
2065
|
chunk({ [IN_PROCESS_SIDE_KEY]: ev })
|
|
2043
2066
|
})
|
|
2067
|
+
if (answer.files?.length) chunk({ [IN_PROCESS_FILES_KEY]: answer.files })
|
|
2044
2068
|
let finish = "stop"
|
|
2045
2069
|
if (answer.toolCalls && answer.toolCalls.length > 0) {
|
|
2046
2070
|
chunk({ choices: [{ index: 0, delta: { tool_calls: wireToolCalls(answer.toolCalls, true) } }] })
|
package/src/media.ts
CHANGED
|
@@ -522,7 +522,7 @@ export const A2E_STYLE: MediaStyle & { _doc?: string } = {
|
|
|
522
522
|
|
|
523
523
|
/**
|
|
524
524
|
* Built-in CLI presets for {@link loadMediaStyle} (ADR 0042): media through a locally installed,
|
|
525
|
-
* logged-in CLI
|
|
525
|
+
* logged-in CLI: the vendor's official CLI in its documented headless mode, signed in by the user; the provider's terms apply. The `*Mock` twins behave the same
|
|
526
526
|
* (name, verbs, args, prompt folding, errors) but never start a process and return a fixed file.
|
|
527
527
|
*/
|
|
528
528
|
export const MediaStyle = {
|
|
@@ -2009,7 +2009,7 @@ const GROK_SPEC: CliSpec = {
|
|
|
2009
2009
|
command: "grok",
|
|
2010
2010
|
templates: {
|
|
2011
2011
|
image: "Generate exactly one image with your image generation tool: {prompt}. Do not write code. Reply only with the file path.",
|
|
2012
|
-
video: "Generate exactly one
|
|
2012
|
+
video: "Generate exactly one video with your video generation tool: {prompt}. Do not write code. Reply only with the file path.",
|
|
2013
2013
|
},
|
|
2014
2014
|
argv: (prompt, cwd, model) => [
|
|
2015
2015
|
"-p", prompt, "--cwd", cwd, "--output-format", "streaming-json", "--yolo", "--no-auto-update",
|