toolnexus 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -0
- package/dist/agents/agent.d.ts +12 -1
- package/dist/agents/agent.js +43 -7
- package/dist/agents/compaction.d.ts +19 -0
- package/dist/agents/compaction.js +26 -4
- package/dist/agents/runtime.d.ts +5 -0
- package/dist/agents/runtime.js +13 -2
- package/dist/batteries.d.ts +223 -0
- package/dist/batteries.js +488 -0
- package/dist/builtin.d.ts +16 -0
- package/dist/builtin.js +315 -17
- package/dist/builtin_extras.d.ts +80 -0
- package/dist/builtin_extras.js +248 -0
- package/dist/classifier.d.ts +14 -0
- package/dist/classifier.js +34 -1
- package/dist/client.d.ts +91 -15
- package/dist/client.js +377 -110
- package/dist/index.d.ts +8 -0
- package/dist/index.js +11 -0
- package/dist/judge.d.ts +98 -0
- package/dist/judge.js +199 -0
- package/dist/mcp.d.ts +32 -1
- package/dist/mcp.js +184 -6
- package/dist/models.d.ts +20 -0
- package/dist/models.js +46 -0
- package/dist/promptcache.d.ts +21 -0
- package/dist/promptcache.js +59 -0
- package/dist/store.d.ts +32 -0
- package/dist/store.js +131 -0
- package/dist/tooltiers.d.ts +49 -0
- package/dist/tooltiers.js +197 -0
- package/dist/types.d.ts +19 -0
- package/package.json +2 -2
- package/src/agents/agent.ts +39 -8
- package/src/agents/compaction.ts +45 -4
- package/src/agents/runtime.ts +16 -2
- package/src/batteries.ts +614 -0
- package/src/builtin.ts +333 -18
- package/src/builtin_extras.ts +285 -0
- package/src/classifier.ts +36 -1
- package/src/client.ts +415 -115
- package/src/index.ts +31 -0
- package/src/judge.ts +305 -0
- package/src/mcp.ts +204 -7
- package/src/models.ts +56 -0
- package/src/promptcache.ts +63 -0
- package/src/store.ts +142 -0
- package/src/tooltiers.ts +221 -0
- package/src/types.ts +20 -0
package/dist/client.js
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
import { pendingOf } from "./types.js";
|
|
2
|
+
import { toOpenAI, toAnthropic } from "./adapters.js";
|
|
3
|
+
import { applyPromptCache } from "./promptcache.js";
|
|
4
|
+
import { buildToolPlan, toolSearch, unwrapCallTool } from "./tooltiers.js";
|
|
2
5
|
import { isRetryableStatus, retryAfterMs } from "./retry.js";
|
|
3
6
|
import { toOpenAIWire, toAnthropicWire, checkPromptParts } from "./wire.js";
|
|
4
7
|
import { finishReasonFor, argsString, openAIMessagesToAnthropic, openAIToolsToAnthropic, openAIToolChoiceToAnthropic, hasSystemMessage, } from "./translate.js";
|
|
@@ -74,13 +77,28 @@ export async function llmHttpError(res) {
|
|
|
74
77
|
/** Default conversation provider — keeps transcripts in memory for the client's lifetime. */
|
|
75
78
|
export class InMemoryConversationStore {
|
|
76
79
|
map = new Map();
|
|
80
|
+
updated = new Map();
|
|
77
81
|
async get(id) {
|
|
78
82
|
const m = this.map.get(id);
|
|
79
83
|
return m ? [...m] : undefined;
|
|
80
84
|
}
|
|
81
85
|
async save(id, messages) {
|
|
82
86
|
this.map.set(id, [...messages]);
|
|
87
|
+
this.updated.set(id, new Date());
|
|
83
88
|
}
|
|
89
|
+
/** Every stored conversation, newest first (ties by id ascending). */
|
|
90
|
+
async list() {
|
|
91
|
+
return sortEntries([...this.updated].map(([id, updatedAt]) => ({ id, updatedAt })));
|
|
92
|
+
}
|
|
93
|
+
/** Remove a conversation; absent ids are not an error. */
|
|
94
|
+
async delete(id) {
|
|
95
|
+
this.map.delete(id);
|
|
96
|
+
this.updated.delete(id);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/** @internal Newest first, ties by id ascending (SPEC.md "Conversation memory"). */
|
|
100
|
+
export function sortEntries(entries) {
|
|
101
|
+
return entries.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime() || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
|
|
84
102
|
}
|
|
85
103
|
function delay(ms, signal) {
|
|
86
104
|
return new Promise((resolve, reject) => {
|
|
@@ -91,12 +109,19 @@ function delay(ms, signal) {
|
|
|
91
109
|
function addUsage(acc, raw, style) {
|
|
92
110
|
if (!raw)
|
|
93
111
|
return;
|
|
112
|
+
const cache = (key, n) => {
|
|
113
|
+
if (typeof n === "number" && n > 0)
|
|
114
|
+
acc[key] = (acc[key] ?? 0) + n;
|
|
115
|
+
};
|
|
94
116
|
if (style === "anthropic") {
|
|
95
117
|
acc.promptTokens += raw.input_tokens ?? 0;
|
|
96
118
|
acc.completionTokens += raw.output_tokens ?? 0;
|
|
97
119
|
acc.totalTokens += (raw.input_tokens ?? 0) + (raw.output_tokens ?? 0);
|
|
120
|
+
cache("cacheReadTokens", raw.cache_read_input_tokens);
|
|
121
|
+
cache("cacheWriteTokens", raw.cache_creation_input_tokens);
|
|
98
122
|
}
|
|
99
123
|
else {
|
|
124
|
+
cache("cacheReadTokens", raw.prompt_tokens_details?.cached_tokens);
|
|
100
125
|
acc.promptTokens += raw.prompt_tokens ?? 0;
|
|
101
126
|
acc.completionTokens += raw.completion_tokens ?? 0;
|
|
102
127
|
acc.totalTokens += raw.total_tokens ?? (raw.prompt_tokens ?? 0) + (raw.completion_tokens ?? 0);
|
|
@@ -110,18 +135,45 @@ function perCall(raw, style) {
|
|
|
110
135
|
return { prompt: raw.input_tokens ?? 0, completion: raw.output_tokens ?? 0 };
|
|
111
136
|
return { prompt: raw.prompt_tokens ?? 0, completion: raw.completion_tokens ?? 0 };
|
|
112
137
|
}
|
|
113
|
-
|
|
138
|
+
/**
|
|
139
|
+
* Pick the API key (SPEC 4C, security). An explicit `apiKey` is used as-is; else `apiKeyEnv`;
|
|
140
|
+
* else ONLY the env var that belongs to the `baseUrl` host. There is no cross-provider fallback:
|
|
141
|
+
* any other host needs `apiKey` or `apiKeyEnv`. The value is never logged.
|
|
142
|
+
*/
|
|
143
|
+
export function resolveKey(opts) {
|
|
114
144
|
if (opts.apiKey)
|
|
115
145
|
return opts.apiKey;
|
|
116
|
-
const
|
|
117
|
-
|
|
118
|
-
(
|
|
119
|
-
|
|
120
|
-
|
|
146
|
+
const envName = opts.apiKeyEnv || keyEnvForBaseUrl(opts.baseUrl);
|
|
147
|
+
if (!envName) {
|
|
148
|
+
throw new Error(`No API key: set apiKey or apiKeyEnv (provider env vars are only used for https://api.openai.com, https://api.anthropic.com and https://openrouter.ai, never for ${JSON.stringify(opts.baseUrl)})`);
|
|
149
|
+
}
|
|
150
|
+
const key = process.env[envName];
|
|
121
151
|
if (!key)
|
|
122
|
-
throw new Error(
|
|
152
|
+
throw new Error(`No API key: set apiKey or ${envName}`);
|
|
123
153
|
return key;
|
|
124
154
|
}
|
|
155
|
+
/** The env var whose key may be sent to `baseUrl`'s host, or "" when none may. */
|
|
156
|
+
export function keyEnvForBaseUrl(baseUrl) {
|
|
157
|
+
const h = hostOf(baseUrl);
|
|
158
|
+
if (h === "api.openai.com")
|
|
159
|
+
return "OPENAI_API_KEY";
|
|
160
|
+
if (h === "api.anthropic.com")
|
|
161
|
+
return "ANTHROPIC_API_KEY";
|
|
162
|
+
if (h === "openrouter.ai" || h.endsWith(".openrouter.ai"))
|
|
163
|
+
return "OPENROUTER_API_KEY";
|
|
164
|
+
return "";
|
|
165
|
+
}
|
|
166
|
+
function hostOf(baseUrl) {
|
|
167
|
+
try {
|
|
168
|
+
const u = new URL(baseUrl);
|
|
169
|
+
if (u.protocol !== "https:")
|
|
170
|
+
return ""; // a provider key is never sent over plain http
|
|
171
|
+
return u.hostname.toLowerCase();
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return "";
|
|
175
|
+
}
|
|
176
|
+
}
|
|
125
177
|
// ---- Prometheus metrics registry (zero-dependency; §8 Observability) ----
|
|
126
178
|
/** FIXED across all ports for byte-parity of `..._duration_seconds` histograms. Seconds. */
|
|
127
179
|
const DURATION_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
@@ -217,6 +269,8 @@ export class Client {
|
|
|
217
269
|
store;
|
|
218
270
|
/** Cumulative Prometheus registry fed by the same events `onMetric` sees. */
|
|
219
271
|
registry = new MetricsRegistry();
|
|
272
|
+
/** The frozen ToolTiers plan per toolkit (SPEC 8 Tool tiers). */
|
|
273
|
+
tierPlans = new WeakMap();
|
|
220
274
|
constructor(opts) {
|
|
221
275
|
this.opts = opts;
|
|
222
276
|
this.store = opts.store ?? new InMemoryConversationStore();
|
|
@@ -235,6 +289,8 @@ export class Client {
|
|
|
235
289
|
* every LLM body-build site so the ordering contract holds identically on all four paths.
|
|
236
290
|
*/
|
|
237
291
|
finalizeBody(body) {
|
|
292
|
+
if (this.opts.promptCache)
|
|
293
|
+
applyPromptCache(body, this.opts.style, this.opts.promptCache);
|
|
238
294
|
const rp = this.opts.requestParams;
|
|
239
295
|
if (rp) {
|
|
240
296
|
for (const [k, v] of Object.entries(rp)) {
|
|
@@ -254,8 +310,8 @@ export class Client {
|
|
|
254
310
|
}
|
|
255
311
|
/** §8 Gap 5. OpenAI-style body; omits tools/tool_choice when the tool list is empty. Key order
|
|
256
312
|
* matches the pre-change body so a no-options call is byte-identical. */
|
|
257
|
-
openaiBody(messages, tools, stream) {
|
|
258
|
-
const b = { model
|
|
313
|
+
openaiBody(messages, tools, stream, model = this.opts.model) {
|
|
314
|
+
const b = { model, messages: toOpenAIWire(messages, this.wireOpts()) };
|
|
259
315
|
if (tools.length) {
|
|
260
316
|
b.tools = tools;
|
|
261
317
|
b.tool_choice = "auto";
|
|
@@ -267,8 +323,8 @@ export class Client {
|
|
|
267
323
|
return this.finalizeBody(b);
|
|
268
324
|
}
|
|
269
325
|
/** §8 Gap 5. Anthropic-style body; omits tools when the tool list is empty. */
|
|
270
|
-
anthropicBody(system, messages, tools, stream) {
|
|
271
|
-
const b = { model
|
|
326
|
+
anthropicBody(system, messages, tools, stream, model = this.opts.model) {
|
|
327
|
+
const b = { model, max_tokens: 4096, system, messages: toAnthropicWire(messages, this.wireOpts()) };
|
|
272
328
|
if (tools.length)
|
|
273
329
|
b.tools = tools;
|
|
274
330
|
if (stream)
|
|
@@ -290,13 +346,13 @@ export class Client {
|
|
|
290
346
|
}
|
|
291
347
|
/** Emit the terminal `run` metric event and build the RunResult. `exhausted` marks a
|
|
292
348
|
* maxTurns loop exit — with no final text that is a LOUD "incomplete", never "done" (§7D). */
|
|
293
|
-
endRun(runStart, text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
294
|
-
this.emit({ event: "run", model
|
|
295
|
-
return this.result(text, messages, toolCalls, turns, usage, exhausted);
|
|
349
|
+
endRun(runStart, model, text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
350
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart });
|
|
351
|
+
return this.result(model, text, messages, toolCalls, turns, usage, exhausted);
|
|
296
352
|
}
|
|
297
353
|
/** Emit a `run` error metric event (once, on a thrown run). */
|
|
298
|
-
emitRunError(runStart, toolCalls, turns, usage, e) {
|
|
299
|
-
this.emit({ event: "run", model
|
|
354
|
+
emitRunError(runStart, model, toolCalls, turns, usage, e) {
|
|
355
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart, error: e instanceof Error ? e.message : String(e) });
|
|
300
356
|
}
|
|
301
357
|
/**
|
|
302
358
|
* Run the loop. `ctx` — and `ctx.toolkit` inside it — are OPTIONAL: an absent toolkit is a
|
|
@@ -334,14 +390,17 @@ export class Client {
|
|
|
334
390
|
if (sys && !hasSystemMessage(messages))
|
|
335
391
|
messages = [{ role: "system", content: sys }, ...messages];
|
|
336
392
|
let tools = declared;
|
|
393
|
+
let turnModel = this.opts.model;
|
|
337
394
|
if (this.opts.hooks?.beforeLLM) {
|
|
338
395
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn: 0 });
|
|
339
396
|
if (ov?.messages)
|
|
340
397
|
messages = ov.messages;
|
|
341
398
|
if (ov?.tools)
|
|
342
399
|
tools = ov.tools;
|
|
400
|
+
if (ov?.model)
|
|
401
|
+
turnModel = ov.model;
|
|
343
402
|
}
|
|
344
|
-
const body = { model:
|
|
403
|
+
const body = { model: turnModel, messages };
|
|
345
404
|
if (tools.length)
|
|
346
405
|
body.tools = tools;
|
|
347
406
|
if (req.toolChoice !== undefined)
|
|
@@ -354,9 +413,9 @@ export class Client {
|
|
|
354
413
|
method: "POST",
|
|
355
414
|
headers: { "content-type": "application/json", authorization: `Bearer ${key}`, ...this.opts.headers },
|
|
356
415
|
body: JSON.stringify(this.finalizeBody(body)),
|
|
357
|
-
}, signal, "openai");
|
|
416
|
+
}, signal, "openai", turnModel);
|
|
358
417
|
if (this.opts.hooks?.afterLLM)
|
|
359
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
418
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn: 0 });
|
|
360
419
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
361
420
|
addUsage(usage, data.usage, "openai");
|
|
362
421
|
const choice = data.choices?.[0];
|
|
@@ -370,7 +429,7 @@ export class Client {
|
|
|
370
429
|
toolCalls,
|
|
371
430
|
finishReason: choice?.finish_reason ?? finishReasonFor(toolCalls.length > 0),
|
|
372
431
|
usage,
|
|
373
|
-
model:
|
|
432
|
+
model: turnModel,
|
|
374
433
|
raw: data,
|
|
375
434
|
};
|
|
376
435
|
}
|
|
@@ -390,15 +449,18 @@ export class Client {
|
|
|
390
449
|
...(req.toolkit ? req.toolkit.toAnthropic() : []),
|
|
391
450
|
...openAIToolsToAnthropic(req.tools),
|
|
392
451
|
];
|
|
452
|
+
let turnModel = this.opts.model;
|
|
393
453
|
if (this.opts.hooks?.beforeLLM) {
|
|
394
454
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn: 0 });
|
|
395
455
|
if (ov?.messages)
|
|
396
456
|
messages = ov.messages;
|
|
397
457
|
if (ov?.tools)
|
|
398
458
|
tools = ov.tools;
|
|
459
|
+
if (ov?.model)
|
|
460
|
+
turnModel = ov.model;
|
|
399
461
|
}
|
|
400
462
|
const body = {
|
|
401
|
-
model:
|
|
463
|
+
model: turnModel,
|
|
402
464
|
max_tokens: req.maxTokens && req.maxTokens > 0 ? req.maxTokens : 4096,
|
|
403
465
|
messages,
|
|
404
466
|
};
|
|
@@ -420,9 +482,9 @@ export class Client {
|
|
|
420
482
|
...this.opts.headers,
|
|
421
483
|
},
|
|
422
484
|
body: JSON.stringify(this.finalizeBody(body)),
|
|
423
|
-
}, signal, "anthropic");
|
|
485
|
+
}, signal, "anthropic", turnModel);
|
|
424
486
|
if (this.opts.hooks?.afterLLM)
|
|
425
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
487
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn: 0 });
|
|
426
488
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
427
489
|
addUsage(usage, data.usage, "anthropic");
|
|
428
490
|
const text = [];
|
|
@@ -443,7 +505,7 @@ export class Client {
|
|
|
443
505
|
toolCalls,
|
|
444
506
|
finishReason: finishReasonFor(toolCalls.length > 0, data.stop_reason),
|
|
445
507
|
usage,
|
|
446
|
-
model:
|
|
508
|
+
model: turnModel,
|
|
447
509
|
raw: data,
|
|
448
510
|
};
|
|
449
511
|
}
|
|
@@ -483,9 +545,24 @@ export class Client {
|
|
|
483
545
|
}
|
|
484
546
|
if (!ctx?.id)
|
|
485
547
|
return this.run(prompt, { toolkit: ctx?.toolkit, signal: ctx?.signal });
|
|
486
|
-
|
|
487
|
-
const
|
|
488
|
-
await this.store.
|
|
548
|
+
checkPromptParts(prompt, this.wireOpts());
|
|
549
|
+
const id = ctx.id;
|
|
550
|
+
const history = (await this.store.get(id)) ?? [];
|
|
551
|
+
// §8: an interrupted turn is saved too — the loop hands back its transcript, repaired.
|
|
552
|
+
let partial;
|
|
553
|
+
const onInterrupt = (m) => { partial = m; };
|
|
554
|
+
let result;
|
|
555
|
+
try {
|
|
556
|
+
result = this.opts.style === "anthropic"
|
|
557
|
+
? await this.runAnthropic(prompt, ctx.toolkit, ctx.signal, history, onInterrupt)
|
|
558
|
+
: await this.runOpenAI(prompt, ctx.toolkit, ctx.signal, history, onInterrupt);
|
|
559
|
+
}
|
|
560
|
+
catch (e) {
|
|
561
|
+
if (partial)
|
|
562
|
+
await this.store.save(id, partial);
|
|
563
|
+
throw e;
|
|
564
|
+
}
|
|
565
|
+
await this.store.save(id, result.messages);
|
|
489
566
|
return result;
|
|
490
567
|
}
|
|
491
568
|
/** A stateful multi-turn conversation that retains history across sends. */
|
|
@@ -500,13 +577,23 @@ export class Client {
|
|
|
500
577
|
async *stream(prompt, ctx) {
|
|
501
578
|
checkPromptParts(prompt, this.wireOpts());
|
|
502
579
|
const history = ctx?.id ? (await this.store.get(ctx.id)) ?? [] : undefined;
|
|
580
|
+
// §8: with an id, an interrupted stream saves its repaired transcript before the error surfaces.
|
|
581
|
+
let partial;
|
|
582
|
+
const onInterrupt = ctx?.id ? (m) => { partial = m; } : undefined;
|
|
503
583
|
const gen = this.opts.style === "anthropic"
|
|
504
|
-
? this.streamAnthropic(prompt, ctx?.toolkit, ctx?.signal, history)
|
|
505
|
-
: this.streamOpenAI(prompt, ctx?.toolkit, ctx?.signal, history);
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
584
|
+
? this.streamAnthropic(prompt, ctx?.toolkit, ctx?.signal, history, onInterrupt)
|
|
585
|
+
: this.streamOpenAI(prompt, ctx?.toolkit, ctx?.signal, history, onInterrupt);
|
|
586
|
+
try {
|
|
587
|
+
for await (const ev of gen) {
|
|
588
|
+
if (ev.type === "done" && ctx?.id)
|
|
589
|
+
await this.store.save(ctx.id, ev.result.messages);
|
|
590
|
+
yield ev;
|
|
591
|
+
}
|
|
592
|
+
}
|
|
593
|
+
catch (e) {
|
|
594
|
+
if (partial && ctx?.id)
|
|
595
|
+
await this.store.save(ctx.id, partial);
|
|
596
|
+
throw e;
|
|
510
597
|
}
|
|
511
598
|
}
|
|
512
599
|
/** §0.10 system message. The toolkit is OPTIONAL — with none there is no skills prompt and
|
|
@@ -581,13 +668,32 @@ export class Client {
|
|
|
581
668
|
throw lastErr;
|
|
582
669
|
}
|
|
583
670
|
/** Run one tool through beforeTool/afterTool hooks (mutate args, short-circuit, transform result). */
|
|
584
|
-
async runTool(toolkit, name, args, id, turn) {
|
|
671
|
+
async runTool(toolkit, name, args, id, turn, signal) {
|
|
672
|
+
if (this.opts.toolTiers && name === "tool_search") {
|
|
673
|
+
// Engine-internal and read-only: no hooks (SPEC 8 Tool tiers).
|
|
674
|
+
const t0 = Date.now();
|
|
675
|
+
const result = toolSearch(toolkit?.tools() ?? [], (n) => toolkit?.get(n), args ?? {});
|
|
676
|
+
this.emitTool(name, "builtin", result, t0);
|
|
677
|
+
return { args, result };
|
|
678
|
+
}
|
|
679
|
+
// call_tool is unwrapped HERE, so hooks, hints and permission gates see the real tool. The
|
|
680
|
+
// returned args keep the wrapper shape (with any beforeTool rewrite inside), so a 10
|
|
681
|
+
// re-run unwraps the same way.
|
|
682
|
+
const inner = this.unwrapCall(name, args);
|
|
683
|
+
if (inner) {
|
|
684
|
+
const r = await this.runToolDirect(toolkit, inner.name, inner.args, id, turn, signal);
|
|
685
|
+
return { args: { name: inner.name, arguments: r.args }, result: r.result };
|
|
686
|
+
}
|
|
687
|
+
return this.runToolDirect(toolkit, name, args, id, turn, signal);
|
|
688
|
+
}
|
|
689
|
+
async runToolDirect(toolkit, name, args, id, turn, signal) {
|
|
585
690
|
const h = this.opts.hooks;
|
|
586
|
-
const
|
|
691
|
+
const tool = toolkit?.get(name);
|
|
692
|
+
const source = tool?.source ?? "custom";
|
|
587
693
|
const t0 = Date.now();
|
|
588
694
|
let a = args;
|
|
589
695
|
if (h?.beforeTool) {
|
|
590
|
-
const ov = await h.beforeTool({ name, args: a, id, turn });
|
|
696
|
+
const ov = await h.beforeTool({ name, args: a, id, turn, ...(tool?.hints ? { hints: tool.hints } : {}) });
|
|
591
697
|
if (ov?.result) {
|
|
592
698
|
// short-circuit (deny/cache/dry-run, or a guard-raised suspension — path B); still a tool
|
|
593
699
|
// call for metrics, but a suspension is classified pending, never an error.
|
|
@@ -599,7 +705,7 @@ export class Client {
|
|
|
599
705
|
}
|
|
600
706
|
// No toolkit ⇒ nothing was declared, so a tool call is the provider inventing one.
|
|
601
707
|
let result = toolkit
|
|
602
|
-
? await toolkit.execute(name, a)
|
|
708
|
+
? await toolkit.execute(name, a, { ...(signal ? { signal } : {}), ...(id ? { callId: id } : {}) })
|
|
603
709
|
: { output: `unknown tool: ${name}`, isError: true };
|
|
604
710
|
// A suspension (§10) is not a real result: skip afterTool's failure path on it — the resolved
|
|
605
711
|
// result (post-waitFor) still flows through afterTool in resolvePending — and never count it as
|
|
@@ -612,13 +718,67 @@ export class Client {
|
|
|
612
718
|
this.emitTool(name, source, result, t0);
|
|
613
719
|
return { args: a, result };
|
|
614
720
|
}
|
|
721
|
+
/** ToolTiers `call_tool {name, arguments}` -> the real call; undefined when tiers are off. */
|
|
722
|
+
unwrapCall(name, args) {
|
|
723
|
+
return this.opts.toolTiers ? unwrapCallTool(name, args) : undefined;
|
|
724
|
+
}
|
|
725
|
+
/** The stream's tool_call event, naming the real tool for a `call_tool` call. */
|
|
726
|
+
callEvent(id, name, args) {
|
|
727
|
+
const inner = this.unwrapCall(name, args);
|
|
728
|
+
return inner ? { type: "tool_call", id, name: inner.name, args: inner.args, via: "call_tool" } : { type: "tool_call", id, name, args };
|
|
729
|
+
}
|
|
730
|
+
resultEvent(id, name, args, result) {
|
|
731
|
+
const inner = this.unwrapCall(name, args);
|
|
732
|
+
return {
|
|
733
|
+
type: "tool_result", id, name: inner ? inner.name : name, output: result.output, isError: result.isError,
|
|
734
|
+
...(result.metadata ? { metadata: result.metadata } : {}),
|
|
735
|
+
...(inner ? { via: "call_tool" } : {}),
|
|
736
|
+
};
|
|
737
|
+
}
|
|
738
|
+
plan(toolkit) {
|
|
739
|
+
if (!this.opts.toolTiers || !toolkit)
|
|
740
|
+
return undefined;
|
|
741
|
+
let p = this.tierPlans.get(toolkit);
|
|
742
|
+
if (!p) {
|
|
743
|
+
p = buildToolPlan(this.opts.toolTiers, toolkit.tools());
|
|
744
|
+
this.tierPlans.set(toolkit, p);
|
|
745
|
+
}
|
|
746
|
+
return p;
|
|
747
|
+
}
|
|
748
|
+
/** The tools array for a request: core tools (sorted) then tool_search and call_tool. Without
|
|
749
|
+
* tiers it is every tool, as before. */
|
|
750
|
+
requestTools(toolkit) {
|
|
751
|
+
if (!toolkit)
|
|
752
|
+
return [];
|
|
753
|
+
const p = this.plan(toolkit);
|
|
754
|
+
if (!p)
|
|
755
|
+
return toolkit.tools();
|
|
756
|
+
const out = [];
|
|
757
|
+
for (const n of p.core) {
|
|
758
|
+
const t = toolkit.get(n);
|
|
759
|
+
if (t)
|
|
760
|
+
out.push(t);
|
|
761
|
+
}
|
|
762
|
+
return [...out, p.search, p.call];
|
|
763
|
+
}
|
|
764
|
+
/** The frozen tier plan for `toolkit` (built on first use): tool name -> tier. Undefined when
|
|
765
|
+
* `toolTiers` is not set. For a host UI or log. */
|
|
766
|
+
toolTierPlan(toolkit) {
|
|
767
|
+
const p = this.plan(toolkit);
|
|
768
|
+
return p ? Object.fromEntries(p.tier) : undefined;
|
|
769
|
+
}
|
|
770
|
+
/** Forget `toolkit`'s plan so the next request rebuilds it - call at a new session start.
|
|
771
|
+
* Mid-session it breaks the cached prefix. */
|
|
772
|
+
resetToolTiers(toolkit) {
|
|
773
|
+
this.tierPlans.delete(toolkit);
|
|
774
|
+
}
|
|
615
775
|
/** Emit the `tool` metric, classifying a §10 suspension as `pending` (isError:false), not a failure. */
|
|
616
776
|
emitTool(name, source, result, t0) {
|
|
617
777
|
const req = pendingOf(result);
|
|
618
778
|
this.emit({ event: "tool", tool: name, source, isError: req ? false : result.isError, ms: Date.now() - t0, ...(req ? { pending: true } : {}) });
|
|
619
779
|
}
|
|
620
780
|
/** One non-streaming LLM call, with an `llm` metric event (ok/error + per-call tokens + ms). */
|
|
621
|
-
async llmCallJson(url, init, signal, style) {
|
|
781
|
+
async llmCallJson(url, init, signal, style, model) {
|
|
622
782
|
const t0 = Date.now();
|
|
623
783
|
try {
|
|
624
784
|
const res = await this.llmFetch(url, init, signal);
|
|
@@ -626,54 +786,54 @@ export class Client {
|
|
|
626
786
|
throw await llmHttpError(res);
|
|
627
787
|
const data = await res.json();
|
|
628
788
|
const tok = perCall(data.usage, style);
|
|
629
|
-
this.emit({ event: "llm", model
|
|
789
|
+
this.emit({ event: "llm", model, status: "ok", ms: Date.now() - t0, promptTokens: tok.prompt, completionTokens: tok.completion });
|
|
630
790
|
return data;
|
|
631
791
|
}
|
|
632
792
|
catch (e) {
|
|
633
|
-
this.emit({ event: "llm", model
|
|
793
|
+
this.emit({ event: "llm", model, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 });
|
|
634
794
|
throw e;
|
|
635
795
|
}
|
|
636
796
|
}
|
|
637
797
|
// The run-scoped timeout timer is owned by these four wrappers, so it is cleared however the
|
|
638
798
|
// run leaves: return, throw, or a consumer abandoning the stream half-way.
|
|
639
|
-
async runOpenAI(prompt, toolkit, external, history) {
|
|
799
|
+
async runOpenAI(prompt, toolkit, external, history, onInterrupt) {
|
|
640
800
|
const signal = this.makeSignal(external);
|
|
641
801
|
try {
|
|
642
|
-
return await this.runOpenAIWithSignal(signal, prompt, toolkit, history);
|
|
802
|
+
return await this.runOpenAIWithSignal(signal, prompt, toolkit, history, onInterrupt);
|
|
643
803
|
}
|
|
644
804
|
finally {
|
|
645
805
|
this.releaseSignal(signal);
|
|
646
806
|
}
|
|
647
807
|
}
|
|
648
|
-
async runAnthropic(prompt, toolkit, external, history) {
|
|
808
|
+
async runAnthropic(prompt, toolkit, external, history, onInterrupt) {
|
|
649
809
|
const signal = this.makeSignal(external);
|
|
650
810
|
try {
|
|
651
|
-
return await this.runAnthropicWithSignal(signal, prompt, toolkit, history);
|
|
811
|
+
return await this.runAnthropicWithSignal(signal, prompt, toolkit, history, onInterrupt);
|
|
652
812
|
}
|
|
653
813
|
finally {
|
|
654
814
|
this.releaseSignal(signal);
|
|
655
815
|
}
|
|
656
816
|
}
|
|
657
|
-
async *streamOpenAI(prompt, toolkit, external, history) {
|
|
817
|
+
async *streamOpenAI(prompt, toolkit, external, history, onInterrupt) {
|
|
658
818
|
const signal = this.makeSignal(external);
|
|
659
819
|
try {
|
|
660
|
-
yield* this.streamOpenAIWithSignal(signal, prompt, toolkit, history);
|
|
820
|
+
yield* this.streamOpenAIWithSignal(signal, prompt, toolkit, history, onInterrupt);
|
|
661
821
|
}
|
|
662
822
|
finally {
|
|
663
823
|
this.releaseSignal(signal);
|
|
664
824
|
}
|
|
665
825
|
}
|
|
666
|
-
async *streamAnthropic(prompt, toolkit, external, history) {
|
|
826
|
+
async *streamAnthropic(prompt, toolkit, external, history, onInterrupt) {
|
|
667
827
|
const signal = this.makeSignal(external);
|
|
668
828
|
try {
|
|
669
|
-
yield* this.streamAnthropicWithSignal(signal, prompt, toolkit, history);
|
|
829
|
+
yield* this.streamAnthropicWithSignal(signal, prompt, toolkit, history, onInterrupt);
|
|
670
830
|
}
|
|
671
831
|
finally {
|
|
672
832
|
this.releaseSignal(signal);
|
|
673
833
|
}
|
|
674
834
|
}
|
|
675
835
|
// ---- OpenAI-style: POST {baseUrl}/chat/completions ----
|
|
676
|
-
async runOpenAIWithSignal(signal, prompt, toolkit, history) {
|
|
836
|
+
async runOpenAIWithSignal(signal, prompt, toolkit, history, onInterrupt) {
|
|
677
837
|
const key = resolveKey(this.opts);
|
|
678
838
|
const runStart = Date.now();
|
|
679
839
|
let messages = history && history.length ? [...history] : [];
|
|
@@ -683,45 +843,56 @@ export class Client {
|
|
|
683
843
|
messages.push({ role: "system", content: system });
|
|
684
844
|
}
|
|
685
845
|
messages.push({ role: "user", content: prompt });
|
|
686
|
-
let tools =
|
|
846
|
+
let tools = toOpenAI(this.requestTools(toolkit));
|
|
687
847
|
const toolCalls = [];
|
|
688
848
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
689
849
|
let turns = 0;
|
|
850
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
851
|
+
let lastModel = this.opts.model;
|
|
852
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
853
|
+
const done = new Map();
|
|
690
854
|
try {
|
|
691
855
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
692
856
|
turns++;
|
|
857
|
+
let turnModel = this.opts.model;
|
|
693
858
|
if (this.opts.hooks?.beforeLLM) {
|
|
694
859
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
695
860
|
if (ov?.messages)
|
|
696
861
|
messages = ov.messages;
|
|
697
862
|
if (ov?.tools)
|
|
698
863
|
tools = ov.tools;
|
|
864
|
+
if (ov?.model)
|
|
865
|
+
turnModel = ov.model;
|
|
699
866
|
}
|
|
867
|
+
lastModel = turnModel;
|
|
700
868
|
const data = await this.llmCallJson(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
701
869
|
method: "POST",
|
|
702
870
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
|
|
703
|
-
body: JSON.stringify(this.openaiBody(messages, tools, false)),
|
|
704
|
-
}, signal, "openai");
|
|
871
|
+
body: JSON.stringify(this.openaiBody(messages, tools, false, turnModel)),
|
|
872
|
+
}, signal, "openai", turnModel);
|
|
705
873
|
addUsage(usage, data.usage, "openai");
|
|
706
874
|
if (this.opts.hooks?.afterLLM)
|
|
707
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
875
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn });
|
|
708
876
|
const msg = data.choices[0].message;
|
|
709
877
|
messages.push(msg);
|
|
710
878
|
const calls = msg.tool_calls ?? [];
|
|
711
879
|
if (calls.length === 0)
|
|
712
|
-
return this.endRun(runStart, msg.content ?? "", messages, toolCalls, turns, usage);
|
|
880
|
+
return this.endRun(runStart, lastModel, msg.content ?? "", messages, toolCalls, turns, usage);
|
|
713
881
|
// execute all tool calls in this turn concurrently (true parallel tool calling)
|
|
714
|
-
|
|
715
|
-
|
|
882
|
+
done.clear();
|
|
883
|
+
const settled = await untilAborted(signal, Promise.all(calls.map(async (call) => {
|
|
884
|
+
let { args, result } = await this.runTool(toolkit, call.function.name, safeJson(call.function.arguments), call.id, turn, signal);
|
|
716
885
|
let halted;
|
|
717
886
|
const req = pendingOf(result);
|
|
718
887
|
if (req) {
|
|
719
|
-
const r = await this.resolvePending(toolkit, call.function.name, args, req, call.id, turn);
|
|
888
|
+
const r = await this.resolvePending(toolkit, call.function.name, args, req, call.id, turn, signal);
|
|
720
889
|
result = r.result;
|
|
721
890
|
halted = r.halted;
|
|
722
891
|
}
|
|
892
|
+
if (!halted && call.id)
|
|
893
|
+
done.set(call.id, result);
|
|
723
894
|
return { call, args, result, halted };
|
|
724
|
-
}));
|
|
895
|
+
})));
|
|
725
896
|
// Record in tool-call order; on the FIRST durable halt, surface it and stop — deterministic,
|
|
726
897
|
// and the later suspensions' placeholder results never enter the transcript (they re-suspend
|
|
727
898
|
// on resume). Mirrors the streaming path.
|
|
@@ -729,42 +900,46 @@ export class Client {
|
|
|
729
900
|
toolCalls.push({ name: s.call.function.name, args: s.args, output: s.result.output, isError: s.result.isError, metadata: s.result.metadata });
|
|
730
901
|
messages.push(toolMessage(s.call.id, s.call.function.name, s.result));
|
|
731
902
|
if (s.halted)
|
|
732
|
-
return this.pendingRun(runStart, s.halted, messages, toolCalls, turns, usage);
|
|
903
|
+
return this.pendingRun(runStart, lastModel, s.halted, messages, toolCalls, turns, usage);
|
|
733
904
|
}
|
|
734
905
|
}
|
|
735
|
-
return this.endRun(runStart, lastText(messages), messages, toolCalls, turns, usage, true);
|
|
906
|
+
return this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true);
|
|
736
907
|
}
|
|
737
908
|
catch (e) {
|
|
738
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
909
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
910
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done));
|
|
739
911
|
throw e;
|
|
740
912
|
}
|
|
741
913
|
}
|
|
742
|
-
result(text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
914
|
+
result(model, text, messages, toolCalls, turns, usage, exhausted = false) {
|
|
743
915
|
const incomplete = exhausted && text === "";
|
|
744
|
-
const r = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model
|
|
916
|
+
const r = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: incomplete ? "incomplete" : "done" };
|
|
745
917
|
if (incomplete)
|
|
746
918
|
r.limit = "maxTurns";
|
|
747
919
|
return r;
|
|
748
920
|
}
|
|
749
921
|
/** §10: a run halted because a tool suspended and no `waitFor` was configured. */
|
|
750
|
-
pendingRun(runStart, request, messages, toolCalls, turns, usage) {
|
|
751
|
-
this.emit({ event: "run", model
|
|
752
|
-
return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model
|
|
922
|
+
pendingRun(runStart, model, request, messages, toolCalls, turns, usage) {
|
|
923
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart });
|
|
924
|
+
return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: "pending", pending: request };
|
|
753
925
|
}
|
|
754
926
|
/**
|
|
755
927
|
* §10: resolve a suspended tool result. Calls `waitFor(request)`, and on `ok` re-executes the
|
|
756
928
|
* tool once with `Context.answer = answer`. Returns the resolved result, or `{ halted }` when
|
|
757
929
|
* no `waitFor` is configured (the run should stop and surface the request).
|
|
758
930
|
*/
|
|
759
|
-
async resolvePending(toolkit, name, args, request, id, turn) {
|
|
931
|
+
async resolvePending(toolkit, name, args, request, id, turn, signal) {
|
|
932
|
+
const inner = this.unwrapCall(name, args);
|
|
933
|
+
if (inner)
|
|
934
|
+
({ name, args } = inner);
|
|
760
935
|
if (!this.opts.waitFor)
|
|
761
936
|
return { result: { output: request.prompt, isError: true }, halted: request };
|
|
762
|
-
const answer = await this.opts.waitFor(request);
|
|
937
|
+
const answer = await this.opts.waitFor(request, { signal: signal ?? new AbortController().signal });
|
|
763
938
|
if (!answer?.ok)
|
|
764
939
|
return { result: { output: `declined/expired: ${request.prompt}`, isError: true } };
|
|
765
940
|
const t0 = Date.now();
|
|
766
941
|
let result = toolkit
|
|
767
|
-
? await toolkit.execute(name, args, { answer })
|
|
942
|
+
? await toolkit.execute(name, args, { answer, ...(signal ? { signal } : {}), ...(id ? { callId: id } : {}) })
|
|
768
943
|
: { output: `unknown tool: ${name}`, isError: true };
|
|
769
944
|
if (this.opts.hooks?.afterTool) {
|
|
770
945
|
const ov = await this.opts.hooks.afterTool({ name, args, result, id, turn });
|
|
@@ -777,27 +952,35 @@ export class Client {
|
|
|
777
952
|
return { result };
|
|
778
953
|
}
|
|
779
954
|
// ---- Anthropic-style: POST {baseUrl}/messages ----
|
|
780
|
-
async runAnthropicWithSignal(signal, prompt, toolkit, history) {
|
|
955
|
+
async runAnthropicWithSignal(signal, prompt, toolkit, history, onInterrupt) {
|
|
781
956
|
const key = resolveKey(this.opts);
|
|
782
957
|
const base = this.opts.baseUrl.replace(/\/$/, "");
|
|
783
958
|
const endpoint = base.endsWith("/v1") ? `${base}/messages` : `${base}/v1/messages`;
|
|
784
959
|
const system = this.system(toolkit);
|
|
785
960
|
const runStart = Date.now();
|
|
786
961
|
let messages = history && history.length ? [...history, { role: "user", content: prompt }] : [{ role: "user", content: prompt }];
|
|
787
|
-
let tools =
|
|
962
|
+
let tools = toAnthropic(this.requestTools(toolkit));
|
|
788
963
|
const toolCalls = [];
|
|
789
964
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
790
965
|
let turns = 0;
|
|
966
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
967
|
+
let lastModel = this.opts.model;
|
|
968
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
969
|
+
const done = new Map();
|
|
791
970
|
try {
|
|
792
971
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
793
972
|
turns++;
|
|
973
|
+
let turnModel = this.opts.model;
|
|
794
974
|
if (this.opts.hooks?.beforeLLM) {
|
|
795
975
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
796
976
|
if (ov?.messages)
|
|
797
977
|
messages = ov.messages;
|
|
798
978
|
if (ov?.tools)
|
|
799
979
|
tools = ov.tools;
|
|
980
|
+
if (ov?.model)
|
|
981
|
+
turnModel = ov.model;
|
|
800
982
|
}
|
|
983
|
+
lastModel = turnModel;
|
|
801
984
|
const data = await this.llmCallJson(endpoint, {
|
|
802
985
|
method: "POST",
|
|
803
986
|
headers: {
|
|
@@ -806,29 +989,32 @@ export class Client {
|
|
|
806
989
|
"Content-Type": "application/json",
|
|
807
990
|
...this.opts.headers,
|
|
808
991
|
},
|
|
809
|
-
body: JSON.stringify(this.anthropicBody(system, messages, tools, false)),
|
|
810
|
-
}, signal, "anthropic");
|
|
992
|
+
body: JSON.stringify(this.anthropicBody(system, messages, tools, false, turnModel)),
|
|
993
|
+
}, signal, "anthropic", turnModel);
|
|
811
994
|
addUsage(usage, data.usage, "anthropic");
|
|
812
995
|
if (this.opts.hooks?.afterLLM)
|
|
813
|
-
await this.opts.hooks.afterLLM({ response: data, model:
|
|
996
|
+
await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn });
|
|
814
997
|
messages.push({ role: "assistant", content: data.content });
|
|
815
998
|
const uses = (data.content ?? []).filter((b) => b.type === "tool_use");
|
|
816
999
|
if (uses.length === 0) {
|
|
817
1000
|
const text = (data.content ?? []).filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
818
|
-
return this.endRun(runStart, text, messages, toolCalls, turns, usage);
|
|
1001
|
+
return this.endRun(runStart, lastModel, text, messages, toolCalls, turns, usage);
|
|
819
1002
|
}
|
|
820
1003
|
// execute all tool_use blocks in this turn concurrently (true parallel tool calling)
|
|
821
|
-
|
|
822
|
-
|
|
1004
|
+
done.clear();
|
|
1005
|
+
const settled = await untilAborted(signal, Promise.all(uses.map(async (use) => {
|
|
1006
|
+
let { args, result } = await this.runTool(toolkit, use.name, use.input ?? {}, use.id, turn, signal);
|
|
823
1007
|
let halted;
|
|
824
1008
|
const req = pendingOf(result);
|
|
825
1009
|
if (req) {
|
|
826
|
-
const r = await this.resolvePending(toolkit, use.name, args, req, use.id, turn);
|
|
1010
|
+
const r = await this.resolvePending(toolkit, use.name, args, req, use.id, turn, signal);
|
|
827
1011
|
result = r.result;
|
|
828
1012
|
halted = r.halted;
|
|
829
1013
|
}
|
|
1014
|
+
if (!halted && use.id)
|
|
1015
|
+
done.set(use.id, result);
|
|
830
1016
|
return { use, args, result, halted };
|
|
831
|
-
}));
|
|
1017
|
+
})));
|
|
832
1018
|
// Record in tool-call order; on the FIRST durable halt, include that tool_result block, stop
|
|
833
1019
|
// building the content, and surface it — deterministic (G3), later suspensions re-suspend on
|
|
834
1020
|
// resume. Mirrors the OpenAI path and the streaming loops.
|
|
@@ -844,17 +1030,18 @@ export class Client {
|
|
|
844
1030
|
}
|
|
845
1031
|
messages.push({ role: "user", content });
|
|
846
1032
|
if (halted)
|
|
847
|
-
return this.pendingRun(runStart, halted, messages, toolCalls, turns, usage);
|
|
1033
|
+
return this.pendingRun(runStart, lastModel, halted, messages, toolCalls, turns, usage);
|
|
848
1034
|
}
|
|
849
|
-
return this.endRun(runStart, "", messages, toolCalls, turns, usage, true);
|
|
1035
|
+
return this.endRun(runStart, lastModel, "", messages, toolCalls, turns, usage, true);
|
|
850
1036
|
}
|
|
851
1037
|
catch (e) {
|
|
852
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
1038
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
1039
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done));
|
|
853
1040
|
throw e;
|
|
854
1041
|
}
|
|
855
1042
|
}
|
|
856
1043
|
// ---- Streaming: OpenAI-style ----
|
|
857
|
-
async *streamOpenAIWithSignal(signal, prompt, toolkit, history) {
|
|
1044
|
+
async *streamOpenAIWithSignal(signal, prompt, toolkit, history, onInterrupt) {
|
|
858
1045
|
const key = resolveKey(this.opts);
|
|
859
1046
|
const runStart = Date.now();
|
|
860
1047
|
let messages = history && history.length ? [...history] : [];
|
|
@@ -864,20 +1051,28 @@ export class Client {
|
|
|
864
1051
|
messages.push({ role: "system", content: system });
|
|
865
1052
|
}
|
|
866
1053
|
messages.push({ role: "user", content: prompt });
|
|
867
|
-
let tools =
|
|
1054
|
+
let tools = toOpenAI(this.requestTools(toolkit));
|
|
868
1055
|
const toolCalls = [];
|
|
869
1056
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
870
1057
|
let turns = 0;
|
|
1058
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1059
|
+
let lastModel = this.opts.model;
|
|
1060
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
1061
|
+
const done = new Map();
|
|
871
1062
|
try {
|
|
872
1063
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
873
1064
|
turns++;
|
|
1065
|
+
let turnModel = this.opts.model;
|
|
874
1066
|
if (this.opts.hooks?.beforeLLM) {
|
|
875
1067
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
876
1068
|
if (ov?.messages)
|
|
877
1069
|
messages = ov.messages;
|
|
878
1070
|
if (ov?.tools)
|
|
879
1071
|
tools = ov.tools;
|
|
1072
|
+
if (ov?.model)
|
|
1073
|
+
turnModel = ov.model;
|
|
880
1074
|
}
|
|
1075
|
+
lastModel = turnModel;
|
|
881
1076
|
const t0 = Date.now();
|
|
882
1077
|
const beforeP = usage.promptTokens, beforeC = usage.completionTokens;
|
|
883
1078
|
let content = "";
|
|
@@ -886,7 +1081,7 @@ export class Client {
|
|
|
886
1081
|
const res = await this.llmFetch(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
887
1082
|
method: "POST",
|
|
888
1083
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
|
|
889
|
-
body: JSON.stringify(this.openaiBody(messages, tools, true)),
|
|
1084
|
+
body: JSON.stringify(this.openaiBody(messages, tools, true, turnModel)),
|
|
890
1085
|
}, signal);
|
|
891
1086
|
if (!res.ok || !res.body)
|
|
892
1087
|
throw await llmHttpError(res);
|
|
@@ -918,72 +1113,87 @@ export class Client {
|
|
|
918
1113
|
}
|
|
919
1114
|
}
|
|
920
1115
|
catch (e) {
|
|
921
|
-
this.emit({ event: "llm", model:
|
|
1116
|
+
this.emit({ event: "llm", model: turnModel, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 });
|
|
922
1117
|
throw e;
|
|
923
1118
|
}
|
|
924
|
-
this.emit({ event: "llm", model:
|
|
1119
|
+
this.emit({ event: "llm", model: turnModel, status: "ok", ms: Date.now() - t0, promptTokens: usage.promptTokens - beforeP, completionTokens: usage.completionTokens - beforeC });
|
|
925
1120
|
if (this.opts.hooks?.afterLLM)
|
|
926
|
-
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model:
|
|
1121
|
+
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model: turnModel, turn });
|
|
927
1122
|
const calls = [...acc.values()];
|
|
928
1123
|
if (calls.length === 0) {
|
|
929
1124
|
messages.push({ role: "assistant", content });
|
|
930
1125
|
yield { type: "usage", usage };
|
|
931
|
-
yield { type: "done", result: this.endRun(runStart, content, messages, toolCalls, turns, usage) };
|
|
1126
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, content, messages, toolCalls, turns, usage) };
|
|
932
1127
|
return;
|
|
933
1128
|
}
|
|
934
1129
|
messages.push({ role: "assistant", content: content || null, tool_calls: calls.map((c) => ({ id: c.id, type: "function", function: { name: c.name, arguments: c.args } })) });
|
|
935
1130
|
for (const c of calls)
|
|
936
|
-
yield
|
|
937
|
-
|
|
1131
|
+
yield this.callEvent(c.id, c.name, safeJson(c.args));
|
|
1132
|
+
done.clear();
|
|
1133
|
+
const settled = await untilAborted(signal, Promise.all(calls.map(async (c) => {
|
|
1134
|
+
const r = await this.runTool(toolkit, c.name, safeJson(c.args), c.id, turn, signal);
|
|
1135
|
+
if (!pendingOf(r.result) && c.id)
|
|
1136
|
+
done.set(c.id, r.result);
|
|
1137
|
+
return r;
|
|
1138
|
+
})));
|
|
938
1139
|
for (let i = 0; i < calls.length; i++) {
|
|
939
1140
|
const c = calls[i];
|
|
940
1141
|
let { args, result } = settled[i];
|
|
941
1142
|
const req = pendingOf(result);
|
|
942
1143
|
if (req) {
|
|
943
1144
|
yield { type: "pending", request: req }; // surface BEFORE waitFor so a channel can push the link
|
|
944
|
-
const r = await this.resolvePending(toolkit, c.name, args, req, c.id, turn);
|
|
1145
|
+
const r = await this.resolvePending(toolkit, c.name, args, req, c.id, turn, signal);
|
|
945
1146
|
result = r.result;
|
|
946
1147
|
if (r.halted) {
|
|
947
1148
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata });
|
|
948
1149
|
messages.push(toolMessage(c.id, c.name, result));
|
|
949
|
-
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) };
|
|
1150
|
+
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
|
|
950
1151
|
return;
|
|
951
1152
|
}
|
|
952
1153
|
}
|
|
953
1154
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata });
|
|
954
1155
|
messages.push(toolMessage(c.id, c.name, result));
|
|
955
|
-
yield
|
|
1156
|
+
yield this.resultEvent(c.id, c.name, safeJson(c.args), result);
|
|
956
1157
|
}
|
|
957
1158
|
}
|
|
958
|
-
yield { type: "done", result: this.endRun(runStart, lastText(messages), messages, toolCalls, turns, usage, true) };
|
|
1159
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true) };
|
|
959
1160
|
}
|
|
960
1161
|
catch (e) {
|
|
961
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
1162
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
1163
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done));
|
|
962
1164
|
throw e;
|
|
963
1165
|
}
|
|
964
1166
|
}
|
|
965
1167
|
// ---- Streaming: Anthropic-style ----
|
|
966
|
-
async *streamAnthropicWithSignal(signal, prompt, toolkit, history) {
|
|
1168
|
+
async *streamAnthropicWithSignal(signal, prompt, toolkit, history, onInterrupt) {
|
|
967
1169
|
const key = resolveKey(this.opts);
|
|
968
1170
|
const runStart = Date.now();
|
|
969
1171
|
const base = this.opts.baseUrl.replace(/\/$/, "");
|
|
970
1172
|
const endpoint = base.endsWith("/v1") ? `${base}/messages` : `${base}/v1/messages`;
|
|
971
1173
|
const system = this.system(toolkit);
|
|
972
1174
|
let messages = history && history.length ? [...history, { role: "user", content: prompt }] : [{ role: "user", content: prompt }];
|
|
973
|
-
let tools =
|
|
1175
|
+
let tools = toAnthropic(this.requestTools(toolkit));
|
|
974
1176
|
const toolCalls = [];
|
|
975
1177
|
const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
|
|
976
1178
|
let turns = 0;
|
|
1179
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1180
|
+
let lastModel = this.opts.model;
|
|
1181
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
1182
|
+
const done = new Map();
|
|
977
1183
|
try {
|
|
978
1184
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
979
1185
|
turns++;
|
|
1186
|
+
let turnModel = this.opts.model;
|
|
980
1187
|
if (this.opts.hooks?.beforeLLM) {
|
|
981
1188
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn });
|
|
982
1189
|
if (ov?.messages)
|
|
983
1190
|
messages = ov.messages;
|
|
984
1191
|
if (ov?.tools)
|
|
985
1192
|
tools = ov.tools;
|
|
1193
|
+
if (ov?.model)
|
|
1194
|
+
turnModel = ov.model;
|
|
986
1195
|
}
|
|
1196
|
+
lastModel = turnModel;
|
|
987
1197
|
const t0 = Date.now();
|
|
988
1198
|
const beforeP = usage.promptTokens, beforeC = usage.completionTokens;
|
|
989
1199
|
const blocks = new Map();
|
|
@@ -992,7 +1202,7 @@ export class Client {
|
|
|
992
1202
|
const res = await this.llmFetch(endpoint, {
|
|
993
1203
|
method: "POST",
|
|
994
1204
|
headers: { "x-api-key": key, "anthropic-version": "2023-06-01", "Content-Type": "application/json", ...this.opts.headers },
|
|
995
|
-
body: JSON.stringify(this.anthropicBody(system, messages, tools, true)),
|
|
1205
|
+
body: JSON.stringify(this.anthropicBody(system, messages, tools, true, turnModel)),
|
|
996
1206
|
}, signal);
|
|
997
1207
|
if (!res.ok || !res.body)
|
|
998
1208
|
throw await llmHttpError(res);
|
|
@@ -1022,24 +1232,30 @@ export class Client {
|
|
|
1022
1232
|
}
|
|
1023
1233
|
}
|
|
1024
1234
|
catch (e) {
|
|
1025
|
-
this.emit({ event: "llm", model:
|
|
1235
|
+
this.emit({ event: "llm", model: turnModel, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 });
|
|
1026
1236
|
throw e;
|
|
1027
1237
|
}
|
|
1028
|
-
this.emit({ event: "llm", model:
|
|
1238
|
+
this.emit({ event: "llm", model: turnModel, status: "ok", ms: Date.now() - t0, promptTokens: usage.promptTokens - beforeP, completionTokens: usage.completionTokens - beforeC });
|
|
1029
1239
|
if (this.opts.hooks?.afterLLM)
|
|
1030
|
-
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model:
|
|
1240
|
+
await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model: turnModel, turn });
|
|
1031
1241
|
const content = [...blocks.values()].map((b) => b.type === "tool_use" ? { type: "tool_use", id: b.id, name: b.name, input: safeJson(b.json || "{}") } : { type: "text", text: b.text });
|
|
1032
1242
|
messages.push({ role: "assistant", content });
|
|
1033
1243
|
const uses = content.filter((b) => b.type === "tool_use");
|
|
1034
1244
|
if (stopReason !== "tool_use" || uses.length === 0) {
|
|
1035
1245
|
const text = content.filter((b) => b.type === "text").map((b) => b.text).join("");
|
|
1036
1246
|
yield { type: "usage", usage };
|
|
1037
|
-
yield { type: "done", result: this.endRun(runStart, text, messages, toolCalls, turns, usage) };
|
|
1247
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, text, messages, toolCalls, turns, usage) };
|
|
1038
1248
|
return;
|
|
1039
1249
|
}
|
|
1040
1250
|
for (const u of uses)
|
|
1041
|
-
yield
|
|
1042
|
-
|
|
1251
|
+
yield this.callEvent(u.id, u.name, u.input);
|
|
1252
|
+
done.clear();
|
|
1253
|
+
const settled = await untilAborted(signal, Promise.all(uses.map(async (u) => {
|
|
1254
|
+
const r = await this.runTool(toolkit, u.name, u.input ?? {}, u.id, turn, signal);
|
|
1255
|
+
if (!pendingOf(r.result) && u.id)
|
|
1256
|
+
done.set(u.id, r.result);
|
|
1257
|
+
return r;
|
|
1258
|
+
})));
|
|
1043
1259
|
const results = [];
|
|
1044
1260
|
for (let i = 0; i < uses.length; i++) {
|
|
1045
1261
|
const u = uses[i];
|
|
@@ -1047,26 +1263,27 @@ export class Client {
|
|
|
1047
1263
|
const req = pendingOf(result);
|
|
1048
1264
|
if (req) {
|
|
1049
1265
|
yield { type: "pending", request: req };
|
|
1050
|
-
const r = await this.resolvePending(toolkit, u.name, args, req, u.id, turn);
|
|
1266
|
+
const r = await this.resolvePending(toolkit, u.name, args, req, u.id, turn, signal);
|
|
1051
1267
|
result = r.result;
|
|
1052
1268
|
if (r.halted) {
|
|
1053
1269
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata });
|
|
1054
1270
|
results.push(toolResultBlock(u.id, result));
|
|
1055
1271
|
messages.push({ role: "user", content: results });
|
|
1056
|
-
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) };
|
|
1272
|
+
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) };
|
|
1057
1273
|
return;
|
|
1058
1274
|
}
|
|
1059
1275
|
}
|
|
1060
1276
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata });
|
|
1061
1277
|
results.push(toolResultBlock(u.id, result));
|
|
1062
|
-
yield
|
|
1278
|
+
yield this.resultEvent(u.id, u.name, u.input, result);
|
|
1063
1279
|
}
|
|
1064
1280
|
messages.push({ role: "user", content: results });
|
|
1065
1281
|
}
|
|
1066
|
-
yield { type: "done", result: this.endRun(runStart, "", messages, toolCalls, turns, usage, true) };
|
|
1282
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, "", messages, toolCalls, turns, usage, true) };
|
|
1067
1283
|
}
|
|
1068
1284
|
catch (e) {
|
|
1069
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e);
|
|
1285
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e);
|
|
1286
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done));
|
|
1070
1287
|
throw e;
|
|
1071
1288
|
}
|
|
1072
1289
|
}
|
|
@@ -1246,6 +1463,56 @@ function toolResultBlock(id, result) {
|
|
|
1246
1463
|
b.parts = result.parts;
|
|
1247
1464
|
return b;
|
|
1248
1465
|
}
|
|
1466
|
+
const INTERRUPTED = { output: "interrupted: the turn ended before this tool returned. Whether its side effects happened is unknown. Do not re-run it automatically; check first.", isError: true };
|
|
1467
|
+
/** Settle with `p`, or reject with the abort reason as soon as `signal` aborts. */
|
|
1468
|
+
function untilAborted(signal, p) {
|
|
1469
|
+
if (signal.aborted)
|
|
1470
|
+
return Promise.reject(signal.reason ?? new Error("aborted"));
|
|
1471
|
+
return new Promise((resolve, reject) => {
|
|
1472
|
+
const onAbort = () => reject(signal.reason ?? new Error("aborted"));
|
|
1473
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
1474
|
+
p.then((v) => { signal.removeEventListener("abort", onAbort); resolve(v); }, (e) => { signal.removeEventListener("abort", onAbort); reject(e); });
|
|
1475
|
+
});
|
|
1476
|
+
}
|
|
1477
|
+
/**
|
|
1478
|
+
* §8 "Interrupted turns are saved": a copy of `messages` in which every tool call of the LAST
|
|
1479
|
+
* assistant message has a result — the finished one from `done`, else `interrupted` (isError) —
|
|
1480
|
+
* in the dialect's own shape, so the saved transcript replays on either API.
|
|
1481
|
+
*/
|
|
1482
|
+
function interruptedTranscript(messages, anthropic, done) {
|
|
1483
|
+
const out = [...messages];
|
|
1484
|
+
let ai = -1;
|
|
1485
|
+
for (let i = out.length - 1; i >= 0; i--)
|
|
1486
|
+
if (out[i]?.role === "assistant") {
|
|
1487
|
+
ai = i;
|
|
1488
|
+
break;
|
|
1489
|
+
}
|
|
1490
|
+
if (ai < 0)
|
|
1491
|
+
return out;
|
|
1492
|
+
const resultFor = (id) => done.get(id) ?? INTERRUPTED;
|
|
1493
|
+
if (!anthropic) {
|
|
1494
|
+
const calls = out[ai].tool_calls ?? [];
|
|
1495
|
+
const have = new Set(out.slice(ai + 1).filter((m) => m?.role === "tool").map((m) => m.tool_call_id));
|
|
1496
|
+
for (const c of calls)
|
|
1497
|
+
if (!have.has(c.id))
|
|
1498
|
+
out.push(toolMessage(c.id, c.function?.name ?? "", resultFor(c.id)));
|
|
1499
|
+
return out;
|
|
1500
|
+
}
|
|
1501
|
+
const uses = Array.isArray(out[ai].content) ? out[ai].content.filter((b) => b?.type === "tool_use") : [];
|
|
1502
|
+
if (!uses.length)
|
|
1503
|
+
return out;
|
|
1504
|
+
const next = out[ai + 1];
|
|
1505
|
+
const blocks = next?.role === "user" && Array.isArray(next.content) ? [...next.content] : [];
|
|
1506
|
+
const have = new Set(blocks.filter((b) => b?.type === "tool_result").map((b) => b.tool_use_id));
|
|
1507
|
+
for (const u of uses)
|
|
1508
|
+
if (!have.has(u.id))
|
|
1509
|
+
blocks.push(toolResultBlock(u.id, resultFor(u.id)));
|
|
1510
|
+
if (next?.role === "user" && Array.isArray(next.content))
|
|
1511
|
+
out[ai + 1] = { ...next, content: blocks };
|
|
1512
|
+
else
|
|
1513
|
+
out.splice(ai + 1, 0, { role: "user", content: blocks });
|
|
1514
|
+
return out;
|
|
1515
|
+
}
|
|
1249
1516
|
function lastText(messages) {
|
|
1250
1517
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1251
1518
|
if (messages[i].role === "assistant" && typeof messages[i].content === "string")
|