toolnexus 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +40 -0
- package/dist/agents/agent.d.ts +12 -1
- package/dist/agents/agent.js +43 -7
- package/dist/agents/compaction.d.ts +19 -0
- package/dist/agents/compaction.js +26 -4
- package/dist/agents/runtime.d.ts +5 -0
- package/dist/agents/runtime.js +13 -2
- package/dist/batteries.d.ts +223 -0
- package/dist/batteries.js +488 -0
- package/dist/builtin.d.ts +16 -0
- package/dist/builtin.js +315 -17
- package/dist/builtin_extras.d.ts +80 -0
- package/dist/builtin_extras.js +248 -0
- package/dist/classifier.d.ts +14 -0
- package/dist/classifier.js +34 -1
- package/dist/client.d.ts +91 -15
- package/dist/client.js +377 -110
- package/dist/index.d.ts +8 -0
- package/dist/index.js +11 -0
- package/dist/judge.d.ts +98 -0
- package/dist/judge.js +199 -0
- package/dist/mcp.d.ts +32 -1
- package/dist/mcp.js +184 -6
- package/dist/models.d.ts +20 -0
- package/dist/models.js +46 -0
- package/dist/promptcache.d.ts +21 -0
- package/dist/promptcache.js +59 -0
- package/dist/store.d.ts +32 -0
- package/dist/store.js +131 -0
- package/dist/tooltiers.d.ts +49 -0
- package/dist/tooltiers.js +197 -0
- package/dist/types.d.ts +19 -0
- package/package.json +2 -2
- package/src/agents/agent.ts +39 -8
- package/src/agents/compaction.ts +45 -4
- package/src/agents/runtime.ts +16 -2
- package/src/batteries.ts +614 -0
- package/src/builtin.ts +333 -18
- package/src/builtin_extras.ts +285 -0
- package/src/classifier.ts +36 -1
- package/src/client.ts +415 -115
- package/src/index.ts +31 -0
- package/src/judge.ts +305 -0
- package/src/mcp.ts +204 -7
- package/src/models.ts +56 -0
- package/src/promptcache.ts +63 -0
- package/src/store.ts +142 -0
- package/src/tooltiers.ts +221 -0
- package/src/types.ts +20 -0
package/src/client.ts
CHANGED
|
@@ -4,8 +4,11 @@
|
|
|
4
4
|
* Toolkit. See ../../SPEC.md §8.
|
|
5
5
|
*/
|
|
6
6
|
import type { Toolkit } from "./toolkit.js"
|
|
7
|
-
import type { ToolResult, Request, Answer } from "./types.js"
|
|
7
|
+
import type { Tool, ToolHints, ToolResult, Request, Answer } from "./types.js"
|
|
8
8
|
import { pendingOf } from "./types.js"
|
|
9
|
+
import { toOpenAI, toAnthropic } from "./adapters.js"
|
|
10
|
+
import { applyPromptCache, type PromptCacheOptions } from "./promptcache.js"
|
|
11
|
+
import { buildToolPlan, toolSearch, unwrapCallTool, type ToolPlan, type ToolTier, type ToolTierOptions } from "./tooltiers.js"
|
|
9
12
|
import { isRetryableStatus, retryAfterMs } from "./retry.js"
|
|
10
13
|
import type { ContentPart, PromptInput, UnsupportedPartMode } from "./content.js"
|
|
11
14
|
import { toOpenAIWire, toAnthropicWire, checkPromptParts, type WireOptions } from "./wire.js"
|
|
@@ -26,7 +29,15 @@ export interface ClientOptions {
|
|
|
26
29
|
baseUrl: string
|
|
27
30
|
style: ClientStyle
|
|
28
31
|
model: string
|
|
32
|
+
/** The key, used as-is. Absent => `apiKeyEnv`, else ONLY the env var that belongs to the
|
|
33
|
+
* `baseUrl` host over https: api.openai.com -> OPENAI_API_KEY, api.anthropic.com ->
|
|
34
|
+
* ANTHROPIC_API_KEY, openrouter.ai (and subdomains) -> OPENROUTER_API_KEY. Any other host (or
|
|
35
|
+
* plain http) needs `apiKey` or `apiKeyEnv` - a key is never sent to a host it does not belong
|
|
36
|
+
* to (SPEC 4C). Never logged. */
|
|
29
37
|
apiKey?: string
|
|
38
|
+
/** The NAME of the env var to read the key from when `apiKey` is absent - for a host not in
|
|
39
|
+
* the table above. Read at call time; the value is never logged. */
|
|
40
|
+
apiKeyEnv?: string
|
|
30
41
|
headers?: Record<string, string>
|
|
31
42
|
systemPrompt?: string
|
|
32
43
|
maxTurns?: number
|
|
@@ -68,7 +79,17 @@ export interface ClientOptions {
|
|
|
68
79
|
* Replaying the leaf's stored transcript instead is DEFERRED to its own change; it would not
|
|
69
80
|
* remove this requirement anyway.
|
|
70
81
|
*/
|
|
71
|
-
|
|
82
|
+
/** The second argument carries the turn's abort signal (SPEC 4C), so a pending ask can end
|
|
83
|
+
* when the turn is cancelled: settle once `ctx.signal` aborts. */
|
|
84
|
+
waitFor?: (request: Request, ctx: { signal: AbortSignal }) => Promise<Answer>
|
|
85
|
+
/** Prompt caching (SPEC 8): anthropic `cache_control` breakpoints on the last tool, the system
|
|
86
|
+
* prompt and the latest message; openai `prompt_cache_key`. Applied before `requestParams`
|
|
87
|
+
* and `bodyTransform`, so both can still override it. Absent => body byte-identical. */
|
|
88
|
+
promptCache?: PromptCacheOptions
|
|
89
|
+
/** Tool tiers (SPEC 8): send only core tool schemas plus a `tool_search` tool that loads the
|
|
90
|
+
* rest on demand (and `call_tool`). The plan is frozen per toolkit until `resetToolTiers`.
|
|
91
|
+
* Absent => every tool is sent, as before. */
|
|
92
|
+
toolTiers?: ToolTierOptions
|
|
72
93
|
/** §8 Gap 1. Extra top-level keys shallow-merged into EVERY LLM body after the client builds
|
|
73
94
|
* its own — a requestParams key WINS on collision (e.g. max_tokens overrides the anthropic
|
|
74
95
|
* default). `messages`/`tools`/`stream` are forbidden here (stripped + warned) — rewrite
|
|
@@ -220,15 +241,37 @@ export interface ConversationStore {
|
|
|
220
241
|
/** Default conversation provider — keeps transcripts in memory for the client's lifetime. */
|
|
221
242
|
export class InMemoryConversationStore implements ConversationStore {
|
|
222
243
|
private readonly map = new Map<string, any[]>()
|
|
244
|
+
private readonly updated = new Map<string, Date>()
|
|
223
245
|
async get(id: string): Promise<any[] | undefined> {
|
|
224
246
|
const m = this.map.get(id)
|
|
225
247
|
return m ? [...m] : undefined
|
|
226
248
|
}
|
|
227
249
|
async save(id: string, messages: any[]): Promise<void> {
|
|
228
250
|
this.map.set(id, [...messages])
|
|
251
|
+
this.updated.set(id, new Date())
|
|
252
|
+
}
|
|
253
|
+
/** Every stored conversation, newest first (ties by id ascending). */
|
|
254
|
+
async list(): Promise<ConversationEntry[]> {
|
|
255
|
+
return sortEntries([...this.updated].map(([id, updatedAt]) => ({ id, updatedAt })))
|
|
256
|
+
}
|
|
257
|
+
/** Remove a conversation; absent ids are not an error. */
|
|
258
|
+
async delete(id: string): Promise<void> {
|
|
259
|
+
this.map.delete(id)
|
|
260
|
+
this.updated.delete(id)
|
|
229
261
|
}
|
|
230
262
|
}
|
|
231
263
|
|
|
264
|
+
/** One `list()` row of a shipped conversation store. */
|
|
265
|
+
export interface ConversationEntry {
|
|
266
|
+
id: string
|
|
267
|
+
updatedAt: Date
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** @internal Newest first, ties by id ascending (SPEC.md "Conversation memory"). */
|
|
271
|
+
export function sortEntries(entries: ConversationEntry[]): ConversationEntry[] {
|
|
272
|
+
return entries.sort((a, b) => b.updatedAt.getTime() - a.updatedAt.getTime() || (a.id < b.id ? -1 : a.id > b.id ? 1 : 0))
|
|
273
|
+
}
|
|
274
|
+
|
|
232
275
|
function delay(ms: number, signal: AbortSignal): Promise<void> {
|
|
233
276
|
return new Promise((resolve, reject) => {
|
|
234
277
|
const t = setTimeout(resolve, ms)
|
|
@@ -240,13 +283,30 @@ function delay(ms: number, signal: AbortSignal): Promise<void> {
|
|
|
240
283
|
* Lifecycle hooks around the agent loop. Each may observe, and (where noted)
|
|
241
284
|
* mutate or short-circuit. All may be async.
|
|
242
285
|
*/
|
|
286
|
+
/** The `beforeLLM` hook event. */
|
|
287
|
+
export interface BeforeLLMEvent {
|
|
288
|
+
messages: any[]
|
|
289
|
+
tools: any[]
|
|
290
|
+
model: string
|
|
291
|
+
turn: number
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** What a `beforeLLM` hook may return. `model` (non-empty) applies to that turn only. */
|
|
295
|
+
export interface LLMOverride {
|
|
296
|
+
messages?: any[]
|
|
297
|
+
tools?: any[]
|
|
298
|
+
model?: string | null
|
|
299
|
+
}
|
|
300
|
+
|
|
243
301
|
export interface Hooks {
|
|
244
|
-
/** Before each model call. Return { messages?, tools? } to replace them
|
|
245
|
-
|
|
302
|
+
/** Before each model call. Return { messages?, tools? } to replace them, and { model } to send a
|
|
303
|
+
* different model for THIS turn only (request body + afterLLM event); absent/null/empty keeps the
|
|
304
|
+
* configured model verbatim (SPEC §8, e.g. a ModelRouterClassifier). */
|
|
305
|
+
beforeLLM?(ev: BeforeLLMEvent): void | LLMOverride | Promise<void | LLMOverride>
|
|
246
306
|
/** After each model call (observe: logging, cost, tracing). */
|
|
247
307
|
afterLLM?(ev: { response: any; model: string; turn: number }): void | Promise<void>
|
|
248
308
|
/** Before a tool runs. Return { result } to SHORT-CIRCUIT (deny/cache), { args } to rewrite. */
|
|
249
|
-
beforeTool?(ev: { name: string; args: Record<string, unknown>; id?: string; turn: number }): void | { args?: Record<string, unknown>; result?: ToolResult } | Promise<void | { args?: Record<string, unknown>; result?: ToolResult }>
|
|
309
|
+
beforeTool?(ev: { name: string; args: Record<string, unknown>; id?: string; turn: number; hints?: ToolHints }): void | { args?: Record<string, unknown>; result?: ToolResult } | Promise<void | { args?: Record<string, unknown>; result?: ToolResult }>
|
|
250
310
|
/** After a tool runs. Return { result } to replace the result. */
|
|
251
311
|
afterTool?(ev: { name: string; args: Record<string, unknown>; result: ToolResult; id?: string; turn: number }): void | { result?: ToolResult } | Promise<void | { result?: ToolResult }>
|
|
252
312
|
}
|
|
@@ -265,6 +325,14 @@ export interface Usage {
|
|
|
265
325
|
promptTokens: number
|
|
266
326
|
completionTokens: number
|
|
267
327
|
totalTokens: number
|
|
328
|
+
/** Prompt tokens served from the provider's prompt cache (anthropic cache_read_input_tokens;
|
|
329
|
+
* openai prompt_tokens_details.cached_tokens). Present once a provider reports any.
|
|
330
|
+
* Under anthropic `promptTokens` EXCLUDES cached tokens: the context sent is
|
|
331
|
+
* promptTokens + cacheReadTokens + cacheWriteTokens. Under openai `promptTokens` already
|
|
332
|
+
* includes cacheReadTokens. */
|
|
333
|
+
cacheReadTokens?: number
|
|
334
|
+
/** Tokens written to the prompt cache (anthropic cache_creation_input_tokens). */
|
|
335
|
+
cacheWriteTokens?: number
|
|
268
336
|
}
|
|
269
337
|
|
|
270
338
|
export interface RunResult {
|
|
@@ -293,19 +361,25 @@ export interface RunResult {
|
|
|
293
361
|
/** Events yielded by client.stream(). */
|
|
294
362
|
export type StreamEvent =
|
|
295
363
|
| { type: "text"; delta: string }
|
|
296
|
-
| { type: "tool_call"; id: string; name: string; args: Record<string, unknown
|
|
297
|
-
| { type: "tool_result"; id: string; name: string; output: string; isError: boolean }
|
|
364
|
+
| { type: "tool_call"; id: string; name: string; args: Record<string, unknown>; via?: "call_tool" }
|
|
365
|
+
| { type: "tool_result"; id: string; name: string; output: string; isError: boolean; metadata?: Record<string, unknown>; via?: "call_tool" }
|
|
298
366
|
| { type: "usage"; usage: Usage }
|
|
299
367
|
| { type: "pending"; request: Request }
|
|
300
368
|
| { type: "done"; result: RunResult }
|
|
301
369
|
|
|
302
370
|
function addUsage(acc: Usage, raw: any, style: ClientStyle): void {
|
|
303
371
|
if (!raw) return
|
|
372
|
+
const cache = (key: "cacheReadTokens" | "cacheWriteTokens", n: unknown) => {
|
|
373
|
+
if (typeof n === "number" && n > 0) acc[key] = (acc[key] ?? 0) + n
|
|
374
|
+
}
|
|
304
375
|
if (style === "anthropic") {
|
|
305
376
|
acc.promptTokens += raw.input_tokens ?? 0
|
|
306
377
|
acc.completionTokens += raw.output_tokens ?? 0
|
|
307
378
|
acc.totalTokens += (raw.input_tokens ?? 0) + (raw.output_tokens ?? 0)
|
|
379
|
+
cache("cacheReadTokens", raw.cache_read_input_tokens)
|
|
380
|
+
cache("cacheWriteTokens", raw.cache_creation_input_tokens)
|
|
308
381
|
} else {
|
|
382
|
+
cache("cacheReadTokens", raw.prompt_tokens_details?.cached_tokens)
|
|
309
383
|
acc.promptTokens += raw.prompt_tokens ?? 0
|
|
310
384
|
acc.completionTokens += raw.completion_tokens ?? 0
|
|
311
385
|
acc.totalTokens += raw.total_tokens ?? (raw.prompt_tokens ?? 0) + (raw.completion_tokens ?? 0)
|
|
@@ -319,18 +393,43 @@ function perCall(raw: any, style: ClientStyle): { prompt: number; completion: nu
|
|
|
319
393
|
return { prompt: raw.prompt_tokens ?? 0, completion: raw.completion_tokens ?? 0 }
|
|
320
394
|
}
|
|
321
395
|
|
|
322
|
-
|
|
396
|
+
/**
|
|
397
|
+
* Pick the API key (SPEC 4C, security). An explicit `apiKey` is used as-is; else `apiKeyEnv`;
|
|
398
|
+
* else ONLY the env var that belongs to the `baseUrl` host. There is no cross-provider fallback:
|
|
399
|
+
* any other host needs `apiKey` or `apiKeyEnv`. The value is never logged.
|
|
400
|
+
*/
|
|
401
|
+
export function resolveKey(opts: Pick<ClientOptions, "apiKey" | "apiKeyEnv" | "baseUrl">): string {
|
|
323
402
|
if (opts.apiKey) return opts.apiKey
|
|
324
|
-
const
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
403
|
+
const envName = opts.apiKeyEnv || keyEnvForBaseUrl(opts.baseUrl)
|
|
404
|
+
if (!envName) {
|
|
405
|
+
throw new Error(
|
|
406
|
+
`No API key: set apiKey or apiKeyEnv (provider env vars are only used for https://api.openai.com, https://api.anthropic.com and https://openrouter.ai, never for ${JSON.stringify(opts.baseUrl)})`,
|
|
407
|
+
)
|
|
408
|
+
}
|
|
409
|
+
const key = process.env[envName]
|
|
410
|
+
if (!key) throw new Error(`No API key: set apiKey or ${envName}`)
|
|
331
411
|
return key
|
|
332
412
|
}
|
|
333
413
|
|
|
414
|
+
/** The env var whose key may be sent to `baseUrl`'s host, or "" when none may. */
|
|
415
|
+
export function keyEnvForBaseUrl(baseUrl: string): string {
|
|
416
|
+
const h = hostOf(baseUrl)
|
|
417
|
+
if (h === "api.openai.com") return "OPENAI_API_KEY"
|
|
418
|
+
if (h === "api.anthropic.com") return "ANTHROPIC_API_KEY"
|
|
419
|
+
if (h === "openrouter.ai" || h.endsWith(".openrouter.ai")) return "OPENROUTER_API_KEY"
|
|
420
|
+
return ""
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
function hostOf(baseUrl: string): string {
|
|
424
|
+
try {
|
|
425
|
+
const u = new URL(baseUrl)
|
|
426
|
+
if (u.protocol !== "https:") return "" // a provider key is never sent over plain http
|
|
427
|
+
return u.hostname.toLowerCase()
|
|
428
|
+
} catch {
|
|
429
|
+
return ""
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
|
|
334
433
|
// ---- Prometheus metrics registry (zero-dependency; §8 Observability) ----
|
|
335
434
|
|
|
336
435
|
/** FIXED across all ports for byte-parity of `..._duration_seconds` histograms. Seconds. */
|
|
@@ -438,6 +537,8 @@ export class Client {
|
|
|
438
537
|
private readonly store: ConversationStore
|
|
439
538
|
/** Cumulative Prometheus registry fed by the same events `onMetric` sees. */
|
|
440
539
|
private readonly registry = new MetricsRegistry()
|
|
540
|
+
/** The frozen ToolTiers plan per toolkit (SPEC 8 Tool tiers). */
|
|
541
|
+
private tierPlans = new WeakMap<Toolkit, ToolPlan>()
|
|
441
542
|
constructor(private readonly opts: ClientOptions) {
|
|
442
543
|
this.store = opts.store ?? new InMemoryConversationStore()
|
|
443
544
|
}
|
|
@@ -457,6 +558,7 @@ export class Client {
|
|
|
457
558
|
* every LLM body-build site so the ordering contract holds identically on all four paths.
|
|
458
559
|
*/
|
|
459
560
|
private finalizeBody(body: Record<string, any>): Record<string, any> {
|
|
561
|
+
if (this.opts.promptCache) applyPromptCache(body, this.opts.style, this.opts.promptCache)
|
|
460
562
|
const rp = this.opts.requestParams
|
|
461
563
|
if (rp) {
|
|
462
564
|
for (const [k, v] of Object.entries(rp)) {
|
|
@@ -476,8 +578,8 @@ export class Client {
|
|
|
476
578
|
|
|
477
579
|
/** §8 Gap 5. OpenAI-style body; omits tools/tool_choice when the tool list is empty. Key order
|
|
478
580
|
* matches the pre-change body so a no-options call is byte-identical. */
|
|
479
|
-
private openaiBody(messages: any[], tools: any[], stream: boolean): Record<string, any> {
|
|
480
|
-
const b: Record<string, any> = { model
|
|
581
|
+
private openaiBody(messages: any[], tools: any[], stream: boolean, model: string = this.opts.model): Record<string, any> {
|
|
582
|
+
const b: Record<string, any> = { model, messages: toOpenAIWire(messages, this.wireOpts()) }
|
|
481
583
|
if (tools.length) {
|
|
482
584
|
b.tools = tools
|
|
483
585
|
b.tool_choice = "auto"
|
|
@@ -490,8 +592,8 @@ export class Client {
|
|
|
490
592
|
}
|
|
491
593
|
|
|
492
594
|
/** §8 Gap 5. Anthropic-style body; omits tools when the tool list is empty. */
|
|
493
|
-
private anthropicBody(system: string, messages: any[], tools: any[], stream: boolean): Record<string, any> {
|
|
494
|
-
const b: Record<string, any> = { model
|
|
595
|
+
private anthropicBody(system: string, messages: any[], tools: any[], stream: boolean, model: string = this.opts.model): Record<string, any> {
|
|
596
|
+
const b: Record<string, any> = { model, max_tokens: 4096, system, messages: toAnthropicWire(messages, this.wireOpts()) }
|
|
495
597
|
if (tools.length) b.tools = tools
|
|
496
598
|
if (stream) b.stream = true
|
|
497
599
|
return this.finalizeBody(b)
|
|
@@ -515,14 +617,14 @@ export class Client {
|
|
|
515
617
|
|
|
516
618
|
/** Emit the terminal `run` metric event and build the RunResult. `exhausted` marks a
|
|
517
619
|
* maxTurns loop exit — with no final text that is a LOUD "incomplete", never "done" (§7D). */
|
|
518
|
-
private endRun(runStart: number, text: string, messages: any[], toolCalls: ToolCallRecord[], turns: number, usage: Usage, exhausted = false): RunResult {
|
|
519
|
-
this.emit({ event: "run", model
|
|
520
|
-
return this.result(text, messages, toolCalls, turns, usage, exhausted)
|
|
620
|
+
private endRun(runStart: number, model: string, text: string, messages: any[], toolCalls: ToolCallRecord[], turns: number, usage: Usage, exhausted = false): RunResult {
|
|
621
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart })
|
|
622
|
+
return this.result(model, text, messages, toolCalls, turns, usage, exhausted)
|
|
521
623
|
}
|
|
522
624
|
|
|
523
625
|
/** Emit a `run` error metric event (once, on a thrown run). */
|
|
524
|
-
private emitRunError(runStart: number, toolCalls: ToolCallRecord[], turns: number, usage: Usage, e: unknown): void {
|
|
525
|
-
this.emit({ event: "run", model
|
|
626
|
+
private emitRunError(runStart: number, model: string, toolCalls: ToolCallRecord[], turns: number, usage: Usage, e: unknown): void {
|
|
627
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart, error: e instanceof Error ? e.message : String(e) })
|
|
526
628
|
}
|
|
527
629
|
|
|
528
630
|
/**
|
|
@@ -563,13 +665,15 @@ export class Client {
|
|
|
563
665
|
if (sys && !hasSystemMessage(messages)) messages = [{ role: "system", content: sys }, ...messages]
|
|
564
666
|
|
|
565
667
|
let tools = declared
|
|
668
|
+
let turnModel = this.opts.model
|
|
566
669
|
if (this.opts.hooks?.beforeLLM) {
|
|
567
670
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn: 0 })
|
|
568
671
|
if (ov?.messages) messages = ov.messages
|
|
569
672
|
if (ov?.tools) tools = ov.tools
|
|
673
|
+
if (ov?.model) turnModel = ov.model
|
|
570
674
|
}
|
|
571
675
|
|
|
572
|
-
const body: Record<string, any> = { model:
|
|
676
|
+
const body: Record<string, any> = { model: turnModel, messages }
|
|
573
677
|
if (tools.length) body.tools = tools
|
|
574
678
|
if (req.toolChoice !== undefined) body.tool_choice = req.toolChoice
|
|
575
679
|
if (req.maxTokens) body.max_tokens = req.maxTokens
|
|
@@ -585,8 +689,9 @@ export class Client {
|
|
|
585
689
|
},
|
|
586
690
|
signal,
|
|
587
691
|
"openai",
|
|
692
|
+
turnModel,
|
|
588
693
|
)
|
|
589
|
-
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model:
|
|
694
|
+
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn: 0 })
|
|
590
695
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
591
696
|
addUsage(usage, data.usage, "openai")
|
|
592
697
|
const choice = data.choices?.[0]
|
|
@@ -600,7 +705,7 @@ export class Client {
|
|
|
600
705
|
toolCalls,
|
|
601
706
|
finishReason: choice?.finish_reason ?? finishReasonFor(toolCalls.length > 0),
|
|
602
707
|
usage,
|
|
603
|
-
model:
|
|
708
|
+
model: turnModel,
|
|
604
709
|
raw: data,
|
|
605
710
|
}
|
|
606
711
|
} finally {
|
|
@@ -622,14 +727,16 @@ export class Client {
|
|
|
622
727
|
...openAIToolsToAnthropic(req.tools),
|
|
623
728
|
]
|
|
624
729
|
|
|
730
|
+
let turnModel = this.opts.model
|
|
625
731
|
if (this.opts.hooks?.beforeLLM) {
|
|
626
732
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn: 0 })
|
|
627
733
|
if (ov?.messages) messages = ov.messages
|
|
628
734
|
if (ov?.tools) tools = ov.tools
|
|
735
|
+
if (ov?.model) turnModel = ov.model
|
|
629
736
|
}
|
|
630
737
|
|
|
631
738
|
const body: Record<string, any> = {
|
|
632
|
-
model:
|
|
739
|
+
model: turnModel,
|
|
633
740
|
max_tokens: req.maxTokens && req.maxTokens > 0 ? req.maxTokens : 4096,
|
|
634
741
|
messages,
|
|
635
742
|
}
|
|
@@ -654,8 +761,9 @@ export class Client {
|
|
|
654
761
|
},
|
|
655
762
|
signal,
|
|
656
763
|
"anthropic",
|
|
764
|
+
turnModel,
|
|
657
765
|
)
|
|
658
|
-
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model:
|
|
766
|
+
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn: 0 })
|
|
659
767
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
660
768
|
addUsage(usage, data.usage, "anthropic")
|
|
661
769
|
const text: string[] = []
|
|
@@ -675,7 +783,7 @@ export class Client {
|
|
|
675
783
|
toolCalls,
|
|
676
784
|
finishReason: finishReasonFor(toolCalls.length > 0, data.stop_reason),
|
|
677
785
|
usage,
|
|
678
|
-
model:
|
|
786
|
+
model: turnModel,
|
|
679
787
|
raw: data,
|
|
680
788
|
}
|
|
681
789
|
} finally {
|
|
@@ -713,9 +821,22 @@ export class Client {
|
|
|
713
821
|
return result!
|
|
714
822
|
}
|
|
715
823
|
if (!ctx?.id) return this.run(prompt, { toolkit: ctx?.toolkit, signal: ctx?.signal })
|
|
716
|
-
|
|
717
|
-
const
|
|
718
|
-
await this.store.
|
|
824
|
+
checkPromptParts(prompt, this.wireOpts())
|
|
825
|
+
const id = ctx.id
|
|
826
|
+
const history = (await this.store.get(id)) ?? []
|
|
827
|
+
// §8: an interrupted turn is saved too — the loop hands back its transcript, repaired.
|
|
828
|
+
let partial: any[] | undefined
|
|
829
|
+
const onInterrupt: OnInterrupt = (m) => { partial = m }
|
|
830
|
+
let result: RunResult
|
|
831
|
+
try {
|
|
832
|
+
result = this.opts.style === "anthropic"
|
|
833
|
+
? await this.runAnthropic(prompt, ctx.toolkit, ctx.signal, history, onInterrupt)
|
|
834
|
+
: await this.runOpenAI(prompt, ctx.toolkit, ctx.signal, history, onInterrupt)
|
|
835
|
+
} catch (e) {
|
|
836
|
+
if (partial) await this.store.save(id, partial)
|
|
837
|
+
throw e
|
|
838
|
+
}
|
|
839
|
+
await this.store.save(id, result.messages)
|
|
719
840
|
return result
|
|
720
841
|
}
|
|
721
842
|
|
|
@@ -732,12 +853,20 @@ export class Client {
|
|
|
732
853
|
async *stream(prompt: PromptInput, ctx?: { toolkit?: Toolkit; id?: string; signal?: AbortSignal }): AsyncGenerator<StreamEvent, void, unknown> {
|
|
733
854
|
checkPromptParts(prompt, this.wireOpts())
|
|
734
855
|
const history = ctx?.id ? (await this.store.get(ctx.id)) ?? [] : undefined
|
|
856
|
+
// §8: with an id, an interrupted stream saves its repaired transcript before the error surfaces.
|
|
857
|
+
let partial: any[] | undefined
|
|
858
|
+
const onInterrupt: OnInterrupt | undefined = ctx?.id ? (m) => { partial = m } : undefined
|
|
735
859
|
const gen = this.opts.style === "anthropic"
|
|
736
|
-
? this.streamAnthropic(prompt, ctx?.toolkit, ctx?.signal, history)
|
|
737
|
-
: this.streamOpenAI(prompt, ctx?.toolkit, ctx?.signal, history)
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
860
|
+
? this.streamAnthropic(prompt, ctx?.toolkit, ctx?.signal, history, onInterrupt)
|
|
861
|
+
: this.streamOpenAI(prompt, ctx?.toolkit, ctx?.signal, history, onInterrupt)
|
|
862
|
+
try {
|
|
863
|
+
for await (const ev of gen) {
|
|
864
|
+
if (ev.type === "done" && ctx?.id) await this.store.save(ctx.id, ev.result.messages)
|
|
865
|
+
yield ev
|
|
866
|
+
}
|
|
867
|
+
} catch (e) {
|
|
868
|
+
if (partial && ctx?.id) await this.store.save(ctx.id, partial)
|
|
869
|
+
throw e
|
|
741
870
|
}
|
|
742
871
|
}
|
|
743
872
|
|
|
@@ -811,13 +940,33 @@ export class Client {
|
|
|
811
940
|
}
|
|
812
941
|
|
|
813
942
|
/** Run one tool through beforeTool/afterTool hooks (mutate args, short-circuit, transform result). */
|
|
814
|
-
private async runTool(toolkit: Toolkit | undefined, name: string, args: Record<string, unknown>, id: string | undefined, turn: number) {
|
|
943
|
+
private async runTool(toolkit: Toolkit | undefined, name: string, args: Record<string, unknown>, id: string | undefined, turn: number, signal?: AbortSignal) {
|
|
944
|
+
if (this.opts.toolTiers && name === "tool_search") {
|
|
945
|
+
// Engine-internal and read-only: no hooks (SPEC 8 Tool tiers).
|
|
946
|
+
const t0 = Date.now()
|
|
947
|
+
const result = toolSearch(toolkit?.tools() ?? [], (n) => toolkit?.get(n), args ?? {})
|
|
948
|
+
this.emitTool(name, "builtin", result, t0)
|
|
949
|
+
return { args, result }
|
|
950
|
+
}
|
|
951
|
+
// call_tool is unwrapped HERE, so hooks, hints and permission gates see the real tool. The
|
|
952
|
+
// returned args keep the wrapper shape (with any beforeTool rewrite inside), so a 10
|
|
953
|
+
// re-run unwraps the same way.
|
|
954
|
+
const inner = this.unwrapCall(name, args)
|
|
955
|
+
if (inner) {
|
|
956
|
+
const r = await this.runToolDirect(toolkit, inner.name, inner.args, id, turn, signal)
|
|
957
|
+
return { args: { name: inner.name, arguments: r.args }, result: r.result }
|
|
958
|
+
}
|
|
959
|
+
return this.runToolDirect(toolkit, name, args, id, turn, signal)
|
|
960
|
+
}
|
|
961
|
+
|
|
962
|
+
private async runToolDirect(toolkit: Toolkit | undefined, name: string, args: Record<string, unknown>, id: string | undefined, turn: number, signal?: AbortSignal) {
|
|
815
963
|
const h = this.opts.hooks
|
|
816
|
-
const
|
|
964
|
+
const tool = toolkit?.get(name)
|
|
965
|
+
const source = tool?.source ?? "custom"
|
|
817
966
|
const t0 = Date.now()
|
|
818
967
|
let a = args
|
|
819
968
|
if (h?.beforeTool) {
|
|
820
|
-
const ov = await h.beforeTool({ name, args: a, id, turn })
|
|
969
|
+
const ov = await h.beforeTool({ name, args: a, id, turn, ...(tool?.hints ? { hints: tool.hints } : {}) })
|
|
821
970
|
if (ov?.result) {
|
|
822
971
|
// short-circuit (deny/cache/dry-run, or a guard-raised suspension — path B); still a tool
|
|
823
972
|
// call for metrics, but a suspension is classified pending, never an error.
|
|
@@ -828,7 +977,7 @@ export class Client {
|
|
|
828
977
|
}
|
|
829
978
|
// No toolkit ⇒ nothing was declared, so a tool call is the provider inventing one.
|
|
830
979
|
let result: ToolResult = toolkit
|
|
831
|
-
? await toolkit.execute(name, a)
|
|
980
|
+
? await toolkit.execute(name, a, { ...(signal ? { signal } : {}), ...(id ? { callId: id } : {}) })
|
|
832
981
|
: { output: `unknown tool: ${name}`, isError: true }
|
|
833
982
|
// A suspension (§10) is not a real result: skip afterTool's failure path on it — the resolved
|
|
834
983
|
// result (post-waitFor) still flows through afterTool in resolvePending — and never count it as
|
|
@@ -841,6 +990,63 @@ export class Client {
|
|
|
841
990
|
return { args: a, result }
|
|
842
991
|
}
|
|
843
992
|
|
|
993
|
+
/** ToolTiers `call_tool {name, arguments}` -> the real call; undefined when tiers are off. */
|
|
994
|
+
private unwrapCall(name: string, args: Record<string, unknown>): { name: string; args: Record<string, unknown> } | undefined {
|
|
995
|
+
return this.opts.toolTiers ? unwrapCallTool(name, args) : undefined
|
|
996
|
+
}
|
|
997
|
+
|
|
998
|
+
/** The stream's tool_call event, naming the real tool for a `call_tool` call. */
|
|
999
|
+
private callEvent(id: string, name: string, args: Record<string, unknown>): StreamEvent {
|
|
1000
|
+
const inner = this.unwrapCall(name, args)
|
|
1001
|
+
return inner ? { type: "tool_call", id, name: inner.name, args: inner.args, via: "call_tool" } : { type: "tool_call", id, name, args }
|
|
1002
|
+
}
|
|
1003
|
+
|
|
1004
|
+
private resultEvent(id: string, name: string, args: Record<string, unknown>, result: ToolResult): StreamEvent {
|
|
1005
|
+
const inner = this.unwrapCall(name, args)
|
|
1006
|
+
return {
|
|
1007
|
+
type: "tool_result", id, name: inner ? inner.name : name, output: result.output, isError: result.isError,
|
|
1008
|
+
...(result.metadata ? { metadata: result.metadata } : {}),
|
|
1009
|
+
...(inner ? { via: "call_tool" as const } : {}),
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
private plan(toolkit: Toolkit | undefined): ToolPlan | undefined {
|
|
1014
|
+
if (!this.opts.toolTiers || !toolkit) return undefined
|
|
1015
|
+
let p = this.tierPlans.get(toolkit)
|
|
1016
|
+
if (!p) {
|
|
1017
|
+
p = buildToolPlan(this.opts.toolTiers, toolkit.tools())
|
|
1018
|
+
this.tierPlans.set(toolkit, p)
|
|
1019
|
+
}
|
|
1020
|
+
return p
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/** The tools array for a request: core tools (sorted) then tool_search and call_tool. Without
|
|
1024
|
+
* tiers it is every tool, as before. */
|
|
1025
|
+
private requestTools(toolkit: Toolkit | undefined): Tool[] {
|
|
1026
|
+
if (!toolkit) return []
|
|
1027
|
+
const p = this.plan(toolkit)
|
|
1028
|
+
if (!p) return toolkit.tools()
|
|
1029
|
+
const out: Tool[] = []
|
|
1030
|
+
for (const n of p.core) {
|
|
1031
|
+
const t = toolkit.get(n)
|
|
1032
|
+
if (t) out.push(t)
|
|
1033
|
+
}
|
|
1034
|
+
return [...out, p.search, p.call]
|
|
1035
|
+
}
|
|
1036
|
+
|
|
1037
|
+
/** The frozen tier plan for `toolkit` (built on first use): tool name -> tier. Undefined when
|
|
1038
|
+
* `toolTiers` is not set. For a host UI or log. */
|
|
1039
|
+
toolTierPlan(toolkit: Toolkit): Record<string, ToolTier> | undefined {
|
|
1040
|
+
const p = this.plan(toolkit)
|
|
1041
|
+
return p ? Object.fromEntries(p.tier) : undefined
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
/** Forget `toolkit`'s plan so the next request rebuilds it - call at a new session start.
|
|
1045
|
+
* Mid-session it breaks the cached prefix. */
|
|
1046
|
+
resetToolTiers(toolkit: Toolkit): void {
|
|
1047
|
+
this.tierPlans.delete(toolkit)
|
|
1048
|
+
}
|
|
1049
|
+
|
|
844
1050
|
/** Emit the `tool` metric, classifying a §10 suspension as `pending` (isError:false), not a failure. */
|
|
845
1051
|
private emitTool(name: string, source: string, result: ToolResult, t0: number) {
|
|
846
1052
|
const req = pendingOf(result)
|
|
@@ -848,61 +1054,61 @@ export class Client {
|
|
|
848
1054
|
}
|
|
849
1055
|
|
|
850
1056
|
/** One non-streaming LLM call, with an `llm` metric event (ok/error + per-call tokens + ms). */
|
|
851
|
-
private async llmCallJson(url: string, init: RequestInit, signal: AbortSignal, style: ClientStyle): Promise<any> {
|
|
1057
|
+
private async llmCallJson(url: string, init: RequestInit, signal: AbortSignal, style: ClientStyle, model: string): Promise<any> {
|
|
852
1058
|
const t0 = Date.now()
|
|
853
1059
|
try {
|
|
854
1060
|
const res = await this.llmFetch(url, init, signal)
|
|
855
1061
|
if (!res.ok) throw await llmHttpError(res)
|
|
856
1062
|
const data: any = await res.json()
|
|
857
1063
|
const tok = perCall(data.usage, style)
|
|
858
|
-
this.emit({ event: "llm", model
|
|
1064
|
+
this.emit({ event: "llm", model, status: "ok", ms: Date.now() - t0, promptTokens: tok.prompt, completionTokens: tok.completion })
|
|
859
1065
|
return data
|
|
860
1066
|
} catch (e) {
|
|
861
|
-
this.emit({ event: "llm", model
|
|
1067
|
+
this.emit({ event: "llm", model, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 })
|
|
862
1068
|
throw e
|
|
863
1069
|
}
|
|
864
1070
|
}
|
|
865
1071
|
|
|
866
1072
|
// The run-scoped timeout timer is owned by these four wrappers, so it is cleared however the
|
|
867
1073
|
// run leaves: return, throw, or a consumer abandoning the stream half-way.
|
|
868
|
-
private async runOpenAI(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[]): Promise<RunResult> {
|
|
1074
|
+
private async runOpenAI(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[], onInterrupt?: OnInterrupt): Promise<RunResult> {
|
|
869
1075
|
const signal = this.makeSignal(external)
|
|
870
1076
|
try {
|
|
871
|
-
return await this.runOpenAIWithSignal(signal, prompt, toolkit, history)
|
|
1077
|
+
return await this.runOpenAIWithSignal(signal, prompt, toolkit, history, onInterrupt)
|
|
872
1078
|
} finally {
|
|
873
1079
|
this.releaseSignal(signal)
|
|
874
1080
|
}
|
|
875
1081
|
}
|
|
876
1082
|
|
|
877
|
-
private async runAnthropic(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[]): Promise<RunResult> {
|
|
1083
|
+
private async runAnthropic(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[], onInterrupt?: OnInterrupt): Promise<RunResult> {
|
|
878
1084
|
const signal = this.makeSignal(external)
|
|
879
1085
|
try {
|
|
880
|
-
return await this.runAnthropicWithSignal(signal, prompt, toolkit, history)
|
|
1086
|
+
return await this.runAnthropicWithSignal(signal, prompt, toolkit, history, onInterrupt)
|
|
881
1087
|
} finally {
|
|
882
1088
|
this.releaseSignal(signal)
|
|
883
1089
|
}
|
|
884
1090
|
}
|
|
885
1091
|
|
|
886
|
-
private async *streamOpenAI(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[]): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1092
|
+
private async *streamOpenAI(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[], onInterrupt?: OnInterrupt): AsyncGenerator<StreamEvent, void, unknown> {
|
|
887
1093
|
const signal = this.makeSignal(external)
|
|
888
1094
|
try {
|
|
889
|
-
yield* this.streamOpenAIWithSignal(signal, prompt, toolkit, history)
|
|
1095
|
+
yield* this.streamOpenAIWithSignal(signal, prompt, toolkit, history, onInterrupt)
|
|
890
1096
|
} finally {
|
|
891
1097
|
this.releaseSignal(signal)
|
|
892
1098
|
}
|
|
893
1099
|
}
|
|
894
1100
|
|
|
895
|
-
private async *streamAnthropic(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[]): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1101
|
+
private async *streamAnthropic(prompt: PromptInput, toolkit: Toolkit | undefined, external?: AbortSignal, history?: any[], onInterrupt?: OnInterrupt): AsyncGenerator<StreamEvent, void, unknown> {
|
|
896
1102
|
const signal = this.makeSignal(external)
|
|
897
1103
|
try {
|
|
898
|
-
yield* this.streamAnthropicWithSignal(signal, prompt, toolkit, history)
|
|
1104
|
+
yield* this.streamAnthropicWithSignal(signal, prompt, toolkit, history, onInterrupt)
|
|
899
1105
|
} finally {
|
|
900
1106
|
this.releaseSignal(signal)
|
|
901
1107
|
}
|
|
902
1108
|
}
|
|
903
1109
|
|
|
904
1110
|
// ---- OpenAI-style: POST {baseUrl}/chat/completions ----
|
|
905
|
-
private async runOpenAIWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[]): Promise<RunResult> {
|
|
1111
|
+
private async runOpenAIWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[], onInterrupt?: OnInterrupt): Promise<RunResult> {
|
|
906
1112
|
const key = resolveKey(this.opts)
|
|
907
1113
|
const runStart = Date.now()
|
|
908
1114
|
let messages: any[] = history && history.length ? [...history] : []
|
|
@@ -911,71 +1117,81 @@ export class Client {
|
|
|
911
1117
|
if (system) messages.push({ role: "system", content: system })
|
|
912
1118
|
}
|
|
913
1119
|
messages.push({ role: "user", content: prompt })
|
|
914
|
-
let tools =
|
|
1120
|
+
let tools = toOpenAI(this.requestTools(toolkit))
|
|
915
1121
|
const toolCalls: ToolCallRecord[] = []
|
|
916
1122
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
917
1123
|
let turns = 0
|
|
1124
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1125
|
+
let lastModel = this.opts.model
|
|
1126
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
1127
|
+
const done = new Map<string, ToolResult>()
|
|
918
1128
|
|
|
919
1129
|
try {
|
|
920
1130
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
921
1131
|
turns++
|
|
1132
|
+
let turnModel = this.opts.model
|
|
922
1133
|
if (this.opts.hooks?.beforeLLM) {
|
|
923
1134
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn })
|
|
924
1135
|
if (ov?.messages) messages = ov.messages
|
|
925
1136
|
if (ov?.tools) tools = ov.tools
|
|
1137
|
+
if (ov?.model) turnModel = ov.model
|
|
926
1138
|
}
|
|
1139
|
+
lastModel = turnModel
|
|
927
1140
|
const data: any = await this.llmCallJson(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
928
1141
|
method: "POST",
|
|
929
1142
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
|
|
930
|
-
body: JSON.stringify(this.openaiBody(messages, tools, false)),
|
|
931
|
-
}, signal, "openai")
|
|
1143
|
+
body: JSON.stringify(this.openaiBody(messages, tools, false, turnModel)),
|
|
1144
|
+
}, signal, "openai", turnModel)
|
|
932
1145
|
addUsage(usage, data.usage, "openai")
|
|
933
|
-
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model:
|
|
1146
|
+
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn })
|
|
934
1147
|
const msg = data.choices[0].message
|
|
935
1148
|
messages.push(msg)
|
|
936
1149
|
const calls = msg.tool_calls ?? []
|
|
937
|
-
if (calls.length === 0) return this.endRun(runStart, msg.content ?? "", messages, toolCalls, turns, usage)
|
|
1150
|
+
if (calls.length === 0) return this.endRun(runStart, lastModel, msg.content ?? "", messages, toolCalls, turns, usage)
|
|
938
1151
|
// execute all tool calls in this turn concurrently (true parallel tool calling)
|
|
939
|
-
|
|
1152
|
+
done.clear()
|
|
1153
|
+
const settled = await untilAborted(signal, Promise.all(
|
|
940
1154
|
calls.map(async (call: any) => {
|
|
941
|
-
let { args, result } = await this.runTool(toolkit, call.function.name, safeJson(call.function.arguments), call.id, turn)
|
|
1155
|
+
let { args, result } = await this.runTool(toolkit, call.function.name, safeJson(call.function.arguments), call.id, turn, signal)
|
|
942
1156
|
let halted: Request | undefined
|
|
943
1157
|
const req = pendingOf(result)
|
|
944
1158
|
if (req) {
|
|
945
|
-
const r = await this.resolvePending(toolkit, call.function.name, args, req, call.id, turn)
|
|
1159
|
+
const r = await this.resolvePending(toolkit, call.function.name, args, req, call.id, turn, signal)
|
|
946
1160
|
result = r.result
|
|
947
1161
|
halted = r.halted
|
|
948
1162
|
}
|
|
1163
|
+
if (!halted && call.id) done.set(call.id, result)
|
|
949
1164
|
return { call, args, result, halted }
|
|
950
1165
|
}),
|
|
951
|
-
)
|
|
1166
|
+
))
|
|
952
1167
|
// Record in tool-call order; on the FIRST durable halt, surface it and stop — deterministic,
|
|
953
1168
|
// and the later suspensions' placeholder results never enter the transcript (they re-suspend
|
|
954
1169
|
// on resume). Mirrors the streaming path.
|
|
955
1170
|
for (const s of settled) {
|
|
956
1171
|
toolCalls.push({ name: s.call.function.name, args: s.args, output: s.result.output, isError: s.result.isError, metadata: s.result.metadata })
|
|
957
1172
|
messages.push(toolMessage(s.call.id, s.call.function.name, s.result))
|
|
958
|
-
if (s.halted) return this.pendingRun(runStart, s.halted, messages, toolCalls, turns, usage)
|
|
1173
|
+
if (s.halted) return this.pendingRun(runStart, lastModel, s.halted, messages, toolCalls, turns, usage)
|
|
959
1174
|
}
|
|
960
1175
|
}
|
|
961
|
-
return this.endRun(runStart, lastText(messages), messages, toolCalls, turns, usage, true)
|
|
1176
|
+
return this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true)
|
|
962
1177
|
} catch (e) {
|
|
963
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e)
|
|
1178
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e)
|
|
1179
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done))
|
|
964
1180
|
throw e
|
|
965
1181
|
}
|
|
966
1182
|
}
|
|
967
1183
|
|
|
968
|
-
private result(text: string, messages: any[], toolCalls: ToolCallRecord[], turns: number, usage: Usage, exhausted = false): RunResult {
|
|
1184
|
+
private result(model: string, text: string, messages: any[], toolCalls: ToolCallRecord[], turns: number, usage: Usage, exhausted = false): RunResult {
|
|
969
1185
|
const incomplete = exhausted && text === ""
|
|
970
|
-
const r: RunResult = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model
|
|
1186
|
+
const r: RunResult = { text, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: incomplete ? "incomplete" : "done" }
|
|
971
1187
|
if (incomplete) r.limit = "maxTurns"
|
|
972
1188
|
return r
|
|
973
1189
|
}
|
|
974
1190
|
|
|
975
1191
|
/** §10: a run halted because a tool suspended and no `waitFor` was configured. */
|
|
976
|
-
private pendingRun(runStart: number, request: Request, messages: any[], toolCalls: ToolCallRecord[], turns: number, usage: Usage): RunResult {
|
|
977
|
-
this.emit({ event: "run", model
|
|
978
|
-
return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model
|
|
1192
|
+
private pendingRun(runStart: number, model: string, request: Request, messages: any[], toolCalls: ToolCallRecord[], turns: number, usage: Usage): RunResult {
|
|
1193
|
+
this.emit({ event: "run", model, turns, toolCalls: toolCalls.length, totalTokens: usage.totalTokens, ms: Date.now() - runStart })
|
|
1194
|
+
return { text: request.prompt, messages, toolCalls, toolCallCount: toolCalls.length, turns, usage, model, status: "pending", pending: request }
|
|
979
1195
|
}
|
|
980
1196
|
|
|
981
1197
|
/**
|
|
@@ -984,14 +1200,16 @@ export class Client {
|
|
|
984
1200
|
* no `waitFor` is configured (the run should stop and surface the request).
|
|
985
1201
|
*/
|
|
986
1202
|
private async resolvePending(
|
|
987
|
-
toolkit: Toolkit | undefined, name: string, args: Record<string, unknown>, request: Request, id: string | undefined, turn: number,
|
|
1203
|
+
toolkit: Toolkit | undefined, name: string, args: Record<string, unknown>, request: Request, id: string | undefined, turn: number, signal?: AbortSignal,
|
|
988
1204
|
): Promise<{ result: ToolResult; halted?: Request }> {
|
|
1205
|
+
const inner = this.unwrapCall(name, args)
|
|
1206
|
+
if (inner) ({ name, args } = inner)
|
|
989
1207
|
if (!this.opts.waitFor) return { result: { output: request.prompt, isError: true }, halted: request }
|
|
990
|
-
const answer = await this.opts.waitFor(request)
|
|
1208
|
+
const answer = await this.opts.waitFor(request, { signal: signal ?? new AbortController().signal })
|
|
991
1209
|
if (!answer?.ok) return { result: { output: `declined/expired: ${request.prompt}`, isError: true } }
|
|
992
1210
|
const t0 = Date.now()
|
|
993
1211
|
let result: ToolResult = toolkit
|
|
994
|
-
? await toolkit.execute(name, args, { answer })
|
|
1212
|
+
? await toolkit.execute(name, args, { answer, ...(signal ? { signal } : {}), ...(id ? { callId: id } : {}) })
|
|
995
1213
|
: { output: `unknown tool: ${name}`, isError: true }
|
|
996
1214
|
if (this.opts.hooks?.afterTool) {
|
|
997
1215
|
const ov = await this.opts.hooks.afterTool({ name, args, result, id, turn })
|
|
@@ -1003,26 +1221,33 @@ export class Client {
|
|
|
1003
1221
|
}
|
|
1004
1222
|
|
|
1005
1223
|
// ---- Anthropic-style: POST {baseUrl}/messages ----
|
|
1006
|
-
private async runAnthropicWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[]): Promise<RunResult> {
|
|
1224
|
+
private async runAnthropicWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[], onInterrupt?: OnInterrupt): Promise<RunResult> {
|
|
1007
1225
|
const key = resolveKey(this.opts)
|
|
1008
1226
|
const base = this.opts.baseUrl.replace(/\/$/, "")
|
|
1009
1227
|
const endpoint = base.endsWith("/v1") ? `${base}/messages` : `${base}/v1/messages`
|
|
1010
1228
|
const system = this.system(toolkit)
|
|
1011
1229
|
const runStart = Date.now()
|
|
1012
1230
|
let messages: any[] = history && history.length ? [...history, { role: "user", content: prompt }] : [{ role: "user", content: prompt }]
|
|
1013
|
-
let tools =
|
|
1231
|
+
let tools = toAnthropic(this.requestTools(toolkit))
|
|
1014
1232
|
const toolCalls: ToolCallRecord[] = []
|
|
1015
1233
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
1016
1234
|
let turns = 0
|
|
1235
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1236
|
+
let lastModel = this.opts.model
|
|
1237
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
1238
|
+
const done = new Map<string, ToolResult>()
|
|
1017
1239
|
|
|
1018
1240
|
try {
|
|
1019
1241
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
1020
1242
|
turns++
|
|
1243
|
+
let turnModel = this.opts.model
|
|
1021
1244
|
if (this.opts.hooks?.beforeLLM) {
|
|
1022
1245
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn })
|
|
1023
1246
|
if (ov?.messages) messages = ov.messages
|
|
1024
1247
|
if (ov?.tools) tools = ov.tools
|
|
1248
|
+
if (ov?.model) turnModel = ov.model
|
|
1025
1249
|
}
|
|
1250
|
+
lastModel = turnModel
|
|
1026
1251
|
const data: any = await this.llmCallJson(endpoint, {
|
|
1027
1252
|
method: "POST",
|
|
1028
1253
|
headers: {
|
|
@@ -1031,30 +1256,32 @@ export class Client {
|
|
|
1031
1256
|
"Content-Type": "application/json",
|
|
1032
1257
|
...this.opts.headers,
|
|
1033
1258
|
},
|
|
1034
|
-
body: JSON.stringify(this.anthropicBody(system, messages, tools, false)),
|
|
1035
|
-
}, signal, "anthropic")
|
|
1259
|
+
body: JSON.stringify(this.anthropicBody(system, messages, tools, false, turnModel)),
|
|
1260
|
+
}, signal, "anthropic", turnModel)
|
|
1036
1261
|
addUsage(usage, data.usage, "anthropic")
|
|
1037
|
-
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model:
|
|
1262
|
+
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: data, model: turnModel, turn })
|
|
1038
1263
|
messages.push({ role: "assistant", content: data.content })
|
|
1039
1264
|
const uses = (data.content ?? []).filter((b: any) => b.type === "tool_use")
|
|
1040
1265
|
if (uses.length === 0) {
|
|
1041
1266
|
const text = (data.content ?? []).filter((b: any) => b.type === "text").map((b: any) => b.text).join("")
|
|
1042
|
-
return this.endRun(runStart, text, messages, toolCalls, turns, usage)
|
|
1267
|
+
return this.endRun(runStart, lastModel, text, messages, toolCalls, turns, usage)
|
|
1043
1268
|
}
|
|
1044
1269
|
// execute all tool_use blocks in this turn concurrently (true parallel tool calling)
|
|
1045
|
-
|
|
1270
|
+
done.clear()
|
|
1271
|
+
const settled = await untilAborted(signal, Promise.all(
|
|
1046
1272
|
uses.map(async (use: any) => {
|
|
1047
|
-
let { args, result } = await this.runTool(toolkit, use.name, use.input ?? {}, use.id, turn)
|
|
1273
|
+
let { args, result } = await this.runTool(toolkit, use.name, use.input ?? {}, use.id, turn, signal)
|
|
1048
1274
|
let halted: Request | undefined
|
|
1049
1275
|
const req = pendingOf(result)
|
|
1050
1276
|
if (req) {
|
|
1051
|
-
const r = await this.resolvePending(toolkit, use.name, args, req, use.id, turn)
|
|
1277
|
+
const r = await this.resolvePending(toolkit, use.name, args, req, use.id, turn, signal)
|
|
1052
1278
|
result = r.result
|
|
1053
1279
|
halted = r.halted
|
|
1054
1280
|
}
|
|
1281
|
+
if (!halted && use.id) done.set(use.id, result)
|
|
1055
1282
|
return { use, args, result, halted }
|
|
1056
1283
|
}),
|
|
1057
|
-
)
|
|
1284
|
+
))
|
|
1058
1285
|
// Record in tool-call order; on the FIRST durable halt, include that tool_result block, stop
|
|
1059
1286
|
// building the content, and surface it — deterministic (G3), later suspensions re-suspend on
|
|
1060
1287
|
// resume. Mirrors the OpenAI path and the streaming loops.
|
|
@@ -1066,17 +1293,18 @@ export class Client {
|
|
|
1066
1293
|
if (s.halted) { halted = s.halted; break }
|
|
1067
1294
|
}
|
|
1068
1295
|
messages.push({ role: "user", content })
|
|
1069
|
-
if (halted) return this.pendingRun(runStart, halted, messages, toolCalls, turns, usage)
|
|
1296
|
+
if (halted) return this.pendingRun(runStart, lastModel, halted, messages, toolCalls, turns, usage)
|
|
1070
1297
|
}
|
|
1071
|
-
return this.endRun(runStart, "", messages, toolCalls, turns, usage, true)
|
|
1298
|
+
return this.endRun(runStart, lastModel, "", messages, toolCalls, turns, usage, true)
|
|
1072
1299
|
} catch (e) {
|
|
1073
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e)
|
|
1300
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e)
|
|
1301
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done))
|
|
1074
1302
|
throw e
|
|
1075
1303
|
}
|
|
1076
1304
|
}
|
|
1077
1305
|
|
|
1078
1306
|
// ---- Streaming: OpenAI-style ----
|
|
1079
|
-
private async *streamOpenAIWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[]): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1307
|
+
private async *streamOpenAIWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[], onInterrupt?: OnInterrupt): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1080
1308
|
const key = resolveKey(this.opts)
|
|
1081
1309
|
const runStart = Date.now()
|
|
1082
1310
|
let messages: any[] = history && history.length ? [...history] : []
|
|
@@ -1085,19 +1313,26 @@ export class Client {
|
|
|
1085
1313
|
if (system) messages.push({ role: "system", content: system })
|
|
1086
1314
|
}
|
|
1087
1315
|
messages.push({ role: "user", content: prompt })
|
|
1088
|
-
let tools =
|
|
1316
|
+
let tools = toOpenAI(this.requestTools(toolkit))
|
|
1089
1317
|
const toolCalls: ToolCallRecord[] = []
|
|
1090
1318
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
1091
1319
|
let turns = 0
|
|
1320
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1321
|
+
let lastModel = this.opts.model
|
|
1322
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
1323
|
+
const done = new Map<string, ToolResult>()
|
|
1092
1324
|
|
|
1093
1325
|
try {
|
|
1094
1326
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
1095
1327
|
turns++
|
|
1328
|
+
let turnModel = this.opts.model
|
|
1096
1329
|
if (this.opts.hooks?.beforeLLM) {
|
|
1097
1330
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn })
|
|
1098
1331
|
if (ov?.messages) messages = ov.messages
|
|
1099
1332
|
if (ov?.tools) tools = ov.tools
|
|
1333
|
+
if (ov?.model) turnModel = ov.model
|
|
1100
1334
|
}
|
|
1335
|
+
lastModel = turnModel
|
|
1101
1336
|
const t0 = Date.now()
|
|
1102
1337
|
const beforeP = usage.promptTokens, beforeC = usage.completionTokens
|
|
1103
1338
|
let content = ""
|
|
@@ -1106,7 +1341,7 @@ export class Client {
|
|
|
1106
1341
|
const res = await this.llmFetch(`${this.opts.baseUrl.replace(/\/$/, "")}/chat/completions`, {
|
|
1107
1342
|
method: "POST",
|
|
1108
1343
|
headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json", ...this.opts.headers },
|
|
1109
|
-
body: JSON.stringify(this.openaiBody(messages, tools, true)),
|
|
1344
|
+
body: JSON.stringify(this.openaiBody(messages, tools, true, turnModel)),
|
|
1110
1345
|
}, signal)
|
|
1111
1346
|
if (!res.ok || !res.body) throw await llmHttpError(res)
|
|
1112
1347
|
for await (const line of sseLines(res.body)) {
|
|
@@ -1127,70 +1362,83 @@ export class Client {
|
|
|
1127
1362
|
}
|
|
1128
1363
|
}
|
|
1129
1364
|
} catch (e) {
|
|
1130
|
-
this.emit({ event: "llm", model:
|
|
1365
|
+
this.emit({ event: "llm", model: turnModel, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 })
|
|
1131
1366
|
throw e
|
|
1132
1367
|
}
|
|
1133
|
-
this.emit({ event: "llm", model:
|
|
1134
|
-
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model:
|
|
1368
|
+
this.emit({ event: "llm", model: turnModel, status: "ok", ms: Date.now() - t0, promptTokens: usage.promptTokens - beforeP, completionTokens: usage.completionTokens - beforeC })
|
|
1369
|
+
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model: turnModel, turn })
|
|
1135
1370
|
|
|
1136
1371
|
const calls = [...acc.values()]
|
|
1137
1372
|
if (calls.length === 0) {
|
|
1138
1373
|
messages.push({ role: "assistant", content })
|
|
1139
1374
|
yield { type: "usage", usage }
|
|
1140
|
-
yield { type: "done", result: this.endRun(runStart, content, messages, toolCalls, turns, usage) }
|
|
1375
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, content, messages, toolCalls, turns, usage) }
|
|
1141
1376
|
return
|
|
1142
1377
|
}
|
|
1143
1378
|
messages.push({ role: "assistant", content: content || null, tool_calls: calls.map((c) => ({ id: c.id, type: "function", function: { name: c.name, arguments: c.args } })) })
|
|
1144
|
-
for (const c of calls) yield
|
|
1145
|
-
|
|
1379
|
+
for (const c of calls) yield this.callEvent(c.id, c.name, safeJson(c.args))
|
|
1380
|
+
done.clear()
|
|
1381
|
+
const settled = await untilAborted(signal, Promise.all(calls.map(async (c) => {
|
|
1382
|
+
const r = await this.runTool(toolkit, c.name, safeJson(c.args), c.id, turn, signal)
|
|
1383
|
+
if (!pendingOf(r.result) && c.id) done.set(c.id, r.result)
|
|
1384
|
+
return r
|
|
1385
|
+
})))
|
|
1146
1386
|
for (let i = 0; i < calls.length; i++) {
|
|
1147
1387
|
const c = calls[i]
|
|
1148
1388
|
let { args, result } = settled[i]
|
|
1149
1389
|
const req = pendingOf(result)
|
|
1150
1390
|
if (req) {
|
|
1151
1391
|
yield { type: "pending", request: req } // surface BEFORE waitFor so a channel can push the link
|
|
1152
|
-
const r = await this.resolvePending(toolkit, c.name, args, req, c.id, turn)
|
|
1392
|
+
const r = await this.resolvePending(toolkit, c.name, args, req, c.id, turn, signal)
|
|
1153
1393
|
result = r.result
|
|
1154
1394
|
if (r.halted) {
|
|
1155
1395
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
1156
1396
|
messages.push(toolMessage(c.id, c.name, result))
|
|
1157
|
-
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) }
|
|
1397
|
+
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) }
|
|
1158
1398
|
return
|
|
1159
1399
|
}
|
|
1160
1400
|
}
|
|
1161
1401
|
toolCalls.push({ name: c.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
1162
1402
|
messages.push(toolMessage(c.id, c.name, result))
|
|
1163
|
-
yield
|
|
1403
|
+
yield this.resultEvent(c.id, c.name, safeJson(c.args), result)
|
|
1164
1404
|
}
|
|
1165
1405
|
}
|
|
1166
|
-
yield { type: "done", result: this.endRun(runStart, lastText(messages), messages, toolCalls, turns, usage, true) }
|
|
1406
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, lastText(messages), messages, toolCalls, turns, usage, true) }
|
|
1167
1407
|
} catch (e) {
|
|
1168
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e)
|
|
1408
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e)
|
|
1409
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done))
|
|
1169
1410
|
throw e
|
|
1170
1411
|
}
|
|
1171
1412
|
}
|
|
1172
1413
|
|
|
1173
1414
|
// ---- Streaming: Anthropic-style ----
|
|
1174
|
-
private async *streamAnthropicWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[]): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1415
|
+
private async *streamAnthropicWithSignal(signal: AbortSignal, prompt: PromptInput, toolkit: Toolkit | undefined, history?: any[], onInterrupt?: OnInterrupt): AsyncGenerator<StreamEvent, void, unknown> {
|
|
1175
1416
|
const key = resolveKey(this.opts)
|
|
1176
1417
|
const runStart = Date.now()
|
|
1177
1418
|
const base = this.opts.baseUrl.replace(/\/$/, "")
|
|
1178
1419
|
const endpoint = base.endsWith("/v1") ? `${base}/messages` : `${base}/v1/messages`
|
|
1179
1420
|
const system = this.system(toolkit)
|
|
1180
1421
|
let messages: any[] = history && history.length ? [...history, { role: "user", content: prompt }] : [{ role: "user", content: prompt }]
|
|
1181
|
-
let tools =
|
|
1422
|
+
let tools = toAnthropic(this.requestTools(toolkit))
|
|
1182
1423
|
const toolCalls: ToolCallRecord[] = []
|
|
1183
1424
|
const usage: Usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }
|
|
1184
1425
|
let turns = 0
|
|
1426
|
+
// The model of the last call this run made (a beforeLLM override when one applied) — §8.
|
|
1427
|
+
let lastModel = this.opts.model
|
|
1428
|
+
// Results finished in the current tool phase, by call id — what an interrupted save keeps.
|
|
1429
|
+
const done = new Map<string, ToolResult>()
|
|
1185
1430
|
|
|
1186
1431
|
try {
|
|
1187
1432
|
for (let turn = 0; turn < (this.opts.maxTurns ?? 10); turn++) {
|
|
1188
1433
|
turns++
|
|
1434
|
+
let turnModel = this.opts.model
|
|
1189
1435
|
if (this.opts.hooks?.beforeLLM) {
|
|
1190
1436
|
const ov = await this.opts.hooks.beforeLLM({ messages, tools, model: this.opts.model, turn })
|
|
1191
1437
|
if (ov?.messages) messages = ov.messages
|
|
1192
1438
|
if (ov?.tools) tools = ov.tools
|
|
1439
|
+
if (ov?.model) turnModel = ov.model
|
|
1193
1440
|
}
|
|
1441
|
+
lastModel = turnModel
|
|
1194
1442
|
const t0 = Date.now()
|
|
1195
1443
|
const beforeP = usage.promptTokens, beforeC = usage.completionTokens
|
|
1196
1444
|
const blocks = new Map<number, { type: string; text?: string; id?: string; name?: string; json?: string }>()
|
|
@@ -1199,7 +1447,7 @@ export class Client {
|
|
|
1199
1447
|
const res = await this.llmFetch(endpoint, {
|
|
1200
1448
|
method: "POST",
|
|
1201
1449
|
headers: { "x-api-key": key, "anthropic-version": "2023-06-01", "Content-Type": "application/json", ...this.opts.headers },
|
|
1202
|
-
body: JSON.stringify(this.anthropicBody(system, messages, tools, true)),
|
|
1450
|
+
body: JSON.stringify(this.anthropicBody(system, messages, tools, true, turnModel)),
|
|
1203
1451
|
}, signal)
|
|
1204
1452
|
if (!res.ok || !res.body) throw await llmHttpError(res)
|
|
1205
1453
|
for await (const line of sseLines(res.body)) {
|
|
@@ -1215,11 +1463,11 @@ export class Client {
|
|
|
1215
1463
|
} else if (j.type === "message_delta") { stopReason = j.delta?.stop_reason ?? stopReason; addUsage(usage, j.usage, "anthropic") }
|
|
1216
1464
|
}
|
|
1217
1465
|
} catch (e) {
|
|
1218
|
-
this.emit({ event: "llm", model:
|
|
1466
|
+
this.emit({ event: "llm", model: turnModel, status: "error", ms: Date.now() - t0, promptTokens: 0, completionTokens: 0 })
|
|
1219
1467
|
throw e
|
|
1220
1468
|
}
|
|
1221
|
-
this.emit({ event: "llm", model:
|
|
1222
|
-
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model:
|
|
1469
|
+
this.emit({ event: "llm", model: turnModel, status: "ok", ms: Date.now() - t0, promptTokens: usage.promptTokens - beforeP, completionTokens: usage.completionTokens - beforeC })
|
|
1470
|
+
if (this.opts.hooks?.afterLLM) await this.opts.hooks.afterLLM({ response: { streamed: true, usage }, model: turnModel, turn })
|
|
1223
1471
|
|
|
1224
1472
|
const content = [...blocks.values()].map((b) => b.type === "tool_use" ? { type: "tool_use", id: b.id, name: b.name, input: safeJson(b.json || "{}") } : { type: "text", text: b.text })
|
|
1225
1473
|
messages.push({ role: "assistant", content })
|
|
@@ -1227,11 +1475,16 @@ export class Client {
|
|
|
1227
1475
|
if (stopReason !== "tool_use" || uses.length === 0) {
|
|
1228
1476
|
const text = content.filter((b: any) => b.type === "text").map((b: any) => b.text).join("")
|
|
1229
1477
|
yield { type: "usage", usage }
|
|
1230
|
-
yield { type: "done", result: this.endRun(runStart, text, messages, toolCalls, turns, usage) }
|
|
1478
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, text, messages, toolCalls, turns, usage) }
|
|
1231
1479
|
return
|
|
1232
1480
|
}
|
|
1233
|
-
for (const u of uses as any[]) yield
|
|
1234
|
-
|
|
1481
|
+
for (const u of uses as any[]) yield this.callEvent(u.id, u.name, u.input)
|
|
1482
|
+
done.clear()
|
|
1483
|
+
const settled = await untilAborted(signal, Promise.all((uses as any[]).map(async (u) => {
|
|
1484
|
+
const r = await this.runTool(toolkit, u.name, u.input ?? {}, u.id, turn, signal)
|
|
1485
|
+
if (!pendingOf(r.result) && u.id) done.set(u.id, r.result)
|
|
1486
|
+
return r
|
|
1487
|
+
})))
|
|
1235
1488
|
const results: any[] = []
|
|
1236
1489
|
for (let i = 0; i < uses.length; i++) {
|
|
1237
1490
|
const u = uses[i] as any
|
|
@@ -1239,25 +1492,26 @@ export class Client {
|
|
|
1239
1492
|
const req = pendingOf(result)
|
|
1240
1493
|
if (req) {
|
|
1241
1494
|
yield { type: "pending", request: req }
|
|
1242
|
-
const r = await this.resolvePending(toolkit, u.name, args, req, u.id, turn)
|
|
1495
|
+
const r = await this.resolvePending(toolkit, u.name, args, req, u.id, turn, signal)
|
|
1243
1496
|
result = r.result
|
|
1244
1497
|
if (r.halted) {
|
|
1245
1498
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
1246
1499
|
results.push(toolResultBlock(u.id, result))
|
|
1247
1500
|
messages.push({ role: "user", content: results })
|
|
1248
|
-
yield { type: "done", result: this.pendingRun(runStart, r.halted, messages, toolCalls, turns, usage) }
|
|
1501
|
+
yield { type: "done", result: this.pendingRun(runStart, lastModel, r.halted, messages, toolCalls, turns, usage) }
|
|
1249
1502
|
return
|
|
1250
1503
|
}
|
|
1251
1504
|
}
|
|
1252
1505
|
toolCalls.push({ name: u.name, args, output: result.output, isError: result.isError, metadata: result.metadata })
|
|
1253
1506
|
results.push(toolResultBlock(u.id, result))
|
|
1254
|
-
yield
|
|
1507
|
+
yield this.resultEvent(u.id, u.name, u.input, result)
|
|
1255
1508
|
}
|
|
1256
1509
|
messages.push({ role: "user", content: results })
|
|
1257
1510
|
}
|
|
1258
|
-
yield { type: "done", result: this.endRun(runStart, "", messages, toolCalls, turns, usage, true) }
|
|
1511
|
+
yield { type: "done", result: this.endRun(runStart, lastModel, "", messages, toolCalls, turns, usage, true) }
|
|
1259
1512
|
} catch (e) {
|
|
1260
|
-
this.emitRunError(runStart, toolCalls, turns, usage, e)
|
|
1513
|
+
this.emitRunError(runStart, lastModel, toolCalls, turns, usage, e)
|
|
1514
|
+
onInterrupt?.(interruptedTranscript(messages, this.opts.style === "anthropic", done))
|
|
1261
1515
|
throw e
|
|
1262
1516
|
}
|
|
1263
1517
|
}
|
|
@@ -1488,6 +1742,52 @@ function toolResultBlock(id: string | undefined, result: ToolResult): any {
|
|
|
1488
1742
|
return b
|
|
1489
1743
|
}
|
|
1490
1744
|
|
|
1745
|
+
/** Receives the repaired transcript of a run that ended in an error or cancellation. */
|
|
1746
|
+
type OnInterrupt = (messages: any[]) => void
|
|
1747
|
+
|
|
1748
|
+
const INTERRUPTED: ToolResult = { output: "interrupted: the turn ended before this tool returned. Whether its side effects happened is unknown. Do not re-run it automatically; check first.", isError: true }
|
|
1749
|
+
|
|
1750
|
+
/** Settle with `p`, or reject with the abort reason as soon as `signal` aborts. */
|
|
1751
|
+
function untilAborted<T>(signal: AbortSignal, p: Promise<T>): Promise<T> {
|
|
1752
|
+
if (signal.aborted) return Promise.reject(signal.reason ?? new Error("aborted"))
|
|
1753
|
+
return new Promise<T>((resolve, reject) => {
|
|
1754
|
+
const onAbort = () => reject(signal.reason ?? new Error("aborted"))
|
|
1755
|
+
signal.addEventListener("abort", onAbort, { once: true })
|
|
1756
|
+
p.then(
|
|
1757
|
+
(v) => { signal.removeEventListener("abort", onAbort); resolve(v) },
|
|
1758
|
+
(e) => { signal.removeEventListener("abort", onAbort); reject(e) },
|
|
1759
|
+
)
|
|
1760
|
+
})
|
|
1761
|
+
}
|
|
1762
|
+
|
|
1763
|
+
/**
|
|
1764
|
+
* §8 "Interrupted turns are saved": a copy of `messages` in which every tool call of the LAST
|
|
1765
|
+
* assistant message has a result — the finished one from `done`, else `interrupted` (isError) —
|
|
1766
|
+
* in the dialect's own shape, so the saved transcript replays on either API.
|
|
1767
|
+
*/
|
|
1768
|
+
function interruptedTranscript(messages: any[], anthropic: boolean, done: Map<string, ToolResult>): any[] {
|
|
1769
|
+
const out = [...messages]
|
|
1770
|
+
let ai = -1
|
|
1771
|
+
for (let i = out.length - 1; i >= 0; i--) if (out[i]?.role === "assistant") { ai = i; break }
|
|
1772
|
+
if (ai < 0) return out
|
|
1773
|
+
const resultFor = (id: string) => done.get(id) ?? INTERRUPTED
|
|
1774
|
+
if (!anthropic) {
|
|
1775
|
+
const calls: any[] = out[ai].tool_calls ?? []
|
|
1776
|
+
const have = new Set(out.slice(ai + 1).filter((m) => m?.role === "tool").map((m) => m.tool_call_id))
|
|
1777
|
+
for (const c of calls) if (!have.has(c.id)) out.push(toolMessage(c.id, c.function?.name ?? "", resultFor(c.id)))
|
|
1778
|
+
return out
|
|
1779
|
+
}
|
|
1780
|
+
const uses = Array.isArray(out[ai].content) ? out[ai].content.filter((b: any) => b?.type === "tool_use") : []
|
|
1781
|
+
if (!uses.length) return out
|
|
1782
|
+
const next = out[ai + 1]
|
|
1783
|
+
const blocks: any[] = next?.role === "user" && Array.isArray(next.content) ? [...next.content] : []
|
|
1784
|
+
const have = new Set(blocks.filter((b) => b?.type === "tool_result").map((b) => b.tool_use_id))
|
|
1785
|
+
for (const u of uses) if (!have.has(u.id)) blocks.push(toolResultBlock(u.id, resultFor(u.id)))
|
|
1786
|
+
if (next?.role === "user" && Array.isArray(next.content)) out[ai + 1] = { ...next, content: blocks }
|
|
1787
|
+
else out.splice(ai + 1, 0, { role: "user", content: blocks })
|
|
1788
|
+
return out
|
|
1789
|
+
}
|
|
1790
|
+
|
|
1491
1791
|
function lastText(messages: any[]): string {
|
|
1492
1792
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
1493
1793
|
if (messages[i].role === "assistant" && typeof messages[i].content === "string") return messages[i].content
|