vern-llm 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/index.cjs +1512 -375
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +893 -222
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +893 -222
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1509 -376
- package/dist/index.js.map +1 -1
- package/package.json +16 -2
package/dist/index.js
CHANGED
|
@@ -1,23 +1,273 @@
|
|
|
1
1
|
import { randomUUID } from "crypto";
|
|
2
2
|
|
|
3
3
|
//#region src/types/errors.ts
|
|
4
|
+
/**
|
|
5
|
+
* Tool contract codes: a model or provider response defect, not a
|
|
6
|
+
* transient provider fault. Deterministic on the wire request, so
|
|
7
|
+
* retrying can't change the outcome and it shouldn't count toward the
|
|
8
|
+
* circuit breaker either. Shared by `LLMError.retryable` below and by
|
|
9
|
+
* `CallExecutor`'s own retry/breaker accounting, so the two can't drift
|
|
10
|
+
* apart.
|
|
11
|
+
*/
|
|
12
|
+
const NON_RETRYABLE_TOOL_CONTRACT_CODES = new Set([
|
|
13
|
+
"unknown_tool",
|
|
14
|
+
"duplicate_tool_call_id",
|
|
15
|
+
"tool_choice_none_violated",
|
|
16
|
+
"unexpected_tool_calls"
|
|
17
|
+
]);
|
|
18
|
+
/**
|
|
19
|
+
* Local rate-limit codes: the call never reached the provider, so it says
|
|
20
|
+
* nothing about the provider's health, and retrying either just requeues
|
|
21
|
+
* behind the same limit (the two queue codes) or can never succeed at all
|
|
22
|
+
* (`rate_limit_capacity_exceeded`). Shared for the same reason as
|
|
23
|
+
* {@link NON_RETRYABLE_TOOL_CONTRACT_CODES}.
|
|
24
|
+
*/
|
|
25
|
+
const LOCAL_RATE_LIMIT_CODES = new Set([
|
|
26
|
+
"rate_limit_queue_full",
|
|
27
|
+
"rate_limit_queue_timeout",
|
|
28
|
+
"rate_limit_capacity_exceeded"
|
|
29
|
+
]);
|
|
30
|
+
/**
|
|
31
|
+
* Types that are never worth retrying on their own: deterministic
|
|
32
|
+
* caller-input, model-response, or cancellation failures rather than a
|
|
33
|
+
* transient provider fault.
|
|
34
|
+
*/
|
|
35
|
+
const NON_RETRYABLE_TYPES = new Set([
|
|
36
|
+
"parse",
|
|
37
|
+
"validation",
|
|
38
|
+
"invalid_params",
|
|
39
|
+
"aborted"
|
|
40
|
+
]);
|
|
41
|
+
/**
|
|
42
|
+
* Shared retryability rule behind both `LLMError.retryable` and
|
|
43
|
+
* `LLMErrorSnapshot.retryable`. Pulled out so the two can't drift apart:
|
|
44
|
+
* a snapshot is a point-in-time copy of an error's fields, and this is
|
|
45
|
+
* one of them, so it has to be computed the same way in both places.
|
|
46
|
+
*/
|
|
47
|
+
function computeRetryable(type, code) {
|
|
48
|
+
if (NON_RETRYABLE_TYPES.has(type)) return false;
|
|
49
|
+
if (code && NON_RETRYABLE_TOOL_CONTRACT_CODES.has(code)) return false;
|
|
50
|
+
if (code && LOCAL_RATE_LIMIT_CODES.has(code)) return false;
|
|
51
|
+
return true;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Returns `issues` unchanged when it can survive `JSON.stringify`.
|
|
55
|
+
* Most `issues` values are VernLLM's own structured shapes (see
|
|
56
|
+
* `LLMErrorIssuesByCode`) and always safe. The one exception is a
|
|
57
|
+
* schema validation failure, where `issues` is a caller supplied
|
|
58
|
+
* `SchemaLike` validator's own `error: unknown`, not controlled by
|
|
59
|
+
* VernLLM and not guaranteed to be circular free. Rather than silently
|
|
60
|
+
* dropping it in that case, this returns a marker string so a reader
|
|
61
|
+
* of serialized output can tell "no issues data" apart from "issues
|
|
62
|
+
* existed but could not be shown".
|
|
63
|
+
*/
|
|
64
|
+
function safeIssues(issues) {
|
|
65
|
+
if (issues === void 0) return void 0;
|
|
66
|
+
try {
|
|
67
|
+
JSON.stringify(issues);
|
|
68
|
+
return issues;
|
|
69
|
+
} catch {
|
|
70
|
+
return "[Unserializable: issues contained a circular reference]";
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Returns a JSON safe, independent copy of `body`, or a marker string if
|
|
75
|
+
* `body` can't survive `JSON.stringify` (e.g. a circular reference). A
|
|
76
|
+
* request body built from adapter-transformed messages is normally
|
|
77
|
+
* always plain data, but tool call arguments or a caller supplied
|
|
78
|
+
* `cause`-adjacent value could in principle carry a circular reference,
|
|
79
|
+
* so this guards the same way `safeIssues` does rather than assuming it
|
|
80
|
+
* can't happen. Unlike `safeIssues`, this clones rather than returning
|
|
81
|
+
* the same reference: the object backing a request body can still be
|
|
82
|
+
* mutated by adapter code between when a request is dispatched and when
|
|
83
|
+
* an attempt is later recorded as failed (e.g. `fromGemini` sets
|
|
84
|
+
* `request.config` in place), so returning the same reference here could
|
|
85
|
+
* make a stored snapshot silently reflect a later, different state than
|
|
86
|
+
* what was actually sent.
|
|
87
|
+
*/
|
|
88
|
+
function safeBody(body) {
|
|
89
|
+
if (body === void 0) return void 0;
|
|
90
|
+
try {
|
|
91
|
+
return JSON.parse(JSON.stringify(body));
|
|
92
|
+
} catch {
|
|
93
|
+
return "[Unserializable: request body contained a circular reference]";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const AUTH_HEADER_NAMES = new Set([
|
|
97
|
+
"authorization",
|
|
98
|
+
"x-api-key",
|
|
99
|
+
"x-goog-api-key",
|
|
100
|
+
"api-key"
|
|
101
|
+
]);
|
|
102
|
+
/** Removes auth headers before a request snapshot is built. Case insensitive on header names. */
|
|
103
|
+
function stripAuthHeaders(headers) {
|
|
104
|
+
if (headers === void 0) return void 0;
|
|
105
|
+
const out = {};
|
|
106
|
+
for (const [key, value] of Object.entries(headers)) if (!AUTH_HEADER_NAMES.has(key.toLowerCase())) out[key] = value;
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Depth cap for `safeAttempts`, guarding against a pathological,
|
|
111
|
+
* self referential `attempts` array. `attempts` is a public
|
|
112
|
+
* `LLMErrorOptions` field, so a caller can construct one by hand; this
|
|
113
|
+
* keeps that path bounded the same way a circular `issues` value is
|
|
114
|
+
* bounded, rather than assuming well formed input.
|
|
115
|
+
*/
|
|
116
|
+
const MAX_ATTEMPTS_DEPTH = 20;
|
|
117
|
+
/**
|
|
118
|
+
* Returns a copy of `attempts` with every nested snapshot's `issues`
|
|
119
|
+
* re-checked through `safeIssues`, recursively through each snapshot's
|
|
120
|
+
* own `attempts`. Needed for two reasons: `safeIssues` returns a safe
|
|
121
|
+
* `issues` value by reference, so a shared object can be mutated into a
|
|
122
|
+
* circular one after the snapshot was created, and `attempts` is a
|
|
123
|
+
* public constructor option, so a caller can hand build a `RetryAttempt`
|
|
124
|
+
* (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
|
|
125
|
+
* in directly, never touching `toSnapshot()` at all. The same applies to
|
|
126
|
+
* `request`: its `body` is re-checked through `safeBody`, and its
|
|
127
|
+
* `headers` are re-stripped through `stripAuthHeaders`, so a hand built
|
|
128
|
+
* `RetryAttempt.request` can't smuggle an auth header past `toSnapshot()`
|
|
129
|
+
* either. Extra fields on an attempt (e.g. `FallbackAttempt`'s
|
|
130
|
+
* `provider`/`model`) are preserved.
|
|
131
|
+
*/
|
|
132
|
+
function safeAttempts(attempts, depth = 0) {
|
|
133
|
+
if (attempts === void 0) return void 0;
|
|
134
|
+
if (depth >= MAX_ATTEMPTS_DEPTH) return [];
|
|
135
|
+
return attempts.map((attempt) => ({
|
|
136
|
+
...attempt,
|
|
137
|
+
error: {
|
|
138
|
+
...attempt.error,
|
|
139
|
+
issues: safeIssues(attempt.error.issues),
|
|
140
|
+
attempts: safeAttempts(attempt.error.attempts, depth + 1)
|
|
141
|
+
},
|
|
142
|
+
request: attempt.request && {
|
|
143
|
+
...attempt.request,
|
|
144
|
+
body: safeBody(attempt.request.body),
|
|
145
|
+
headers: stripAuthHeaders(attempt.request.headers)
|
|
146
|
+
}
|
|
147
|
+
}));
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Builds a point-in-time, plain data copy of one attempt's outgoing
|
|
151
|
+
* request. Mirrors `LLMError.toSnapshot()`: never thrown or dispatched
|
|
152
|
+
* again, safe to serialize and store. A plain function rather than a
|
|
153
|
+
* method, since unlike `LLMError` a request has no throwable identity or
|
|
154
|
+
* derived state worth wrapping in a class.
|
|
155
|
+
*
|
|
156
|
+
* `startedAt` is optional so existing call sites (and tests) that don't
|
|
157
|
+
* care about exact timing keep working, but a caller that has a real
|
|
158
|
+
* capture time should always pass it: this function may run well after
|
|
159
|
+
* the request was actually dispatched (e.g. `callExecutor` only builds
|
|
160
|
+
* the snapshot once an attempt has failed), so defaulting to `Date.now()`
|
|
161
|
+
* here would record failure-handling time, not request-start time.
|
|
162
|
+
*/
|
|
163
|
+
function toRequestSnapshot(provider, model, body, headers, startedAt = Date.now()) {
|
|
164
|
+
return {
|
|
165
|
+
provider,
|
|
166
|
+
model,
|
|
167
|
+
body: safeBody(body),
|
|
168
|
+
headers: stripAuthHeaders(headers),
|
|
169
|
+
startedAt
|
|
170
|
+
};
|
|
171
|
+
}
|
|
4
172
|
var LLMError = class extends Error {
|
|
5
|
-
|
|
173
|
+
status;
|
|
174
|
+
issues;
|
|
175
|
+
cause;
|
|
176
|
+
retryAfterMs;
|
|
177
|
+
/** Stable discriminator within `type`. Absent on errors predating it. */
|
|
178
|
+
code;
|
|
179
|
+
/** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
|
|
180
|
+
attempts;
|
|
181
|
+
constructor(message, type, options = {}) {
|
|
6
182
|
super(message);
|
|
7
183
|
this.type = type;
|
|
8
|
-
this.status = status;
|
|
9
|
-
this.issues = issues;
|
|
10
|
-
this.cause = cause;
|
|
11
|
-
this.retryAfterMs = retryAfterMs;
|
|
12
|
-
this.code = code;
|
|
13
184
|
this.name = "LLMError";
|
|
185
|
+
this.status = options.status;
|
|
186
|
+
this.issues = options.issues;
|
|
187
|
+
this.cause = options.cause;
|
|
188
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
189
|
+
this.code = options.code;
|
|
190
|
+
this.attempts = options.attempts;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Computed purely from `type`/`code`, independent of any specific call's
|
|
194
|
+
* `nonRetryableStatus` list. False for `parse`/`validation`/
|
|
195
|
+
* `invalid_params`/`aborted` types (the caller's own input, the model's
|
|
196
|
+
* own response, or intentional cancellation, none of which are the
|
|
197
|
+
* provider being unhealthy), the tool contract codes, and the local
|
|
198
|
+
* rate limit codes. Subclasses (see `FallbackExhaustedError`) may
|
|
199
|
+
* override this when `type` alone carries no retry signal.
|
|
200
|
+
*/
|
|
201
|
+
get retryable() {
|
|
202
|
+
return computeRetryable(this.type, this.code);
|
|
203
|
+
}
|
|
204
|
+
/**
|
|
205
|
+
* Copies this error's fields into an {@link LLMErrorSnapshot}, for
|
|
206
|
+
* recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
|
|
207
|
+
* captured here since a snapshot has no getter of its own. `cause` is
|
|
208
|
+
* not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
|
|
209
|
+
* nested `attempts` entry's own `issues` go through `safeAttempts`,
|
|
210
|
+
* since a schema validation failure's `issues` is a caller supplied
|
|
211
|
+
* value, not controlled by VernLLM, and `attempts` is itself a public
|
|
212
|
+
* constructor option a caller can hand build.
|
|
213
|
+
*/
|
|
214
|
+
toSnapshot() {
|
|
215
|
+
return {
|
|
216
|
+
message: this.message,
|
|
217
|
+
type: this.type,
|
|
218
|
+
status: this.status,
|
|
219
|
+
issues: safeIssues(this.issues),
|
|
220
|
+
retryAfterMs: this.retryAfterMs,
|
|
221
|
+
code: this.code,
|
|
222
|
+
retryable: this.retryable,
|
|
223
|
+
attempts: safeAttempts(this.attempts)
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* Controls what `JSON.stringify(err)` produces. Omits `cause` for the
|
|
228
|
+
* same reason `toSnapshot()` does: `cause` is `unknown` and never
|
|
229
|
+
* validated by VernLLM, and some SDK errors carry circular structures
|
|
230
|
+
* `JSON.stringify` cannot serialize at all. Read `err.cause` directly
|
|
231
|
+
* instead. `issues`, including every nested `attempts` entry's own
|
|
232
|
+
* `issues`, goes through `safeAttempts` for the same reason: a schema
|
|
233
|
+
* validation failure's `issues` is caller supplied and not guaranteed
|
|
234
|
+
* circular free. Also includes `message` and `retryable`, which a
|
|
235
|
+
* plain property walk would otherwise miss: `message` is
|
|
236
|
+
* non-enumerable on `Error`, and `retryable` is a getter, not an own
|
|
237
|
+
* property.
|
|
238
|
+
*/
|
|
239
|
+
toJSON() {
|
|
240
|
+
return {
|
|
241
|
+
name: this.name,
|
|
242
|
+
message: this.message,
|
|
243
|
+
type: this.type,
|
|
244
|
+
status: this.status,
|
|
245
|
+
issues: safeIssues(this.issues),
|
|
246
|
+
retryAfterMs: this.retryAfterMs,
|
|
247
|
+
code: this.code,
|
|
248
|
+
retryable: this.retryable,
|
|
249
|
+
attempts: safeAttempts(this.attempts)
|
|
250
|
+
};
|
|
14
251
|
}
|
|
15
|
-
/** Every tool contract failure in one response, when there is more than one. */
|
|
16
|
-
toolIssues;
|
|
17
252
|
};
|
|
18
253
|
function isLLMError(err) {
|
|
19
254
|
return err instanceof LLMError;
|
|
20
255
|
}
|
|
256
|
+
/**
|
|
257
|
+
* Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
|
|
258
|
+
* `code` to, for any code listed there. `code` stays the only discriminator
|
|
259
|
+
* VernLLM uses; this just gives that existing check a typed return instead
|
|
260
|
+
* of requiring a manual cast of `issues`:
|
|
261
|
+
*
|
|
262
|
+
* ```ts
|
|
263
|
+
* if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
|
|
264
|
+
* console.log(err.issues.names); // string[], no cast needed
|
|
265
|
+
* }
|
|
266
|
+
* ```
|
|
267
|
+
*/
|
|
268
|
+
function hasIssues(err, code) {
|
|
269
|
+
return err.code === code && err.issues !== void 0;
|
|
270
|
+
}
|
|
21
271
|
|
|
22
272
|
//#endregion
|
|
23
273
|
//#region src/types/cache.ts
|
|
@@ -147,7 +397,12 @@ function isToolCallResult(result) {
|
|
|
147
397
|
//#endregion
|
|
148
398
|
//#region src/types/fallback.ts
|
|
149
399
|
/** Tool contract failures are the model ignoring the request, not a sick provider: repeating it elsewhere can't help. */
|
|
150
|
-
const TOOL_CONTRACT_CODES = new Set([
|
|
400
|
+
const TOOL_CONTRACT_CODES = new Set([
|
|
401
|
+
"unknown_tool",
|
|
402
|
+
"duplicate_tool_call_id",
|
|
403
|
+
"tool_choice_none_violated",
|
|
404
|
+
"unexpected_tool_calls"
|
|
405
|
+
]);
|
|
151
406
|
/**
|
|
152
407
|
* The default `fallbackOn` policy. Exported so a caller can wrap rather
|
|
153
408
|
* than replace it, e.g. `fallbackOn: (e, ctx) => myCheck(e) ? 'stop' : defaultFallbackOn(e, ctx)`.
|
|
@@ -170,10 +425,30 @@ const defaultFallbackOn = (error) => {
|
|
|
170
425
|
var FallbackExhaustedError = class extends LLMError {
|
|
171
426
|
constructor(attempts) {
|
|
172
427
|
const last = attempts[attempts.length - 1]?.error;
|
|
173
|
-
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`,
|
|
428
|
+
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`, "fallback_exhausted", {
|
|
429
|
+
status: last?.status,
|
|
430
|
+
cause: last,
|
|
431
|
+
retryAfterMs: last?.retryAfterMs,
|
|
432
|
+
code: "fallback_exhausted",
|
|
433
|
+
attempts
|
|
434
|
+
});
|
|
174
435
|
this.attempts = attempts;
|
|
175
436
|
}
|
|
437
|
+
/**
|
|
438
|
+
* `type: 'fallback_exhausted'` by itself says nothing about whether
|
|
439
|
+
* retrying could help; the reason the last target failed does. Defers to
|
|
440
|
+
* that attempt's own `retryable` instead of anything about this class's
|
|
441
|
+
* own type.
|
|
442
|
+
*/
|
|
443
|
+
get retryable() {
|
|
444
|
+
const last = this.attempts[this.attempts.length - 1]?.error;
|
|
445
|
+
return last ? last.retryable : super.retryable;
|
|
446
|
+
}
|
|
176
447
|
};
|
|
448
|
+
/** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
|
|
449
|
+
function isFallbackExhaustedError(err) {
|
|
450
|
+
return err instanceof FallbackExhaustedError;
|
|
451
|
+
}
|
|
177
452
|
|
|
178
453
|
//#endregion
|
|
179
454
|
//#region src/internal/execution/usage.utils.ts
|
|
@@ -195,7 +470,7 @@ async function reserve(params, coalesced, signal) {
|
|
|
195
470
|
return true;
|
|
196
471
|
} catch (error) {
|
|
197
472
|
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
198
|
-
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded",
|
|
473
|
+
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded", { cause: error });
|
|
199
474
|
}
|
|
200
475
|
}
|
|
201
476
|
/**
|
|
@@ -395,13 +670,30 @@ var CacheOrchestrator = class {
|
|
|
395
670
|
*/
|
|
396
671
|
async deleteCache(key) {
|
|
397
672
|
if (!this.cache.delete) return;
|
|
398
|
-
|
|
673
|
+
try {
|
|
674
|
+
await this.cache.delete(await this.resolveCacheKey(key));
|
|
675
|
+
} catch (error) {
|
|
676
|
+
this.logger.warn(`[VernLLM] cache delete failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
677
|
+
}
|
|
399
678
|
}
|
|
400
679
|
/** Logs a failed refundUsage attempt via the configured logger. */
|
|
401
680
|
logRefundError(logMessage, error) {
|
|
402
681
|
this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
|
|
403
682
|
}
|
|
404
683
|
/**
|
|
684
|
+
* Reads from the cache, treating a failed adapter read as a miss rather
|
|
685
|
+
* than letting it fail the call. The request still falls through to a
|
|
686
|
+
* real provider call, but that fallback is now logged instead of silent.
|
|
687
|
+
*/
|
|
688
|
+
async getCached(key) {
|
|
689
|
+
try {
|
|
690
|
+
return await this.cache.get(key);
|
|
691
|
+
} catch (error) {
|
|
692
|
+
this.logger.warn(`[VernLLM] cache read failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
693
|
+
return { hit: false };
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
405
697
|
* Internal cache primitive around caller-supplied logic. Concurrent misses
|
|
406
698
|
* for the same `cacheKey` share a single in-flight call, avoiding cache
|
|
407
699
|
* stampedes.
|
|
@@ -421,7 +713,7 @@ var CacheOrchestrator = class {
|
|
|
421
713
|
...params,
|
|
422
714
|
cacheKey: resolvedKey
|
|
423
715
|
};
|
|
424
|
-
const cached = await this.
|
|
716
|
+
const cached = await this.getCached(resolvedKey);
|
|
425
717
|
if (cached.hit) return cached.value;
|
|
426
718
|
const existing = this.inFlight.get(resolvedKey);
|
|
427
719
|
if (existing) return withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
@@ -442,7 +734,7 @@ var CacheOrchestrator = class {
|
|
|
442
734
|
try {
|
|
443
735
|
await this.cache.set(params.cacheKey, result, params.ttl);
|
|
444
736
|
} catch (error) {
|
|
445
|
-
this.logger.
|
|
737
|
+
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
446
738
|
}
|
|
447
739
|
return result;
|
|
448
740
|
}
|
|
@@ -471,7 +763,7 @@ var CacheOrchestrator = class {
|
|
|
471
763
|
...params,
|
|
472
764
|
cacheKey: resolvedKey
|
|
473
765
|
};
|
|
474
|
-
const cached = await this.
|
|
766
|
+
const cached = await this.getCached(resolvedKey);
|
|
475
767
|
if (cached.hit) {
|
|
476
768
|
const value = cached.value;
|
|
477
769
|
return {
|
|
@@ -520,7 +812,7 @@ var CacheOrchestrator = class {
|
|
|
520
812
|
try {
|
|
521
813
|
await this.cache.set(params.cacheKey, value, params.ttl);
|
|
522
814
|
} catch (error) {
|
|
523
|
-
this.logger.
|
|
815
|
+
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
524
816
|
}
|
|
525
817
|
return value;
|
|
526
818
|
}, (error) => {
|
|
@@ -562,6 +854,7 @@ var CircuitBreaker = class {
|
|
|
562
854
|
threshold;
|
|
563
855
|
cooldownMs;
|
|
564
856
|
onStateChange;
|
|
857
|
+
/** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
|
|
565
858
|
isolateByModel;
|
|
566
859
|
sharedBucket = newBucket();
|
|
567
860
|
bucketsByModel = new Map();
|
|
@@ -607,12 +900,12 @@ var CircuitBreaker = class {
|
|
|
607
900
|
if (bucket.state === "closed") return;
|
|
608
901
|
if (bucket.state === "open") {
|
|
609
902
|
const elapsed = Date.now() - bucket.openedAt;
|
|
610
|
-
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open");
|
|
903
|
+
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open", { code: "circuit_cooling_down" });
|
|
611
904
|
bucket.trialInFlight = true;
|
|
612
905
|
this.transition(bucket, "half-open", model);
|
|
613
906
|
return;
|
|
614
907
|
}
|
|
615
|
-
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open");
|
|
908
|
+
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open", { code: "circuit_trial_in_flight" });
|
|
616
909
|
bucket.trialInFlight = true;
|
|
617
910
|
}
|
|
618
911
|
recordSuccess(model) {
|
|
@@ -647,6 +940,33 @@ var CircuitBreaker = class {
|
|
|
647
940
|
getState(model) {
|
|
648
941
|
return this.lookupBucket(model)?.state ?? "closed";
|
|
649
942
|
}
|
|
943
|
+
/**
|
|
944
|
+
* Manually opens the circuit, as if `threshold` consecutive failures had
|
|
945
|
+
* just happened, e.g. to pull a provider out of rotation ahead of known
|
|
946
|
+
* maintenance. Resets the cooldown window from now, same as a real
|
|
947
|
+
* threshold-crossing failure would, and clears any in-flight half-open
|
|
948
|
+
* trial since it no longer applies once the circuit is (re)opened.
|
|
949
|
+
*/
|
|
950
|
+
open(model) {
|
|
951
|
+
const bucket = this.ensureBucketFor(model);
|
|
952
|
+
bucket.openedAt = Date.now();
|
|
953
|
+
bucket.trialInFlight = false;
|
|
954
|
+
this.transition(bucket, "open", model);
|
|
955
|
+
}
|
|
956
|
+
/**
|
|
957
|
+
* Manually closes the circuit and resets its failure count, e.g. once a
|
|
958
|
+
* provider is confirmed healthy again without waiting out the cooldown.
|
|
959
|
+
* Mirrors `recordSuccess`'s bookkeeping (including dropping the
|
|
960
|
+
* per-model bucket under `isolateByModel`, once idle) but without
|
|
961
|
+
* requiring an actual successful call first.
|
|
962
|
+
*/
|
|
963
|
+
close(model) {
|
|
964
|
+
const bucket = this.ensureBucketFor(model);
|
|
965
|
+
bucket.consecutiveFailures = 0;
|
|
966
|
+
bucket.trialInFlight = false;
|
|
967
|
+
this.transition(bucket, "closed", model);
|
|
968
|
+
if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
|
|
969
|
+
}
|
|
650
970
|
};
|
|
651
971
|
|
|
652
972
|
//#endregion
|
|
@@ -765,7 +1085,7 @@ async function withTimeout(fn, timeoutMs, externalSignal) {
|
|
|
765
1085
|
try {
|
|
766
1086
|
return await fn(signal);
|
|
767
1087
|
} catch (err) {
|
|
768
|
-
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout");
|
|
1088
|
+
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout", { code: "request_timeout" });
|
|
769
1089
|
throw err;
|
|
770
1090
|
} finally {
|
|
771
1091
|
clearTimeout(timer);
|
|
@@ -798,7 +1118,7 @@ function withChunkIdleTimeout(next, timeoutMs, onIdle, logger) {
|
|
|
798
1118
|
const timer = setTimeout(() => {
|
|
799
1119
|
settled = true;
|
|
800
1120
|
onIdle?.();
|
|
801
|
-
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout"));
|
|
1121
|
+
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout", { code: "idle_timeout" }));
|
|
802
1122
|
}, clampTimeoutMs(activeTimeoutMs));
|
|
803
1123
|
next().then((result) => {
|
|
804
1124
|
clearTimeout(timer);
|
|
@@ -963,20 +1283,99 @@ function describeError(err) {
|
|
|
963
1283
|
} catch {}
|
|
964
1284
|
return formatSafely(err);
|
|
965
1285
|
}
|
|
966
|
-
/**
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
1286
|
+
/**
|
|
1287
|
+
* Maps an HTTP status to its corresponding `LLMErrorCode`, derived purely
|
|
1288
|
+
* from the status itself so it applies the same way regardless of which
|
|
1289
|
+
* adapter or client raised the error. Used both when building a fresh
|
|
1290
|
+
* `LLMError` and when filling in a `code` on an already-normalized one
|
|
1291
|
+
* that doesn't have one yet, so the two paths can't drift apart.
|
|
1292
|
+
*/
|
|
1293
|
+
function codeForStatus(status) {
|
|
1294
|
+
switch (status) {
|
|
1295
|
+
case 429: return "provider_rate_limited";
|
|
1296
|
+
case 401: return "authentication";
|
|
1297
|
+
case 403: return "authorization";
|
|
1298
|
+
case 404: return "not_found";
|
|
1299
|
+
case 413: return "payload_too_large";
|
|
1300
|
+
default: return status >= 500 ? "server_error" : void 0;
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Whether a provider's error response actually contains anything a person
|
|
1305
|
+
* could act on. Some providers return a non-2xx status with **no body at
|
|
1306
|
+
* all** for certain field-validation failures (Mistral's OpenAI-compatible
|
|
1307
|
+
* endpoint does this, for example, when a request includes a field the
|
|
1308
|
+
* target model doesn't support). SDKs built on top of `openai` render that
|
|
1309
|
+
* specific case as a message like `"400 status code (no body)"`.
|
|
1310
|
+
*
|
|
1311
|
+
* Derived from the object's own `error`/`message` fields directly, rather
|
|
1312
|
+
* than from whatever `describeError` rendered, because `describeError`
|
|
1313
|
+
* falls back to serializing the *whole* thrown value when neither field is
|
|
1314
|
+
* present or meaningful. That fallback is local echo (e.g. just the
|
|
1315
|
+
* `status` a caller passed in), not provider diagnostic content, and
|
|
1316
|
+
* treating it as "detail" defeats the whole point of this check.
|
|
1317
|
+
*/
|
|
1318
|
+
const NO_BODY_MESSAGE_PATTERN = /\(no body\)/i;
|
|
1319
|
+
function isEmptyObject(value) {
|
|
1320
|
+
return Object.keys(value).length === 0;
|
|
1321
|
+
}
|
|
1322
|
+
function hasNoDiagnosticDetail(error) {
|
|
1323
|
+
if (error && typeof error === "object") {
|
|
1324
|
+
const { error: errorField, message } = error;
|
|
1325
|
+
if (errorField !== void 0 && errorField !== null) {
|
|
1326
|
+
const isEmptyString = typeof errorField === "string" && errorField.trim().length === 0;
|
|
1327
|
+
const isEmptyStruct = typeof errorField === "object" && isEmptyObject(errorField);
|
|
1328
|
+
if (!isEmptyString && !isEmptyStruct) return false;
|
|
1329
|
+
}
|
|
1330
|
+
if (typeof message === "string") {
|
|
1331
|
+
const trimmed = message.trim();
|
|
1332
|
+
return trimmed.length === 0 || NO_BODY_MESSAGE_PATTERN.test(trimmed);
|
|
973
1333
|
}
|
|
1334
|
+
return true;
|
|
1335
|
+
}
|
|
1336
|
+
return true;
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1339
|
+
* Converts any thrown value into a well-typed LLMError. `attempts`, when
|
|
1340
|
+
* given, is the accumulated record of every attempt made before `error`
|
|
1341
|
+
* was thrown; it's passed straight into the constructed error's options
|
|
1342
|
+
* rather than assigned onto the error afterward, so `attempts` is always
|
|
1343
|
+
* settled once, through the constructor, like every other field on
|
|
1344
|
+
* `LLMError`.
|
|
1345
|
+
*/
|
|
1346
|
+
function normalizeError(error, signal, attempts) {
|
|
1347
|
+
if (signal?.aborted) return new LLMError("LLM request aborted", "aborted", { attempts });
|
|
1348
|
+
if (error instanceof LLMError) {
|
|
1349
|
+
if (error.code === void 0 && error.status !== void 0) error.code = codeForStatus(error.status);
|
|
1350
|
+
if (error.attempts === void 0 && attempts !== void 0) error.attempts = attempts;
|
|
974
1351
|
return error;
|
|
975
1352
|
}
|
|
976
1353
|
const status = extractStatus(error);
|
|
977
1354
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
978
|
-
if (status !== void 0)
|
|
979
|
-
|
|
1355
|
+
if (status !== void 0) {
|
|
1356
|
+
const description = describeError(error);
|
|
1357
|
+
const code = codeForStatus(status);
|
|
1358
|
+
const isRequestValidationStatus = code === void 0;
|
|
1359
|
+
const message = hasNoDiagnosticDetail(error) ? isRequestValidationStatus ? `LLM request failed with status ${status} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.` : `LLM request failed with status ${status} and no error detail from the provider.` : `LLM request failed: ${description}`;
|
|
1360
|
+
return new LLMError(message, "api", {
|
|
1361
|
+
status,
|
|
1362
|
+
cause: error,
|
|
1363
|
+
retryAfterMs,
|
|
1364
|
+
code,
|
|
1365
|
+
attempts
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1368
|
+
if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
|
|
1369
|
+
cause: error,
|
|
1370
|
+
retryAfterMs,
|
|
1371
|
+
code: "connection_failed",
|
|
1372
|
+
attempts
|
|
1373
|
+
});
|
|
1374
|
+
return new LLMError("LLM request failed", "unknown", {
|
|
1375
|
+
cause: error,
|
|
1376
|
+
retryAfterMs,
|
|
1377
|
+
attempts
|
|
1378
|
+
});
|
|
980
1379
|
}
|
|
981
1380
|
|
|
982
1381
|
//#endregion
|
|
@@ -1025,7 +1424,7 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1025
1424
|
try {
|
|
1026
1425
|
parsedArgs = wc.function.arguments.trim() ? JSON.parse(wc.function.arguments) : {};
|
|
1027
1426
|
} catch {
|
|
1028
|
-
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse");
|
|
1427
|
+
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse", { code: "tool_arguments_parse_failed" });
|
|
1029
1428
|
}
|
|
1030
1429
|
return {
|
|
1031
1430
|
id: wc.id,
|
|
@@ -1038,11 +1437,22 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1038
1437
|
//#endregion
|
|
1039
1438
|
//#region src/internal/execution/requestBuilder.ts
|
|
1040
1439
|
/**
|
|
1440
|
+
* Serializes `ConversationTurn` assistant content for the wire. Strings
|
|
1441
|
+
* pass through unchanged. Parsed JSON values are `JSON.stringify`'d.
|
|
1442
|
+
*/
|
|
1443
|
+
function serializeAssistantContent(content) {
|
|
1444
|
+
return typeof content === "string" ? content : JSON.stringify(content);
|
|
1445
|
+
}
|
|
1446
|
+
/**
|
|
1041
1447
|
* Builds the wire request object for one call, applying per-instance
|
|
1042
1448
|
* defaults (model, max tokens, temperature) and per-call overrides.
|
|
1043
|
-
* Owns every
|
|
1044
|
-
* history alternation, duplicate/empty tool lists,
|
|
1045
|
-
* real tool.
|
|
1449
|
+
* Owns every check that depends only on the caller's own input shape, not
|
|
1450
|
+
* on execution: history alternation, duplicate/empty tool lists,
|
|
1451
|
+
* `toolChoice` naming a real tool. All deterministic on the call site's
|
|
1452
|
+
* own input and never touch the network, so every throw here is
|
|
1453
|
+
* `type: 'invalid_params'`, not `'validation'` (which is reserved for the
|
|
1454
|
+
* model/provider's own response failing a contract check). Has no
|
|
1455
|
+
* knowledge of retry, timeouts, or the breaker, only
|
|
1046
1456
|
* the three defaults a `FallbackTarget` can override per-target (see the
|
|
1047
1457
|
* `defaultMaxTokens`/`defaultTemperature` overrides in the fallback
|
|
1048
1458
|
* design), which is what keeps it separable from `CallExecutor`.
|
|
@@ -1051,16 +1461,24 @@ var RequestBuilder = class {
|
|
|
1051
1461
|
model;
|
|
1052
1462
|
defaultMaxTokens;
|
|
1053
1463
|
defaultTemperature;
|
|
1464
|
+
defaultReasoningEffort;
|
|
1465
|
+
defaultBudgetTokens;
|
|
1466
|
+
supportsJsonObjectMode;
|
|
1054
1467
|
constructor(options) {
|
|
1055
1468
|
this.model = options.model;
|
|
1056
1469
|
this.defaultMaxTokens = options.defaultMaxTokens;
|
|
1057
1470
|
this.defaultTemperature = options.defaultTemperature;
|
|
1471
|
+
this.defaultReasoningEffort = options.defaultReasoningEffort;
|
|
1472
|
+
this.defaultBudgetTokens = options.defaultBudgetTokens;
|
|
1473
|
+
this.supportsJsonObjectMode = options.supportsJsonObjectMode;
|
|
1058
1474
|
}
|
|
1059
1475
|
/** Applies per-call defaults and shapes params into the client's request object. */
|
|
1060
1476
|
build(params) {
|
|
1061
|
-
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model,
|
|
1477
|
+
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, jsonSchema, tools, toolChoice } = params;
|
|
1062
1478
|
const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
|
|
1063
|
-
|
|
1479
|
+
const reasoningEffort = params.reasoningEffort === void 0 ? this.defaultReasoningEffort : params.reasoningEffort;
|
|
1480
|
+
const budgetTokens = params.budgetTokens === void 0 ? this.defaultBudgetTokens : params.budgetTokens;
|
|
1481
|
+
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
|
|
1064
1482
|
if (tools) {
|
|
1065
1483
|
const seen = new Set();
|
|
1066
1484
|
const duplicates = new Set();
|
|
@@ -1068,13 +1486,26 @@ var RequestBuilder = class {
|
|
|
1068
1486
|
if (seen.has(tool.name)) duplicates.add(tool.name);
|
|
1069
1487
|
seen.add(tool.name);
|
|
1070
1488
|
}
|
|
1071
|
-
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "
|
|
1489
|
+
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "invalid_params", {
|
|
1490
|
+
code: "duplicate_tool_names",
|
|
1491
|
+
issues: { names: [...duplicates] }
|
|
1492
|
+
});
|
|
1072
1493
|
}
|
|
1073
|
-
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "
|
|
1074
|
-
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1494
|
+
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "invalid_params");
|
|
1495
|
+
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "invalid_params", {
|
|
1496
|
+
code: "unknown_tool_choice",
|
|
1497
|
+
issues: {
|
|
1498
|
+
requested: toolChoice.name,
|
|
1499
|
+
available: tools.map((t) => t.name)
|
|
1500
|
+
}
|
|
1501
|
+
});
|
|
1502
|
+
const jsonModeExplicit = params.jsonMode;
|
|
1503
|
+
const jsonMode = jsonModeExplicit ?? (tools ? false : true);
|
|
1504
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === true) throw new LLMError("jsonMode: true was set explicitly, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.", "invalid_params");
|
|
1505
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 && params.schema) throw new LLMError("`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.", "invalid_params");
|
|
1506
|
+
const jsonModeEffective = !this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 ? false : jsonMode;
|
|
1507
|
+
const useJson = jsonModeEffective || Boolean(jsonSchema);
|
|
1508
|
+
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
|
|
1078
1509
|
const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
|
|
1079
1510
|
this.validateHistory(history);
|
|
1080
1511
|
const request = {
|
|
@@ -1083,6 +1514,7 @@ var RequestBuilder = class {
|
|
|
1083
1514
|
max_tokens: maxTokens,
|
|
1084
1515
|
...responseFormat ? { response_format: responseFormat } : {},
|
|
1085
1516
|
...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
|
|
1517
|
+
...budgetTokens !== void 0 && budgetTokens !== null ? { budget_tokens: budgetTokens } : {},
|
|
1086
1518
|
...tools ? { tools: toWireTools(tools) } : {},
|
|
1087
1519
|
...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
|
|
1088
1520
|
messages: [
|
|
@@ -1111,29 +1543,47 @@ var RequestBuilder = class {
|
|
|
1111
1543
|
let previousTurn;
|
|
1112
1544
|
for (const [index, turn] of history.entries()) {
|
|
1113
1545
|
if (turn.role === "tool") {
|
|
1114
|
-
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "
|
|
1115
|
-
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "
|
|
1546
|
+
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "invalid_params");
|
|
1547
|
+
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "invalid_params");
|
|
1116
1548
|
const requestedIds = new Set(previousTurn.toolCalls.map((tc) => tc.id));
|
|
1117
1549
|
const resultIds = turn.toolResults.map((tr) => tr.toolCallId);
|
|
1118
1550
|
const unknownIds = resultIds.filter((id) => !requestedIds.has(id));
|
|
1119
|
-
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "
|
|
1551
|
+
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "invalid_params", {
|
|
1552
|
+
code: "unknown_tool_result_ids",
|
|
1553
|
+
issues: {
|
|
1554
|
+
historyIndex: index,
|
|
1555
|
+
ids: unknownIds
|
|
1556
|
+
}
|
|
1557
|
+
});
|
|
1120
1558
|
const seenIds = new Set();
|
|
1121
1559
|
const duplicateIds = new Set();
|
|
1122
1560
|
for (const id of resultIds) {
|
|
1123
1561
|
if (seenIds.has(id)) duplicateIds.add(id);
|
|
1124
1562
|
seenIds.add(id);
|
|
1125
1563
|
}
|
|
1126
|
-
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "
|
|
1564
|
+
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "invalid_params", {
|
|
1565
|
+
code: "duplicate_tool_result_ids",
|
|
1566
|
+
issues: {
|
|
1567
|
+
historyIndex: index,
|
|
1568
|
+
ids: [...duplicateIds]
|
|
1569
|
+
}
|
|
1570
|
+
});
|
|
1127
1571
|
const missingIds = [...requestedIds].filter((id) => !resultIds.includes(id));
|
|
1128
|
-
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "
|
|
1572
|
+
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "invalid_params", {
|
|
1573
|
+
code: "missing_tool_results",
|
|
1574
|
+
issues: {
|
|
1575
|
+
historyIndex: index,
|
|
1576
|
+
ids: missingIds
|
|
1577
|
+
}
|
|
1578
|
+
});
|
|
1129
1579
|
} else {
|
|
1130
|
-
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "
|
|
1131
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "
|
|
1580
|
+
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "invalid_params");
|
|
1581
|
+
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "invalid_params");
|
|
1132
1582
|
}
|
|
1133
1583
|
previousTurn = turn;
|
|
1134
1584
|
}
|
|
1135
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "
|
|
1136
|
-
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "
|
|
1585
|
+
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "invalid_params");
|
|
1586
|
+
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "invalid_params");
|
|
1137
1587
|
}
|
|
1138
1588
|
/** Maps VernLLM's app-facing `ToolChoice` onto the OpenAI-shaped wire `tool_choice`. */
|
|
1139
1589
|
buildWireToolChoice(toolChoice) {
|
|
@@ -1160,9 +1610,13 @@ var RequestBuilder = class {
|
|
|
1160
1610
|
}));
|
|
1161
1611
|
if (turn.role === "assistant" && turn.toolCalls?.length) return [{
|
|
1162
1612
|
role: "assistant",
|
|
1163
|
-
...turn.content ? { content: turn.content } : {},
|
|
1613
|
+
...turn.content !== void 0 ? { content: serializeAssistantContent(turn.content) } : {},
|
|
1164
1614
|
tool_calls: toWireToolCalls(turn.toolCalls)
|
|
1165
1615
|
}];
|
|
1616
|
+
if (turn.role === "assistant") return [{
|
|
1617
|
+
role: "assistant",
|
|
1618
|
+
content: serializeAssistantContent(turn.content === void 0 ? "" : turn.content)
|
|
1619
|
+
}];
|
|
1166
1620
|
return [{
|
|
1167
1621
|
role: turn.role,
|
|
1168
1622
|
content: turn.content ?? ""
|
|
@@ -1296,10 +1750,12 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1296
1750
|
complete: wireChunk.complete
|
|
1297
1751
|
});
|
|
1298
1752
|
} else if (wireChunk.type === "usage") {
|
|
1753
|
+
const reasoningTokens = wireChunk.usage.completion_tokens_details?.reasoning_tokens;
|
|
1299
1754
|
usage = {
|
|
1300
1755
|
promptTokens: wireChunk.usage.prompt_tokens ?? 0,
|
|
1301
1756
|
completionTokens: wireChunk.usage.completion_tokens ?? 0,
|
|
1302
1757
|
totalTokens: wireChunk.usage.total_tokens ?? 0,
|
|
1758
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
1303
1759
|
requestId,
|
|
1304
1760
|
model,
|
|
1305
1761
|
provider: providerName,
|
|
@@ -1353,6 +1809,16 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1353
1809
|
//#endregion
|
|
1354
1810
|
//#region src/internal/execution/callExecutor.ts
|
|
1355
1811
|
/**
|
|
1812
|
+
* Identity function with its own parameter, used only to sidestep a TS
|
|
1813
|
+
* quirk: a `let` reassigned solely inside a nested closure (like
|
|
1814
|
+
* `retryWithBackoff`'s `onRequest`) gets narrowed to `undefined` at the
|
|
1815
|
+
* point it was last synchronously assigned, which would otherwise make
|
|
1816
|
+
* `lastRequestForAttempt` read as `never` at the point it's used below.
|
|
1817
|
+
*/
|
|
1818
|
+
function passThroughRequestSnapshot(snapshot) {
|
|
1819
|
+
return snapshot;
|
|
1820
|
+
}
|
|
1821
|
+
/**
|
|
1356
1822
|
* Everything one provider target needs to attempt a call: request
|
|
1357
1823
|
* building, retry with backoff, the per-target breaker, the per-target
|
|
1358
1824
|
* limiter. Never exported publicly. `VernLLM` holds one per target and
|
|
@@ -1395,12 +1861,27 @@ var CallExecutor = class {
|
|
|
1395
1861
|
this.requestBuilder = new RequestBuilder({
|
|
1396
1862
|
model,
|
|
1397
1863
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
1398
|
-
defaultTemperature: options.defaultTemperature
|
|
1864
|
+
defaultTemperature: options.defaultTemperature,
|
|
1865
|
+
defaultReasoningEffort: options.defaultReasoningEffort,
|
|
1866
|
+
defaultBudgetTokens: options.defaultBudgetTokens,
|
|
1867
|
+
supportsJsonObjectMode: client.supportsJsonObjectMode ?? true
|
|
1399
1868
|
});
|
|
1400
1869
|
}
|
|
1401
1870
|
getCircuitState(model) {
|
|
1402
1871
|
return this.breaker?.getState(model);
|
|
1403
1872
|
}
|
|
1873
|
+
/** Whether this target's breaker tracks failures per model. `false` if no breaker is configured. */
|
|
1874
|
+
get isolateByModel() {
|
|
1875
|
+
return this.breaker?.isolateByModel ?? false;
|
|
1876
|
+
}
|
|
1877
|
+
/** Manually opens this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1878
|
+
openCircuit(model) {
|
|
1879
|
+
this.breaker?.open(model);
|
|
1880
|
+
}
|
|
1881
|
+
/** Manually closes this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1882
|
+
closeCircuit(model) {
|
|
1883
|
+
this.breaker?.close(model);
|
|
1884
|
+
}
|
|
1404
1885
|
/**
|
|
1405
1886
|
* Throws if the breaker is open for this target/model, exactly like the
|
|
1406
1887
|
* check `run`/`runStream` used to make internally. Exposed so `VernLLM`
|
|
@@ -1422,10 +1903,11 @@ var CallExecutor = class {
|
|
|
1422
1903
|
*/
|
|
1423
1904
|
async run(params, requestId, onAttempt) {
|
|
1424
1905
|
const model = params.model ?? this.model;
|
|
1906
|
+
const attempts = [];
|
|
1425
1907
|
try {
|
|
1426
|
-
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt);
|
|
1908
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1427
1909
|
} catch (error) {
|
|
1428
|
-
const normalized = normalizeError(error, params.signal);
|
|
1910
|
+
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1429
1911
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1430
1912
|
this.logger.debug(`[VernLLM:${requestId}] error:\n${this.redactText(describeError(error))}`);
|
|
1431
1913
|
throw normalized;
|
|
@@ -1434,10 +1916,11 @@ var CallExecutor = class {
|
|
|
1434
1916
|
/** Streaming counterpart to `run`. Mirrors the old streaming branch of `VernLLM.call`. */
|
|
1435
1917
|
async runStream(params, requestId, onAttempt) {
|
|
1436
1918
|
const model = params.model ?? this.model;
|
|
1919
|
+
const attempts = [];
|
|
1437
1920
|
try {
|
|
1438
|
-
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt);
|
|
1921
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeStreamCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1439
1922
|
} catch (error) {
|
|
1440
|
-
const normalized = normalizeError(error, params.signal);
|
|
1923
|
+
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1441
1924
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1442
1925
|
this.logger.debug(`[VernLLM:${requestId}] stream-open error:\n${this.redactText(describeError(error))}`);
|
|
1443
1926
|
throw normalized;
|
|
@@ -1450,8 +1933,9 @@ var CallExecutor = class {
|
|
|
1450
1933
|
* set. Throws on an empty response (no text and no tool_calls) so the
|
|
1451
1934
|
* retry loop treats it like any other transient failure.
|
|
1452
1935
|
*/
|
|
1453
|
-
async executeCall(params, requestId, attempt) {
|
|
1936
|
+
async executeCall(params, requestId, attempt, onRequest) {
|
|
1454
1937
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1938
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1455
1939
|
let release;
|
|
1456
1940
|
if (this.limiter) {
|
|
1457
1941
|
const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
|
|
@@ -1505,11 +1989,11 @@ var CallExecutor = class {
|
|
|
1505
1989
|
finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt) {
|
|
1506
1990
|
try {
|
|
1507
1991
|
const content = rawContent?.trim();
|
|
1508
|
-
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api");
|
|
1992
|
+
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api", { code: "empty_response" });
|
|
1509
1993
|
this.logger.debug(`[VernLLM:${requestId}] output:\n${this.redactedOutput(content, wireToolCalls).slice(0, 800)}`);
|
|
1510
1994
|
if (wireToolCalls?.length) {
|
|
1511
|
-
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "
|
|
1512
|
-
if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "
|
|
1995
|
+
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "validation", { code: "unexpected_tool_calls" });
|
|
1996
|
+
if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "validation", { code: "tool_choice_none_violated" });
|
|
1513
1997
|
const toolCalls = parseWireToolCalls(wireToolCalls);
|
|
1514
1998
|
this.validateToolCallArguments(toolCalls, params.tools);
|
|
1515
1999
|
this.breaker?.recordSuccess(model);
|
|
@@ -1559,10 +2043,14 @@ var CallExecutor = class {
|
|
|
1559
2043
|
* not on the first chunk arriving, so a connection that opens but then
|
|
1560
2044
|
* dies mid-stream isn't masked as a success (see `buildStreamResult`).
|
|
1561
2045
|
*/
|
|
1562
|
-
async executeStreamCall(params, requestId, attempt) {
|
|
2046
|
+
async executeStreamCall(params, requestId, attempt, onRequest) {
|
|
1563
2047
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
2048
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1564
2049
|
const completions = this.client.chat.completions;
|
|
1565
|
-
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "
|
|
2050
|
+
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
|
|
2051
|
+
code: "unsupported_capability",
|
|
2052
|
+
issues: { capability: "createStream" }
|
|
2053
|
+
});
|
|
1566
2054
|
const createStream = completions.createStream.bind(completions);
|
|
1567
2055
|
let release;
|
|
1568
2056
|
if (this.limiter) {
|
|
@@ -1623,13 +2111,13 @@ var CallExecutor = class {
|
|
|
1623
2111
|
* `argumentsSchema`, if present.
|
|
1624
2112
|
*
|
|
1625
2113
|
* Contract failures (unknown name, duplicate id) are collected across
|
|
1626
|
-
* every call and thrown together
|
|
1627
|
-
*
|
|
1628
|
-
*
|
|
1629
|
-
*
|
|
1630
|
-
*
|
|
1631
|
-
*
|
|
1632
|
-
*
|
|
2114
|
+
* every call and thrown together as one `type: 'validation'` error with
|
|
2115
|
+
* `issues: ToolIssue[]`, since retrying a request that already has these
|
|
2116
|
+
* errors cannot help (excluded from retry by `type`) and a caller fixing
|
|
2117
|
+
* them wants to see every one, not just the first. Schema failures keep
|
|
2118
|
+
* the original single-error, `type: 'validation'` shape rather than being
|
|
2119
|
+
* folded into the aggregate, since they're a distinct failure kind from
|
|
2120
|
+
* the contract failures above.
|
|
1633
2121
|
*/
|
|
1634
2122
|
validateToolCallArguments(toolCalls, tools) {
|
|
1635
2123
|
const known = new Map(tools.map((t) => [t.name, t]));
|
|
@@ -1651,28 +2139,51 @@ var CallExecutor = class {
|
|
|
1651
2139
|
if (toolIssues.length > 0) {
|
|
1652
2140
|
const unknownTool = toolIssues.find((i) => i.code === "unknown_tool");
|
|
1653
2141
|
const primary = unknownTool ? `Model requested tool "${unknownTool.name}", which was not in the tools offered ([${[...known.keys()].join(", ")}]).` : `Duplicate tool call id "${toolIssues[0].toolCallId}" in the model's response.`;
|
|
1654
|
-
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
2142
|
+
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see error.issues.)` : primary;
|
|
2143
|
+
throw new LLMError(message, "validation", {
|
|
2144
|
+
code: unknownTool ? "unknown_tool" : "duplicate_tool_call_id",
|
|
2145
|
+
issues: toolIssues
|
|
2146
|
+
});
|
|
1658
2147
|
}
|
|
1659
2148
|
for (const call of toolCalls) {
|
|
1660
2149
|
const definition = known.get(call.name);
|
|
1661
2150
|
if (!definition?.argumentsSchema) continue;
|
|
1662
2151
|
const result = definition.argumentsSchema.safeParse(call.arguments);
|
|
1663
|
-
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation",
|
|
2152
|
+
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation", { issues: result.error });
|
|
1664
2153
|
}
|
|
1665
2154
|
}
|
|
1666
|
-
/**
|
|
1667
|
-
|
|
2155
|
+
/**
|
|
2156
|
+
* Runs `fn`, retrying with backoff according to `shouldRetry`. When
|
|
2157
|
+
* `attempts` is given, every failed attempt that is actually followed by
|
|
2158
|
+
* a retry is recorded, in order. This mirrors `LLMError.attempts`'s
|
|
2159
|
+
* contract: every attempt made before this error was thrown. The
|
|
2160
|
+
* terminal failure is never pushed since it isn't a prior attempt, it
|
|
2161
|
+
* is the error being thrown. `attempts` stays empty when nothing was
|
|
2162
|
+
* retried, so no separate bookkeeping is needed at the call sites.
|
|
2163
|
+
* Each failure is recorded as a snapshot (`LLMError.toSnapshot()`),
|
|
2164
|
+
* not the live `LLMError`, per `RetryAttempt`'s contract.
|
|
2165
|
+
*/
|
|
2166
|
+
async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
|
|
1668
2167
|
let lastError;
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
2168
|
+
let lastRequestForAttempt;
|
|
2169
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
2170
|
+
lastRequestForAttempt = void 0;
|
|
2171
|
+
try {
|
|
2172
|
+
if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
|
|
2173
|
+
onAttempt?.();
|
|
2174
|
+
return await fn(attempt, (req) => {
|
|
2175
|
+
lastRequestForAttempt = req;
|
|
2176
|
+
});
|
|
2177
|
+
} catch (error) {
|
|
2178
|
+
lastError = error;
|
|
2179
|
+
const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
|
|
2180
|
+
if (!willRetry) break;
|
|
2181
|
+
attempts?.push({
|
|
2182
|
+
index: attempt,
|
|
2183
|
+
error: normalizeError(error, signal).toSnapshot(),
|
|
2184
|
+
request: passThroughRequestSnapshot(lastRequestForAttempt)
|
|
2185
|
+
});
|
|
2186
|
+
}
|
|
1676
2187
|
}
|
|
1677
2188
|
throw lastError;
|
|
1678
2189
|
}
|
|
@@ -1684,10 +2195,12 @@ var CallExecutor = class {
|
|
|
1684
2195
|
*/
|
|
1685
2196
|
extractUsage(response, requestId, model) {
|
|
1686
2197
|
if (!response.usage) return void 0;
|
|
2198
|
+
const reasoningTokens = response.usage.completion_tokens_details?.reasoning_tokens;
|
|
1687
2199
|
return {
|
|
1688
2200
|
promptTokens: response.usage.prompt_tokens ?? 0,
|
|
1689
2201
|
completionTokens: response.usage.completion_tokens ?? 0,
|
|
1690
2202
|
totalTokens: response.usage.total_tokens ?? 0,
|
|
2203
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
1691
2204
|
requestId,
|
|
1692
2205
|
model,
|
|
1693
2206
|
provider: this.providerName,
|
|
@@ -1742,7 +2255,7 @@ var CallExecutor = class {
|
|
|
1742
2255
|
if (parsed === null || parsed === void 0) throw new LLMError("Invalid JSON response", "parse");
|
|
1743
2256
|
if (!schema) return parsed;
|
|
1744
2257
|
const result = schema.safeParse(parsed);
|
|
1745
|
-
if (!result.success) throw new LLMError("Schema validation failed", "validation",
|
|
2258
|
+
if (!result.success) throw new LLMError("Schema validation failed", "validation", { issues: result.error });
|
|
1746
2259
|
return result.data;
|
|
1747
2260
|
}
|
|
1748
2261
|
/**
|
|
@@ -1755,7 +2268,7 @@ var CallExecutor = class {
|
|
|
1755
2268
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
1756
2269
|
const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
|
|
1757
2270
|
const retryAfterHonored = retryAfterMs !== void 0;
|
|
1758
|
-
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${delay}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
2271
|
+
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${Math.ceil(delay)}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
1759
2272
|
this.reportEvent({
|
|
1760
2273
|
kind: "retry",
|
|
1761
2274
|
requestId,
|
|
@@ -1769,15 +2282,10 @@ var CallExecutor = class {
|
|
|
1769
2282
|
});
|
|
1770
2283
|
await waitForRetry(delay, signal);
|
|
1771
2284
|
}
|
|
1772
|
-
isNonRetryableToolContractError(error) {
|
|
1773
|
-
return error instanceof LLMError && (error.code === "unknown_tool" || error.code === "duplicate_tool_call_id" || error.code === "tool_choice_none_violated");
|
|
1774
|
-
}
|
|
1775
2285
|
/** Decides whether a failed attempt is worth retrying. */
|
|
1776
2286
|
shouldRetry(error, signal) {
|
|
1777
2287
|
if (signal?.aborted) return false;
|
|
1778
|
-
if (error instanceof LLMError &&
|
|
1779
|
-
if (error instanceof LLMError && error.code === "local_rate_limit") return false;
|
|
1780
|
-
if (this.isNonRetryableToolContractError(error)) return false;
|
|
2288
|
+
if (error instanceof LLMError && !error.retryable) return false;
|
|
1781
2289
|
const status = extractStatus(error);
|
|
1782
2290
|
return !(status !== void 0 && this.nonRetryableStatus.includes(status));
|
|
1783
2291
|
}
|
|
@@ -1787,16 +2295,48 @@ var CallExecutor = class {
|
|
|
1787
2295
|
* reusing a call id, or a provider ignoring `toolChoice: 'none'` isn't
|
|
1788
2296
|
* the provider being unhealthy, it's a model/provider response defect
|
|
1789
2297
|
* that will very likely recur regardless of provider health, so it
|
|
1790
|
-
* shouldn't push a healthy provider's circuit toward opening.
|
|
1791
|
-
*
|
|
1792
|
-
*
|
|
2298
|
+
* shouldn't push a healthy provider's circuit toward opening. Same for
|
|
2299
|
+
* a caller-input bug or a local rate-limit rejection: neither ever
|
|
2300
|
+
* reached the provider at all. This is exactly what `LLMError.retryable`
|
|
2301
|
+
* already excludes, so this defers to it directly.
|
|
1793
2302
|
*/
|
|
1794
2303
|
countsTowardBreaker(error) {
|
|
1795
|
-
|
|
1796
|
-
return true;
|
|
2304
|
+
return error.retryable;
|
|
1797
2305
|
}
|
|
1798
2306
|
};
|
|
1799
2307
|
|
|
2308
|
+
//#endregion
|
|
2309
|
+
//#region src/internal/logger.utils.ts
|
|
2310
|
+
/**
|
|
2311
|
+
* Wraps a `Logger` so a throwing implementation can never break the call
|
|
2312
|
+
* it's trying to describe. `logger` is user-supplied (`VernLLMOptions.logger`),
|
|
2313
|
+
* so a custom logger that ships to a file, Datadog, etc. can throw for
|
|
2314
|
+
* reasons unrelated to VernLLM. Wrap once at construction so every
|
|
2315
|
+
* downstream `this.logger.warn(...)` call stays as-is and is safe by
|
|
2316
|
+
* construction, instead of guarding each call site individually.
|
|
2317
|
+
*/
|
|
2318
|
+
function createSafeLogger(logger) {
|
|
2319
|
+
return {
|
|
2320
|
+
debug: safe(logger, "debug"),
|
|
2321
|
+
warn: safe(logger, "warn"),
|
|
2322
|
+
error: safe(logger, "error")
|
|
2323
|
+
};
|
|
2324
|
+
}
|
|
2325
|
+
function safe(logger, method) {
|
|
2326
|
+
const fn = logger[method].bind(logger);
|
|
2327
|
+
return (...args) => {
|
|
2328
|
+
try {
|
|
2329
|
+
swallowRejection(fn(...args));
|
|
2330
|
+
} catch {}
|
|
2331
|
+
};
|
|
2332
|
+
}
|
|
2333
|
+
function isPromiseLike(value) {
|
|
2334
|
+
return typeof value?.then === "function";
|
|
2335
|
+
}
|
|
2336
|
+
function swallowRejection(result) {
|
|
2337
|
+
if (isPromiseLike(result)) Promise.resolve(result).catch(() => {});
|
|
2338
|
+
}
|
|
2339
|
+
|
|
1800
2340
|
//#endregion
|
|
1801
2341
|
//#region src/logger.ts
|
|
1802
2342
|
/**
|
|
@@ -1937,8 +2477,8 @@ var RateLimiter = class {
|
|
|
1937
2477
|
*/
|
|
1938
2478
|
async acquire(estimatedTokens, signal) {
|
|
1939
2479
|
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
1940
|
-
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "
|
|
1941
|
-
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "
|
|
2480
|
+
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "invalid_params");
|
|
2481
|
+
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "rate_limited", { code: "rate_limit_capacity_exceeded" });
|
|
1942
2482
|
if (this.queue.length === 0) {
|
|
1943
2483
|
const attempt = this.tryAcquireBuckets(estimatedTokens);
|
|
1944
2484
|
if (attempt.ok) return {
|
|
@@ -1951,7 +2491,7 @@ var RateLimiter = class {
|
|
|
1951
2491
|
return this.enqueue(estimatedTokens, void 0, signal);
|
|
1952
2492
|
}
|
|
1953
2493
|
queueFullError() {
|
|
1954
|
-
return new LLMError("Rate limit queue is full", "
|
|
2494
|
+
return new LLMError("Rate limit queue is full", "rate_limited", { code: "rate_limit_queue_full" });
|
|
1955
2495
|
}
|
|
1956
2496
|
enqueue(estimatedTokens, initialReason, signal) {
|
|
1957
2497
|
return new Promise((resolvePromise, rejectPromise) => {
|
|
@@ -1984,7 +2524,7 @@ var RateLimiter = class {
|
|
|
1984
2524
|
if (index !== -1) this.queue.splice(index, 1);
|
|
1985
2525
|
};
|
|
1986
2526
|
if (this.maxQueueMs > 0) queueTimer = setTimeout(() => {
|
|
1987
|
-
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "
|
|
2527
|
+
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "rate_limited", { code: "rate_limit_queue_timeout" }));
|
|
1988
2528
|
}, this.maxQueueMs);
|
|
1989
2529
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1990
2530
|
this.queue.push(waiter);
|
|
@@ -2084,7 +2624,7 @@ var RateLimiter = class {
|
|
|
2084
2624
|
//#endregion
|
|
2085
2625
|
//#region src/vernLLM.ts
|
|
2086
2626
|
/**
|
|
2087
|
-
* A
|
|
2627
|
+
* A LLM call framework for resilience, observability and control. This is VernLLM!
|
|
2088
2628
|
*
|
|
2089
2629
|
* Adds retry with backoff and jitter, per-attempt timeouts, an optional
|
|
2090
2630
|
* circuit breaker, JSON parsing with optional schema validation, usage
|
|
@@ -2118,12 +2658,14 @@ var VernLLM = class {
|
|
|
2118
2658
|
* `nonRetryableStatus` `[400, 401, 403, 404, 422]`, `debug` false.
|
|
2119
2659
|
*/
|
|
2120
2660
|
constructor(options) {
|
|
2121
|
-
this.logger = options.logger ?? new ConsoleLogger(options.debug ?? false);
|
|
2661
|
+
this.logger = createSafeLogger(options.logger ?? new ConsoleLogger(options.debug ?? false));
|
|
2122
2662
|
const providerName = options.name ?? "primary";
|
|
2123
2663
|
this.cacheOrchestrator = new CacheOrchestrator(options.cache ?? new InMemoryCacheAdapter(), this.logger);
|
|
2124
2664
|
this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
|
|
2125
2665
|
this.reportEvent = makeEventReporter(options.onEvent, this.logger);
|
|
2126
2666
|
const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
|
|
2667
|
+
const primaryDefaultReasoningEffort = options.defaultReasoningEffort;
|
|
2668
|
+
const primaryDefaultBudgetTokens = options.defaultBudgetTokens;
|
|
2127
2669
|
const primaryTarget = {
|
|
2128
2670
|
client: options.client,
|
|
2129
2671
|
model: options.model,
|
|
@@ -2134,6 +2676,8 @@ var VernLLM = class {
|
|
|
2134
2676
|
baseDelayMs: options.baseDelayMs,
|
|
2135
2677
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
2136
2678
|
defaultTemperature: primaryDefaultTemperature,
|
|
2679
|
+
defaultReasoningEffort: primaryDefaultReasoningEffort,
|
|
2680
|
+
defaultBudgetTokens: primaryDefaultBudgetTokens,
|
|
2137
2681
|
nonRetryableStatus: options.nonRetryableStatus,
|
|
2138
2682
|
circuitBreaker: options.circuitBreaker,
|
|
2139
2683
|
rateLimit: options.rateLimit
|
|
@@ -2151,6 +2695,8 @@ var VernLLM = class {
|
|
|
2151
2695
|
baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
|
|
2152
2696
|
defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
|
|
2153
2697
|
defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
|
|
2698
|
+
defaultReasoningEffort: target.defaultReasoningEffort === void 0 ? primaryDefaultReasoningEffort : target.defaultReasoningEffort,
|
|
2699
|
+
defaultBudgetTokens: target.defaultBudgetTokens === void 0 ? primaryDefaultBudgetTokens : target.defaultBudgetTokens,
|
|
2154
2700
|
nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
|
|
2155
2701
|
400,
|
|
2156
2702
|
401,
|
|
@@ -2211,7 +2757,7 @@ var VernLLM = class {
|
|
|
2211
2757
|
index: i - 1,
|
|
2212
2758
|
provider: executor.providerName,
|
|
2213
2759
|
model: params.model ?? executor.model,
|
|
2214
|
-
error: normalized
|
|
2760
|
+
error: normalized.toSnapshot()
|
|
2215
2761
|
});
|
|
2216
2762
|
const isLast = i === this.executors.length - 1;
|
|
2217
2763
|
const policyDecision = this.fallbackOn(normalized, { isLastTarget: isLast });
|
|
@@ -2230,7 +2776,7 @@ var VernLLM = class {
|
|
|
2230
2776
|
});
|
|
2231
2777
|
}
|
|
2232
2778
|
}
|
|
2233
|
-
throw new LLMError("No provider targets configured", "
|
|
2779
|
+
throw new LLMError("No provider targets configured", "invalid_params");
|
|
2234
2780
|
}
|
|
2235
2781
|
async call(params) {
|
|
2236
2782
|
if (params.signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
@@ -2276,7 +2822,7 @@ var VernLLM = class {
|
|
|
2276
2822
|
async cachedCall(params) {
|
|
2277
2823
|
const { call: callParams,...cacheParams } = params;
|
|
2278
2824
|
const restCallParams = callParams;
|
|
2279
|
-
if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "
|
|
2825
|
+
if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "invalid_params");
|
|
2280
2826
|
if (restCallParams.stream) {
|
|
2281
2827
|
const streamParams = restCallParams;
|
|
2282
2828
|
return this.cacheOrchestrator.runCachedStream({
|
|
@@ -2290,36 +2836,107 @@ var VernLLM = class {
|
|
|
2290
2836
|
});
|
|
2291
2837
|
}
|
|
2292
2838
|
/**
|
|
2293
|
-
* @param
|
|
2294
|
-
* model
|
|
2295
|
-
*
|
|
2296
|
-
*
|
|
2297
|
-
*
|
|
2298
|
-
*
|
|
2839
|
+
* @param target.index Which target to read. Defaults to the primary.
|
|
2840
|
+
* @param target.model Which model bucket to read, if the target isolates by model.
|
|
2841
|
+
* @returns The breaker state, or `undefined` if that target has no breaker.
|
|
2842
|
+
* @throws {RangeError} If `target.index` names no target. Lets a real
|
|
2843
|
+
* target with no breaker (`undefined`) stay distinguishable from a
|
|
2844
|
+
* target that doesn't exist.
|
|
2299
2845
|
*/
|
|
2300
|
-
getCircuitState(
|
|
2301
|
-
|
|
2846
|
+
getCircuitState(target) {
|
|
2847
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "getCircuitState");
|
|
2848
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "getCircuitState");
|
|
2849
|
+
return executor.getCircuitState(target?.model ?? executor.model);
|
|
2302
2850
|
}
|
|
2303
2851
|
/**
|
|
2304
|
-
* @param model
|
|
2305
|
-
* target's
|
|
2306
|
-
* Ignored otherwise. Omit for the shared circuit (the default) or, under
|
|
2307
|
-
* isolation, the state of calls that didn't resolve a model.
|
|
2308
|
-
* @returns The current circuit state for every target in declaration
|
|
2309
|
-
* order, including the primary and all fallback targets. Each entry
|
|
2310
|
-
* includes the target's provider name, chain index, whether it is a
|
|
2311
|
-
* fallback, and its circuit state, or undefined if that target has no
|
|
2312
|
-
* circuit breaker configured.
|
|
2852
|
+
* @param model Which model bucket to read, for targets that isolate by model.
|
|
2853
|
+
* @returns Every target's state, in chain order.
|
|
2313
2854
|
*/
|
|
2314
2855
|
getCircuitStates(model) {
|
|
2315
2856
|
return this.executors.map((executor, index) => ({
|
|
2316
2857
|
provider: executor.providerName,
|
|
2317
2858
|
index,
|
|
2318
2859
|
isFallback: index > 0,
|
|
2319
|
-
|
|
2860
|
+
isolateByModel: executor.isolateByModel,
|
|
2861
|
+
state: executor.getCircuitState(model ?? executor.model)
|
|
2320
2862
|
}));
|
|
2321
2863
|
}
|
|
2864
|
+
/**
|
|
2865
|
+
* Manually opens a target's breaker, e.g. to pull a provider out of
|
|
2866
|
+
* rotation ahead of known maintenance instead of waiting for it to fail.
|
|
2867
|
+
*
|
|
2868
|
+
* @param target.index Which target to open. Defaults to the primary.
|
|
2869
|
+
* @param target.model Which model bucket to open, if the target isolates by model.
|
|
2870
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
2871
|
+
*/
|
|
2872
|
+
openCircuit(target) {
|
|
2873
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "openCircuit");
|
|
2874
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "openCircuit");
|
|
2875
|
+
executor.openCircuit(target?.model ?? executor.model);
|
|
2876
|
+
}
|
|
2877
|
+
/**
|
|
2878
|
+
* Manually closes a target's breaker, e.g. once a provider is confirmed
|
|
2879
|
+
* healthy again without waiting out the cooldown.
|
|
2880
|
+
*
|
|
2881
|
+
* @param target.index Which target to close. Defaults to the primary.
|
|
2882
|
+
* @param target.model Which model bucket to close, if the target isolates by model.
|
|
2883
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
2884
|
+
*/
|
|
2885
|
+
closeCircuit(target) {
|
|
2886
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "closeCircuit");
|
|
2887
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "closeCircuit");
|
|
2888
|
+
executor.closeCircuit(target?.model ?? executor.model);
|
|
2889
|
+
}
|
|
2890
|
+
/** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
|
|
2891
|
+
resolveExecutor(index, caller) {
|
|
2892
|
+
const executor = this.executors[index];
|
|
2893
|
+
if (!executor) throw new RangeError(`${caller}: no target at index ${index} (chain has ${this.executors.length} target${this.executors.length === 1 ? "" : "s"})`);
|
|
2894
|
+
return executor;
|
|
2895
|
+
}
|
|
2896
|
+
/** Warns when `model` can't do anything on this target, so it's never silently ignored. */
|
|
2897
|
+
warnIfModelUnsupported(isolateByModel, model, caller) {
|
|
2898
|
+
if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
|
|
2899
|
+
}
|
|
2322
2900
|
};
|
|
2901
|
+
/**
|
|
2902
|
+
* Identity function preserving `params`'s own precise type, unlike a `:
|
|
2903
|
+
* CallParams<T>` annotation, which would widen `tools` away and break the
|
|
2904
|
+
* `ConditionalToolCallParams<T>` overload for `tools: someCondition ?
|
|
2905
|
+
* [tool] : undefined`. Use it when you need `call()` params in a named,
|
|
2906
|
+
* reusable variable; skip it when you can pass the object inline.
|
|
2907
|
+
*
|
|
2908
|
+
* ```ts
|
|
2909
|
+
* const params = defineCallParams({
|
|
2910
|
+
* userContent: 'What is the weather?',
|
|
2911
|
+
* tools: someCondition ? [weatherTool] : undefined,
|
|
2912
|
+
* });
|
|
2913
|
+
* const result = await llm.call(params);
|
|
2914
|
+
* // result: unknown | CallWithToolsResult<unknown>, same as inline
|
|
2915
|
+
* ```
|
|
2916
|
+
*
|
|
2917
|
+
* `T` isn't a parameter here; pin it via `llm.call<T>(params)` as usual.
|
|
2918
|
+
* `defineCachedCallParams` is the `cachedCall()` counterpart.
|
|
2919
|
+
*/
|
|
2920
|
+
function defineCallParams(params) {
|
|
2921
|
+
return params;
|
|
2922
|
+
}
|
|
2923
|
+
/**
|
|
2924
|
+
* The `cachedCall()` counterpart to `defineCallParams`: preserves the
|
|
2925
|
+
* whole `{ cacheKey, ttl, call }` object, `call.tools` included, in one
|
|
2926
|
+
* named variable.
|
|
2927
|
+
*
|
|
2928
|
+
* ```ts
|
|
2929
|
+
* const params = defineCachedCallParams({
|
|
2930
|
+
* cacheKey: 'weather-ny',
|
|
2931
|
+
* ttl: 60,
|
|
2932
|
+
* call: { userContent: 'What is the weather?', tools: someCondition ? [weatherTool] : undefined },
|
|
2933
|
+
* });
|
|
2934
|
+
* const result = await llm.cachedCall(params);
|
|
2935
|
+
* ```
|
|
2936
|
+
*/
|
|
2937
|
+
function defineCachedCallParams(params) {
|
|
2938
|
+
return params;
|
|
2939
|
+
}
|
|
2323
2940
|
|
|
2324
2941
|
//#endregion
|
|
2325
2942
|
//#region src/adapters/internal/sse.ts
|
|
@@ -2358,7 +2975,7 @@ async function* parseSseStream(source) {
|
|
|
2358
2975
|
try {
|
|
2359
2976
|
text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
|
|
2360
2977
|
} catch (cause) {
|
|
2361
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse",
|
|
2978
|
+
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2362
2979
|
}
|
|
2363
2980
|
buffer = (buffer + text).replace(/\r\n/g, "\n").replace(/\r(?!$)/g, "\n");
|
|
2364
2981
|
let boundary$1 = buffer.indexOf("\n\n");
|
|
@@ -2374,7 +2991,7 @@ async function* parseSseStream(source) {
|
|
|
2374
2991
|
try {
|
|
2375
2992
|
buffer += decoder.decode();
|
|
2376
2993
|
} catch (cause) {
|
|
2377
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse",
|
|
2994
|
+
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2378
2995
|
}
|
|
2379
2996
|
buffer = buffer.replace(/\r$/, "\n");
|
|
2380
2997
|
let boundary = buffer.indexOf("\n\n");
|
|
@@ -2419,7 +3036,10 @@ function parseSseFrame(frame) {
|
|
|
2419
3036
|
try {
|
|
2420
3037
|
return JSON.parse(data);
|
|
2421
3038
|
} catch (cause) {
|
|
2422
|
-
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse",
|
|
3039
|
+
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse", {
|
|
3040
|
+
cause,
|
|
3041
|
+
code: "stream_frame_invalid"
|
|
3042
|
+
});
|
|
2423
3043
|
}
|
|
2424
3044
|
}
|
|
2425
3045
|
|
|
@@ -2439,13 +3059,14 @@ const SUPPORTED_IMAGE_MIME_TYPES = [
|
|
|
2439
3059
|
];
|
|
2440
3060
|
/**
|
|
2441
3061
|
* Validates an `ImageBlock.mimeType` against the shared supported set.
|
|
2442
|
-
* Throws a non-retryable `LLMError('
|
|
2443
|
-
* mimeType is a
|
|
2444
|
-
* the same
|
|
3062
|
+
* Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
|
|
3063
|
+
* mimeType is a bug in the caller's own input, deterministic before any
|
|
3064
|
+
* request is built, the same class of failure as every other check in
|
|
3065
|
+
* `RequestBuilder`.
|
|
2445
3066
|
*/
|
|
2446
3067
|
function assertSupportedImageMimeType(mimeType) {
|
|
2447
3068
|
if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
|
|
2448
|
-
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "
|
|
3069
|
+
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "invalid_params");
|
|
2449
3070
|
}
|
|
2450
3071
|
|
|
2451
3072
|
//#endregion
|
|
@@ -2456,6 +3077,275 @@ function supportsNativeStructuredOutput(model, override) {
|
|
|
2456
3077
|
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
2457
3078
|
}
|
|
2458
3079
|
|
|
3080
|
+
//#endregion
|
|
3081
|
+
//#region src/adapters/internal/reasoningBudget.utils.ts
|
|
3082
|
+
const DEFAULT_EFFORT_TOKENS = {
|
|
3083
|
+
minimal: 1024,
|
|
3084
|
+
low: 4096,
|
|
3085
|
+
medium: 16e3,
|
|
3086
|
+
high: 32e3
|
|
3087
|
+
};
|
|
3088
|
+
/**
|
|
3089
|
+
* Merges a caller-supplied partial override over `DEFAULT_EFFORT_TOKENS`.
|
|
3090
|
+
* Called once per adapter instance (not per request), so a per-instance
|
|
3091
|
+
* override only needs to specify the tiers it actually wants to change.
|
|
3092
|
+
*
|
|
3093
|
+
* Throws `LLMError('invalid_params')` if the override doesn't keep the
|
|
3094
|
+
* tiers in strictly ascending order (`minimal < low < medium < high`).
|
|
3095
|
+
* `budgetTokensToEffort` buckets by walking the tiers low to high and
|
|
3096
|
+
* returning on the first one a value is `<=`, so an unordered table (e.g.
|
|
3097
|
+
* `low` above `medium`) wouldn't just produce a "wrong" bucket, it would
|
|
3098
|
+
* make some tiers unreachable outright, silently, with no signal to the
|
|
3099
|
+
* caller that their override doesn't do what they think it does.
|
|
3100
|
+
*/
|
|
3101
|
+
function resolveEffortTokenTable(override) {
|
|
3102
|
+
if (!override) return DEFAULT_EFFORT_TOKENS;
|
|
3103
|
+
const table = {
|
|
3104
|
+
...DEFAULT_EFFORT_TOKENS,
|
|
3105
|
+
...override
|
|
3106
|
+
};
|
|
3107
|
+
if (!(table.minimal < table.low && table.low < table.medium && table.medium < table.high)) throw new LLMError(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(table)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`, "invalid_params");
|
|
3108
|
+
return table;
|
|
3109
|
+
}
|
|
3110
|
+
/** Converts a `reasoningEffort` tier into the nearest `budgetTokens` value. */
|
|
3111
|
+
function effortToBudgetTokens(effort, table = DEFAULT_EFFORT_TOKENS) {
|
|
3112
|
+
return table[effort];
|
|
3113
|
+
}
|
|
3114
|
+
/**
|
|
3115
|
+
* Converts a raw `budgetTokens` value into the nearest `reasoningEffort`
|
|
3116
|
+
* tier, for providers that only understand tiers. Buckets by the same
|
|
3117
|
+
* `table` `effortToBudgetTokens` produces its values from, so the two
|
|
3118
|
+
* functions agree with each other at the boundary values, as long as the
|
|
3119
|
+
* same (possibly overridden) table is passed to both. A value strictly
|
|
3120
|
+
* between two tiers (e.g. 4097, one above the default `low`) rounds up to
|
|
3121
|
+
* the next tier it's still `<=`, i.e. `medium` here, not down to `low`.
|
|
3122
|
+
*/
|
|
3123
|
+
function budgetTokensToEffort(budgetTokens, table = DEFAULT_EFFORT_TOKENS) {
|
|
3124
|
+
if (budgetTokens <= table.minimal) return "minimal";
|
|
3125
|
+
if (budgetTokens <= table.low) return "low";
|
|
3126
|
+
if (budgetTokens <= table.medium) return "medium";
|
|
3127
|
+
return "high";
|
|
3128
|
+
}
|
|
3129
|
+
/**
|
|
3130
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3131
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3132
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3133
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3134
|
+
* non-Opus model id.
|
|
3135
|
+
*/
|
|
3136
|
+
/**
|
|
3137
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3138
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3139
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3140
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3141
|
+
* non-Opus model id.
|
|
3142
|
+
*
|
|
3143
|
+
* Anthropic model ids sometimes carry a trailing snapshot date instead of
|
|
3144
|
+
* (or in addition to) an explicit minor version, e.g. the real, still-
|
|
3145
|
+
* supported base `"claude-opus-4-20250514"` (no `.7`-style minor at all,
|
|
3146
|
+
* just a date suffix directly after the major version). Read naively,
|
|
3147
|
+
* `20250514` looks like a minor version far above any real threshold and
|
|
3148
|
+
* would misclassify this pre-4.6 model as adaptive-only. Snapshot dates
|
|
3149
|
+
* are always 8 digits (`YYYYMMDD`); a real minor version never is, so an
|
|
3150
|
+
* 8+ digit second segment is treated as a date, not a minor version.
|
|
3151
|
+
*/
|
|
3152
|
+
function parseOpusVersion(model) {
|
|
3153
|
+
const match = /opus-(\d+)(?:-(\d+))?/.exec(model);
|
|
3154
|
+
if (!match) return null;
|
|
3155
|
+
const minorStr = match[2];
|
|
3156
|
+
const minor = minorStr === void 0 || minorStr.length >= 8 ? 0 : Number(minorStr);
|
|
3157
|
+
return [Number(match[1]), minor];
|
|
3158
|
+
}
|
|
3159
|
+
/**
|
|
3160
|
+
* Default rule for whether `model` only supports adaptive thinking
|
|
3161
|
+
* (`thinking: { type: 'adaptive' }`) and returns a 400 for manual,
|
|
3162
|
+
* budget-based thinking (`thinking: { type: 'enabled', budget_tokens }`):
|
|
3163
|
+
* Claude Opus 4.7 and later (matched as a version threshold, so 4.8, 4.9,
|
|
3164
|
+
* 5, and every future Opus point release are covered automatically,
|
|
3165
|
+
* without a new list entry per release), and every Claude 5 tier model
|
|
3166
|
+
* outside the Opus family (Sonnet 5, Fable 5, Mythos 5, Mythos Preview).
|
|
3167
|
+
* `mythos` alone is enough to catch both Mythos names without listing
|
|
3168
|
+
* each separately.
|
|
3169
|
+
*
|
|
3170
|
+
* Necessarily best-effort: a new model family with its own name (not
|
|
3171
|
+
* `opus-*`, not `sonnet-5`/`fable-5`/`mythos-*`) still needs a code
|
|
3172
|
+
* update here, or a caller-supplied `adaptiveOnlyModels` override (see
|
|
3173
|
+
* `isAdaptiveOnlyModel`) covering it in the meantime.
|
|
3174
|
+
*/
|
|
3175
|
+
function isDefaultAdaptiveOnly(model) {
|
|
3176
|
+
const opusVersion = parseOpusVersion(model);
|
|
3177
|
+
if (opusVersion) {
|
|
3178
|
+
const [major, minor] = opusVersion;
|
|
3179
|
+
return major > 4 || major === 4 && minor >= 7;
|
|
3180
|
+
}
|
|
3181
|
+
return [
|
|
3182
|
+
"sonnet-5",
|
|
3183
|
+
"fable-5",
|
|
3184
|
+
"mythos"
|
|
3185
|
+
].some((s) => model.includes(s));
|
|
3186
|
+
}
|
|
3187
|
+
/**
|
|
3188
|
+
* Whether `model` is adaptive-only, per the built-in rule above, or per a
|
|
3189
|
+
* caller-supplied `adaptiveOnlyModels` override. The override is
|
|
3190
|
+
* additive, not a replacement: it can mark an *additional* model as
|
|
3191
|
+
* adaptive-only (useful for a model family this package doesn't know
|
|
3192
|
+
* about yet), but it can't un-mark one the built-in rule already caught,
|
|
3193
|
+
* since a caller correcting a false negative is the only direction that
|
|
3194
|
+
* needs covering, a false positive here would mean this package is
|
|
3195
|
+
* simply wrong and needs its own fix, not a per-caller workaround.
|
|
3196
|
+
*/
|
|
3197
|
+
function isAdaptiveOnlyModel(model, override) {
|
|
3198
|
+
if (isDefaultAdaptiveOnly(model)) return true;
|
|
3199
|
+
if (!override) return false;
|
|
3200
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3201
|
+
}
|
|
3202
|
+
/** Whether `model` is known to support manual, budget-based thinking. */
|
|
3203
|
+
function supportsManualThinkingBudget(model, override) {
|
|
3204
|
+
return !isAdaptiveOnlyModel(model, override);
|
|
3205
|
+
}
|
|
3206
|
+
/**
|
|
3207
|
+
* Anthropic (and Claude models on Bedrock) require `budget_tokens` to be
|
|
3208
|
+
* at least 1024 and strictly less than `max_tokens`, since the thinking
|
|
3209
|
+
* budget and the reply share the same `max_tokens` ceiling. VernLLM's own
|
|
3210
|
+
* default `maxTokens` is 1000 (see `RequestBuilder`'s `defaultMaxTokens`),
|
|
3211
|
+
* below the 1024 floor, so the *default* `minimal` tier (1024 tokens) is
|
|
3212
|
+
* silently invalid against the *default* `max_tokens` unless a caller
|
|
3213
|
+
* happens to raise one or the other. Checked here, once, right before a
|
|
3214
|
+
* `thinking` block would be built, rather than left for Anthropic's own
|
|
3215
|
+
* 400 to explain after a real network round trip.
|
|
3216
|
+
*/
|
|
3217
|
+
function assertValidClaudeBudgetTokens(budgetTokens, maxTokens) {
|
|
3218
|
+
if (budgetTokens < 1024) throw new LLMError(`budgetTokens (${budgetTokens}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`, "invalid_params");
|
|
3219
|
+
if (budgetTokens >= maxTokens) throw new LLMError(`budgetTokens (${budgetTokens}) must be less than maxTokens (${maxTokens}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`, "invalid_params");
|
|
3220
|
+
}
|
|
3221
|
+
/**
|
|
3222
|
+
* Anthropic rejects any form of `thinking` (manual `budget_tokens` or
|
|
3223
|
+
* adaptive) combined with a `tool_choice` that forces tool use, a forced
|
|
3224
|
+
* single tool or "must call some tool", with a 400: `"Thinking may not be
|
|
3225
|
+
* enabled when tool_choice forces tool use."` Auto/none (or no tools at
|
|
3226
|
+
* all) are unaffected, thinking only conflicts with a choice that removes
|
|
3227
|
+
* the model's ability to just reply with text. This is a Claude-model
|
|
3228
|
+
* constraint, not specific to the Anthropic API's own wire shape, so it
|
|
3229
|
+
* applies identically to Claude models called through Bedrock's Converse
|
|
3230
|
+
* API, which forwards `thinking` under `additionalModelRequestFields` but
|
|
3231
|
+
* is still talking to the same underlying model.
|
|
3232
|
+
*
|
|
3233
|
+
* This combination can arise two ways: a caller explicitly sets both
|
|
3234
|
+
* `budgetTokens`/`reasoningEffort` and a forced `toolChoice`, or, more
|
|
3235
|
+
* subtly (Anthropic adapter only), a caller sets `jsonSchema` on a model
|
|
3236
|
+
* without native structured output support, which silently forces a
|
|
3237
|
+
* single synthetic tool call to emulate it, with no `tool_choice` of the
|
|
3238
|
+
* caller's own in sight. Both end up resolving to a forced tool choice by
|
|
3239
|
+
* the time each adapter calls this, so checking the adapter's own
|
|
3240
|
+
* already-resolved choice (rather than the caller's raw
|
|
3241
|
+
* `params.tool_choice`) catches both, right before a `thinking` block
|
|
3242
|
+
* would be built, rather than left for Anthropic's own 400 to explain
|
|
3243
|
+
* after a real network round trip.
|
|
3244
|
+
*
|
|
3245
|
+
* Takes a plain description of the forced choice rather than either
|
|
3246
|
+
* adapter's own wire shape (Anthropic SDK's `{ type: 'tool' | 'any', ... }`
|
|
3247
|
+
* vs Converse's `{ tool: {...} } | { any: {} }`), so both adapters can
|
|
3248
|
+
* share one check without either shape leaking into this file. Pass
|
|
3249
|
+
* `undefined` when the resolved choice is `auto`/`none`/unset, forcing
|
|
3250
|
+
* nothing.
|
|
3251
|
+
*/
|
|
3252
|
+
function assertNoForcedToolChoiceWithThinking(forcedChoiceDescription) {
|
|
3253
|
+
if (!forcedChoiceDescription) return;
|
|
3254
|
+
throw new LLMError(`budgetTokens/reasoningEffort was set alongside ${forcedChoiceDescription}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`, "invalid_params");
|
|
3255
|
+
}
|
|
3256
|
+
/**
|
|
3257
|
+
* Maps VernLLM's four-tier `reasoningEffort` onto Anthropic's five-tier
|
|
3258
|
+
* adaptive effort. `xhigh` and `max` have no VernLLM-side equivalent and
|
|
3259
|
+
* are unreachable through this mapping; a caller who wants either has to
|
|
3260
|
+
* target Anthropic/Bedrock-specific behavior already, so there's no gap
|
|
3261
|
+
* the shared `CallParams` surface needs to cover for a first pass.
|
|
3262
|
+
*/
|
|
3263
|
+
function toClaudeAdaptiveEffort(effort) {
|
|
3264
|
+
return effort === "minimal" ? "low" : effort;
|
|
3265
|
+
}
|
|
3266
|
+
/** Converts VernLLM's `reasoningEffort` directly into Gemini's `ThinkingLevel` enum value. */
|
|
3267
|
+
function toGeminiThinkingLevel(effort, model) {
|
|
3268
|
+
return clampGeminiThinkingLevel(model, effort.toUpperCase());
|
|
3269
|
+
}
|
|
3270
|
+
/**
|
|
3271
|
+
* Parses a Gemini model id's minor version, e.g. `"gemini-3.1-pro"` -> `1`,
|
|
3272
|
+
* `"gemini-3-pro"` -> `0` (no explicit minor). Only meaningful alongside
|
|
3273
|
+
* `parseGeminiMajorVersion`.
|
|
3274
|
+
*/
|
|
3275
|
+
function parseGeminiMinorVersion(model) {
|
|
3276
|
+
const match = /gemini-\d+\.(\d+)/.exec(model);
|
|
3277
|
+
return match ? Number(match[1]) : 0;
|
|
3278
|
+
}
|
|
3279
|
+
/**
|
|
3280
|
+
* Some Gemini 3 "Pro" tier models accept a narrower set of `thinkingLevel`
|
|
3281
|
+
* values than VernLLM's four tiers map onto, confirmed against real API
|
|
3282
|
+
* 400s and Google's own migration guidance, not assumed:
|
|
3283
|
+
* - Gemini 3 Pro (major 3, minor 0, e.g. `"gemini-3-pro-preview"`): only
|
|
3284
|
+
* `LOW` and `HIGH`; `MEDIUM` returns a 400 ("Thinking level MEDIUM is
|
|
3285
|
+
* not supported for this model").
|
|
3286
|
+
* - Gemini 3.1 Pro (major 3, minor >= 1): `LOW`/`MEDIUM`/`HIGH`, no
|
|
3287
|
+
* `MINIMAL`, Google's own docs point users toward a Flash-tier model
|
|
3288
|
+
* instead for the lowest setting.
|
|
3289
|
+
* - Every Flash-tier Gemini 3+ model accepts the full four levels, no
|
|
3290
|
+
* clamping needed, matched by this function simply not applying to
|
|
3291
|
+
* anything without `"pro"` in the model id.
|
|
3292
|
+
*
|
|
3293
|
+
* Clamped automatically rather than left to error, since `reasoningEffort`
|
|
3294
|
+
* is a per-call value, a caller hitting this isn't misconfiguring an
|
|
3295
|
+
* instance once, they're getting an intermittent-looking failure on
|
|
3296
|
+
* whichever specific call happened to pick an unsupported tier. Necessarily
|
|
3297
|
+
* best-effort: a future Pro-tier release could add back a level this rule
|
|
3298
|
+
* still clamps, or clamp one this rule doesn't yet know to touch.
|
|
3299
|
+
*/
|
|
3300
|
+
function clampGeminiThinkingLevel(model, level) {
|
|
3301
|
+
if (!model.includes("pro")) return level;
|
|
3302
|
+
const major = parseGeminiMajorVersion(model);
|
|
3303
|
+
if (major === null || major < 3) return level;
|
|
3304
|
+
const minor = parseGeminiMinorVersion(model);
|
|
3305
|
+
if (minor === 0) return level === "HIGH" ? "HIGH" : "LOW";
|
|
3306
|
+
return level === "MINIMAL" ? "LOW" : level;
|
|
3307
|
+
}
|
|
3308
|
+
/**
|
|
3309
|
+
* Parses a Gemini model id's major generation number, e.g.
|
|
3310
|
+
* `"gemini-3.1-flash-lite"` -> `3`, `"gemini-2.5-flash"` -> `2`. Not
|
|
3311
|
+
* anchored, so a Vertex-prefixed or otherwise decorated id still matches.
|
|
3312
|
+
* Returns `null` for a non-Gemini model id.
|
|
3313
|
+
*/
|
|
3314
|
+
function parseGeminiMajorVersion(model) {
|
|
3315
|
+
const match = /gemini-(\d+)/.exec(model);
|
|
3316
|
+
return match ? Number(match[1]) : null;
|
|
3317
|
+
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Default rule for whether `model` uses `thinkingLevel` instead of
|
|
3320
|
+
* `thinkingBudget`: every Gemini 3 series model and later, matched as a
|
|
3321
|
+
* version threshold so 3.1, 3.5, 3.6, and every future Gemini 3.x or
|
|
3322
|
+
* later release are covered automatically, without a new entry per
|
|
3323
|
+
* release, same reasoning as `isDefaultAdaptiveOnly`'s Opus threshold.
|
|
3324
|
+
* Gemini 2.5 and earlier still use `thinkingBudget`.
|
|
3325
|
+
*
|
|
3326
|
+
* `thinkingBudget` is still *accepted* on Gemini 3 for backward
|
|
3327
|
+
* compatibility, per Google's own docs, but "may result in unexpected
|
|
3328
|
+
* performance" there, so this rule switches VernLLM's own default
|
|
3329
|
+
* behavior over rather than leaving it on the old field indefinitely.
|
|
3330
|
+
*/
|
|
3331
|
+
function isDefaultThinkingLevelModel(model) {
|
|
3332
|
+
const major = parseGeminiMajorVersion(model);
|
|
3333
|
+
return major !== null && major >= 3;
|
|
3334
|
+
}
|
|
3335
|
+
/**
|
|
3336
|
+
* Whether `model` uses `thinkingLevel`, per the built-in version
|
|
3337
|
+
* threshold above, or per a caller-supplied `thinkingLevelModels`
|
|
3338
|
+
* override. Additive, not a replacement, same reasoning as
|
|
3339
|
+
* `isAdaptiveOnlyModel`: an override can mark an *additional* model as
|
|
3340
|
+
* using `thinkingLevel` (a model family this package doesn't recognize
|
|
3341
|
+
* yet), it can't un-mark one the built-in threshold already caught.
|
|
3342
|
+
*/
|
|
3343
|
+
function usesGeminiThinkingLevel(model, override) {
|
|
3344
|
+
if (isDefaultThinkingLevelModel(model)) return true;
|
|
3345
|
+
if (!override) return false;
|
|
3346
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3347
|
+
}
|
|
3348
|
+
|
|
2459
3349
|
//#endregion
|
|
2460
3350
|
//#region src/adapters/anthropic.ts
|
|
2461
3351
|
/**
|
|
@@ -2540,7 +3430,7 @@ function buildAnthropicTools(tools, toolChoiceParam) {
|
|
|
2540
3430
|
* `params.tools` are left for the normal, non-forced tool-call handling
|
|
2541
3431
|
* both `create` and `createStream` already do when `toolName` is unset.
|
|
2542
3432
|
*/
|
|
2543
|
-
function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
3433
|
+
function buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
2544
3434
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2545
3435
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2546
3436
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -2548,8 +3438,8 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2548
3438
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
2549
3439
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
2550
3440
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
|
|
3441
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic's native output_config.format on covered models, or a forced single tool call otherwise).", "validation");
|
|
2551
3442
|
let toolName;
|
|
2552
|
-
let jsonInstruction;
|
|
2553
3443
|
let outputFormat;
|
|
2554
3444
|
let tools;
|
|
2555
3445
|
let toolChoice;
|
|
@@ -2572,20 +3462,42 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2572
3462
|
type: "tool",
|
|
2573
3463
|
name: toolName
|
|
2574
3464
|
};
|
|
2575
|
-
}
|
|
3465
|
+
}
|
|
2576
3466
|
if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
|
|
2577
|
-
|
|
3467
|
+
let thinking;
|
|
3468
|
+
let effort;
|
|
3469
|
+
if (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0) {
|
|
3470
|
+
assertNoForcedToolChoiceWithThinking(toolChoice?.type === "tool" ? `toolChoice forcing the "${toolChoice.name}" tool` : toolChoice?.type === "any" ? "toolChoice: 'required' (Anthropic's \"any\" tool_choice)" : void 0);
|
|
3471
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
3472
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
3473
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
3474
|
+
thinking = {
|
|
3475
|
+
type: "enabled",
|
|
3476
|
+
budget_tokens: budgetTokens
|
|
3477
|
+
};
|
|
3478
|
+
} else {
|
|
3479
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
3480
|
+
thinking = { type: "adaptive" };
|
|
3481
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
3482
|
+
}
|
|
3483
|
+
}
|
|
3484
|
+
const system = systemMessage?.content;
|
|
3485
|
+
const temperature = thinking ? void 0 : params.temperature;
|
|
2578
3486
|
const body = {
|
|
2579
3487
|
model: params.model,
|
|
2580
3488
|
max_tokens: params.max_tokens,
|
|
2581
|
-
...
|
|
3489
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
2582
3490
|
system: system || void 0,
|
|
2583
3491
|
messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
|
|
2584
3492
|
...tools ? {
|
|
2585
3493
|
tools,
|
|
2586
3494
|
tool_choice: toolChoice
|
|
2587
3495
|
} : {},
|
|
2588
|
-
...outputFormat ? { output_config: {
|
|
3496
|
+
...outputFormat || effort ? { output_config: {
|
|
3497
|
+
...outputFormat ? { format: outputFormat } : {},
|
|
3498
|
+
...effort ? { effort } : {}
|
|
3499
|
+
} } : {},
|
|
3500
|
+
...thinking ? { thinking } : {}
|
|
2589
3501
|
};
|
|
2590
3502
|
return {
|
|
2591
3503
|
body,
|
|
@@ -2614,103 +3526,113 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2614
3526
|
* schema matching applies only when `strict: true` is forwarded and
|
|
2615
3527
|
* supported.
|
|
2616
3528
|
*
|
|
2617
|
-
* `response_format: json_object` (
|
|
2618
|
-
*
|
|
2619
|
-
*
|
|
2620
|
-
*
|
|
2621
|
-
*
|
|
3529
|
+
* `response_format: json_object` throws `LLMError('validation')`. Anthropic
|
|
3530
|
+
* has no API-level field that mechanically guarantees JSON output the way
|
|
3531
|
+
* OpenAI's `json_object` mode does; the only way to emulate it was a
|
|
3532
|
+
* system-prompt instruction with no actual enforcement behind it, a
|
|
3533
|
+
* guarantee this adapter no longer pretends to make. Use `jsonSchema`
|
|
3534
|
+
* instead, which maps to a real constraint either way (native
|
|
3535
|
+
* `output_config.format` or a forced tool call).
|
|
2622
3536
|
*/
|
|
2623
3537
|
function fromAnthropic(anthropicClient, options) {
|
|
2624
3538
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3539
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3540
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
2625
3541
|
const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
|
|
2626
|
-
return {
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
if (
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
choices: [{ message: {
|
|
2651
|
-
content: text,
|
|
2652
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
2653
|
-
} }],
|
|
2654
|
-
usage: {
|
|
2655
|
-
prompt_tokens: response.usage?.input_tokens,
|
|
2656
|
-
completion_tokens: response.usage?.output_tokens,
|
|
2657
|
-
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
|
|
3542
|
+
return {
|
|
3543
|
+
supportsJsonObjectMode: false,
|
|
3544
|
+
chat: { completions: {
|
|
3545
|
+
async create(params, options$1) {
|
|
3546
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3547
|
+
const response = await anthropicClient.messages.create(body, options$1);
|
|
3548
|
+
let text;
|
|
3549
|
+
let wireToolCalls;
|
|
3550
|
+
if (toolName) {
|
|
3551
|
+
const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
|
|
3552
|
+
if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3553
|
+
if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
|
|
3554
|
+
text = JSON.stringify(toolUse.input);
|
|
3555
|
+
} else {
|
|
3556
|
+
text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
|
|
3557
|
+
const toolUses = response.content.filter((block) => block.type === "tool_use");
|
|
3558
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
|
|
3559
|
+
id: block.id,
|
|
3560
|
+
type: "function",
|
|
3561
|
+
function: {
|
|
3562
|
+
name: block.name,
|
|
3563
|
+
arguments: JSON.stringify(block.input ?? {})
|
|
3564
|
+
}
|
|
3565
|
+
}));
|
|
2658
3566
|
}
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
2671
|
-
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
2672
|
-
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
2673
|
-
blockKinds.set(event.index, kind);
|
|
2674
|
-
if (kind === "json-tool") sawJsonTool = true;
|
|
2675
|
-
else if (!toolName) yield {
|
|
2676
|
-
type: "tool_call_delta",
|
|
2677
|
-
index: event.index,
|
|
2678
|
-
id: event.content_block.id,
|
|
2679
|
-
name: event.content_block.name
|
|
3567
|
+
return {
|
|
3568
|
+
choices: [{ message: {
|
|
3569
|
+
content: text,
|
|
3570
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3571
|
+
} }],
|
|
3572
|
+
usage: {
|
|
3573
|
+
prompt_tokens: response.usage?.input_tokens,
|
|
3574
|
+
completion_tokens: response.usage?.output_tokens,
|
|
3575
|
+
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0),
|
|
3576
|
+
...response.usage?.output_tokens_details?.thinking_tokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usage.output_tokens_details.thinking_tokens } } : {}
|
|
3577
|
+
}
|
|
2680
3578
|
};
|
|
2681
|
-
}
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
3579
|
+
},
|
|
3580
|
+
async *createStream(params, options$1) {
|
|
3581
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3582
|
+
const stream = await rawMessagesCreate({
|
|
3583
|
+
...body,
|
|
3584
|
+
stream: true
|
|
3585
|
+
}, options$1);
|
|
3586
|
+
const blockKinds = new Map();
|
|
3587
|
+
let inputTokens = 0;
|
|
3588
|
+
let sawJsonTool = false;
|
|
3589
|
+
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3590
|
+
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3591
|
+
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3592
|
+
blockKinds.set(event.index, kind);
|
|
3593
|
+
if (kind === "json-tool") sawJsonTool = true;
|
|
2694
3594
|
else if (!toolName) yield {
|
|
2695
3595
|
type: "tool_call_delta",
|
|
2696
3596
|
index: event.index,
|
|
2697
|
-
|
|
3597
|
+
id: event.content_block.id,
|
|
3598
|
+
name: event.content_block.name
|
|
2698
3599
|
};
|
|
2699
|
-
}
|
|
2700
|
-
|
|
2701
|
-
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
3600
|
+
} else blockKinds.set(event.index, "text");
|
|
3601
|
+
else if (event.type === "content_block_delta") {
|
|
3602
|
+
if (event.delta.type === "text_delta") {
|
|
3603
|
+
if (!toolName) yield {
|
|
3604
|
+
type: "text-delta",
|
|
3605
|
+
delta: event.delta.text
|
|
3606
|
+
};
|
|
3607
|
+
} else if (event.delta.type === "input_json_delta") {
|
|
3608
|
+
const kind = blockKinds.get(event.index);
|
|
3609
|
+
if (kind === "json-tool") yield {
|
|
3610
|
+
type: "text-delta",
|
|
3611
|
+
delta: event.delta.partial_json
|
|
3612
|
+
};
|
|
3613
|
+
else if (!toolName) yield {
|
|
3614
|
+
type: "tool_call_delta",
|
|
3615
|
+
index: event.index,
|
|
3616
|
+
argumentsDelta: event.delta.partial_json
|
|
3617
|
+
};
|
|
2708
3618
|
}
|
|
2709
|
-
}
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
3619
|
+
} else if (event.type === "message_delta") {
|
|
3620
|
+
const outputTokens = event.usage?.output_tokens ?? 0;
|
|
3621
|
+
const thinkingTokens = event.usage?.output_tokens_details?.thinking_tokens;
|
|
3622
|
+
yield {
|
|
3623
|
+
type: "usage",
|
|
3624
|
+
usage: {
|
|
3625
|
+
prompt_tokens: inputTokens,
|
|
3626
|
+
completion_tokens: outputTokens,
|
|
3627
|
+
total_tokens: inputTokens + outputTokens,
|
|
3628
|
+
...thinkingTokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: thinkingTokens } } : {}
|
|
3629
|
+
}
|
|
3630
|
+
};
|
|
3631
|
+
} else if (event.type === "ping") yield { type: "ping" };
|
|
3632
|
+
if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3633
|
+
}
|
|
3634
|
+
} }
|
|
3635
|
+
};
|
|
2714
3636
|
}
|
|
2715
3637
|
/**
|
|
2716
3638
|
* Anthropic requires strict role alternation, so the per-wire-message
|
|
@@ -2758,7 +3680,7 @@ function toAnthropicMessage(m) {
|
|
|
2758
3680
|
try {
|
|
2759
3681
|
input = tc.function.arguments.trim() ? JSON.parse(tc.function.arguments) : {};
|
|
2760
3682
|
} catch (cause) {
|
|
2761
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation",
|
|
3683
|
+
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
2762
3684
|
}
|
|
2763
3685
|
if (input === null || Array.isArray(input) || typeof input !== "object") throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) arguments must be a JSON object.`, "validation");
|
|
2764
3686
|
blocks.push({
|
|
@@ -2839,17 +3761,31 @@ function parseToolArguments(text, toolName) {
|
|
|
2839
3761
|
try {
|
|
2840
3762
|
parsed = text.trim() ? JSON.parse(text) : {};
|
|
2841
3763
|
} catch (cause) {
|
|
2842
|
-
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "
|
|
3764
|
+
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "parse", {
|
|
3765
|
+
cause,
|
|
3766
|
+
code: "tool_arguments_parse_failed"
|
|
3767
|
+
});
|
|
2843
3768
|
}
|
|
2844
3769
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
|
|
2845
3770
|
return parsed;
|
|
2846
3771
|
}
|
|
3772
|
+
/**
|
|
3773
|
+
* Parses a wire tool message's `content` into the object Gemini's
|
|
3774
|
+
* `functionResponse.response` expects. Gemini (and the real SDK's
|
|
3775
|
+
* `FunctionResponse.response` type) requires an object, so a result that
|
|
3776
|
+
* parses to something other than a plain JSON object (a string, number,
|
|
3777
|
+
* array, or unparseable text) is wrapped under an `output` key, mirroring
|
|
3778
|
+
* Gemini's own documented convention for non-object function results.
|
|
3779
|
+
*/
|
|
2847
3780
|
function parseToolResult(text) {
|
|
3781
|
+
let parsed;
|
|
2848
3782
|
try {
|
|
2849
|
-
|
|
3783
|
+
parsed = text.trim() ? JSON.parse(text) : "";
|
|
2850
3784
|
} catch {
|
|
2851
|
-
|
|
3785
|
+
parsed = text;
|
|
2852
3786
|
}
|
|
3787
|
+
if (parsed && !Array.isArray(parsed) && typeof parsed === "object") return parsed;
|
|
3788
|
+
return { output: parsed };
|
|
2853
3789
|
}
|
|
2854
3790
|
/**
|
|
2855
3791
|
* Gemini expects the results of everything the model asked for in one turn
|
|
@@ -2878,7 +3814,7 @@ function mergeConsecutiveFunctionResponses(contents) {
|
|
|
2878
3814
|
* `abortSignal` is folded into `config` by the caller (`create`/
|
|
2879
3815
|
* `createStream`), once the request options are available.
|
|
2880
3816
|
*/
|
|
2881
|
-
function buildGeminiRequest(params) {
|
|
3817
|
+
function buildGeminiRequest(params, effortTokenTable, thinkingLevelModels) {
|
|
2882
3818
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2883
3819
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2884
3820
|
const wantsJson = Boolean(params.response_format);
|
|
@@ -2903,51 +3839,37 @@ function buildGeminiRequest(params) {
|
|
|
2903
3839
|
})) }];
|
|
2904
3840
|
config.toolConfig = toGeminiToolConfig(params.tool_choice);
|
|
2905
3841
|
}
|
|
3842
|
+
if (usesGeminiThinkingLevel(params.model, thinkingLevelModels)) {
|
|
3843
|
+
const effortTier = params.reasoning_effort ?? (params.budget_tokens !== void 0 ? budgetTokensToEffort(params.budget_tokens, effortTokenTable) : void 0);
|
|
3844
|
+
if (effortTier !== void 0) config.thinkingConfig = { thinkingLevel: toGeminiThinkingLevel(effortTier, params.model) };
|
|
3845
|
+
} else {
|
|
3846
|
+
const thinkingBudget = params.budget_tokens ?? (params.reasoning_effort ? effortToBudgetTokens(params.reasoning_effort, effortTokenTable) : void 0);
|
|
3847
|
+
if (thinkingBudget !== void 0) config.thinkingConfig = { thinkingBudget };
|
|
3848
|
+
}
|
|
2906
3849
|
return {
|
|
2907
3850
|
model: params.model,
|
|
2908
3851
|
contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
|
|
2909
3852
|
config
|
|
2910
3853
|
};
|
|
2911
3854
|
}
|
|
2912
|
-
|
|
2913
|
-
|
|
2914
|
-
|
|
2915
|
-
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
*
|
|
2923
|
-
* `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
|
|
2924
|
-
* `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
|
|
2925
|
-
* `responseSchema` and `tools` in the same request natively, so both are
|
|
2926
|
-
* set independently here and no special-casing is needed for the
|
|
2927
|
-
* combination, unlike `fromAnthropic`/`fromBedrock`.
|
|
2928
|
-
*
|
|
2929
|
-
* `createStream` calls `generateContentStream` (optional on `GeminiClient`
|
|
2930
|
-
*, required only if the caller sets `stream: true`) and translates each
|
|
2931
|
-
* partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
|
|
2932
|
-
* Gemini's own function-calling API doesn't stream tool-call arguments
|
|
2933
|
-
* incrementally: a `functionCall` part always arrives whole in one chunk,
|
|
2934
|
-
* so each one is emitted as a single, complete `tool_call_delta` (a
|
|
2935
|
-
* one-shot "delta" containing the full arguments) rather than accumulated
|
|
2936
|
-
* fragments, that's a real difference in the underlying API, not
|
|
2937
|
-
* something this adapter can smooth over. `usageMetadata` is (per Gemini's
|
|
2938
|
-
* own behavior) only reliably present on the last chunk, so the `usage`
|
|
2939
|
-
* `WireStreamChunk` is emitted once, after the stream completes, from
|
|
2940
|
-
* whichever chunk's `usageMetadata` was seen last.
|
|
2941
|
-
*/
|
|
2942
|
-
function fromGemini(geminiClient) {
|
|
3855
|
+
function fromGemini(client, options) {
|
|
3856
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3857
|
+
const thinkingLevelModels = options?.thinkingLevelModels;
|
|
3858
|
+
const resolved = client.models ?? client;
|
|
3859
|
+
if (typeof resolved.generateContent !== "function") throw new LLMError("fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).", "invalid_params", {
|
|
3860
|
+
code: "unsupported_capability",
|
|
3861
|
+
issues: { capability: "generateContent" }
|
|
3862
|
+
});
|
|
3863
|
+
const generateContent = resolved.generateContent.bind(resolved);
|
|
3864
|
+
const generateContentStream = typeof resolved.generateContentStream === "function" ? resolved.generateContentStream.bind(resolved) : void 0;
|
|
2943
3865
|
return { chat: { completions: {
|
|
2944
|
-
async create(params, options) {
|
|
2945
|
-
const request = buildGeminiRequest(params);
|
|
3866
|
+
async create(params, options$1) {
|
|
3867
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
2946
3868
|
request.config = {
|
|
2947
3869
|
...request.config,
|
|
2948
|
-
abortSignal: options.signal
|
|
3870
|
+
abortSignal: options$1.signal
|
|
2949
3871
|
};
|
|
2950
|
-
const response = await
|
|
3872
|
+
const response = await generateContent(request);
|
|
2951
3873
|
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2952
3874
|
const text = parts.map((p) => p.text ?? "").join("");
|
|
2953
3875
|
const functionCalls = parts.filter((p) => p.functionCall);
|
|
@@ -2968,18 +3890,22 @@ function fromGemini(geminiClient) {
|
|
|
2968
3890
|
usage: {
|
|
2969
3891
|
prompt_tokens: response.usageMetadata?.promptTokenCount,
|
|
2970
3892
|
completion_tokens: response.usageMetadata?.candidatesTokenCount,
|
|
2971
|
-
total_tokens: response.usageMetadata?.totalTokenCount
|
|
3893
|
+
total_tokens: response.usageMetadata?.totalTokenCount,
|
|
3894
|
+
...response.usageMetadata?.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usageMetadata.thoughtsTokenCount } } : {}
|
|
2972
3895
|
}
|
|
2973
3896
|
};
|
|
2974
3897
|
},
|
|
2975
|
-
async *createStream(params, options) {
|
|
2976
|
-
if (!
|
|
2977
|
-
|
|
3898
|
+
async *createStream(params, options$1) {
|
|
3899
|
+
if (!generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
|
|
3900
|
+
code: "unsupported_capability",
|
|
3901
|
+
issues: { capability: "generateContentStream" }
|
|
3902
|
+
});
|
|
3903
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
2978
3904
|
request.config = {
|
|
2979
3905
|
...request.config,
|
|
2980
|
-
abortSignal: options.signal
|
|
3906
|
+
abortSignal: options$1.signal
|
|
2981
3907
|
};
|
|
2982
|
-
const stream = await
|
|
3908
|
+
const stream = await generateContentStream(request);
|
|
2983
3909
|
let toolCallIndex = 0;
|
|
2984
3910
|
let lastUsage;
|
|
2985
3911
|
for await (const chunk of stream) {
|
|
@@ -3008,7 +3934,8 @@ function fromGemini(geminiClient) {
|
|
|
3008
3934
|
usage: {
|
|
3009
3935
|
prompt_tokens: lastUsage.promptTokenCount,
|
|
3010
3936
|
completion_tokens: lastUsage.candidatesTokenCount,
|
|
3011
|
-
total_tokens: lastUsage.totalTokenCount
|
|
3937
|
+
total_tokens: lastUsage.totalTokenCount,
|
|
3938
|
+
...lastUsage.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: lastUsage.thoughtsTokenCount } } : {}
|
|
3012
3939
|
}
|
|
3013
3940
|
};
|
|
3014
3941
|
}
|
|
@@ -3017,6 +3944,19 @@ function fromGemini(geminiClient) {
|
|
|
3017
3944
|
|
|
3018
3945
|
//#endregion
|
|
3019
3946
|
//#region src/adapters/bedrock.ts
|
|
3947
|
+
/**
|
|
3948
|
+
* Default heuristic for whether a Bedrock model id is a Claude model,
|
|
3949
|
+
* matching AWS's own `anthropic.claude-*`/`us.anthropic.claude-*` naming.
|
|
3950
|
+
* Only used to decide whether a reasoning token budget is worth forwarding
|
|
3951
|
+
* through `additionalModelRequestFields`, not a general capability check,
|
|
3952
|
+
* so a plain substring match is enough, no override hook needed the way
|
|
3953
|
+
* `nativeStructuredOutputModels`/`toolUseSupportedModels` have one: a
|
|
3954
|
+
* false positive here just sends an inert extra field, not a request that
|
|
3955
|
+
* fails outright.
|
|
3956
|
+
*/
|
|
3957
|
+
function isClaudeModel(model) {
|
|
3958
|
+
return model.includes("claude");
|
|
3959
|
+
}
|
|
3020
3960
|
/** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
|
|
3021
3961
|
function toBedrockImageFormat(mimeType) {
|
|
3022
3962
|
switch (assertSupportedImageMimeType(mimeType)) {
|
|
@@ -3083,7 +4023,7 @@ function buildBedrockToolConfig(tools, toolChoiceParam) {
|
|
|
3083
4023
|
* normal, non-forced tool-call handling both `create` and `createStream`
|
|
3084
4024
|
* already do when `toolName` is unset.
|
|
3085
4025
|
*/
|
|
3086
|
-
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
|
|
4026
|
+
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
3087
4027
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3088
4028
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3089
4029
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -3091,8 +4031,8 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3091
4031
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
3092
4032
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
3093
4033
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
|
|
4034
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse's native outputConfig.textFormat on covered models, or a forced tool call otherwise).", "validation");
|
|
3094
4035
|
let toolName;
|
|
3095
|
-
let jsonInstruction;
|
|
3096
4036
|
let toolConfig;
|
|
3097
4037
|
let outputConfig;
|
|
3098
4038
|
if (jsonSchema && isNative) {
|
|
@@ -3117,23 +4057,48 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3117
4057
|
} }],
|
|
3118
4058
|
toolChoice: { tool: { name: toolName } }
|
|
3119
4059
|
};
|
|
3120
|
-
}
|
|
4060
|
+
}
|
|
3121
4061
|
if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
|
|
3122
4062
|
if (jsonSchema && toolConfig && toolUseSupportedModels) {
|
|
3123
4063
|
const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
|
|
3124
|
-
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "
|
|
4064
|
+
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "invalid_params", {
|
|
4065
|
+
code: "unsupported_capability",
|
|
4066
|
+
issues: { capability: "toolUseSupportedModels" }
|
|
4067
|
+
});
|
|
4068
|
+
}
|
|
4069
|
+
let additionalModelRequestFields;
|
|
4070
|
+
let effort;
|
|
4071
|
+
if (isClaudeModel(params.model) && (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0)) {
|
|
4072
|
+
const forcedToolChoice = toolConfig?.toolChoice;
|
|
4073
|
+
assertNoForcedToolChoiceWithThinking(forcedToolChoice && "tool" in forcedToolChoice ? `toolChoice forcing the "${forcedToolChoice.tool?.name}" tool` : forcedToolChoice && "any" in forcedToolChoice ? "toolChoice: 'required' (Converse's \"any\" tool_choice)" : void 0);
|
|
4074
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
4075
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
4076
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
4077
|
+
additionalModelRequestFields = { thinking: {
|
|
4078
|
+
type: "enabled",
|
|
4079
|
+
budget_tokens: budgetTokens
|
|
4080
|
+
} };
|
|
4081
|
+
} else {
|
|
4082
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
4083
|
+
additionalModelRequestFields = { thinking: { type: "adaptive" } };
|
|
4084
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
4085
|
+
}
|
|
3125
4086
|
}
|
|
3126
|
-
const
|
|
4087
|
+
const temperature = additionalModelRequestFields ? void 0 : params.temperature;
|
|
3127
4088
|
const request = {
|
|
3128
4089
|
modelId: params.model,
|
|
3129
4090
|
messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
|
|
3130
|
-
system:
|
|
4091
|
+
system: systemMessage?.content ? [{ text: systemMessage.content }] : void 0,
|
|
3131
4092
|
inferenceConfig: {
|
|
3132
|
-
...
|
|
4093
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
3133
4094
|
maxTokens: params.max_tokens
|
|
3134
4095
|
},
|
|
3135
4096
|
...toolConfig ? { toolConfig } : {},
|
|
3136
|
-
...outputConfig ? { outputConfig
|
|
4097
|
+
...outputConfig || effort ? { outputConfig: {
|
|
4098
|
+
...outputConfig ?? {},
|
|
4099
|
+
...effort ? { effort } : {}
|
|
4100
|
+
} } : {},
|
|
4101
|
+
...additionalModelRequestFields ? { additionalModelRequestFields } : {}
|
|
3137
4102
|
};
|
|
3138
4103
|
return {
|
|
3139
4104
|
request,
|
|
@@ -3141,6 +4106,120 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3141
4106
|
};
|
|
3142
4107
|
}
|
|
3143
4108
|
/**
|
|
4109
|
+
* Distinguishes a real AWS SDK v3 client (`.send(command)`) from a
|
|
4110
|
+
* hand-written `BedrockConverseClient` (`.converse(params)`) purely
|
|
4111
|
+
* structurally, so `fromBedrock` can accept either without the caller
|
|
4112
|
+
* saying which one they're passing. The two shapes don't overlap: nothing
|
|
4113
|
+
* implementing `.converse()` would also need `.send()`.
|
|
4114
|
+
*/
|
|
4115
|
+
function isAwsSendClient(client) {
|
|
4116
|
+
return typeof client.send === "function";
|
|
4117
|
+
}
|
|
4118
|
+
/**
|
|
4119
|
+
* Narrows one raw AWS stream event down to VernLLM's intentionally minimal
|
|
4120
|
+
* `BedrockConverseStreamEvent` union. Returns `undefined` if the event
|
|
4121
|
+
* isn't one of the kinds this adapter models.
|
|
4122
|
+
*
|
|
4123
|
+
* AWS's real `ConverseStreamOutput` type is a strictly larger union than
|
|
4124
|
+
* `BedrockConverseStreamEvent`. On top of every member modeled here, it
|
|
4125
|
+
* also includes a generated `$unknown` member, AWS's forward-compatibility
|
|
4126
|
+
* escape hatch for event kinds added to the service after this SDK version
|
|
4127
|
+
* was generated. A blind type assertion from one union to the other would
|
|
4128
|
+
* compile, but would let `$unknown` (or any other future member) reach
|
|
4129
|
+
* `fromBedrock`'s event-handling loop unnarrowed, as if it were one of the
|
|
4130
|
+
* kinds actually handled there.
|
|
4131
|
+
*
|
|
4132
|
+
* Returning `undefined` for anything unrecognized, filtered out by
|
|
4133
|
+
* `normalizeBedrockEventStream` below, keeps two guarantees. AWS SDK
|
|
4134
|
+
* generated types never leak into `fromBedrock`'s application code, only
|
|
4135
|
+
* this module's own `BedrockConverseStreamEvent` shape does. An event kind
|
|
4136
|
+
* this adapter doesn't yet know about is silently skipped, the same
|
|
4137
|
+
* forward-compatible behavior AWS's own `$unknown` convention implies,
|
|
4138
|
+
* rather than crashing the stream or being misrouted into a handler that
|
|
4139
|
+
* doesn't actually match its shape.
|
|
4140
|
+
*/
|
|
4141
|
+
function normalizeBedrockStreamEvent(raw) {
|
|
4142
|
+
if ("messageStart" in raw) return { messageStart: raw.messageStart };
|
|
4143
|
+
if ("contentBlockStart" in raw) return { contentBlockStart: raw.contentBlockStart };
|
|
4144
|
+
if ("contentBlockDelta" in raw) return { contentBlockDelta: raw.contentBlockDelta };
|
|
4145
|
+
if ("contentBlockStop" in raw) return { contentBlockStop: raw.contentBlockStop };
|
|
4146
|
+
if ("messageStop" in raw) return { messageStop: raw.messageStop };
|
|
4147
|
+
if ("metadata" in raw) return { metadata: raw.metadata };
|
|
4148
|
+
if ("internalServerException" in raw) return { internalServerException: raw.internalServerException };
|
|
4149
|
+
if ("modelStreamErrorException" in raw) return { modelStreamErrorException: raw.modelStreamErrorException };
|
|
4150
|
+
if ("validationException" in raw) return { validationException: raw.validationException };
|
|
4151
|
+
if ("throttlingException" in raw) return { throttlingException: raw.throttlingException };
|
|
4152
|
+
if ("serviceUnavailableException" in raw) return { serviceUnavailableException: raw.serviceUnavailableException };
|
|
4153
|
+
return void 0;
|
|
4154
|
+
}
|
|
4155
|
+
/**
|
|
4156
|
+
* Wraps a raw AWS event stream, narrowing each event through
|
|
4157
|
+
* `normalizeBedrockStreamEvent` and filtering out anything that doesn't
|
|
4158
|
+
* map onto `BedrockConverseStreamEvent`. `fromBedrock`'s event loop only
|
|
4159
|
+
* ever sees the shapes it actually models.
|
|
4160
|
+
*/
|
|
4161
|
+
async function* normalizeBedrockEventStream(rawStream) {
|
|
4162
|
+
for await (const raw of rawStream) {
|
|
4163
|
+
const event = normalizeBedrockStreamEvent(raw);
|
|
4164
|
+
if (event) yield event;
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
/**
|
|
4168
|
+
* Adapts a real AWS SDK v3 client (anything with `.send()`, matching
|
|
4169
|
+
* `BedrockRuntimeClient`) into a `BedrockConverseClient`, so `fromBedrock`
|
|
4170
|
+
* can accept either without a hand-written `.converse()`/`.converseStream()`
|
|
4171
|
+
* wrapper. Internally does what that wrapper would: `client.send(new
|
|
4172
|
+
* ConverseCommand(params))`, `client.send(new
|
|
4173
|
+
* ConverseStreamCommand(params))`.
|
|
4174
|
+
*
|
|
4175
|
+
* `@aws-sdk/client-bedrock-runtime` is intentionally not a dependency (not
|
|
4176
|
+
* even a peer dependency) of this package. `vern-llm` otherwise has zero
|
|
4177
|
+
* runtime dependencies, and every other adapter works the same way:
|
|
4178
|
+
* structural typing over whatever client the caller already has. Instead,
|
|
4179
|
+
* `ConverseCommand`/`ConverseStreamCommand` are pulled in with a dynamic
|
|
4180
|
+
* `import()` the first time either method actually runs, and memoized
|
|
4181
|
+
* after that. Nothing is added to `package.json`, static or peer.
|
|
4182
|
+
* Bundlers only pull the AWS SDK in for code paths that actually pass a
|
|
4183
|
+
* raw AWS client to `fromBedrock`; a hand-written `BedrockConverseClient`
|
|
4184
|
+
* stays unaffected. If `@aws-sdk/client-bedrock-runtime` isn't installed,
|
|
4185
|
+
* the failure is a clear `LLMError` naming exactly what's missing, at the
|
|
4186
|
+
* moment it's needed, rather than a silent peer-dependency warning at
|
|
4187
|
+
* install time or a raw "Cannot find module" a caller has to trace back
|
|
4188
|
+
* themselves.
|
|
4189
|
+
*
|
|
4190
|
+
* Also closes two structural gaps between AWS's generated types and
|
|
4191
|
+
* `BedrockConverseClient`. AWS's `ConverseStreamCommandOutput.stream` is
|
|
4192
|
+
* optional, a response may not include it. This throws a clear `LLMError`
|
|
4193
|
+
* instead of letting `undefined` reach `fromBedrock`'s `for await` loop.
|
|
4194
|
+
* AWS's `ConverseStreamOutput` union is larger than
|
|
4195
|
+
* `BedrockConverseStreamEvent`, it includes a generated `$unknown` member.
|
|
4196
|
+
* Every event is narrowed through `normalizeBedrockStreamEvent` before it
|
|
4197
|
+
* reaches application code, instead of being asserted wholesale from one
|
|
4198
|
+
* type to the other.
|
|
4199
|
+
*/
|
|
4200
|
+
function wrapAwsSendClient(client) {
|
|
4201
|
+
let commandsPromise;
|
|
4202
|
+
function loadCommands() {
|
|
4203
|
+
commandsPromise ??= import("@aws-sdk/client-bedrock-runtime").then((mod) => mod, (cause) => {
|
|
4204
|
+
commandsPromise = void 0;
|
|
4205
|
+
throw new LLMError("fromBedrock requires \"@aws-sdk/client-bedrock-runtime\" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.", "validation", { cause });
|
|
4206
|
+
});
|
|
4207
|
+
return commandsPromise;
|
|
4208
|
+
}
|
|
4209
|
+
return {
|
|
4210
|
+
converse: async (params, requestOptions) => {
|
|
4211
|
+
const { ConverseCommand } = await loadCommands();
|
|
4212
|
+
return client.send(new ConverseCommand(params), { abortSignal: requestOptions.signal });
|
|
4213
|
+
},
|
|
4214
|
+
converseStream: async (params, requestOptions) => {
|
|
4215
|
+
const { ConverseStreamCommand } = await loadCommands();
|
|
4216
|
+
const result = await client.send(new ConverseStreamCommand(params), { abortSignal: requestOptions.signal });
|
|
4217
|
+
if (!result.stream) throw new LLMError("Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.", "api", { code: "server_error" });
|
|
4218
|
+
return { stream: normalizeBedrockEventStream(result.stream) };
|
|
4219
|
+
}
|
|
4220
|
+
};
|
|
4221
|
+
}
|
|
4222
|
+
/**
|
|
3144
4223
|
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
3145
4224
|
* interface VernLLM uses for OpenAI/Groq. The Converse API is unified
|
|
3146
4225
|
* across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
|
|
@@ -3148,6 +4227,16 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3148
4227
|
* regardless of which underlying model `modelId` points at, as long as
|
|
3149
4228
|
* that model supports Converse (most current-generation ones do)
|
|
3150
4229
|
*
|
|
4230
|
+
* `bedrockClient` accepts either a hand-written `BedrockConverseClient`
|
|
4231
|
+
* (a `.converse()`/`.converseStream()` wrapper you provide) or a real AWS
|
|
4232
|
+
* SDK v3 client (anything with `.send()`, matching `BedrockRuntimeClient`)
|
|
4233
|
+
* directly, detected structurally. Passing a raw AWS client skips the
|
|
4234
|
+
* hand-written wrapper entirely, internally doing what it would
|
|
4235
|
+
* (`send(new ConverseCommand(...))`, `send(new
|
|
4236
|
+
* ConverseStreamCommand(...))`). See `wrapAwsSendClient` for how that path
|
|
4237
|
+
* is implemented, including why `@aws-sdk/client-bedrock-runtime` stays
|
|
4238
|
+
* out of this package's dependencies either way.
|
|
4239
|
+
*
|
|
3151
4240
|
* `response_format: json_schema`, on a model covered by
|
|
3152
4241
|
* `options.nativeStructuredOutputModels` (opt-in, unset by default), is
|
|
3153
4242
|
* sent as `outputConfig.textFormat`, its own request field, independent of
|
|
@@ -3169,12 +4258,15 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3169
4258
|
* `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
|
|
3170
4259
|
* unsupported model surfaces Bedrock's raw error unchanged.
|
|
3171
4260
|
*
|
|
3172
|
-
* `response_format: json_object` (
|
|
3173
|
-
*
|
|
3174
|
-
*
|
|
3175
|
-
*
|
|
3176
|
-
*
|
|
3177
|
-
*
|
|
4261
|
+
* `response_format: json_object` throws `LLMError('validation')`: Converse
|
|
4262
|
+
* has no field that mechanically guarantees JSON output, and the only way
|
|
4263
|
+
* to emulate it was an unenforced system-prompt instruction, a guarantee
|
|
4264
|
+
* this adapter no longer pretends to make. Use `jsonSchema` instead.
|
|
4265
|
+
* `reasoning_effort` (no Converse equivalent) is converted to a token
|
|
4266
|
+
* budget and forwarded via `additionalModelRequestFields` for Claude
|
|
4267
|
+
* models only; `budget_tokens` is forwarded the same way directly. Both
|
|
4268
|
+
* are silently dropped for non-Claude models, which have no equivalent
|
|
4269
|
+
* field to reach for. See `adapters/internal/reasoningBudget.utils.ts`.
|
|
3178
4270
|
*
|
|
3179
4271
|
* `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
|
|
3180
4272
|
* `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
|
|
@@ -3192,104 +4284,122 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3192
4284
|
* `create` branch above unwraps it.
|
|
3193
4285
|
*/
|
|
3194
4286
|
function fromBedrock(bedrockClient, options) {
|
|
4287
|
+
const client = isAwsSendClient(bedrockClient) ? wrapAwsSendClient(bedrockClient) : bedrockClient;
|
|
3195
4288
|
const toolUseSupportedModels = options?.toolUseSupportedModels;
|
|
3196
4289
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
const
|
|
3205
|
-
text
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
const
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3227
|
-
} }],
|
|
3228
|
-
usage: {
|
|
3229
|
-
prompt_tokens: response.usage?.inputTokens,
|
|
3230
|
-
completion_tokens: response.usage?.outputTokens,
|
|
3231
|
-
total_tokens: response.usage?.totalTokens
|
|
4290
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
4291
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
4292
|
+
return {
|
|
4293
|
+
supportsJsonObjectMode: false,
|
|
4294
|
+
chat: { completions: {
|
|
4295
|
+
async create(params, requestOptions) {
|
|
4296
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4297
|
+
const response = await client.converse(request, requestOptions);
|
|
4298
|
+
let text;
|
|
4299
|
+
let wireToolCalls;
|
|
4300
|
+
if (toolName) {
|
|
4301
|
+
const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
|
|
4302
|
+
text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
|
|
4303
|
+
} else {
|
|
4304
|
+
const blocks = response.output?.message?.content ?? [];
|
|
4305
|
+
text = blocks.map((c) => c.text ?? "").join("");
|
|
4306
|
+
const toolUses = blocks.filter((block) => Boolean(block.toolUse));
|
|
4307
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
|
|
4308
|
+
const toolUse = block.toolUse;
|
|
4309
|
+
if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
|
|
4310
|
+
return {
|
|
4311
|
+
id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
|
|
4312
|
+
type: "function",
|
|
4313
|
+
function: {
|
|
4314
|
+
name: toolUse.name,
|
|
4315
|
+
arguments: JSON.stringify(toolUse.input ?? {})
|
|
4316
|
+
}
|
|
4317
|
+
};
|
|
4318
|
+
});
|
|
3232
4319
|
}
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
3244
|
-
blockKinds.set(contentBlockIndex, kind);
|
|
3245
|
-
if (kind === "tool_use" && !toolName) yield {
|
|
3246
|
-
type: "tool_call_delta",
|
|
3247
|
-
index: contentBlockIndex,
|
|
3248
|
-
id: start.toolUse.toolUseId,
|
|
3249
|
-
name: start.toolUse.name
|
|
3250
|
-
};
|
|
3251
|
-
} else blockKinds.set(contentBlockIndex, "text");
|
|
3252
|
-
} else if ("contentBlockDelta" in event) {
|
|
3253
|
-
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
3254
|
-
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3255
|
-
type: "text-delta",
|
|
3256
|
-
delta: delta.text
|
|
4320
|
+
return {
|
|
4321
|
+
choices: [{ message: {
|
|
4322
|
+
content: text,
|
|
4323
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
4324
|
+
} }],
|
|
4325
|
+
usage: {
|
|
4326
|
+
prompt_tokens: response.usage?.inputTokens,
|
|
4327
|
+
completion_tokens: response.usage?.outputTokens,
|
|
4328
|
+
total_tokens: response.usage?.totalTokens
|
|
4329
|
+
}
|
|
3257
4330
|
};
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
4331
|
+
},
|
|
4332
|
+
async *createStream(params, requestOptions) {
|
|
4333
|
+
if (!client.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
|
|
4334
|
+
code: "unsupported_capability",
|
|
4335
|
+
issues: { capability: "converseStream" }
|
|
4336
|
+
});
|
|
4337
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4338
|
+
const { stream } = await client.converseStream(request, requestOptions);
|
|
4339
|
+
const blockKinds = new Map();
|
|
4340
|
+
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
4341
|
+
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
4342
|
+
if (start?.toolUse) {
|
|
4343
|
+
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
4344
|
+
blockKinds.set(contentBlockIndex, kind);
|
|
4345
|
+
if (kind === "tool_use" && !toolName) yield {
|
|
4346
|
+
type: "tool_call_delta",
|
|
4347
|
+
index: contentBlockIndex,
|
|
4348
|
+
id: start.toolUse.toolUseId,
|
|
4349
|
+
name: start.toolUse.name
|
|
4350
|
+
};
|
|
4351
|
+
} else blockKinds.set(contentBlockIndex, "text");
|
|
4352
|
+
} else if ("contentBlockDelta" in event) {
|
|
4353
|
+
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
4354
|
+
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3261
4355
|
type: "text-delta",
|
|
3262
|
-
delta: delta.
|
|
3263
|
-
};
|
|
3264
|
-
else if (!toolName) yield {
|
|
3265
|
-
type: "tool_call_delta",
|
|
3266
|
-
index: contentBlockIndex,
|
|
3267
|
-
argumentsDelta: delta.toolUse.input
|
|
4356
|
+
delta: delta.text
|
|
3268
4357
|
};
|
|
4358
|
+
else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
|
|
4359
|
+
const kind = blockKinds.get(contentBlockIndex);
|
|
4360
|
+
if (kind === "json-tool") yield {
|
|
4361
|
+
type: "text-delta",
|
|
4362
|
+
delta: delta.toolUse.input
|
|
4363
|
+
};
|
|
4364
|
+
else if (!toolName) yield {
|
|
4365
|
+
type: "tool_call_delta",
|
|
4366
|
+
index: contentBlockIndex,
|
|
4367
|
+
argumentsDelta: delta.toolUse.input
|
|
4368
|
+
};
|
|
4369
|
+
}
|
|
4370
|
+
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
4371
|
+
type: "usage",
|
|
4372
|
+
usage: {
|
|
4373
|
+
prompt_tokens: event.metadata.usage.inputTokens,
|
|
4374
|
+
completion_tokens: event.metadata.usage.outputTokens,
|
|
4375
|
+
total_tokens: event.metadata.usage.totalTokens
|
|
4376
|
+
}
|
|
4377
|
+
};
|
|
4378
|
+
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
|
|
4379
|
+
status: 429,
|
|
4380
|
+
code: "provider_rate_limited"
|
|
4381
|
+
});
|
|
4382
|
+
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
4383
|
+
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
4384
|
+
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
4385
|
+
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
4386
|
+
throw new LLMError(detail, "api", {
|
|
4387
|
+
status,
|
|
4388
|
+
code: status >= 500 ? "server_error" : void 0
|
|
4389
|
+
});
|
|
3269
4390
|
}
|
|
3270
|
-
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
3271
|
-
type: "usage",
|
|
3272
|
-
usage: {
|
|
3273
|
-
prompt_tokens: event.metadata.usage.inputTokens,
|
|
3274
|
-
completion_tokens: event.metadata.usage.outputTokens,
|
|
3275
|
-
total_tokens: event.metadata.usage.totalTokens
|
|
3276
|
-
}
|
|
3277
|
-
};
|
|
3278
|
-
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", 429);
|
|
3279
|
-
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
3280
|
-
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
3281
|
-
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
3282
|
-
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
3283
|
-
throw new LLMError(detail, "api", status);
|
|
3284
4391
|
}
|
|
3285
|
-
}
|
|
3286
|
-
}
|
|
4392
|
+
} }
|
|
4393
|
+
};
|
|
3287
4394
|
}
|
|
3288
4395
|
/** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
|
|
3289
4396
|
function toBedrockToolChoice(toolChoice) {
|
|
3290
4397
|
if (!toolChoice || toolChoice === "auto") return { auto: {} };
|
|
3291
4398
|
if (toolChoice === "required") return { any: {} };
|
|
3292
|
-
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "
|
|
4399
|
+
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "invalid_params", {
|
|
4400
|
+
code: "unsupported_capability",
|
|
4401
|
+
issues: { capability: "toolChoice: 'none'" }
|
|
4402
|
+
});
|
|
3293
4403
|
return { tool: { name: toolChoice.function.name } };
|
|
3294
4404
|
}
|
|
3295
4405
|
/**
|
|
@@ -3314,7 +4424,7 @@ function toBedrockMessage(m) {
|
|
|
3314
4424
|
else try {
|
|
3315
4425
|
input = JSON.parse(tc.function.arguments);
|
|
3316
4426
|
} catch (cause) {
|
|
3317
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation",
|
|
4427
|
+
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
3318
4428
|
}
|
|
3319
4429
|
blocks.push({ toolUse: {
|
|
3320
4430
|
toolUseId: tc.id,
|
|
@@ -3485,8 +4595,14 @@ function fromFetch(config) {
|
|
|
3485
4595
|
};
|
|
3486
4596
|
},
|
|
3487
4597
|
async *createStream(params, options) {
|
|
3488
|
-
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "
|
|
3489
|
-
|
|
4598
|
+
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "invalid_params", {
|
|
4599
|
+
code: "unsupported_capability",
|
|
4600
|
+
issues: { capability: "mapStreamEvent" }
|
|
4601
|
+
});
|
|
4602
|
+
if (config.request && !config.requestStream) throw new LLMError("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.", "invalid_params", {
|
|
4603
|
+
code: "unsupported_capability",
|
|
4604
|
+
issues: { capability: "requestStream" }
|
|
4605
|
+
});
|
|
3490
4606
|
const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
|
|
3491
4607
|
const requestStream = config.requestStream ?? defaultRequestStream;
|
|
3492
4608
|
const parseFrames = config.parseStreamFrames ?? parseSseStream;
|
|
@@ -3513,6 +4629,22 @@ function fromFetch(config) {
|
|
|
3513
4629
|
//#endregion
|
|
3514
4630
|
//#region src/adapters/openaiCompatible.ts
|
|
3515
4631
|
/**
|
|
4632
|
+
* OpenAI's wire format only understands `reasoning_effort`, not a raw
|
|
4633
|
+
* token budget. When the caller set `reasoningEffort`, it's already on
|
|
4634
|
+
* `params` and passed through unchanged, this function does nothing.
|
|
4635
|
+
* When only `budgetTokens` was set, it's converted to the nearest tier
|
|
4636
|
+
* and `budget_tokens` is dropped, since OpenAI's API would otherwise
|
|
4637
|
+
* silently ignore an unrecognized field.
|
|
4638
|
+
*/
|
|
4639
|
+
function applyReasoningBudget(params, effortTokenTable) {
|
|
4640
|
+
if (params.budget_tokens === void 0) return params;
|
|
4641
|
+
const { budget_tokens,...rest } = params;
|
|
4642
|
+
return rest.reasoning_effort !== void 0 ? rest : {
|
|
4643
|
+
...rest,
|
|
4644
|
+
reasoning_effort: budgetTokensToEffort(budget_tokens, effortTokenTable)
|
|
4645
|
+
};
|
|
4646
|
+
}
|
|
4647
|
+
/**
|
|
3516
4648
|
* Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
|
|
3517
4649
|
* array. Text blocks become `{ type: 'text', text }`; image blocks become
|
|
3518
4650
|
* `{ type: 'image_url', image_url: { url } }` with the base64 payload
|
|
@@ -3578,23 +4710,24 @@ function* toWireStreamChunks(chunk) {
|
|
|
3578
4710
|
function fromOpenAICompatible(client, options = {}) {
|
|
3579
4711
|
const raw = client;
|
|
3580
4712
|
const { supportsStreamUsage = true } = options;
|
|
4713
|
+
const effortTokenTable = resolveEffortTokenTable(options.reasoningEffortTokens);
|
|
3581
4714
|
const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
|
|
3582
4715
|
return { chat: { completions: {
|
|
3583
4716
|
async create(params, options$1) {
|
|
3584
4717
|
const messages = toOpenAIMessages(params);
|
|
3585
|
-
return raw.chat.completions.create({
|
|
4718
|
+
return raw.chat.completions.create(applyReasoningBudget({
|
|
3586
4719
|
...params,
|
|
3587
4720
|
messages
|
|
3588
|
-
}, options$1);
|
|
4721
|
+
}, effortTokenTable), options$1);
|
|
3589
4722
|
},
|
|
3590
4723
|
async *createStream(params, options$1) {
|
|
3591
4724
|
const messages = toOpenAIMessages(params);
|
|
3592
|
-
const stream = await rawCreate({
|
|
4725
|
+
const stream = await rawCreate(applyReasoningBudget({
|
|
3593
4726
|
...params,
|
|
3594
4727
|
messages,
|
|
3595
4728
|
stream: true,
|
|
3596
4729
|
...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
|
|
3597
|
-
}, options$1);
|
|
4730
|
+
}, effortTokenTable), options$1);
|
|
3598
4731
|
for await (const chunk of stream) yield* toWireStreamChunks(chunk);
|
|
3599
4732
|
}
|
|
3600
4733
|
} } };
|
|
@@ -3706,5 +4839,5 @@ const fromAtlasCloud = fromOpenAICompatible;
|
|
|
3706
4839
|
const from01AI = fromOpenAICompatible;
|
|
3707
4840
|
|
|
3708
4841
|
//#endregion
|
|
3709
|
-
export { CircuitBreaker, ConsoleLogger, FallbackExhaustedError, InMemoryCacheAdapter, LLMError, NormalizedCacheAdapter, RateLimiter, SSE_PING, TieredCacheAdapter, VernLLM, defaultEstimateTokens, defaultFallbackOn, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, isLLMError, isToolCallResult, parseSseStream };
|
|
4842
|
+
export { CircuitBreaker, ConsoleLogger, FallbackExhaustedError, InMemoryCacheAdapter, LLMError, NormalizedCacheAdapter, RateLimiter, SSE_PING, TieredCacheAdapter, VernLLM, defaultEstimateTokens, defaultFallbackOn, defineCachedCallParams, defineCallParams, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, hasIssues, isFallbackExhaustedError, isLLMError, isToolCallResult, parseSseStream };
|
|
3710
4843
|
//# sourceMappingURL=index.js.map
|