vern-llm 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/index.cjs +1512 -375
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +893 -222
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +893 -222
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1509 -376
- package/dist/index.js.map +1 -1
- package/package.json +16 -2
package/dist/index.cjs
CHANGED
|
@@ -25,23 +25,273 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
25
25
|
const crypto = __toESM(require("crypto"));
|
|
26
26
|
|
|
27
27
|
//#region src/types/errors.ts
|
|
28
|
+
/**
|
|
29
|
+
* Tool contract codes: a model or provider response defect, not a
|
|
30
|
+
* transient provider fault. Deterministic on the wire request, so
|
|
31
|
+
* retrying can't change the outcome and it shouldn't count toward the
|
|
32
|
+
* circuit breaker either. Shared by `LLMError.retryable` below and by
|
|
33
|
+
* `CallExecutor`'s own retry/breaker accounting, so the two can't drift
|
|
34
|
+
* apart.
|
|
35
|
+
*/
|
|
36
|
+
const NON_RETRYABLE_TOOL_CONTRACT_CODES = new Set([
|
|
37
|
+
"unknown_tool",
|
|
38
|
+
"duplicate_tool_call_id",
|
|
39
|
+
"tool_choice_none_violated",
|
|
40
|
+
"unexpected_tool_calls"
|
|
41
|
+
]);
|
|
42
|
+
/**
|
|
43
|
+
* Local rate-limit codes: the call never reached the provider, so it says
|
|
44
|
+
* nothing about the provider's health, and retrying either just requeues
|
|
45
|
+
* behind the same limit (the two queue codes) or can never succeed at all
|
|
46
|
+
* (`rate_limit_capacity_exceeded`). Shared for the same reason as
|
|
47
|
+
* {@link NON_RETRYABLE_TOOL_CONTRACT_CODES}.
|
|
48
|
+
*/
|
|
49
|
+
const LOCAL_RATE_LIMIT_CODES = new Set([
|
|
50
|
+
"rate_limit_queue_full",
|
|
51
|
+
"rate_limit_queue_timeout",
|
|
52
|
+
"rate_limit_capacity_exceeded"
|
|
53
|
+
]);
|
|
54
|
+
/**
|
|
55
|
+
* Types that are never worth retrying on their own: deterministic
|
|
56
|
+
* caller-input, model-response, or cancellation failures rather than a
|
|
57
|
+
* transient provider fault.
|
|
58
|
+
*/
|
|
59
|
+
const NON_RETRYABLE_TYPES = new Set([
|
|
60
|
+
"parse",
|
|
61
|
+
"validation",
|
|
62
|
+
"invalid_params",
|
|
63
|
+
"aborted"
|
|
64
|
+
]);
|
|
65
|
+
/**
|
|
66
|
+
* Shared retryability rule behind both `LLMError.retryable` and
|
|
67
|
+
* `LLMErrorSnapshot.retryable`. Pulled out so the two can't drift apart:
|
|
68
|
+
* a snapshot is a point-in-time copy of an error's fields, and this is
|
|
69
|
+
* one of them, so it has to be computed the same way in both places.
|
|
70
|
+
*/
|
|
71
|
+
function computeRetryable(type, code) {
|
|
72
|
+
if (NON_RETRYABLE_TYPES.has(type)) return false;
|
|
73
|
+
if (code && NON_RETRYABLE_TOOL_CONTRACT_CODES.has(code)) return false;
|
|
74
|
+
if (code && LOCAL_RATE_LIMIT_CODES.has(code)) return false;
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Returns `issues` unchanged when it can survive `JSON.stringify`.
|
|
79
|
+
* Most `issues` values are VernLLM's own structured shapes (see
|
|
80
|
+
* `LLMErrorIssuesByCode`) and always safe. The one exception is a
|
|
81
|
+
* schema validation failure, where `issues` is a caller supplied
|
|
82
|
+
* `SchemaLike` validator's own `error: unknown`, not controlled by
|
|
83
|
+
* VernLLM and not guaranteed to be circular free. Rather than silently
|
|
84
|
+
* dropping it in that case, this returns a marker string so a reader
|
|
85
|
+
* of serialized output can tell "no issues data" apart from "issues
|
|
86
|
+
* existed but could not be shown".
|
|
87
|
+
*/
|
|
88
|
+
function safeIssues(issues) {
|
|
89
|
+
if (issues === void 0) return void 0;
|
|
90
|
+
try {
|
|
91
|
+
JSON.stringify(issues);
|
|
92
|
+
return issues;
|
|
93
|
+
} catch {
|
|
94
|
+
return "[Unserializable: issues contained a circular reference]";
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Returns a JSON safe, independent copy of `body`, or a marker string if
|
|
99
|
+
* `body` can't survive `JSON.stringify` (e.g. a circular reference). A
|
|
100
|
+
* request body built from adapter-transformed messages is normally
|
|
101
|
+
* always plain data, but tool call arguments or a caller supplied
|
|
102
|
+
* `cause`-adjacent value could in principle carry a circular reference,
|
|
103
|
+
* so this guards the same way `safeIssues` does rather than assuming it
|
|
104
|
+
* can't happen. Unlike `safeIssues`, this clones rather than returning
|
|
105
|
+
* the same reference: the object backing a request body can still be
|
|
106
|
+
* mutated by adapter code between when a request is dispatched and when
|
|
107
|
+
* an attempt is later recorded as failed (e.g. `fromGemini` sets
|
|
108
|
+
* `request.config` in place), so returning the same reference here could
|
|
109
|
+
* make a stored snapshot silently reflect a later, different state than
|
|
110
|
+
* what was actually sent.
|
|
111
|
+
*/
|
|
112
|
+
function safeBody(body) {
|
|
113
|
+
if (body === void 0) return void 0;
|
|
114
|
+
try {
|
|
115
|
+
return JSON.parse(JSON.stringify(body));
|
|
116
|
+
} catch {
|
|
117
|
+
return "[Unserializable: request body contained a circular reference]";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const AUTH_HEADER_NAMES = new Set([
|
|
121
|
+
"authorization",
|
|
122
|
+
"x-api-key",
|
|
123
|
+
"x-goog-api-key",
|
|
124
|
+
"api-key"
|
|
125
|
+
]);
|
|
126
|
+
/** Removes auth headers before a request snapshot is built. Case insensitive on header names. */
|
|
127
|
+
function stripAuthHeaders(headers) {
|
|
128
|
+
if (headers === void 0) return void 0;
|
|
129
|
+
const out = {};
|
|
130
|
+
for (const [key, value] of Object.entries(headers)) if (!AUTH_HEADER_NAMES.has(key.toLowerCase())) out[key] = value;
|
|
131
|
+
return out;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Depth cap for `safeAttempts`, guarding against a pathological,
|
|
135
|
+
* self referential `attempts` array. `attempts` is a public
|
|
136
|
+
* `LLMErrorOptions` field, so a caller can construct one by hand; this
|
|
137
|
+
* keeps that path bounded the same way a circular `issues` value is
|
|
138
|
+
* bounded, rather than assuming well formed input.
|
|
139
|
+
*/
|
|
140
|
+
const MAX_ATTEMPTS_DEPTH = 20;
|
|
141
|
+
/**
|
|
142
|
+
* Returns a copy of `attempts` with every nested snapshot's `issues`
|
|
143
|
+
* re-checked through `safeIssues`, recursively through each snapshot's
|
|
144
|
+
* own `attempts`. Needed for two reasons: `safeIssues` returns a safe
|
|
145
|
+
* `issues` value by reference, so a shared object can be mutated into a
|
|
146
|
+
* circular one after the snapshot was created, and `attempts` is a
|
|
147
|
+
* public constructor option, so a caller can hand build a `RetryAttempt`
|
|
148
|
+
* (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
|
|
149
|
+
* in directly, never touching `toSnapshot()` at all. The same applies to
|
|
150
|
+
* `request`: its `body` is re-checked through `safeBody`, and its
|
|
151
|
+
* `headers` are re-stripped through `stripAuthHeaders`, so a hand built
|
|
152
|
+
* `RetryAttempt.request` can't smuggle an auth header past `toSnapshot()`
|
|
153
|
+
* either. Extra fields on an attempt (e.g. `FallbackAttempt`'s
|
|
154
|
+
* `provider`/`model`) are preserved.
|
|
155
|
+
*/
|
|
156
|
+
function safeAttempts(attempts, depth = 0) {
|
|
157
|
+
if (attempts === void 0) return void 0;
|
|
158
|
+
if (depth >= MAX_ATTEMPTS_DEPTH) return [];
|
|
159
|
+
return attempts.map((attempt) => ({
|
|
160
|
+
...attempt,
|
|
161
|
+
error: {
|
|
162
|
+
...attempt.error,
|
|
163
|
+
issues: safeIssues(attempt.error.issues),
|
|
164
|
+
attempts: safeAttempts(attempt.error.attempts, depth + 1)
|
|
165
|
+
},
|
|
166
|
+
request: attempt.request && {
|
|
167
|
+
...attempt.request,
|
|
168
|
+
body: safeBody(attempt.request.body),
|
|
169
|
+
headers: stripAuthHeaders(attempt.request.headers)
|
|
170
|
+
}
|
|
171
|
+
}));
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Builds a point-in-time, plain data copy of one attempt's outgoing
|
|
175
|
+
* request. Mirrors `LLMError.toSnapshot()`: never thrown or dispatched
|
|
176
|
+
* again, safe to serialize and store. A plain function rather than a
|
|
177
|
+
* method, since unlike `LLMError` a request has no throwable identity or
|
|
178
|
+
* derived state worth wrapping in a class.
|
|
179
|
+
*
|
|
180
|
+
* `startedAt` is optional so existing call sites (and tests) that don't
|
|
181
|
+
* care about exact timing keep working, but a caller that has a real
|
|
182
|
+
* capture time should always pass it: this function may run well after
|
|
183
|
+
* the request was actually dispatched (e.g. `callExecutor` only builds
|
|
184
|
+
* the snapshot once an attempt has failed), so defaulting to `Date.now()`
|
|
185
|
+
* here would record failure-handling time, not request-start time.
|
|
186
|
+
*/
|
|
187
|
+
function toRequestSnapshot(provider, model, body, headers, startedAt = Date.now()) {
|
|
188
|
+
return {
|
|
189
|
+
provider,
|
|
190
|
+
model,
|
|
191
|
+
body: safeBody(body),
|
|
192
|
+
headers: stripAuthHeaders(headers),
|
|
193
|
+
startedAt
|
|
194
|
+
};
|
|
195
|
+
}
|
|
28
196
|
var LLMError = class extends Error {
|
|
29
|
-
|
|
197
|
+
status;
|
|
198
|
+
issues;
|
|
199
|
+
cause;
|
|
200
|
+
retryAfterMs;
|
|
201
|
+
/** Stable discriminator within `type`. Absent on errors predating it. */
|
|
202
|
+
code;
|
|
203
|
+
/** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
|
|
204
|
+
attempts;
|
|
205
|
+
constructor(message, type, options = {}) {
|
|
30
206
|
super(message);
|
|
31
207
|
this.type = type;
|
|
32
|
-
this.status = status;
|
|
33
|
-
this.issues = issues;
|
|
34
|
-
this.cause = cause;
|
|
35
|
-
this.retryAfterMs = retryAfterMs;
|
|
36
|
-
this.code = code;
|
|
37
208
|
this.name = "LLMError";
|
|
209
|
+
this.status = options.status;
|
|
210
|
+
this.issues = options.issues;
|
|
211
|
+
this.cause = options.cause;
|
|
212
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
213
|
+
this.code = options.code;
|
|
214
|
+
this.attempts = options.attempts;
|
|
215
|
+
}
|
|
216
|
+
/**
|
|
217
|
+
* Computed purely from `type`/`code`, independent of any specific call's
|
|
218
|
+
* `nonRetryableStatus` list. False for `parse`/`validation`/
|
|
219
|
+
* `invalid_params`/`aborted` types (the caller's own input, the model's
|
|
220
|
+
* own response, or intentional cancellation, none of which are the
|
|
221
|
+
* provider being unhealthy), the tool contract codes, and the local
|
|
222
|
+
* rate limit codes. Subclasses (see `FallbackExhaustedError`) may
|
|
223
|
+
* override this when `type` alone carries no retry signal.
|
|
224
|
+
*/
|
|
225
|
+
get retryable() {
|
|
226
|
+
return computeRetryable(this.type, this.code);
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Copies this error's fields into an {@link LLMErrorSnapshot}, for
|
|
230
|
+
* recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
|
|
231
|
+
* captured here since a snapshot has no getter of its own. `cause` is
|
|
232
|
+
* not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
|
|
233
|
+
* nested `attempts` entry's own `issues` go through `safeAttempts`,
|
|
234
|
+
* since a schema validation failure's `issues` is a caller supplied
|
|
235
|
+
* value, not controlled by VernLLM, and `attempts` is itself a public
|
|
236
|
+
* constructor option a caller can hand build.
|
|
237
|
+
*/
|
|
238
|
+
toSnapshot() {
|
|
239
|
+
return {
|
|
240
|
+
message: this.message,
|
|
241
|
+
type: this.type,
|
|
242
|
+
status: this.status,
|
|
243
|
+
issues: safeIssues(this.issues),
|
|
244
|
+
retryAfterMs: this.retryAfterMs,
|
|
245
|
+
code: this.code,
|
|
246
|
+
retryable: this.retryable,
|
|
247
|
+
attempts: safeAttempts(this.attempts)
|
|
248
|
+
};
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Controls what `JSON.stringify(err)` produces. Omits `cause` for the
|
|
252
|
+
* same reason `toSnapshot()` does: `cause` is `unknown` and never
|
|
253
|
+
* validated by VernLLM, and some SDK errors carry circular structures
|
|
254
|
+
* `JSON.stringify` cannot serialize at all. Read `err.cause` directly
|
|
255
|
+
* instead. `issues`, including every nested `attempts` entry's own
|
|
256
|
+
* `issues`, goes through `safeAttempts` for the same reason: a schema
|
|
257
|
+
* validation failure's `issues` is caller supplied and not guaranteed
|
|
258
|
+
* circular free. Also includes `message` and `retryable`, which a
|
|
259
|
+
* plain property walk would otherwise miss: `message` is
|
|
260
|
+
* non-enumerable on `Error`, and `retryable` is a getter, not an own
|
|
261
|
+
* property.
|
|
262
|
+
*/
|
|
263
|
+
toJSON() {
|
|
264
|
+
return {
|
|
265
|
+
name: this.name,
|
|
266
|
+
message: this.message,
|
|
267
|
+
type: this.type,
|
|
268
|
+
status: this.status,
|
|
269
|
+
issues: safeIssues(this.issues),
|
|
270
|
+
retryAfterMs: this.retryAfterMs,
|
|
271
|
+
code: this.code,
|
|
272
|
+
retryable: this.retryable,
|
|
273
|
+
attempts: safeAttempts(this.attempts)
|
|
274
|
+
};
|
|
38
275
|
}
|
|
39
|
-
/** Every tool contract failure in one response, when there is more than one. */
|
|
40
|
-
toolIssues;
|
|
41
276
|
};
|
|
42
277
|
function isLLMError(err) {
|
|
43
278
|
return err instanceof LLMError;
|
|
44
279
|
}
|
|
280
|
+
/**
|
|
281
|
+
* Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
|
|
282
|
+
* `code` to, for any code listed there. `code` stays the only discriminator
|
|
283
|
+
* VernLLM uses; this just gives that existing check a typed return instead
|
|
284
|
+
* of requiring a manual cast of `issues`:
|
|
285
|
+
*
|
|
286
|
+
* ```ts
|
|
287
|
+
* if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
|
|
288
|
+
* console.log(err.issues.names); // string[], no cast needed
|
|
289
|
+
* }
|
|
290
|
+
* ```
|
|
291
|
+
*/
|
|
292
|
+
function hasIssues(err, code) {
|
|
293
|
+
return err.code === code && err.issues !== void 0;
|
|
294
|
+
}
|
|
45
295
|
|
|
46
296
|
//#endregion
|
|
47
297
|
//#region src/types/cache.ts
|
|
@@ -171,7 +421,12 @@ function isToolCallResult(result) {
|
|
|
171
421
|
//#endregion
|
|
172
422
|
//#region src/types/fallback.ts
|
|
173
423
|
/** Tool contract failures are the model ignoring the request, not a sick provider: repeating it elsewhere can't help. */
|
|
174
|
-
const TOOL_CONTRACT_CODES = new Set([
|
|
424
|
+
const TOOL_CONTRACT_CODES = new Set([
|
|
425
|
+
"unknown_tool",
|
|
426
|
+
"duplicate_tool_call_id",
|
|
427
|
+
"tool_choice_none_violated",
|
|
428
|
+
"unexpected_tool_calls"
|
|
429
|
+
]);
|
|
175
430
|
/**
|
|
176
431
|
* The default `fallbackOn` policy. Exported so a caller can wrap rather
|
|
177
432
|
* than replace it, e.g. `fallbackOn: (e, ctx) => myCheck(e) ? 'stop' : defaultFallbackOn(e, ctx)`.
|
|
@@ -194,10 +449,30 @@ const defaultFallbackOn = (error) => {
|
|
|
194
449
|
var FallbackExhaustedError = class extends LLMError {
|
|
195
450
|
constructor(attempts) {
|
|
196
451
|
const last = attempts[attempts.length - 1]?.error;
|
|
197
|
-
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`,
|
|
452
|
+
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`, "fallback_exhausted", {
|
|
453
|
+
status: last?.status,
|
|
454
|
+
cause: last,
|
|
455
|
+
retryAfterMs: last?.retryAfterMs,
|
|
456
|
+
code: "fallback_exhausted",
|
|
457
|
+
attempts
|
|
458
|
+
});
|
|
198
459
|
this.attempts = attempts;
|
|
199
460
|
}
|
|
461
|
+
/**
|
|
462
|
+
* `type: 'fallback_exhausted'` by itself says nothing about whether
|
|
463
|
+
* retrying could help; the reason the last target failed does. Defers to
|
|
464
|
+
* that attempt's own `retryable` instead of anything about this class's
|
|
465
|
+
* own type.
|
|
466
|
+
*/
|
|
467
|
+
get retryable() {
|
|
468
|
+
const last = this.attempts[this.attempts.length - 1]?.error;
|
|
469
|
+
return last ? last.retryable : super.retryable;
|
|
470
|
+
}
|
|
200
471
|
};
|
|
472
|
+
/** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
|
|
473
|
+
function isFallbackExhaustedError(err) {
|
|
474
|
+
return err instanceof FallbackExhaustedError;
|
|
475
|
+
}
|
|
201
476
|
|
|
202
477
|
//#endregion
|
|
203
478
|
//#region src/internal/execution/usage.utils.ts
|
|
@@ -219,7 +494,7 @@ async function reserve(params, coalesced, signal) {
|
|
|
219
494
|
return true;
|
|
220
495
|
} catch (error) {
|
|
221
496
|
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
222
|
-
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded",
|
|
497
|
+
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded", { cause: error });
|
|
223
498
|
}
|
|
224
499
|
}
|
|
225
500
|
/**
|
|
@@ -419,13 +694,30 @@ var CacheOrchestrator = class {
|
|
|
419
694
|
*/
|
|
420
695
|
async deleteCache(key) {
|
|
421
696
|
if (!this.cache.delete) return;
|
|
422
|
-
|
|
697
|
+
try {
|
|
698
|
+
await this.cache.delete(await this.resolveCacheKey(key));
|
|
699
|
+
} catch (error) {
|
|
700
|
+
this.logger.warn(`[VernLLM] cache delete failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
701
|
+
}
|
|
423
702
|
}
|
|
424
703
|
/** Logs a failed refundUsage attempt via the configured logger. */
|
|
425
704
|
logRefundError(logMessage, error) {
|
|
426
705
|
this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
|
|
427
706
|
}
|
|
428
707
|
/**
|
|
708
|
+
* Reads from the cache, treating a failed adapter read as a miss rather
|
|
709
|
+
* than letting it fail the call. The request still falls through to a
|
|
710
|
+
* real provider call, but that fallback is now logged instead of silent.
|
|
711
|
+
*/
|
|
712
|
+
async getCached(key) {
|
|
713
|
+
try {
|
|
714
|
+
return await this.cache.get(key);
|
|
715
|
+
} catch (error) {
|
|
716
|
+
this.logger.warn(`[VernLLM] cache read failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
717
|
+
return { hit: false };
|
|
718
|
+
}
|
|
719
|
+
}
|
|
720
|
+
/**
|
|
429
721
|
* Internal cache primitive around caller-supplied logic. Concurrent misses
|
|
430
722
|
* for the same `cacheKey` share a single in-flight call, avoiding cache
|
|
431
723
|
* stampedes.
|
|
@@ -445,7 +737,7 @@ var CacheOrchestrator = class {
|
|
|
445
737
|
...params,
|
|
446
738
|
cacheKey: resolvedKey
|
|
447
739
|
};
|
|
448
|
-
const cached = await this.
|
|
740
|
+
const cached = await this.getCached(resolvedKey);
|
|
449
741
|
if (cached.hit) return cached.value;
|
|
450
742
|
const existing = this.inFlight.get(resolvedKey);
|
|
451
743
|
if (existing) return withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
@@ -466,7 +758,7 @@ var CacheOrchestrator = class {
|
|
|
466
758
|
try {
|
|
467
759
|
await this.cache.set(params.cacheKey, result, params.ttl);
|
|
468
760
|
} catch (error) {
|
|
469
|
-
this.logger.
|
|
761
|
+
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
470
762
|
}
|
|
471
763
|
return result;
|
|
472
764
|
}
|
|
@@ -495,7 +787,7 @@ var CacheOrchestrator = class {
|
|
|
495
787
|
...params,
|
|
496
788
|
cacheKey: resolvedKey
|
|
497
789
|
};
|
|
498
|
-
const cached = await this.
|
|
790
|
+
const cached = await this.getCached(resolvedKey);
|
|
499
791
|
if (cached.hit) {
|
|
500
792
|
const value = cached.value;
|
|
501
793
|
return {
|
|
@@ -544,7 +836,7 @@ var CacheOrchestrator = class {
|
|
|
544
836
|
try {
|
|
545
837
|
await this.cache.set(params.cacheKey, value, params.ttl);
|
|
546
838
|
} catch (error) {
|
|
547
|
-
this.logger.
|
|
839
|
+
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
548
840
|
}
|
|
549
841
|
return value;
|
|
550
842
|
}, (error) => {
|
|
@@ -586,6 +878,7 @@ var CircuitBreaker = class {
|
|
|
586
878
|
threshold;
|
|
587
879
|
cooldownMs;
|
|
588
880
|
onStateChange;
|
|
881
|
+
/** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
|
|
589
882
|
isolateByModel;
|
|
590
883
|
sharedBucket = newBucket();
|
|
591
884
|
bucketsByModel = new Map();
|
|
@@ -631,12 +924,12 @@ var CircuitBreaker = class {
|
|
|
631
924
|
if (bucket.state === "closed") return;
|
|
632
925
|
if (bucket.state === "open") {
|
|
633
926
|
const elapsed = Date.now() - bucket.openedAt;
|
|
634
|
-
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open");
|
|
927
|
+
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open", { code: "circuit_cooling_down" });
|
|
635
928
|
bucket.trialInFlight = true;
|
|
636
929
|
this.transition(bucket, "half-open", model);
|
|
637
930
|
return;
|
|
638
931
|
}
|
|
639
|
-
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open");
|
|
932
|
+
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open", { code: "circuit_trial_in_flight" });
|
|
640
933
|
bucket.trialInFlight = true;
|
|
641
934
|
}
|
|
642
935
|
recordSuccess(model) {
|
|
@@ -671,6 +964,33 @@ var CircuitBreaker = class {
|
|
|
671
964
|
getState(model) {
|
|
672
965
|
return this.lookupBucket(model)?.state ?? "closed";
|
|
673
966
|
}
|
|
967
|
+
/**
|
|
968
|
+
* Manually opens the circuit, as if `threshold` consecutive failures had
|
|
969
|
+
* just happened, e.g. to pull a provider out of rotation ahead of known
|
|
970
|
+
* maintenance. Resets the cooldown window from now, same as a real
|
|
971
|
+
* threshold-crossing failure would, and clears any in-flight half-open
|
|
972
|
+
* trial since it no longer applies once the circuit is (re)opened.
|
|
973
|
+
*/
|
|
974
|
+
open(model) {
|
|
975
|
+
const bucket = this.ensureBucketFor(model);
|
|
976
|
+
bucket.openedAt = Date.now();
|
|
977
|
+
bucket.trialInFlight = false;
|
|
978
|
+
this.transition(bucket, "open", model);
|
|
979
|
+
}
|
|
980
|
+
/**
|
|
981
|
+
* Manually closes the circuit and resets its failure count, e.g. once a
|
|
982
|
+
* provider is confirmed healthy again without waiting out the cooldown.
|
|
983
|
+
* Mirrors `recordSuccess`'s bookkeeping (including dropping the
|
|
984
|
+
* per-model bucket under `isolateByModel`, once idle) but without
|
|
985
|
+
* requiring an actual successful call first.
|
|
986
|
+
*/
|
|
987
|
+
close(model) {
|
|
988
|
+
const bucket = this.ensureBucketFor(model);
|
|
989
|
+
bucket.consecutiveFailures = 0;
|
|
990
|
+
bucket.trialInFlight = false;
|
|
991
|
+
this.transition(bucket, "closed", model);
|
|
992
|
+
if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
|
|
993
|
+
}
|
|
674
994
|
};
|
|
675
995
|
|
|
676
996
|
//#endregion
|
|
@@ -789,7 +1109,7 @@ async function withTimeout(fn, timeoutMs, externalSignal) {
|
|
|
789
1109
|
try {
|
|
790
1110
|
return await fn(signal);
|
|
791
1111
|
} catch (err) {
|
|
792
|
-
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout");
|
|
1112
|
+
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout", { code: "request_timeout" });
|
|
793
1113
|
throw err;
|
|
794
1114
|
} finally {
|
|
795
1115
|
clearTimeout(timer);
|
|
@@ -822,7 +1142,7 @@ function withChunkIdleTimeout(next, timeoutMs, onIdle, logger) {
|
|
|
822
1142
|
const timer = setTimeout(() => {
|
|
823
1143
|
settled = true;
|
|
824
1144
|
onIdle?.();
|
|
825
|
-
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout"));
|
|
1145
|
+
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout", { code: "idle_timeout" }));
|
|
826
1146
|
}, clampTimeoutMs(activeTimeoutMs));
|
|
827
1147
|
next().then((result) => {
|
|
828
1148
|
clearTimeout(timer);
|
|
@@ -987,20 +1307,99 @@ function describeError(err) {
|
|
|
987
1307
|
} catch {}
|
|
988
1308
|
return formatSafely(err);
|
|
989
1309
|
}
|
|
990
|
-
/**
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
1310
|
+
/**
|
|
1311
|
+
* Maps an HTTP status to its corresponding `LLMErrorCode`, derived purely
|
|
1312
|
+
* from the status itself so it applies the same way regardless of which
|
|
1313
|
+
* adapter or client raised the error. Used both when building a fresh
|
|
1314
|
+
* `LLMError` and when filling in a `code` on an already-normalized one
|
|
1315
|
+
* that doesn't have one yet, so the two paths can't drift apart.
|
|
1316
|
+
*/
|
|
1317
|
+
function codeForStatus(status) {
|
|
1318
|
+
switch (status) {
|
|
1319
|
+
case 429: return "provider_rate_limited";
|
|
1320
|
+
case 401: return "authentication";
|
|
1321
|
+
case 403: return "authorization";
|
|
1322
|
+
case 404: return "not_found";
|
|
1323
|
+
case 413: return "payload_too_large";
|
|
1324
|
+
default: return status >= 500 ? "server_error" : void 0;
|
|
1325
|
+
}
|
|
1326
|
+
}
|
|
1327
|
+
/**
|
|
1328
|
+
* Whether a provider's error response actually contains anything a person
|
|
1329
|
+
* could act on. Some providers return a non-2xx status with **no body at
|
|
1330
|
+
* all** for certain field-validation failures (Mistral's OpenAI-compatible
|
|
1331
|
+
* endpoint does this, for example, when a request includes a field the
|
|
1332
|
+
* target model doesn't support). SDKs built on top of `openai` render that
|
|
1333
|
+
* specific case as a message like `"400 status code (no body)"`.
|
|
1334
|
+
*
|
|
1335
|
+
* Derived from the object's own `error`/`message` fields directly, rather
|
|
1336
|
+
* than from whatever `describeError` rendered, because `describeError`
|
|
1337
|
+
* falls back to serializing the *whole* thrown value when neither field is
|
|
1338
|
+
* present or meaningful. That fallback is local echo (e.g. just the
|
|
1339
|
+
* `status` a caller passed in), not provider diagnostic content, and
|
|
1340
|
+
* treating it as "detail" defeats the whole point of this check.
|
|
1341
|
+
*/
|
|
1342
|
+
const NO_BODY_MESSAGE_PATTERN = /\(no body\)/i;
|
|
1343
|
+
function isEmptyObject(value) {
|
|
1344
|
+
return Object.keys(value).length === 0;
|
|
1345
|
+
}
|
|
1346
|
+
function hasNoDiagnosticDetail(error) {
|
|
1347
|
+
if (error && typeof error === "object") {
|
|
1348
|
+
const { error: errorField, message } = error;
|
|
1349
|
+
if (errorField !== void 0 && errorField !== null) {
|
|
1350
|
+
const isEmptyString = typeof errorField === "string" && errorField.trim().length === 0;
|
|
1351
|
+
const isEmptyStruct = typeof errorField === "object" && isEmptyObject(errorField);
|
|
1352
|
+
if (!isEmptyString && !isEmptyStruct) return false;
|
|
1353
|
+
}
|
|
1354
|
+
if (typeof message === "string") {
|
|
1355
|
+
const trimmed = message.trim();
|
|
1356
|
+
return trimmed.length === 0 || NO_BODY_MESSAGE_PATTERN.test(trimmed);
|
|
997
1357
|
}
|
|
1358
|
+
return true;
|
|
1359
|
+
}
|
|
1360
|
+
return true;
|
|
1361
|
+
}
|
|
1362
|
+
/**
|
|
1363
|
+
* Converts any thrown value into a well-typed LLMError. `attempts`, when
|
|
1364
|
+
* given, is the accumulated record of every attempt made before `error`
|
|
1365
|
+
* was thrown; it's passed straight into the constructed error's options
|
|
1366
|
+
* rather than assigned onto the error afterward, so `attempts` is always
|
|
1367
|
+
* settled once, through the constructor, like every other field on
|
|
1368
|
+
* `LLMError`.
|
|
1369
|
+
*/
|
|
1370
|
+
function normalizeError(error, signal, attempts) {
|
|
1371
|
+
if (signal?.aborted) return new LLMError("LLM request aborted", "aborted", { attempts });
|
|
1372
|
+
if (error instanceof LLMError) {
|
|
1373
|
+
if (error.code === void 0 && error.status !== void 0) error.code = codeForStatus(error.status);
|
|
1374
|
+
if (error.attempts === void 0 && attempts !== void 0) error.attempts = attempts;
|
|
998
1375
|
return error;
|
|
999
1376
|
}
|
|
1000
1377
|
const status = extractStatus(error);
|
|
1001
1378
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
1002
|
-
if (status !== void 0)
|
|
1003
|
-
|
|
1379
|
+
if (status !== void 0) {
|
|
1380
|
+
const description = describeError(error);
|
|
1381
|
+
const code = codeForStatus(status);
|
|
1382
|
+
const isRequestValidationStatus = code === void 0;
|
|
1383
|
+
const message = hasNoDiagnosticDetail(error) ? isRequestValidationStatus ? `LLM request failed with status ${status} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.` : `LLM request failed with status ${status} and no error detail from the provider.` : `LLM request failed: ${description}`;
|
|
1384
|
+
return new LLMError(message, "api", {
|
|
1385
|
+
status,
|
|
1386
|
+
cause: error,
|
|
1387
|
+
retryAfterMs,
|
|
1388
|
+
code,
|
|
1389
|
+
attempts
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1392
|
+
if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
|
|
1393
|
+
cause: error,
|
|
1394
|
+
retryAfterMs,
|
|
1395
|
+
code: "connection_failed",
|
|
1396
|
+
attempts
|
|
1397
|
+
});
|
|
1398
|
+
return new LLMError("LLM request failed", "unknown", {
|
|
1399
|
+
cause: error,
|
|
1400
|
+
retryAfterMs,
|
|
1401
|
+
attempts
|
|
1402
|
+
});
|
|
1004
1403
|
}
|
|
1005
1404
|
|
|
1006
1405
|
//#endregion
|
|
@@ -1049,7 +1448,7 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1049
1448
|
try {
|
|
1050
1449
|
parsedArgs = wc.function.arguments.trim() ? JSON.parse(wc.function.arguments) : {};
|
|
1051
1450
|
} catch {
|
|
1052
|
-
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse");
|
|
1451
|
+
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse", { code: "tool_arguments_parse_failed" });
|
|
1053
1452
|
}
|
|
1054
1453
|
return {
|
|
1055
1454
|
id: wc.id,
|
|
@@ -1062,11 +1461,22 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1062
1461
|
//#endregion
|
|
1063
1462
|
//#region src/internal/execution/requestBuilder.ts
|
|
1064
1463
|
/**
|
|
1464
|
+
* Serializes `ConversationTurn` assistant content for the wire. Strings
|
|
1465
|
+
* pass through unchanged. Parsed JSON values are `JSON.stringify`'d.
|
|
1466
|
+
*/
|
|
1467
|
+
function serializeAssistantContent(content) {
|
|
1468
|
+
return typeof content === "string" ? content : JSON.stringify(content);
|
|
1469
|
+
}
|
|
1470
|
+
/**
|
|
1065
1471
|
* Builds the wire request object for one call, applying per-instance
|
|
1066
1472
|
* defaults (model, max tokens, temperature) and per-call overrides.
|
|
1067
|
-
* Owns every
|
|
1068
|
-
* history alternation, duplicate/empty tool lists,
|
|
1069
|
-
* real tool.
|
|
1473
|
+
* Owns every check that depends only on the caller's own input shape, not
|
|
1474
|
+
* on execution: history alternation, duplicate/empty tool lists,
|
|
1475
|
+
* `toolChoice` naming a real tool. All deterministic on the call site's
|
|
1476
|
+
* own input and never touch the network, so every throw here is
|
|
1477
|
+
* `type: 'invalid_params'`, not `'validation'` (which is reserved for the
|
|
1478
|
+
* model/provider's own response failing a contract check). Has no
|
|
1479
|
+
* knowledge of retry, timeouts, or the breaker, only
|
|
1070
1480
|
* the three defaults a `FallbackTarget` can override per-target (see the
|
|
1071
1481
|
* `defaultMaxTokens`/`defaultTemperature` overrides in the fallback
|
|
1072
1482
|
* design), which is what keeps it separable from `CallExecutor`.
|
|
@@ -1075,16 +1485,24 @@ var RequestBuilder = class {
|
|
|
1075
1485
|
model;
|
|
1076
1486
|
defaultMaxTokens;
|
|
1077
1487
|
defaultTemperature;
|
|
1488
|
+
defaultReasoningEffort;
|
|
1489
|
+
defaultBudgetTokens;
|
|
1490
|
+
supportsJsonObjectMode;
|
|
1078
1491
|
constructor(options) {
|
|
1079
1492
|
this.model = options.model;
|
|
1080
1493
|
this.defaultMaxTokens = options.defaultMaxTokens;
|
|
1081
1494
|
this.defaultTemperature = options.defaultTemperature;
|
|
1495
|
+
this.defaultReasoningEffort = options.defaultReasoningEffort;
|
|
1496
|
+
this.defaultBudgetTokens = options.defaultBudgetTokens;
|
|
1497
|
+
this.supportsJsonObjectMode = options.supportsJsonObjectMode;
|
|
1082
1498
|
}
|
|
1083
1499
|
/** Applies per-call defaults and shapes params into the client's request object. */
|
|
1084
1500
|
build(params) {
|
|
1085
|
-
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model,
|
|
1501
|
+
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, jsonSchema, tools, toolChoice } = params;
|
|
1086
1502
|
const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
|
|
1087
|
-
|
|
1503
|
+
const reasoningEffort = params.reasoningEffort === void 0 ? this.defaultReasoningEffort : params.reasoningEffort;
|
|
1504
|
+
const budgetTokens = params.budgetTokens === void 0 ? this.defaultBudgetTokens : params.budgetTokens;
|
|
1505
|
+
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
|
|
1088
1506
|
if (tools) {
|
|
1089
1507
|
const seen = new Set();
|
|
1090
1508
|
const duplicates = new Set();
|
|
@@ -1092,13 +1510,26 @@ var RequestBuilder = class {
|
|
|
1092
1510
|
if (seen.has(tool.name)) duplicates.add(tool.name);
|
|
1093
1511
|
seen.add(tool.name);
|
|
1094
1512
|
}
|
|
1095
|
-
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "
|
|
1513
|
+
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "invalid_params", {
|
|
1514
|
+
code: "duplicate_tool_names",
|
|
1515
|
+
issues: { names: [...duplicates] }
|
|
1516
|
+
});
|
|
1096
1517
|
}
|
|
1097
|
-
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "
|
|
1098
|
-
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1518
|
+
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "invalid_params");
|
|
1519
|
+
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "invalid_params", {
|
|
1520
|
+
code: "unknown_tool_choice",
|
|
1521
|
+
issues: {
|
|
1522
|
+
requested: toolChoice.name,
|
|
1523
|
+
available: tools.map((t) => t.name)
|
|
1524
|
+
}
|
|
1525
|
+
});
|
|
1526
|
+
const jsonModeExplicit = params.jsonMode;
|
|
1527
|
+
const jsonMode = jsonModeExplicit ?? (tools ? false : true);
|
|
1528
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === true) throw new LLMError("jsonMode: true was set explicitly, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.", "invalid_params");
|
|
1529
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 && params.schema) throw new LLMError("`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.", "invalid_params");
|
|
1530
|
+
const jsonModeEffective = !this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 ? false : jsonMode;
|
|
1531
|
+
const useJson = jsonModeEffective || Boolean(jsonSchema);
|
|
1532
|
+
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
|
|
1102
1533
|
const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
|
|
1103
1534
|
this.validateHistory(history);
|
|
1104
1535
|
const request = {
|
|
@@ -1107,6 +1538,7 @@ var RequestBuilder = class {
|
|
|
1107
1538
|
max_tokens: maxTokens,
|
|
1108
1539
|
...responseFormat ? { response_format: responseFormat } : {},
|
|
1109
1540
|
...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
|
|
1541
|
+
...budgetTokens !== void 0 && budgetTokens !== null ? { budget_tokens: budgetTokens } : {},
|
|
1110
1542
|
...tools ? { tools: toWireTools(tools) } : {},
|
|
1111
1543
|
...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
|
|
1112
1544
|
messages: [
|
|
@@ -1135,29 +1567,47 @@ var RequestBuilder = class {
|
|
|
1135
1567
|
let previousTurn;
|
|
1136
1568
|
for (const [index, turn] of history.entries()) {
|
|
1137
1569
|
if (turn.role === "tool") {
|
|
1138
|
-
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "
|
|
1139
|
-
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "
|
|
1570
|
+
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "invalid_params");
|
|
1571
|
+
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "invalid_params");
|
|
1140
1572
|
const requestedIds = new Set(previousTurn.toolCalls.map((tc) => tc.id));
|
|
1141
1573
|
const resultIds = turn.toolResults.map((tr) => tr.toolCallId);
|
|
1142
1574
|
const unknownIds = resultIds.filter((id) => !requestedIds.has(id));
|
|
1143
|
-
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "
|
|
1575
|
+
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "invalid_params", {
|
|
1576
|
+
code: "unknown_tool_result_ids",
|
|
1577
|
+
issues: {
|
|
1578
|
+
historyIndex: index,
|
|
1579
|
+
ids: unknownIds
|
|
1580
|
+
}
|
|
1581
|
+
});
|
|
1144
1582
|
const seenIds = new Set();
|
|
1145
1583
|
const duplicateIds = new Set();
|
|
1146
1584
|
for (const id of resultIds) {
|
|
1147
1585
|
if (seenIds.has(id)) duplicateIds.add(id);
|
|
1148
1586
|
seenIds.add(id);
|
|
1149
1587
|
}
|
|
1150
|
-
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "
|
|
1588
|
+
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "invalid_params", {
|
|
1589
|
+
code: "duplicate_tool_result_ids",
|
|
1590
|
+
issues: {
|
|
1591
|
+
historyIndex: index,
|
|
1592
|
+
ids: [...duplicateIds]
|
|
1593
|
+
}
|
|
1594
|
+
});
|
|
1151
1595
|
const missingIds = [...requestedIds].filter((id) => !resultIds.includes(id));
|
|
1152
|
-
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "
|
|
1596
|
+
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "invalid_params", {
|
|
1597
|
+
code: "missing_tool_results",
|
|
1598
|
+
issues: {
|
|
1599
|
+
historyIndex: index,
|
|
1600
|
+
ids: missingIds
|
|
1601
|
+
}
|
|
1602
|
+
});
|
|
1153
1603
|
} else {
|
|
1154
|
-
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "
|
|
1155
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "
|
|
1604
|
+
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "invalid_params");
|
|
1605
|
+
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "invalid_params");
|
|
1156
1606
|
}
|
|
1157
1607
|
previousTurn = turn;
|
|
1158
1608
|
}
|
|
1159
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "
|
|
1160
|
-
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "
|
|
1609
|
+
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "invalid_params");
|
|
1610
|
+
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "invalid_params");
|
|
1161
1611
|
}
|
|
1162
1612
|
/** Maps VernLLM's app-facing `ToolChoice` onto the OpenAI-shaped wire `tool_choice`. */
|
|
1163
1613
|
buildWireToolChoice(toolChoice) {
|
|
@@ -1184,9 +1634,13 @@ var RequestBuilder = class {
|
|
|
1184
1634
|
}));
|
|
1185
1635
|
if (turn.role === "assistant" && turn.toolCalls?.length) return [{
|
|
1186
1636
|
role: "assistant",
|
|
1187
|
-
...turn.content ? { content: turn.content } : {},
|
|
1637
|
+
...turn.content !== void 0 ? { content: serializeAssistantContent(turn.content) } : {},
|
|
1188
1638
|
tool_calls: toWireToolCalls(turn.toolCalls)
|
|
1189
1639
|
}];
|
|
1640
|
+
if (turn.role === "assistant") return [{
|
|
1641
|
+
role: "assistant",
|
|
1642
|
+
content: serializeAssistantContent(turn.content === void 0 ? "" : turn.content)
|
|
1643
|
+
}];
|
|
1190
1644
|
return [{
|
|
1191
1645
|
role: turn.role,
|
|
1192
1646
|
content: turn.content ?? ""
|
|
@@ -1320,10 +1774,12 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1320
1774
|
complete: wireChunk.complete
|
|
1321
1775
|
});
|
|
1322
1776
|
} else if (wireChunk.type === "usage") {
|
|
1777
|
+
const reasoningTokens = wireChunk.usage.completion_tokens_details?.reasoning_tokens;
|
|
1323
1778
|
usage = {
|
|
1324
1779
|
promptTokens: wireChunk.usage.prompt_tokens ?? 0,
|
|
1325
1780
|
completionTokens: wireChunk.usage.completion_tokens ?? 0,
|
|
1326
1781
|
totalTokens: wireChunk.usage.total_tokens ?? 0,
|
|
1782
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
1327
1783
|
requestId,
|
|
1328
1784
|
model,
|
|
1329
1785
|
provider: providerName,
|
|
@@ -1377,6 +1833,16 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1377
1833
|
//#endregion
|
|
1378
1834
|
//#region src/internal/execution/callExecutor.ts
|
|
1379
1835
|
/**
|
|
1836
|
+
* Identity function with its own parameter, used only to sidestep a TS
|
|
1837
|
+
* quirk: a `let` reassigned solely inside a nested closure (like
|
|
1838
|
+
* `retryWithBackoff`'s `onRequest`) gets narrowed to `undefined` at the
|
|
1839
|
+
* point it was last synchronously assigned, which would otherwise make
|
|
1840
|
+
* `lastRequestForAttempt` read as `never` at the point it's used below.
|
|
1841
|
+
*/
|
|
1842
|
+
function passThroughRequestSnapshot(snapshot) {
|
|
1843
|
+
return snapshot;
|
|
1844
|
+
}
|
|
1845
|
+
/**
|
|
1380
1846
|
* Everything one provider target needs to attempt a call: request
|
|
1381
1847
|
* building, retry with backoff, the per-target breaker, the per-target
|
|
1382
1848
|
* limiter. Never exported publicly. `VernLLM` holds one per target and
|
|
@@ -1419,12 +1885,27 @@ var CallExecutor = class {
|
|
|
1419
1885
|
this.requestBuilder = new RequestBuilder({
|
|
1420
1886
|
model,
|
|
1421
1887
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
1422
|
-
defaultTemperature: options.defaultTemperature
|
|
1888
|
+
defaultTemperature: options.defaultTemperature,
|
|
1889
|
+
defaultReasoningEffort: options.defaultReasoningEffort,
|
|
1890
|
+
defaultBudgetTokens: options.defaultBudgetTokens,
|
|
1891
|
+
supportsJsonObjectMode: client.supportsJsonObjectMode ?? true
|
|
1423
1892
|
});
|
|
1424
1893
|
}
|
|
1425
1894
|
getCircuitState(model) {
|
|
1426
1895
|
return this.breaker?.getState(model);
|
|
1427
1896
|
}
|
|
1897
|
+
/** Whether this target's breaker tracks failures per model. `false` if no breaker is configured. */
|
|
1898
|
+
get isolateByModel() {
|
|
1899
|
+
return this.breaker?.isolateByModel ?? false;
|
|
1900
|
+
}
|
|
1901
|
+
/** Manually opens this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1902
|
+
openCircuit(model) {
|
|
1903
|
+
this.breaker?.open(model);
|
|
1904
|
+
}
|
|
1905
|
+
/** Manually closes this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1906
|
+
closeCircuit(model) {
|
|
1907
|
+
this.breaker?.close(model);
|
|
1908
|
+
}
|
|
1428
1909
|
/**
|
|
1429
1910
|
* Throws if the breaker is open for this target/model, exactly like the
|
|
1430
1911
|
* check `run`/`runStream` used to make internally. Exposed so `VernLLM`
|
|
@@ -1446,10 +1927,11 @@ var CallExecutor = class {
|
|
|
1446
1927
|
*/
|
|
1447
1928
|
async run(params, requestId, onAttempt) {
|
|
1448
1929
|
const model = params.model ?? this.model;
|
|
1930
|
+
const attempts = [];
|
|
1449
1931
|
try {
|
|
1450
|
-
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt);
|
|
1932
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1451
1933
|
} catch (error) {
|
|
1452
|
-
const normalized = normalizeError(error, params.signal);
|
|
1934
|
+
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1453
1935
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1454
1936
|
this.logger.debug(`[VernLLM:${requestId}] error:\n${this.redactText(describeError(error))}`);
|
|
1455
1937
|
throw normalized;
|
|
@@ -1458,10 +1940,11 @@ var CallExecutor = class {
|
|
|
1458
1940
|
/** Streaming counterpart to `run`. Mirrors the old streaming branch of `VernLLM.call`. */
|
|
1459
1941
|
async runStream(params, requestId, onAttempt) {
|
|
1460
1942
|
const model = params.model ?? this.model;
|
|
1943
|
+
const attempts = [];
|
|
1461
1944
|
try {
|
|
1462
|
-
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt);
|
|
1945
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeStreamCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1463
1946
|
} catch (error) {
|
|
1464
|
-
const normalized = normalizeError(error, params.signal);
|
|
1947
|
+
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1465
1948
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1466
1949
|
this.logger.debug(`[VernLLM:${requestId}] stream-open error:\n${this.redactText(describeError(error))}`);
|
|
1467
1950
|
throw normalized;
|
|
@@ -1474,8 +1957,9 @@ var CallExecutor = class {
|
|
|
1474
1957
|
* set. Throws on an empty response (no text and no tool_calls) so the
|
|
1475
1958
|
* retry loop treats it like any other transient failure.
|
|
1476
1959
|
*/
|
|
1477
|
-
async executeCall(params, requestId, attempt) {
|
|
1960
|
+
async executeCall(params, requestId, attempt, onRequest) {
|
|
1478
1961
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1962
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1479
1963
|
let release;
|
|
1480
1964
|
if (this.limiter) {
|
|
1481
1965
|
const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
|
|
@@ -1529,11 +2013,11 @@ var CallExecutor = class {
|
|
|
1529
2013
|
finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt) {
|
|
1530
2014
|
try {
|
|
1531
2015
|
const content = rawContent?.trim();
|
|
1532
|
-
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api");
|
|
2016
|
+
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api", { code: "empty_response" });
|
|
1533
2017
|
this.logger.debug(`[VernLLM:${requestId}] output:\n${this.redactedOutput(content, wireToolCalls).slice(0, 800)}`);
|
|
1534
2018
|
if (wireToolCalls?.length) {
|
|
1535
|
-
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "
|
|
1536
|
-
if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "
|
|
2019
|
+
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "validation", { code: "unexpected_tool_calls" });
|
|
2020
|
+
if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "validation", { code: "tool_choice_none_violated" });
|
|
1537
2021
|
const toolCalls = parseWireToolCalls(wireToolCalls);
|
|
1538
2022
|
this.validateToolCallArguments(toolCalls, params.tools);
|
|
1539
2023
|
this.breaker?.recordSuccess(model);
|
|
@@ -1583,10 +2067,14 @@ var CallExecutor = class {
|
|
|
1583
2067
|
* not on the first chunk arriving, so a connection that opens but then
|
|
1584
2068
|
* dies mid-stream isn't masked as a success (see `buildStreamResult`).
|
|
1585
2069
|
*/
|
|
1586
|
-
async executeStreamCall(params, requestId, attempt) {
|
|
2070
|
+
async executeStreamCall(params, requestId, attempt, onRequest) {
|
|
1587
2071
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
2072
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1588
2073
|
const completions = this.client.chat.completions;
|
|
1589
|
-
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "
|
|
2074
|
+
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
|
|
2075
|
+
code: "unsupported_capability",
|
|
2076
|
+
issues: { capability: "createStream" }
|
|
2077
|
+
});
|
|
1590
2078
|
const createStream = completions.createStream.bind(completions);
|
|
1591
2079
|
let release;
|
|
1592
2080
|
if (this.limiter) {
|
|
@@ -1647,13 +2135,13 @@ var CallExecutor = class {
|
|
|
1647
2135
|
* `argumentsSchema`, if present.
|
|
1648
2136
|
*
|
|
1649
2137
|
* Contract failures (unknown name, duplicate id) are collected across
|
|
1650
|
-
* every call and thrown together
|
|
1651
|
-
*
|
|
1652
|
-
*
|
|
1653
|
-
*
|
|
1654
|
-
*
|
|
1655
|
-
*
|
|
1656
|
-
*
|
|
2138
|
+
* every call and thrown together as one `type: 'validation'` error with
|
|
2139
|
+
* `issues: ToolIssue[]`, since retrying a request that already has these
|
|
2140
|
+
* errors cannot help (excluded from retry by `type`) and a caller fixing
|
|
2141
|
+
* them wants to see every one, not just the first. Schema failures keep
|
|
2142
|
+
* the original single-error, `type: 'validation'` shape rather than being
|
|
2143
|
+
* folded into the aggregate, since they're a distinct failure kind from
|
|
2144
|
+
* the contract failures above.
|
|
1657
2145
|
*/
|
|
1658
2146
|
validateToolCallArguments(toolCalls, tools) {
|
|
1659
2147
|
const known = new Map(tools.map((t) => [t.name, t]));
|
|
@@ -1675,28 +2163,51 @@ var CallExecutor = class {
|
|
|
1675
2163
|
if (toolIssues.length > 0) {
|
|
1676
2164
|
const unknownTool = toolIssues.find((i) => i.code === "unknown_tool");
|
|
1677
2165
|
const primary = unknownTool ? `Model requested tool "${unknownTool.name}", which was not in the tools offered ([${[...known.keys()].join(", ")}]).` : `Duplicate tool call id "${toolIssues[0].toolCallId}" in the model's response.`;
|
|
1678
|
-
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
2166
|
+
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see error.issues.)` : primary;
|
|
2167
|
+
throw new LLMError(message, "validation", {
|
|
2168
|
+
code: unknownTool ? "unknown_tool" : "duplicate_tool_call_id",
|
|
2169
|
+
issues: toolIssues
|
|
2170
|
+
});
|
|
1682
2171
|
}
|
|
1683
2172
|
for (const call of toolCalls) {
|
|
1684
2173
|
const definition = known.get(call.name);
|
|
1685
2174
|
if (!definition?.argumentsSchema) continue;
|
|
1686
2175
|
const result = definition.argumentsSchema.safeParse(call.arguments);
|
|
1687
|
-
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation",
|
|
2176
|
+
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation", { issues: result.error });
|
|
1688
2177
|
}
|
|
1689
2178
|
}
|
|
1690
|
-
/**
|
|
1691
|
-
|
|
2179
|
+
/**
|
|
2180
|
+
* Runs `fn`, retrying with backoff according to `shouldRetry`. When
|
|
2181
|
+
* `attempts` is given, every failed attempt that is actually followed by
|
|
2182
|
+
* a retry is recorded, in order. This mirrors `LLMError.attempts`'s
|
|
2183
|
+
* contract: every attempt made before this error was thrown. The
|
|
2184
|
+
* terminal failure is never pushed since it isn't a prior attempt, it
|
|
2185
|
+
* is the error being thrown. `attempts` stays empty when nothing was
|
|
2186
|
+
* retried, so no separate bookkeeping is needed at the call sites.
|
|
2187
|
+
* Each failure is recorded as a snapshot (`LLMError.toSnapshot()`),
|
|
2188
|
+
* not the live `LLMError`, per `RetryAttempt`'s contract.
|
|
2189
|
+
*/
|
|
2190
|
+
async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
|
|
1692
2191
|
let lastError;
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1697
|
-
|
|
1698
|
-
|
|
1699
|
-
|
|
2192
|
+
let lastRequestForAttempt;
|
|
2193
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
2194
|
+
lastRequestForAttempt = void 0;
|
|
2195
|
+
try {
|
|
2196
|
+
if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
|
|
2197
|
+
onAttempt?.();
|
|
2198
|
+
return await fn(attempt, (req) => {
|
|
2199
|
+
lastRequestForAttempt = req;
|
|
2200
|
+
});
|
|
2201
|
+
} catch (error) {
|
|
2202
|
+
lastError = error;
|
|
2203
|
+
const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
|
|
2204
|
+
if (!willRetry) break;
|
|
2205
|
+
attempts?.push({
|
|
2206
|
+
index: attempt,
|
|
2207
|
+
error: normalizeError(error, signal).toSnapshot(),
|
|
2208
|
+
request: passThroughRequestSnapshot(lastRequestForAttempt)
|
|
2209
|
+
});
|
|
2210
|
+
}
|
|
1700
2211
|
}
|
|
1701
2212
|
throw lastError;
|
|
1702
2213
|
}
|
|
@@ -1708,10 +2219,12 @@ var CallExecutor = class {
|
|
|
1708
2219
|
*/
|
|
1709
2220
|
extractUsage(response, requestId, model) {
|
|
1710
2221
|
if (!response.usage) return void 0;
|
|
2222
|
+
const reasoningTokens = response.usage.completion_tokens_details?.reasoning_tokens;
|
|
1711
2223
|
return {
|
|
1712
2224
|
promptTokens: response.usage.prompt_tokens ?? 0,
|
|
1713
2225
|
completionTokens: response.usage.completion_tokens ?? 0,
|
|
1714
2226
|
totalTokens: response.usage.total_tokens ?? 0,
|
|
2227
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
1715
2228
|
requestId,
|
|
1716
2229
|
model,
|
|
1717
2230
|
provider: this.providerName,
|
|
@@ -1766,7 +2279,7 @@ var CallExecutor = class {
|
|
|
1766
2279
|
if (parsed === null || parsed === void 0) throw new LLMError("Invalid JSON response", "parse");
|
|
1767
2280
|
if (!schema) return parsed;
|
|
1768
2281
|
const result = schema.safeParse(parsed);
|
|
1769
|
-
if (!result.success) throw new LLMError("Schema validation failed", "validation",
|
|
2282
|
+
if (!result.success) throw new LLMError("Schema validation failed", "validation", { issues: result.error });
|
|
1770
2283
|
return result.data;
|
|
1771
2284
|
}
|
|
1772
2285
|
/**
|
|
@@ -1779,7 +2292,7 @@ var CallExecutor = class {
|
|
|
1779
2292
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
1780
2293
|
const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
|
|
1781
2294
|
const retryAfterHonored = retryAfterMs !== void 0;
|
|
1782
|
-
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${delay}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
2295
|
+
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${Math.ceil(delay)}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
1783
2296
|
this.reportEvent({
|
|
1784
2297
|
kind: "retry",
|
|
1785
2298
|
requestId,
|
|
@@ -1793,15 +2306,10 @@ var CallExecutor = class {
|
|
|
1793
2306
|
});
|
|
1794
2307
|
await waitForRetry(delay, signal);
|
|
1795
2308
|
}
|
|
1796
|
-
isNonRetryableToolContractError(error) {
|
|
1797
|
-
return error instanceof LLMError && (error.code === "unknown_tool" || error.code === "duplicate_tool_call_id" || error.code === "tool_choice_none_violated");
|
|
1798
|
-
}
|
|
1799
2309
|
/** Decides whether a failed attempt is worth retrying. */
|
|
1800
2310
|
shouldRetry(error, signal) {
|
|
1801
2311
|
if (signal?.aborted) return false;
|
|
1802
|
-
if (error instanceof LLMError &&
|
|
1803
|
-
if (error instanceof LLMError && error.code === "local_rate_limit") return false;
|
|
1804
|
-
if (this.isNonRetryableToolContractError(error)) return false;
|
|
2312
|
+
if (error instanceof LLMError && !error.retryable) return false;
|
|
1805
2313
|
const status = extractStatus(error);
|
|
1806
2314
|
return !(status !== void 0 && this.nonRetryableStatus.includes(status));
|
|
1807
2315
|
}
|
|
@@ -1811,16 +2319,48 @@ var CallExecutor = class {
|
|
|
1811
2319
|
* reusing a call id, or a provider ignoring `toolChoice: 'none'` isn't
|
|
1812
2320
|
* the provider being unhealthy, it's a model/provider response defect
|
|
1813
2321
|
* that will very likely recur regardless of provider health, so it
|
|
1814
|
-
* shouldn't push a healthy provider's circuit toward opening.
|
|
1815
|
-
*
|
|
1816
|
-
*
|
|
2322
|
+
* shouldn't push a healthy provider's circuit toward opening. Same for
|
|
2323
|
+
* a caller-input bug or a local rate-limit rejection: neither ever
|
|
2324
|
+
* reached the provider at all. This is exactly what `LLMError.retryable`
|
|
2325
|
+
* already excludes, so this defers to it directly.
|
|
1817
2326
|
*/
|
|
1818
2327
|
countsTowardBreaker(error) {
|
|
1819
|
-
|
|
1820
|
-
return true;
|
|
2328
|
+
return error.retryable;
|
|
1821
2329
|
}
|
|
1822
2330
|
};
|
|
1823
2331
|
|
|
2332
|
+
//#endregion
|
|
2333
|
+
//#region src/internal/logger.utils.ts
|
|
2334
|
+
/**
|
|
2335
|
+
* Wraps a `Logger` so a throwing implementation can never break the call
|
|
2336
|
+
* it's trying to describe. `logger` is user-supplied (`VernLLMOptions.logger`),
|
|
2337
|
+
* so a custom logger that ships to a file, Datadog, etc. can throw for
|
|
2338
|
+
* reasons unrelated to VernLLM. Wrap once at construction so every
|
|
2339
|
+
* downstream `this.logger.warn(...)` call stays as-is and is safe by
|
|
2340
|
+
* construction, instead of guarding each call site individually.
|
|
2341
|
+
*/
|
|
2342
|
+
function createSafeLogger(logger) {
|
|
2343
|
+
return {
|
|
2344
|
+
debug: safe(logger, "debug"),
|
|
2345
|
+
warn: safe(logger, "warn"),
|
|
2346
|
+
error: safe(logger, "error")
|
|
2347
|
+
};
|
|
2348
|
+
}
|
|
2349
|
+
function safe(logger, method) {
|
|
2350
|
+
const fn = logger[method].bind(logger);
|
|
2351
|
+
return (...args) => {
|
|
2352
|
+
try {
|
|
2353
|
+
swallowRejection(fn(...args));
|
|
2354
|
+
} catch {}
|
|
2355
|
+
};
|
|
2356
|
+
}
|
|
2357
|
+
function isPromiseLike(value) {
|
|
2358
|
+
return typeof value?.then === "function";
|
|
2359
|
+
}
|
|
2360
|
+
function swallowRejection(result) {
|
|
2361
|
+
if (isPromiseLike(result)) Promise.resolve(result).catch(() => {});
|
|
2362
|
+
}
|
|
2363
|
+
|
|
1824
2364
|
//#endregion
|
|
1825
2365
|
//#region src/logger.ts
|
|
1826
2366
|
/**
|
|
@@ -1961,8 +2501,8 @@ var RateLimiter = class {
|
|
|
1961
2501
|
*/
|
|
1962
2502
|
async acquire(estimatedTokens, signal) {
|
|
1963
2503
|
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
1964
|
-
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "
|
|
1965
|
-
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "
|
|
2504
|
+
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "invalid_params");
|
|
2505
|
+
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "rate_limited", { code: "rate_limit_capacity_exceeded" });
|
|
1966
2506
|
if (this.queue.length === 0) {
|
|
1967
2507
|
const attempt = this.tryAcquireBuckets(estimatedTokens);
|
|
1968
2508
|
if (attempt.ok) return {
|
|
@@ -1975,7 +2515,7 @@ var RateLimiter = class {
|
|
|
1975
2515
|
return this.enqueue(estimatedTokens, void 0, signal);
|
|
1976
2516
|
}
|
|
1977
2517
|
queueFullError() {
|
|
1978
|
-
return new LLMError("Rate limit queue is full", "
|
|
2518
|
+
return new LLMError("Rate limit queue is full", "rate_limited", { code: "rate_limit_queue_full" });
|
|
1979
2519
|
}
|
|
1980
2520
|
enqueue(estimatedTokens, initialReason, signal) {
|
|
1981
2521
|
return new Promise((resolvePromise, rejectPromise) => {
|
|
@@ -2008,7 +2548,7 @@ var RateLimiter = class {
|
|
|
2008
2548
|
if (index !== -1) this.queue.splice(index, 1);
|
|
2009
2549
|
};
|
|
2010
2550
|
if (this.maxQueueMs > 0) queueTimer = setTimeout(() => {
|
|
2011
|
-
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "
|
|
2551
|
+
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "rate_limited", { code: "rate_limit_queue_timeout" }));
|
|
2012
2552
|
}, this.maxQueueMs);
|
|
2013
2553
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
2014
2554
|
this.queue.push(waiter);
|
|
@@ -2108,7 +2648,7 @@ var RateLimiter = class {
|
|
|
2108
2648
|
//#endregion
|
|
2109
2649
|
//#region src/vernLLM.ts
|
|
2110
2650
|
/**
|
|
2111
|
-
* A
|
|
2651
|
+
* A LLM call framework for resilience, observability and control. This is VernLLM!
|
|
2112
2652
|
*
|
|
2113
2653
|
* Adds retry with backoff and jitter, per-attempt timeouts, an optional
|
|
2114
2654
|
* circuit breaker, JSON parsing with optional schema validation, usage
|
|
@@ -2142,12 +2682,14 @@ var VernLLM = class {
|
|
|
2142
2682
|
* `nonRetryableStatus` `[400, 401, 403, 404, 422]`, `debug` false.
|
|
2143
2683
|
*/
|
|
2144
2684
|
constructor(options) {
|
|
2145
|
-
this.logger = options.logger ?? new ConsoleLogger(options.debug ?? false);
|
|
2685
|
+
this.logger = createSafeLogger(options.logger ?? new ConsoleLogger(options.debug ?? false));
|
|
2146
2686
|
const providerName = options.name ?? "primary";
|
|
2147
2687
|
this.cacheOrchestrator = new CacheOrchestrator(options.cache ?? new InMemoryCacheAdapter(), this.logger);
|
|
2148
2688
|
this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
|
|
2149
2689
|
this.reportEvent = makeEventReporter(options.onEvent, this.logger);
|
|
2150
2690
|
const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
|
|
2691
|
+
const primaryDefaultReasoningEffort = options.defaultReasoningEffort;
|
|
2692
|
+
const primaryDefaultBudgetTokens = options.defaultBudgetTokens;
|
|
2151
2693
|
const primaryTarget = {
|
|
2152
2694
|
client: options.client,
|
|
2153
2695
|
model: options.model,
|
|
@@ -2158,6 +2700,8 @@ var VernLLM = class {
|
|
|
2158
2700
|
baseDelayMs: options.baseDelayMs,
|
|
2159
2701
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
2160
2702
|
defaultTemperature: primaryDefaultTemperature,
|
|
2703
|
+
defaultReasoningEffort: primaryDefaultReasoningEffort,
|
|
2704
|
+
defaultBudgetTokens: primaryDefaultBudgetTokens,
|
|
2161
2705
|
nonRetryableStatus: options.nonRetryableStatus,
|
|
2162
2706
|
circuitBreaker: options.circuitBreaker,
|
|
2163
2707
|
rateLimit: options.rateLimit
|
|
@@ -2175,6 +2719,8 @@ var VernLLM = class {
|
|
|
2175
2719
|
baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
|
|
2176
2720
|
defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
|
|
2177
2721
|
defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
|
|
2722
|
+
defaultReasoningEffort: target.defaultReasoningEffort === void 0 ? primaryDefaultReasoningEffort : target.defaultReasoningEffort,
|
|
2723
|
+
defaultBudgetTokens: target.defaultBudgetTokens === void 0 ? primaryDefaultBudgetTokens : target.defaultBudgetTokens,
|
|
2178
2724
|
nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
|
|
2179
2725
|
400,
|
|
2180
2726
|
401,
|
|
@@ -2235,7 +2781,7 @@ var VernLLM = class {
|
|
|
2235
2781
|
index: i - 1,
|
|
2236
2782
|
provider: executor.providerName,
|
|
2237
2783
|
model: params.model ?? executor.model,
|
|
2238
|
-
error: normalized
|
|
2784
|
+
error: normalized.toSnapshot()
|
|
2239
2785
|
});
|
|
2240
2786
|
const isLast = i === this.executors.length - 1;
|
|
2241
2787
|
const policyDecision = this.fallbackOn(normalized, { isLastTarget: isLast });
|
|
@@ -2254,7 +2800,7 @@ var VernLLM = class {
|
|
|
2254
2800
|
});
|
|
2255
2801
|
}
|
|
2256
2802
|
}
|
|
2257
|
-
throw new LLMError("No provider targets configured", "
|
|
2803
|
+
throw new LLMError("No provider targets configured", "invalid_params");
|
|
2258
2804
|
}
|
|
2259
2805
|
async call(params) {
|
|
2260
2806
|
if (params.signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
@@ -2300,7 +2846,7 @@ var VernLLM = class {
|
|
|
2300
2846
|
async cachedCall(params) {
|
|
2301
2847
|
const { call: callParams,...cacheParams } = params;
|
|
2302
2848
|
const restCallParams = callParams;
|
|
2303
|
-
if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "
|
|
2849
|
+
if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "invalid_params");
|
|
2304
2850
|
if (restCallParams.stream) {
|
|
2305
2851
|
const streamParams = restCallParams;
|
|
2306
2852
|
return this.cacheOrchestrator.runCachedStream({
|
|
@@ -2314,36 +2860,107 @@ var VernLLM = class {
|
|
|
2314
2860
|
});
|
|
2315
2861
|
}
|
|
2316
2862
|
/**
|
|
2317
|
-
* @param
|
|
2318
|
-
* model
|
|
2319
|
-
*
|
|
2320
|
-
*
|
|
2321
|
-
*
|
|
2322
|
-
*
|
|
2863
|
+
* @param target.index Which target to read. Defaults to the primary.
|
|
2864
|
+
* @param target.model Which model bucket to read, if the target isolates by model.
|
|
2865
|
+
* @returns The breaker state, or `undefined` if that target has no breaker.
|
|
2866
|
+
* @throws {RangeError} If `target.index` names no target. Lets a real
|
|
2867
|
+
* target with no breaker (`undefined`) stay distinguishable from a
|
|
2868
|
+
* target that doesn't exist.
|
|
2323
2869
|
*/
|
|
2324
|
-
getCircuitState(
|
|
2325
|
-
|
|
2870
|
+
getCircuitState(target) {
|
|
2871
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "getCircuitState");
|
|
2872
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "getCircuitState");
|
|
2873
|
+
return executor.getCircuitState(target?.model ?? executor.model);
|
|
2326
2874
|
}
|
|
2327
2875
|
/**
|
|
2328
|
-
* @param model
|
|
2329
|
-
* target's
|
|
2330
|
-
* Ignored otherwise. Omit for the shared circuit (the default) or, under
|
|
2331
|
-
* isolation, the state of calls that didn't resolve a model.
|
|
2332
|
-
* @returns The current circuit state for every target in declaration
|
|
2333
|
-
* order, including the primary and all fallback targets. Each entry
|
|
2334
|
-
* includes the target's provider name, chain index, whether it is a
|
|
2335
|
-
* fallback, and its circuit state, or undefined if that target has no
|
|
2336
|
-
* circuit breaker configured.
|
|
2876
|
+
* @param model Which model bucket to read, for targets that isolate by model.
|
|
2877
|
+
* @returns Every target's state, in chain order.
|
|
2337
2878
|
*/
|
|
2338
2879
|
getCircuitStates(model) {
|
|
2339
2880
|
return this.executors.map((executor, index) => ({
|
|
2340
2881
|
provider: executor.providerName,
|
|
2341
2882
|
index,
|
|
2342
2883
|
isFallback: index > 0,
|
|
2343
|
-
|
|
2884
|
+
isolateByModel: executor.isolateByModel,
|
|
2885
|
+
state: executor.getCircuitState(model ?? executor.model)
|
|
2344
2886
|
}));
|
|
2345
2887
|
}
|
|
2888
|
+
/**
|
|
2889
|
+
* Manually opens a target's breaker, e.g. to pull a provider out of
|
|
2890
|
+
* rotation ahead of known maintenance instead of waiting for it to fail.
|
|
2891
|
+
*
|
|
2892
|
+
* @param target.index Which target to open. Defaults to the primary.
|
|
2893
|
+
* @param target.model Which model bucket to open, if the target isolates by model.
|
|
2894
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
2895
|
+
*/
|
|
2896
|
+
openCircuit(target) {
|
|
2897
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "openCircuit");
|
|
2898
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "openCircuit");
|
|
2899
|
+
executor.openCircuit(target?.model ?? executor.model);
|
|
2900
|
+
}
|
|
2901
|
+
/**
|
|
2902
|
+
* Manually closes a target's breaker, e.g. once a provider is confirmed
|
|
2903
|
+
* healthy again without waiting out the cooldown.
|
|
2904
|
+
*
|
|
2905
|
+
* @param target.index Which target to close. Defaults to the primary.
|
|
2906
|
+
* @param target.model Which model bucket to close, if the target isolates by model.
|
|
2907
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
2908
|
+
*/
|
|
2909
|
+
closeCircuit(target) {
|
|
2910
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "closeCircuit");
|
|
2911
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "closeCircuit");
|
|
2912
|
+
executor.closeCircuit(target?.model ?? executor.model);
|
|
2913
|
+
}
|
|
2914
|
+
/** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
|
|
2915
|
+
resolveExecutor(index, caller) {
|
|
2916
|
+
const executor = this.executors[index];
|
|
2917
|
+
if (!executor) throw new RangeError(`${caller}: no target at index ${index} (chain has ${this.executors.length} target${this.executors.length === 1 ? "" : "s"})`);
|
|
2918
|
+
return executor;
|
|
2919
|
+
}
|
|
2920
|
+
/** Warns when `model` can't do anything on this target, so it's never silently ignored. */
|
|
2921
|
+
warnIfModelUnsupported(isolateByModel, model, caller) {
|
|
2922
|
+
if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
|
|
2923
|
+
}
|
|
2346
2924
|
};
|
|
2925
|
+
/**
|
|
2926
|
+
* Identity function preserving `params`'s own precise type, unlike a `:
|
|
2927
|
+
* CallParams<T>` annotation, which would widen `tools` away and break the
|
|
2928
|
+
* `ConditionalToolCallParams<T>` overload for `tools: someCondition ?
|
|
2929
|
+
* [tool] : undefined`. Use it when you need `call()` params in a named,
|
|
2930
|
+
* reusable variable; skip it when you can pass the object inline.
|
|
2931
|
+
*
|
|
2932
|
+
* ```ts
|
|
2933
|
+
* const params = defineCallParams({
|
|
2934
|
+
* userContent: 'What is the weather?',
|
|
2935
|
+
* tools: someCondition ? [weatherTool] : undefined,
|
|
2936
|
+
* });
|
|
2937
|
+
* const result = await llm.call(params);
|
|
2938
|
+
* // result: unknown | CallWithToolsResult<unknown>, same as inline
|
|
2939
|
+
* ```
|
|
2940
|
+
*
|
|
2941
|
+
* `T` isn't a parameter here; pin it via `llm.call<T>(params)` as usual.
|
|
2942
|
+
* `defineCachedCallParams` is the `cachedCall()` counterpart.
|
|
2943
|
+
*/
|
|
2944
|
+
function defineCallParams(params) {
|
|
2945
|
+
return params;
|
|
2946
|
+
}
|
|
2947
|
+
/**
|
|
2948
|
+
* The `cachedCall()` counterpart to `defineCallParams`: preserves the
|
|
2949
|
+
* whole `{ cacheKey, ttl, call }` object, `call.tools` included, in one
|
|
2950
|
+
* named variable.
|
|
2951
|
+
*
|
|
2952
|
+
* ```ts
|
|
2953
|
+
* const params = defineCachedCallParams({
|
|
2954
|
+
* cacheKey: 'weather-ny',
|
|
2955
|
+
* ttl: 60,
|
|
2956
|
+
* call: { userContent: 'What is the weather?', tools: someCondition ? [weatherTool] : undefined },
|
|
2957
|
+
* });
|
|
2958
|
+
* const result = await llm.cachedCall(params);
|
|
2959
|
+
* ```
|
|
2960
|
+
*/
|
|
2961
|
+
function defineCachedCallParams(params) {
|
|
2962
|
+
return params;
|
|
2963
|
+
}
|
|
2347
2964
|
|
|
2348
2965
|
//#endregion
|
|
2349
2966
|
//#region src/adapters/internal/sse.ts
|
|
@@ -2382,7 +2999,7 @@ async function* parseSseStream(source) {
|
|
|
2382
2999
|
try {
|
|
2383
3000
|
text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
|
|
2384
3001
|
} catch (cause) {
|
|
2385
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse",
|
|
3002
|
+
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2386
3003
|
}
|
|
2387
3004
|
buffer = (buffer + text).replace(/\r\n/g, "\n").replace(/\r(?!$)/g, "\n");
|
|
2388
3005
|
let boundary$1 = buffer.indexOf("\n\n");
|
|
@@ -2398,7 +3015,7 @@ async function* parseSseStream(source) {
|
|
|
2398
3015
|
try {
|
|
2399
3016
|
buffer += decoder.decode();
|
|
2400
3017
|
} catch (cause) {
|
|
2401
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse",
|
|
3018
|
+
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2402
3019
|
}
|
|
2403
3020
|
buffer = buffer.replace(/\r$/, "\n");
|
|
2404
3021
|
let boundary = buffer.indexOf("\n\n");
|
|
@@ -2443,7 +3060,10 @@ function parseSseFrame(frame) {
|
|
|
2443
3060
|
try {
|
|
2444
3061
|
return JSON.parse(data);
|
|
2445
3062
|
} catch (cause) {
|
|
2446
|
-
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse",
|
|
3063
|
+
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse", {
|
|
3064
|
+
cause,
|
|
3065
|
+
code: "stream_frame_invalid"
|
|
3066
|
+
});
|
|
2447
3067
|
}
|
|
2448
3068
|
}
|
|
2449
3069
|
|
|
@@ -2463,13 +3083,14 @@ const SUPPORTED_IMAGE_MIME_TYPES = [
|
|
|
2463
3083
|
];
|
|
2464
3084
|
/**
|
|
2465
3085
|
* Validates an `ImageBlock.mimeType` against the shared supported set.
|
|
2466
|
-
* Throws a non-retryable `LLMError('
|
|
2467
|
-
* mimeType is a
|
|
2468
|
-
* the same
|
|
3086
|
+
* Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
|
|
3087
|
+
* mimeType is a bug in the caller's own input, deterministic before any
|
|
3088
|
+
* request is built, the same class of failure as every other check in
|
|
3089
|
+
* `RequestBuilder`.
|
|
2469
3090
|
*/
|
|
2470
3091
|
function assertSupportedImageMimeType(mimeType) {
|
|
2471
3092
|
if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
|
|
2472
|
-
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "
|
|
3093
|
+
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "invalid_params");
|
|
2473
3094
|
}
|
|
2474
3095
|
|
|
2475
3096
|
//#endregion
|
|
@@ -2480,6 +3101,275 @@ function supportsNativeStructuredOutput(model, override) {
|
|
|
2480
3101
|
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
2481
3102
|
}
|
|
2482
3103
|
|
|
3104
|
+
//#endregion
|
|
3105
|
+
//#region src/adapters/internal/reasoningBudget.utils.ts
|
|
3106
|
+
const DEFAULT_EFFORT_TOKENS = {
|
|
3107
|
+
minimal: 1024,
|
|
3108
|
+
low: 4096,
|
|
3109
|
+
medium: 16e3,
|
|
3110
|
+
high: 32e3
|
|
3111
|
+
};
|
|
3112
|
+
/**
|
|
3113
|
+
* Merges a caller-supplied partial override over `DEFAULT_EFFORT_TOKENS`.
|
|
3114
|
+
* Called once per adapter instance (not per request), so a per-instance
|
|
3115
|
+
* override only needs to specify the tiers it actually wants to change.
|
|
3116
|
+
*
|
|
3117
|
+
* Throws `LLMError('invalid_params')` if the override doesn't keep the
|
|
3118
|
+
* tiers in strictly ascending order (`minimal < low < medium < high`).
|
|
3119
|
+
* `budgetTokensToEffort` buckets by walking the tiers low to high and
|
|
3120
|
+
* returning on the first one a value is `<=`, so an unordered table (e.g.
|
|
3121
|
+
* `low` above `medium`) wouldn't just produce a "wrong" bucket, it would
|
|
3122
|
+
* make some tiers unreachable outright, silently, with no signal to the
|
|
3123
|
+
* caller that their override doesn't do what they think it does.
|
|
3124
|
+
*/
|
|
3125
|
+
function resolveEffortTokenTable(override) {
|
|
3126
|
+
if (!override) return DEFAULT_EFFORT_TOKENS;
|
|
3127
|
+
const table = {
|
|
3128
|
+
...DEFAULT_EFFORT_TOKENS,
|
|
3129
|
+
...override
|
|
3130
|
+
};
|
|
3131
|
+
if (!(table.minimal < table.low && table.low < table.medium && table.medium < table.high)) throw new LLMError(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(table)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`, "invalid_params");
|
|
3132
|
+
return table;
|
|
3133
|
+
}
|
|
3134
|
+
/** Converts a `reasoningEffort` tier into the nearest `budgetTokens` value. */
|
|
3135
|
+
function effortToBudgetTokens(effort, table = DEFAULT_EFFORT_TOKENS) {
|
|
3136
|
+
return table[effort];
|
|
3137
|
+
}
|
|
3138
|
+
/**
|
|
3139
|
+
* Converts a raw `budgetTokens` value into the nearest `reasoningEffort`
|
|
3140
|
+
* tier, for providers that only understand tiers. Buckets by the same
|
|
3141
|
+
* `table` `effortToBudgetTokens` produces its values from, so the two
|
|
3142
|
+
* functions agree with each other at the boundary values, as long as the
|
|
3143
|
+
* same (possibly overridden) table is passed to both. A value strictly
|
|
3144
|
+
* between two tiers (e.g. 4097, one above the default `low`) rounds up to
|
|
3145
|
+
* the next tier it's still `<=`, i.e. `medium` here, not down to `low`.
|
|
3146
|
+
*/
|
|
3147
|
+
function budgetTokensToEffort(budgetTokens, table = DEFAULT_EFFORT_TOKENS) {
|
|
3148
|
+
if (budgetTokens <= table.minimal) return "minimal";
|
|
3149
|
+
if (budgetTokens <= table.low) return "low";
|
|
3150
|
+
if (budgetTokens <= table.medium) return "medium";
|
|
3151
|
+
return "high";
|
|
3152
|
+
}
|
|
3153
|
+
/**
|
|
3154
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3155
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3156
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3157
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3158
|
+
* non-Opus model id.
|
|
3159
|
+
*/
|
|
3160
|
+
/**
|
|
3161
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3162
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3163
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3164
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3165
|
+
* non-Opus model id.
|
|
3166
|
+
*
|
|
3167
|
+
* Anthropic model ids sometimes carry a trailing snapshot date instead of
|
|
3168
|
+
* (or in addition to) an explicit minor version, e.g. the real, still-
|
|
3169
|
+
* supported base `"claude-opus-4-20250514"` (no `.7`-style minor at all,
|
|
3170
|
+
* just a date suffix directly after the major version). Read naively,
|
|
3171
|
+
* `20250514` looks like a minor version far above any real threshold and
|
|
3172
|
+
* would misclassify this pre-4.6 model as adaptive-only. Snapshot dates
|
|
3173
|
+
* are always 8 digits (`YYYYMMDD`); a real minor version never is, so an
|
|
3174
|
+
* 8+ digit second segment is treated as a date, not a minor version.
|
|
3175
|
+
*/
|
|
3176
|
+
function parseOpusVersion(model) {
|
|
3177
|
+
const match = /opus-(\d+)(?:-(\d+))?/.exec(model);
|
|
3178
|
+
if (!match) return null;
|
|
3179
|
+
const minorStr = match[2];
|
|
3180
|
+
const minor = minorStr === void 0 || minorStr.length >= 8 ? 0 : Number(minorStr);
|
|
3181
|
+
return [Number(match[1]), minor];
|
|
3182
|
+
}
|
|
3183
|
+
/**
|
|
3184
|
+
* Default rule for whether `model` only supports adaptive thinking
|
|
3185
|
+
* (`thinking: { type: 'adaptive' }`) and returns a 400 for manual,
|
|
3186
|
+
* budget-based thinking (`thinking: { type: 'enabled', budget_tokens }`):
|
|
3187
|
+
* Claude Opus 4.7 and later (matched as a version threshold, so 4.8, 4.9,
|
|
3188
|
+
* 5, and every future Opus point release are covered automatically,
|
|
3189
|
+
* without a new list entry per release), and every Claude 5 tier model
|
|
3190
|
+
* outside the Opus family (Sonnet 5, Fable 5, Mythos 5, Mythos Preview).
|
|
3191
|
+
* `mythos` alone is enough to catch both Mythos names without listing
|
|
3192
|
+
* each separately.
|
|
3193
|
+
*
|
|
3194
|
+
* Necessarily best-effort: a new model family with its own name (not
|
|
3195
|
+
* `opus-*`, not `sonnet-5`/`fable-5`/`mythos-*`) still needs a code
|
|
3196
|
+
* update here, or a caller-supplied `adaptiveOnlyModels` override (see
|
|
3197
|
+
* `isAdaptiveOnlyModel`) covering it in the meantime.
|
|
3198
|
+
*/
|
|
3199
|
+
function isDefaultAdaptiveOnly(model) {
|
|
3200
|
+
const opusVersion = parseOpusVersion(model);
|
|
3201
|
+
if (opusVersion) {
|
|
3202
|
+
const [major, minor] = opusVersion;
|
|
3203
|
+
return major > 4 || major === 4 && minor >= 7;
|
|
3204
|
+
}
|
|
3205
|
+
return [
|
|
3206
|
+
"sonnet-5",
|
|
3207
|
+
"fable-5",
|
|
3208
|
+
"mythos"
|
|
3209
|
+
].some((s) => model.includes(s));
|
|
3210
|
+
}
|
|
3211
|
+
/**
|
|
3212
|
+
* Whether `model` is adaptive-only, per the built-in rule above, or per a
|
|
3213
|
+
* caller-supplied `adaptiveOnlyModels` override. The override is
|
|
3214
|
+
* additive, not a replacement: it can mark an *additional* model as
|
|
3215
|
+
* adaptive-only (useful for a model family this package doesn't know
|
|
3216
|
+
* about yet), but it can't un-mark one the built-in rule already caught,
|
|
3217
|
+
* since a caller correcting a false negative is the only direction that
|
|
3218
|
+
* needs covering, a false positive here would mean this package is
|
|
3219
|
+
* simply wrong and needs its own fix, not a per-caller workaround.
|
|
3220
|
+
*/
|
|
3221
|
+
function isAdaptiveOnlyModel(model, override) {
|
|
3222
|
+
if (isDefaultAdaptiveOnly(model)) return true;
|
|
3223
|
+
if (!override) return false;
|
|
3224
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3225
|
+
}
|
|
3226
|
+
/** Whether `model` is known to support manual, budget-based thinking. */
|
|
3227
|
+
function supportsManualThinkingBudget(model, override) {
|
|
3228
|
+
return !isAdaptiveOnlyModel(model, override);
|
|
3229
|
+
}
|
|
3230
|
+
/**
|
|
3231
|
+
* Anthropic (and Claude models on Bedrock) require `budget_tokens` to be
|
|
3232
|
+
* at least 1024 and strictly less than `max_tokens`, since the thinking
|
|
3233
|
+
* budget and the reply share the same `max_tokens` ceiling. VernLLM's own
|
|
3234
|
+
* default `maxTokens` is 1000 (see `RequestBuilder`'s `defaultMaxTokens`),
|
|
3235
|
+
* below the 1024 floor, so the *default* `minimal` tier (1024 tokens) is
|
|
3236
|
+
* silently invalid against the *default* `max_tokens` unless a caller
|
|
3237
|
+
* happens to raise one or the other. Checked here, once, right before a
|
|
3238
|
+
* `thinking` block would be built, rather than left for Anthropic's own
|
|
3239
|
+
* 400 to explain after a real network round trip.
|
|
3240
|
+
*/
|
|
3241
|
+
function assertValidClaudeBudgetTokens(budgetTokens, maxTokens) {
|
|
3242
|
+
if (budgetTokens < 1024) throw new LLMError(`budgetTokens (${budgetTokens}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`, "invalid_params");
|
|
3243
|
+
if (budgetTokens >= maxTokens) throw new LLMError(`budgetTokens (${budgetTokens}) must be less than maxTokens (${maxTokens}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`, "invalid_params");
|
|
3244
|
+
}
|
|
3245
|
+
/**
|
|
3246
|
+
* Anthropic rejects any form of `thinking` (manual `budget_tokens` or
|
|
3247
|
+
* adaptive) combined with a `tool_choice` that forces tool use, a forced
|
|
3248
|
+
* single tool or "must call some tool", with a 400: `"Thinking may not be
|
|
3249
|
+
* enabled when tool_choice forces tool use."` Auto/none (or no tools at
|
|
3250
|
+
* all) are unaffected, thinking only conflicts with a choice that removes
|
|
3251
|
+
* the model's ability to just reply with text. This is a Claude-model
|
|
3252
|
+
* constraint, not specific to the Anthropic API's own wire shape, so it
|
|
3253
|
+
* applies identically to Claude models called through Bedrock's Converse
|
|
3254
|
+
* API, which forwards `thinking` under `additionalModelRequestFields` but
|
|
3255
|
+
* is still talking to the same underlying model.
|
|
3256
|
+
*
|
|
3257
|
+
* This combination can arise two ways: a caller explicitly sets both
|
|
3258
|
+
* `budgetTokens`/`reasoningEffort` and a forced `toolChoice`, or, more
|
|
3259
|
+
* subtly (Anthropic adapter only), a caller sets `jsonSchema` on a model
|
|
3260
|
+
* without native structured output support, which silently forces a
|
|
3261
|
+
* single synthetic tool call to emulate it, with no `tool_choice` of the
|
|
3262
|
+
* caller's own in sight. Both end up resolving to a forced tool choice by
|
|
3263
|
+
* the time each adapter calls this, so checking the adapter's own
|
|
3264
|
+
* already-resolved choice (rather than the caller's raw
|
|
3265
|
+
* `params.tool_choice`) catches both, right before a `thinking` block
|
|
3266
|
+
* would be built, rather than left for Anthropic's own 400 to explain
|
|
3267
|
+
* after a real network round trip.
|
|
3268
|
+
*
|
|
3269
|
+
* Takes a plain description of the forced choice rather than either
|
|
3270
|
+
* adapter's own wire shape (Anthropic SDK's `{ type: 'tool' | 'any', ... }`
|
|
3271
|
+
* vs Converse's `{ tool: {...} } | { any: {} }`), so both adapters can
|
|
3272
|
+
* share one check without either shape leaking into this file. Pass
|
|
3273
|
+
* `undefined` when the resolved choice is `auto`/`none`/unset, forcing
|
|
3274
|
+
* nothing.
|
|
3275
|
+
*/
|
|
3276
|
+
function assertNoForcedToolChoiceWithThinking(forcedChoiceDescription) {
|
|
3277
|
+
if (!forcedChoiceDescription) return;
|
|
3278
|
+
throw new LLMError(`budgetTokens/reasoningEffort was set alongside ${forcedChoiceDescription}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`, "invalid_params");
|
|
3279
|
+
}
|
|
3280
|
+
/**
|
|
3281
|
+
* Maps VernLLM's four-tier `reasoningEffort` onto Anthropic's five-tier
|
|
3282
|
+
* adaptive effort. `xhigh` and `max` have no VernLLM-side equivalent and
|
|
3283
|
+
* are unreachable through this mapping; a caller who wants either has to
|
|
3284
|
+
* target Anthropic/Bedrock-specific behavior already, so there's no gap
|
|
3285
|
+
* the shared `CallParams` surface needs to cover for a first pass.
|
|
3286
|
+
*/
|
|
3287
|
+
function toClaudeAdaptiveEffort(effort) {
|
|
3288
|
+
return effort === "minimal" ? "low" : effort;
|
|
3289
|
+
}
|
|
3290
|
+
/** Converts VernLLM's `reasoningEffort` directly into Gemini's `ThinkingLevel` enum value. */
|
|
3291
|
+
function toGeminiThinkingLevel(effort, model) {
|
|
3292
|
+
return clampGeminiThinkingLevel(model, effort.toUpperCase());
|
|
3293
|
+
}
|
|
3294
|
+
/**
|
|
3295
|
+
* Parses a Gemini model id's minor version, e.g. `"gemini-3.1-pro"` -> `1`,
|
|
3296
|
+
* `"gemini-3-pro"` -> `0` (no explicit minor). Only meaningful alongside
|
|
3297
|
+
* `parseGeminiMajorVersion`.
|
|
3298
|
+
*/
|
|
3299
|
+
function parseGeminiMinorVersion(model) {
|
|
3300
|
+
const match = /gemini-\d+\.(\d+)/.exec(model);
|
|
3301
|
+
return match ? Number(match[1]) : 0;
|
|
3302
|
+
}
|
|
3303
|
+
/**
|
|
3304
|
+
* Some Gemini 3 "Pro" tier models accept a narrower set of `thinkingLevel`
|
|
3305
|
+
* values than VernLLM's four tiers map onto, confirmed against real API
|
|
3306
|
+
* 400s and Google's own migration guidance, not assumed:
|
|
3307
|
+
* - Gemini 3 Pro (major 3, minor 0, e.g. `"gemini-3-pro-preview"`): only
|
|
3308
|
+
* `LOW` and `HIGH`; `MEDIUM` returns a 400 ("Thinking level MEDIUM is
|
|
3309
|
+
* not supported for this model").
|
|
3310
|
+
* - Gemini 3.1 Pro (major 3, minor >= 1): `LOW`/`MEDIUM`/`HIGH`, no
|
|
3311
|
+
* `MINIMAL`, Google's own docs point users toward a Flash-tier model
|
|
3312
|
+
* instead for the lowest setting.
|
|
3313
|
+
* - Every Flash-tier Gemini 3+ model accepts the full four levels, no
|
|
3314
|
+
* clamping needed, matched by this function simply not applying to
|
|
3315
|
+
* anything without `"pro"` in the model id.
|
|
3316
|
+
*
|
|
3317
|
+
* Clamped automatically rather than left to error, since `reasoningEffort`
|
|
3318
|
+
* is a per-call value, a caller hitting this isn't misconfiguring an
|
|
3319
|
+
* instance once, they're getting an intermittent-looking failure on
|
|
3320
|
+
* whichever specific call happened to pick an unsupported tier. Necessarily
|
|
3321
|
+
* best-effort: a future Pro-tier release could add back a level this rule
|
|
3322
|
+
* still clamps, or clamp one this rule doesn't yet know to touch.
|
|
3323
|
+
*/
|
|
3324
|
+
function clampGeminiThinkingLevel(model, level) {
|
|
3325
|
+
if (!model.includes("pro")) return level;
|
|
3326
|
+
const major = parseGeminiMajorVersion(model);
|
|
3327
|
+
if (major === null || major < 3) return level;
|
|
3328
|
+
const minor = parseGeminiMinorVersion(model);
|
|
3329
|
+
if (minor === 0) return level === "HIGH" ? "HIGH" : "LOW";
|
|
3330
|
+
return level === "MINIMAL" ? "LOW" : level;
|
|
3331
|
+
}
|
|
3332
|
+
/**
|
|
3333
|
+
* Parses a Gemini model id's major generation number, e.g.
|
|
3334
|
+
* `"gemini-3.1-flash-lite"` -> `3`, `"gemini-2.5-flash"` -> `2`. Not
|
|
3335
|
+
* anchored, so a Vertex-prefixed or otherwise decorated id still matches.
|
|
3336
|
+
* Returns `null` for a non-Gemini model id.
|
|
3337
|
+
*/
|
|
3338
|
+
function parseGeminiMajorVersion(model) {
|
|
3339
|
+
const match = /gemini-(\d+)/.exec(model);
|
|
3340
|
+
return match ? Number(match[1]) : null;
|
|
3341
|
+
}
|
|
3342
|
+
/**
|
|
3343
|
+
* Default rule for whether `model` uses `thinkingLevel` instead of
|
|
3344
|
+
* `thinkingBudget`: every Gemini 3 series model and later, matched as a
|
|
3345
|
+
* version threshold so 3.1, 3.5, 3.6, and every future Gemini 3.x or
|
|
3346
|
+
* later release are covered automatically, without a new entry per
|
|
3347
|
+
* release, same reasoning as `isDefaultAdaptiveOnly`'s Opus threshold.
|
|
3348
|
+
* Gemini 2.5 and earlier still use `thinkingBudget`.
|
|
3349
|
+
*
|
|
3350
|
+
* `thinkingBudget` is still *accepted* on Gemini 3 for backward
|
|
3351
|
+
* compatibility, per Google's own docs, but "may result in unexpected
|
|
3352
|
+
* performance" there, so this rule switches VernLLM's own default
|
|
3353
|
+
* behavior over rather than leaving it on the old field indefinitely.
|
|
3354
|
+
*/
|
|
3355
|
+
function isDefaultThinkingLevelModel(model) {
|
|
3356
|
+
const major = parseGeminiMajorVersion(model);
|
|
3357
|
+
return major !== null && major >= 3;
|
|
3358
|
+
}
|
|
3359
|
+
/**
|
|
3360
|
+
* Whether `model` uses `thinkingLevel`, per the built-in version
|
|
3361
|
+
* threshold above, or per a caller-supplied `thinkingLevelModels`
|
|
3362
|
+
* override. Additive, not a replacement, same reasoning as
|
|
3363
|
+
* `isAdaptiveOnlyModel`: an override can mark an *additional* model as
|
|
3364
|
+
* using `thinkingLevel` (a model family this package doesn't recognize
|
|
3365
|
+
* yet), it can't un-mark one the built-in threshold already caught.
|
|
3366
|
+
*/
|
|
3367
|
+
function usesGeminiThinkingLevel(model, override) {
|
|
3368
|
+
if (isDefaultThinkingLevelModel(model)) return true;
|
|
3369
|
+
if (!override) return false;
|
|
3370
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3371
|
+
}
|
|
3372
|
+
|
|
2483
3373
|
//#endregion
|
|
2484
3374
|
//#region src/adapters/anthropic.ts
|
|
2485
3375
|
/**
|
|
@@ -2564,7 +3454,7 @@ function buildAnthropicTools(tools, toolChoiceParam) {
|
|
|
2564
3454
|
* `params.tools` are left for the normal, non-forced tool-call handling
|
|
2565
3455
|
* both `create` and `createStream` already do when `toolName` is unset.
|
|
2566
3456
|
*/
|
|
2567
|
-
function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
3457
|
+
function buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
2568
3458
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2569
3459
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2570
3460
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -2572,8 +3462,8 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2572
3462
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
2573
3463
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
2574
3464
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
|
|
3465
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic's native output_config.format on covered models, or a forced single tool call otherwise).", "validation");
|
|
2575
3466
|
let toolName;
|
|
2576
|
-
let jsonInstruction;
|
|
2577
3467
|
let outputFormat;
|
|
2578
3468
|
let tools;
|
|
2579
3469
|
let toolChoice;
|
|
@@ -2596,20 +3486,42 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2596
3486
|
type: "tool",
|
|
2597
3487
|
name: toolName
|
|
2598
3488
|
};
|
|
2599
|
-
}
|
|
3489
|
+
}
|
|
2600
3490
|
if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
|
|
2601
|
-
|
|
3491
|
+
let thinking;
|
|
3492
|
+
let effort;
|
|
3493
|
+
if (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0) {
|
|
3494
|
+
assertNoForcedToolChoiceWithThinking(toolChoice?.type === "tool" ? `toolChoice forcing the "${toolChoice.name}" tool` : toolChoice?.type === "any" ? "toolChoice: 'required' (Anthropic's \"any\" tool_choice)" : void 0);
|
|
3495
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
3496
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
3497
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
3498
|
+
thinking = {
|
|
3499
|
+
type: "enabled",
|
|
3500
|
+
budget_tokens: budgetTokens
|
|
3501
|
+
};
|
|
3502
|
+
} else {
|
|
3503
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
3504
|
+
thinking = { type: "adaptive" };
|
|
3505
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
const system = systemMessage?.content;
|
|
3509
|
+
const temperature = thinking ? void 0 : params.temperature;
|
|
2602
3510
|
const body = {
|
|
2603
3511
|
model: params.model,
|
|
2604
3512
|
max_tokens: params.max_tokens,
|
|
2605
|
-
...
|
|
3513
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
2606
3514
|
system: system || void 0,
|
|
2607
3515
|
messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
|
|
2608
3516
|
...tools ? {
|
|
2609
3517
|
tools,
|
|
2610
3518
|
tool_choice: toolChoice
|
|
2611
3519
|
} : {},
|
|
2612
|
-
...outputFormat ? { output_config: {
|
|
3520
|
+
...outputFormat || effort ? { output_config: {
|
|
3521
|
+
...outputFormat ? { format: outputFormat } : {},
|
|
3522
|
+
...effort ? { effort } : {}
|
|
3523
|
+
} } : {},
|
|
3524
|
+
...thinking ? { thinking } : {}
|
|
2613
3525
|
};
|
|
2614
3526
|
return {
|
|
2615
3527
|
body,
|
|
@@ -2638,103 +3550,113 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2638
3550
|
* schema matching applies only when `strict: true` is forwarded and
|
|
2639
3551
|
* supported.
|
|
2640
3552
|
*
|
|
2641
|
-
* `response_format: json_object` (
|
|
2642
|
-
*
|
|
2643
|
-
*
|
|
2644
|
-
*
|
|
2645
|
-
*
|
|
3553
|
+
* `response_format: json_object` throws `LLMError('validation')`. Anthropic
|
|
3554
|
+
* has no API-level field that mechanically guarantees JSON output the way
|
|
3555
|
+
* OpenAI's `json_object` mode does; the only way to emulate it was a
|
|
3556
|
+
* system-prompt instruction with no actual enforcement behind it, a
|
|
3557
|
+
* guarantee this adapter no longer pretends to make. Use `jsonSchema`
|
|
3558
|
+
* instead, which maps to a real constraint either way (native
|
|
3559
|
+
* `output_config.format` or a forced tool call).
|
|
2646
3560
|
*/
|
|
2647
3561
|
function fromAnthropic(anthropicClient, options) {
|
|
2648
3562
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3563
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3564
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
2649
3565
|
const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
|
|
2650
|
-
return {
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
if (
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
|
|
2672
|
-
|
|
2673
|
-
|
|
2674
|
-
choices: [{ message: {
|
|
2675
|
-
content: text,
|
|
2676
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
2677
|
-
} }],
|
|
2678
|
-
usage: {
|
|
2679
|
-
prompt_tokens: response.usage?.input_tokens,
|
|
2680
|
-
completion_tokens: response.usage?.output_tokens,
|
|
2681
|
-
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
|
|
3566
|
+
return {
|
|
3567
|
+
supportsJsonObjectMode: false,
|
|
3568
|
+
chat: { completions: {
|
|
3569
|
+
async create(params, options$1) {
|
|
3570
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3571
|
+
const response = await anthropicClient.messages.create(body, options$1);
|
|
3572
|
+
let text;
|
|
3573
|
+
let wireToolCalls;
|
|
3574
|
+
if (toolName) {
|
|
3575
|
+
const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
|
|
3576
|
+
if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3577
|
+
if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
|
|
3578
|
+
text = JSON.stringify(toolUse.input);
|
|
3579
|
+
} else {
|
|
3580
|
+
text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
|
|
3581
|
+
const toolUses = response.content.filter((block) => block.type === "tool_use");
|
|
3582
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
|
|
3583
|
+
id: block.id,
|
|
3584
|
+
type: "function",
|
|
3585
|
+
function: {
|
|
3586
|
+
name: block.name,
|
|
3587
|
+
arguments: JSON.stringify(block.input ?? {})
|
|
3588
|
+
}
|
|
3589
|
+
}));
|
|
2682
3590
|
}
|
|
2683
|
-
|
|
2684
|
-
|
|
2685
|
-
|
|
2686
|
-
|
|
2687
|
-
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
|
|
2692
|
-
|
|
2693
|
-
|
|
2694
|
-
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
2695
|
-
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
2696
|
-
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
2697
|
-
blockKinds.set(event.index, kind);
|
|
2698
|
-
if (kind === "json-tool") sawJsonTool = true;
|
|
2699
|
-
else if (!toolName) yield {
|
|
2700
|
-
type: "tool_call_delta",
|
|
2701
|
-
index: event.index,
|
|
2702
|
-
id: event.content_block.id,
|
|
2703
|
-
name: event.content_block.name
|
|
3591
|
+
return {
|
|
3592
|
+
choices: [{ message: {
|
|
3593
|
+
content: text,
|
|
3594
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3595
|
+
} }],
|
|
3596
|
+
usage: {
|
|
3597
|
+
prompt_tokens: response.usage?.input_tokens,
|
|
3598
|
+
completion_tokens: response.usage?.output_tokens,
|
|
3599
|
+
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0),
|
|
3600
|
+
...response.usage?.output_tokens_details?.thinking_tokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usage.output_tokens_details.thinking_tokens } } : {}
|
|
3601
|
+
}
|
|
2704
3602
|
};
|
|
2705
|
-
}
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
3603
|
+
},
|
|
3604
|
+
async *createStream(params, options$1) {
|
|
3605
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3606
|
+
const stream = await rawMessagesCreate({
|
|
3607
|
+
...body,
|
|
3608
|
+
stream: true
|
|
3609
|
+
}, options$1);
|
|
3610
|
+
const blockKinds = new Map();
|
|
3611
|
+
let inputTokens = 0;
|
|
3612
|
+
let sawJsonTool = false;
|
|
3613
|
+
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3614
|
+
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3615
|
+
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3616
|
+
blockKinds.set(event.index, kind);
|
|
3617
|
+
if (kind === "json-tool") sawJsonTool = true;
|
|
2718
3618
|
else if (!toolName) yield {
|
|
2719
3619
|
type: "tool_call_delta",
|
|
2720
3620
|
index: event.index,
|
|
2721
|
-
|
|
3621
|
+
id: event.content_block.id,
|
|
3622
|
+
name: event.content_block.name
|
|
2722
3623
|
};
|
|
2723
|
-
}
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
3624
|
+
} else blockKinds.set(event.index, "text");
|
|
3625
|
+
else if (event.type === "content_block_delta") {
|
|
3626
|
+
if (event.delta.type === "text_delta") {
|
|
3627
|
+
if (!toolName) yield {
|
|
3628
|
+
type: "text-delta",
|
|
3629
|
+
delta: event.delta.text
|
|
3630
|
+
};
|
|
3631
|
+
} else if (event.delta.type === "input_json_delta") {
|
|
3632
|
+
const kind = blockKinds.get(event.index);
|
|
3633
|
+
if (kind === "json-tool") yield {
|
|
3634
|
+
type: "text-delta",
|
|
3635
|
+
delta: event.delta.partial_json
|
|
3636
|
+
};
|
|
3637
|
+
else if (!toolName) yield {
|
|
3638
|
+
type: "tool_call_delta",
|
|
3639
|
+
index: event.index,
|
|
3640
|
+
argumentsDelta: event.delta.partial_json
|
|
3641
|
+
};
|
|
2732
3642
|
}
|
|
2733
|
-
}
|
|
2734
|
-
|
|
2735
|
-
|
|
2736
|
-
|
|
2737
|
-
|
|
3643
|
+
} else if (event.type === "message_delta") {
|
|
3644
|
+
const outputTokens = event.usage?.output_tokens ?? 0;
|
|
3645
|
+
const thinkingTokens = event.usage?.output_tokens_details?.thinking_tokens;
|
|
3646
|
+
yield {
|
|
3647
|
+
type: "usage",
|
|
3648
|
+
usage: {
|
|
3649
|
+
prompt_tokens: inputTokens,
|
|
3650
|
+
completion_tokens: outputTokens,
|
|
3651
|
+
total_tokens: inputTokens + outputTokens,
|
|
3652
|
+
...thinkingTokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: thinkingTokens } } : {}
|
|
3653
|
+
}
|
|
3654
|
+
};
|
|
3655
|
+
} else if (event.type === "ping") yield { type: "ping" };
|
|
3656
|
+
if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3657
|
+
}
|
|
3658
|
+
} }
|
|
3659
|
+
};
|
|
2738
3660
|
}
|
|
2739
3661
|
/**
|
|
2740
3662
|
* Anthropic requires strict role alternation, so the per-wire-message
|
|
@@ -2782,7 +3704,7 @@ function toAnthropicMessage(m) {
|
|
|
2782
3704
|
try {
|
|
2783
3705
|
input = tc.function.arguments.trim() ? JSON.parse(tc.function.arguments) : {};
|
|
2784
3706
|
} catch (cause) {
|
|
2785
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation",
|
|
3707
|
+
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
2786
3708
|
}
|
|
2787
3709
|
if (input === null || Array.isArray(input) || typeof input !== "object") throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) arguments must be a JSON object.`, "validation");
|
|
2788
3710
|
blocks.push({
|
|
@@ -2863,17 +3785,31 @@ function parseToolArguments(text, toolName) {
|
|
|
2863
3785
|
try {
|
|
2864
3786
|
parsed = text.trim() ? JSON.parse(text) : {};
|
|
2865
3787
|
} catch (cause) {
|
|
2866
|
-
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "
|
|
3788
|
+
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "parse", {
|
|
3789
|
+
cause,
|
|
3790
|
+
code: "tool_arguments_parse_failed"
|
|
3791
|
+
});
|
|
2867
3792
|
}
|
|
2868
3793
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
|
|
2869
3794
|
return parsed;
|
|
2870
3795
|
}
|
|
3796
|
+
/**
|
|
3797
|
+
* Parses a wire tool message's `content` into the object Gemini's
|
|
3798
|
+
* `functionResponse.response` expects. Gemini (and the real SDK's
|
|
3799
|
+
* `FunctionResponse.response` type) requires an object, so a result that
|
|
3800
|
+
* parses to something other than a plain JSON object (a string, number,
|
|
3801
|
+
* array, or unparseable text) is wrapped under an `output` key, mirroring
|
|
3802
|
+
* Gemini's own documented convention for non-object function results.
|
|
3803
|
+
*/
|
|
2871
3804
|
function parseToolResult(text) {
|
|
3805
|
+
let parsed;
|
|
2872
3806
|
try {
|
|
2873
|
-
|
|
3807
|
+
parsed = text.trim() ? JSON.parse(text) : "";
|
|
2874
3808
|
} catch {
|
|
2875
|
-
|
|
3809
|
+
parsed = text;
|
|
2876
3810
|
}
|
|
3811
|
+
if (parsed && !Array.isArray(parsed) && typeof parsed === "object") return parsed;
|
|
3812
|
+
return { output: parsed };
|
|
2877
3813
|
}
|
|
2878
3814
|
/**
|
|
2879
3815
|
* Gemini expects the results of everything the model asked for in one turn
|
|
@@ -2902,7 +3838,7 @@ function mergeConsecutiveFunctionResponses(contents) {
|
|
|
2902
3838
|
* `abortSignal` is folded into `config` by the caller (`create`/
|
|
2903
3839
|
* `createStream`), once the request options are available.
|
|
2904
3840
|
*/
|
|
2905
|
-
function buildGeminiRequest(params) {
|
|
3841
|
+
function buildGeminiRequest(params, effortTokenTable, thinkingLevelModels) {
|
|
2906
3842
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2907
3843
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2908
3844
|
const wantsJson = Boolean(params.response_format);
|
|
@@ -2927,51 +3863,37 @@ function buildGeminiRequest(params) {
|
|
|
2927
3863
|
})) }];
|
|
2928
3864
|
config.toolConfig = toGeminiToolConfig(params.tool_choice);
|
|
2929
3865
|
}
|
|
3866
|
+
if (usesGeminiThinkingLevel(params.model, thinkingLevelModels)) {
|
|
3867
|
+
const effortTier = params.reasoning_effort ?? (params.budget_tokens !== void 0 ? budgetTokensToEffort(params.budget_tokens, effortTokenTable) : void 0);
|
|
3868
|
+
if (effortTier !== void 0) config.thinkingConfig = { thinkingLevel: toGeminiThinkingLevel(effortTier, params.model) };
|
|
3869
|
+
} else {
|
|
3870
|
+
const thinkingBudget = params.budget_tokens ?? (params.reasoning_effort ? effortToBudgetTokens(params.reasoning_effort, effortTokenTable) : void 0);
|
|
3871
|
+
if (thinkingBudget !== void 0) config.thinkingConfig = { thinkingBudget };
|
|
3872
|
+
}
|
|
2930
3873
|
return {
|
|
2931
3874
|
model: params.model,
|
|
2932
3875
|
contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
|
|
2933
3876
|
config
|
|
2934
3877
|
};
|
|
2935
3878
|
}
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
2940
|
-
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
2944
|
-
|
|
2945
|
-
|
|
2946
|
-
*
|
|
2947
|
-
* `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
|
|
2948
|
-
* `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
|
|
2949
|
-
* `responseSchema` and `tools` in the same request natively, so both are
|
|
2950
|
-
* set independently here and no special-casing is needed for the
|
|
2951
|
-
* combination, unlike `fromAnthropic`/`fromBedrock`.
|
|
2952
|
-
*
|
|
2953
|
-
* `createStream` calls `generateContentStream` (optional on `GeminiClient`
|
|
2954
|
-
*, required only if the caller sets `stream: true`) and translates each
|
|
2955
|
-
* partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
|
|
2956
|
-
* Gemini's own function-calling API doesn't stream tool-call arguments
|
|
2957
|
-
* incrementally: a `functionCall` part always arrives whole in one chunk,
|
|
2958
|
-
* so each one is emitted as a single, complete `tool_call_delta` (a
|
|
2959
|
-
* one-shot "delta" containing the full arguments) rather than accumulated
|
|
2960
|
-
* fragments, that's a real difference in the underlying API, not
|
|
2961
|
-
* something this adapter can smooth over. `usageMetadata` is (per Gemini's
|
|
2962
|
-
* own behavior) only reliably present on the last chunk, so the `usage`
|
|
2963
|
-
* `WireStreamChunk` is emitted once, after the stream completes, from
|
|
2964
|
-
* whichever chunk's `usageMetadata` was seen last.
|
|
2965
|
-
*/
|
|
2966
|
-
function fromGemini(geminiClient) {
|
|
3879
|
+
function fromGemini(client, options) {
|
|
3880
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3881
|
+
const thinkingLevelModels = options?.thinkingLevelModels;
|
|
3882
|
+
const resolved = client.models ?? client;
|
|
3883
|
+
if (typeof resolved.generateContent !== "function") throw new LLMError("fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).", "invalid_params", {
|
|
3884
|
+
code: "unsupported_capability",
|
|
3885
|
+
issues: { capability: "generateContent" }
|
|
3886
|
+
});
|
|
3887
|
+
const generateContent = resolved.generateContent.bind(resolved);
|
|
3888
|
+
const generateContentStream = typeof resolved.generateContentStream === "function" ? resolved.generateContentStream.bind(resolved) : void 0;
|
|
2967
3889
|
return { chat: { completions: {
|
|
2968
|
-
async create(params, options) {
|
|
2969
|
-
const request = buildGeminiRequest(params);
|
|
3890
|
+
async create(params, options$1) {
|
|
3891
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
2970
3892
|
request.config = {
|
|
2971
3893
|
...request.config,
|
|
2972
|
-
abortSignal: options.signal
|
|
3894
|
+
abortSignal: options$1.signal
|
|
2973
3895
|
};
|
|
2974
|
-
const response = await
|
|
3896
|
+
const response = await generateContent(request);
|
|
2975
3897
|
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
2976
3898
|
const text = parts.map((p) => p.text ?? "").join("");
|
|
2977
3899
|
const functionCalls = parts.filter((p) => p.functionCall);
|
|
@@ -2992,18 +3914,22 @@ function fromGemini(geminiClient) {
|
|
|
2992
3914
|
usage: {
|
|
2993
3915
|
prompt_tokens: response.usageMetadata?.promptTokenCount,
|
|
2994
3916
|
completion_tokens: response.usageMetadata?.candidatesTokenCount,
|
|
2995
|
-
total_tokens: response.usageMetadata?.totalTokenCount
|
|
3917
|
+
total_tokens: response.usageMetadata?.totalTokenCount,
|
|
3918
|
+
...response.usageMetadata?.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usageMetadata.thoughtsTokenCount } } : {}
|
|
2996
3919
|
}
|
|
2997
3920
|
};
|
|
2998
3921
|
},
|
|
2999
|
-
async *createStream(params, options) {
|
|
3000
|
-
if (!
|
|
3001
|
-
|
|
3922
|
+
async *createStream(params, options$1) {
|
|
3923
|
+
if (!generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
|
|
3924
|
+
code: "unsupported_capability",
|
|
3925
|
+
issues: { capability: "generateContentStream" }
|
|
3926
|
+
});
|
|
3927
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
3002
3928
|
request.config = {
|
|
3003
3929
|
...request.config,
|
|
3004
|
-
abortSignal: options.signal
|
|
3930
|
+
abortSignal: options$1.signal
|
|
3005
3931
|
};
|
|
3006
|
-
const stream = await
|
|
3932
|
+
const stream = await generateContentStream(request);
|
|
3007
3933
|
let toolCallIndex = 0;
|
|
3008
3934
|
let lastUsage;
|
|
3009
3935
|
for await (const chunk of stream) {
|
|
@@ -3032,7 +3958,8 @@ function fromGemini(geminiClient) {
|
|
|
3032
3958
|
usage: {
|
|
3033
3959
|
prompt_tokens: lastUsage.promptTokenCount,
|
|
3034
3960
|
completion_tokens: lastUsage.candidatesTokenCount,
|
|
3035
|
-
total_tokens: lastUsage.totalTokenCount
|
|
3961
|
+
total_tokens: lastUsage.totalTokenCount,
|
|
3962
|
+
...lastUsage.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: lastUsage.thoughtsTokenCount } } : {}
|
|
3036
3963
|
}
|
|
3037
3964
|
};
|
|
3038
3965
|
}
|
|
@@ -3041,6 +3968,19 @@ function fromGemini(geminiClient) {
|
|
|
3041
3968
|
|
|
3042
3969
|
//#endregion
|
|
3043
3970
|
//#region src/adapters/bedrock.ts
|
|
3971
|
+
/**
|
|
3972
|
+
* Default heuristic for whether a Bedrock model id is a Claude model,
|
|
3973
|
+
* matching AWS's own `anthropic.claude-*`/`us.anthropic.claude-*` naming.
|
|
3974
|
+
* Only used to decide whether a reasoning token budget is worth forwarding
|
|
3975
|
+
* through `additionalModelRequestFields`, not a general capability check,
|
|
3976
|
+
* so a plain substring match is enough, no override hook needed the way
|
|
3977
|
+
* `nativeStructuredOutputModels`/`toolUseSupportedModels` have one: a
|
|
3978
|
+
* false positive here just sends an inert extra field, not a request that
|
|
3979
|
+
* fails outright.
|
|
3980
|
+
*/
|
|
3981
|
+
function isClaudeModel(model) {
|
|
3982
|
+
return model.includes("claude");
|
|
3983
|
+
}
|
|
3044
3984
|
/** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
|
|
3045
3985
|
function toBedrockImageFormat(mimeType) {
|
|
3046
3986
|
switch (assertSupportedImageMimeType(mimeType)) {
|
|
@@ -3107,7 +4047,7 @@ function buildBedrockToolConfig(tools, toolChoiceParam) {
|
|
|
3107
4047
|
* normal, non-forced tool-call handling both `create` and `createStream`
|
|
3108
4048
|
* already do when `toolName` is unset.
|
|
3109
4049
|
*/
|
|
3110
|
-
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
|
|
4050
|
+
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
3111
4051
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3112
4052
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3113
4053
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -3115,8 +4055,8 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3115
4055
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
3116
4056
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
3117
4057
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
|
|
4058
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse's native outputConfig.textFormat on covered models, or a forced tool call otherwise).", "validation");
|
|
3118
4059
|
let toolName;
|
|
3119
|
-
let jsonInstruction;
|
|
3120
4060
|
let toolConfig;
|
|
3121
4061
|
let outputConfig;
|
|
3122
4062
|
if (jsonSchema && isNative) {
|
|
@@ -3141,23 +4081,48 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3141
4081
|
} }],
|
|
3142
4082
|
toolChoice: { tool: { name: toolName } }
|
|
3143
4083
|
};
|
|
3144
|
-
}
|
|
4084
|
+
}
|
|
3145
4085
|
if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
|
|
3146
4086
|
if (jsonSchema && toolConfig && toolUseSupportedModels) {
|
|
3147
4087
|
const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
|
|
3148
|
-
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "
|
|
4088
|
+
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "invalid_params", {
|
|
4089
|
+
code: "unsupported_capability",
|
|
4090
|
+
issues: { capability: "toolUseSupportedModels" }
|
|
4091
|
+
});
|
|
4092
|
+
}
|
|
4093
|
+
let additionalModelRequestFields;
|
|
4094
|
+
let effort;
|
|
4095
|
+
if (isClaudeModel(params.model) && (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0)) {
|
|
4096
|
+
const forcedToolChoice = toolConfig?.toolChoice;
|
|
4097
|
+
assertNoForcedToolChoiceWithThinking(forcedToolChoice && "tool" in forcedToolChoice ? `toolChoice forcing the "${forcedToolChoice.tool?.name}" tool` : forcedToolChoice && "any" in forcedToolChoice ? "toolChoice: 'required' (Converse's \"any\" tool_choice)" : void 0);
|
|
4098
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
4099
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
4100
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
4101
|
+
additionalModelRequestFields = { thinking: {
|
|
4102
|
+
type: "enabled",
|
|
4103
|
+
budget_tokens: budgetTokens
|
|
4104
|
+
} };
|
|
4105
|
+
} else {
|
|
4106
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
4107
|
+
additionalModelRequestFields = { thinking: { type: "adaptive" } };
|
|
4108
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
4109
|
+
}
|
|
3149
4110
|
}
|
|
3150
|
-
const
|
|
4111
|
+
const temperature = additionalModelRequestFields ? void 0 : params.temperature;
|
|
3151
4112
|
const request = {
|
|
3152
4113
|
modelId: params.model,
|
|
3153
4114
|
messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
|
|
3154
|
-
system:
|
|
4115
|
+
system: systemMessage?.content ? [{ text: systemMessage.content }] : void 0,
|
|
3155
4116
|
inferenceConfig: {
|
|
3156
|
-
...
|
|
4117
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
3157
4118
|
maxTokens: params.max_tokens
|
|
3158
4119
|
},
|
|
3159
4120
|
...toolConfig ? { toolConfig } : {},
|
|
3160
|
-
...outputConfig ? { outputConfig
|
|
4121
|
+
...outputConfig || effort ? { outputConfig: {
|
|
4122
|
+
...outputConfig ?? {},
|
|
4123
|
+
...effort ? { effort } : {}
|
|
4124
|
+
} } : {},
|
|
4125
|
+
...additionalModelRequestFields ? { additionalModelRequestFields } : {}
|
|
3161
4126
|
};
|
|
3162
4127
|
return {
|
|
3163
4128
|
request,
|
|
@@ -3165,6 +4130,120 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3165
4130
|
};
|
|
3166
4131
|
}
|
|
3167
4132
|
/**
|
|
4133
|
+
* Distinguishes a real AWS SDK v3 client (`.send(command)`) from a
|
|
4134
|
+
* hand-written `BedrockConverseClient` (`.converse(params)`) purely
|
|
4135
|
+
* structurally, so `fromBedrock` can accept either without the caller
|
|
4136
|
+
* saying which one they're passing. The two shapes don't overlap: nothing
|
|
4137
|
+
* implementing `.converse()` would also need `.send()`.
|
|
4138
|
+
*/
|
|
4139
|
+
function isAwsSendClient(client) {
|
|
4140
|
+
return typeof client.send === "function";
|
|
4141
|
+
}
|
|
4142
|
+
/**
|
|
4143
|
+
* Narrows one raw AWS stream event down to VernLLM's intentionally minimal
|
|
4144
|
+
* `BedrockConverseStreamEvent` union. Returns `undefined` if the event
|
|
4145
|
+
* isn't one of the kinds this adapter models.
|
|
4146
|
+
*
|
|
4147
|
+
* AWS's real `ConverseStreamOutput` type is a strictly larger union than
|
|
4148
|
+
* `BedrockConverseStreamEvent`. On top of every member modeled here, it
|
|
4149
|
+
* also includes a generated `$unknown` member, AWS's forward-compatibility
|
|
4150
|
+
* escape hatch for event kinds added to the service after this SDK version
|
|
4151
|
+
* was generated. A blind type assertion from one union to the other would
|
|
4152
|
+
* compile, but would let `$unknown` (or any other future member) reach
|
|
4153
|
+
* `fromBedrock`'s event-handling loop unnarrowed, as if it were one of the
|
|
4154
|
+
* kinds actually handled there.
|
|
4155
|
+
*
|
|
4156
|
+
* Returning `undefined` for anything unrecognized, filtered out by
|
|
4157
|
+
* `normalizeBedrockEventStream` below, keeps two guarantees. AWS SDK
|
|
4158
|
+
* generated types never leak into `fromBedrock`'s application code, only
|
|
4159
|
+
* this module's own `BedrockConverseStreamEvent` shape does. An event kind
|
|
4160
|
+
* this adapter doesn't yet know about is silently skipped, the same
|
|
4161
|
+
* forward-compatible behavior AWS's own `$unknown` convention implies,
|
|
4162
|
+
* rather than crashing the stream or being misrouted into a handler that
|
|
4163
|
+
* doesn't actually match its shape.
|
|
4164
|
+
*/
|
|
4165
|
+
function normalizeBedrockStreamEvent(raw) {
|
|
4166
|
+
if ("messageStart" in raw) return { messageStart: raw.messageStart };
|
|
4167
|
+
if ("contentBlockStart" in raw) return { contentBlockStart: raw.contentBlockStart };
|
|
4168
|
+
if ("contentBlockDelta" in raw) return { contentBlockDelta: raw.contentBlockDelta };
|
|
4169
|
+
if ("contentBlockStop" in raw) return { contentBlockStop: raw.contentBlockStop };
|
|
4170
|
+
if ("messageStop" in raw) return { messageStop: raw.messageStop };
|
|
4171
|
+
if ("metadata" in raw) return { metadata: raw.metadata };
|
|
4172
|
+
if ("internalServerException" in raw) return { internalServerException: raw.internalServerException };
|
|
4173
|
+
if ("modelStreamErrorException" in raw) return { modelStreamErrorException: raw.modelStreamErrorException };
|
|
4174
|
+
if ("validationException" in raw) return { validationException: raw.validationException };
|
|
4175
|
+
if ("throttlingException" in raw) return { throttlingException: raw.throttlingException };
|
|
4176
|
+
if ("serviceUnavailableException" in raw) return { serviceUnavailableException: raw.serviceUnavailableException };
|
|
4177
|
+
return void 0;
|
|
4178
|
+
}
|
|
4179
|
+
/**
|
|
4180
|
+
* Wraps a raw AWS event stream, narrowing each event through
|
|
4181
|
+
* `normalizeBedrockStreamEvent` and filtering out anything that doesn't
|
|
4182
|
+
* map onto `BedrockConverseStreamEvent`. `fromBedrock`'s event loop only
|
|
4183
|
+
* ever sees the shapes it actually models.
|
|
4184
|
+
*/
|
|
4185
|
+
async function* normalizeBedrockEventStream(rawStream) {
|
|
4186
|
+
for await (const raw of rawStream) {
|
|
4187
|
+
const event = normalizeBedrockStreamEvent(raw);
|
|
4188
|
+
if (event) yield event;
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
/**
|
|
4192
|
+
* Adapts a real AWS SDK v3 client (anything with `.send()`, matching
|
|
4193
|
+
* `BedrockRuntimeClient`) into a `BedrockConverseClient`, so `fromBedrock`
|
|
4194
|
+
* can accept either without a hand-written `.converse()`/`.converseStream()`
|
|
4195
|
+
* wrapper. Internally does what that wrapper would: `client.send(new
|
|
4196
|
+
* ConverseCommand(params))`, `client.send(new
|
|
4197
|
+
* ConverseStreamCommand(params))`.
|
|
4198
|
+
*
|
|
4199
|
+
* `@aws-sdk/client-bedrock-runtime` is intentionally not a dependency (not
|
|
4200
|
+
* even a peer dependency) of this package. `vern-llm` otherwise has zero
|
|
4201
|
+
* runtime dependencies, and every other adapter works the same way:
|
|
4202
|
+
* structural typing over whatever client the caller already has. Instead,
|
|
4203
|
+
* `ConverseCommand`/`ConverseStreamCommand` are pulled in with a dynamic
|
|
4204
|
+
* `import()` the first time either method actually runs, and memoized
|
|
4205
|
+
* after that. Nothing is added to `package.json`, static or peer.
|
|
4206
|
+
* Bundlers only pull the AWS SDK in for code paths that actually pass a
|
|
4207
|
+
* raw AWS client to `fromBedrock`; a hand-written `BedrockConverseClient`
|
|
4208
|
+
* stays unaffected. If `@aws-sdk/client-bedrock-runtime` isn't installed,
|
|
4209
|
+
* the failure is a clear `LLMError` naming exactly what's missing, at the
|
|
4210
|
+
* moment it's needed, rather than a silent peer-dependency warning at
|
|
4211
|
+
* install time or a raw "Cannot find module" a caller has to trace back
|
|
4212
|
+
* themselves.
|
|
4213
|
+
*
|
|
4214
|
+
* Also closes two structural gaps between AWS's generated types and
|
|
4215
|
+
* `BedrockConverseClient`. AWS's `ConverseStreamCommandOutput.stream` is
|
|
4216
|
+
* optional, a response may not include it. This throws a clear `LLMError`
|
|
4217
|
+
* instead of letting `undefined` reach `fromBedrock`'s `for await` loop.
|
|
4218
|
+
* AWS's `ConverseStreamOutput` union is larger than
|
|
4219
|
+
* `BedrockConverseStreamEvent`, it includes a generated `$unknown` member.
|
|
4220
|
+
* Every event is narrowed through `normalizeBedrockStreamEvent` before it
|
|
4221
|
+
* reaches application code, instead of being asserted wholesale from one
|
|
4222
|
+
* type to the other.
|
|
4223
|
+
*/
|
|
4224
|
+
function wrapAwsSendClient(client) {
|
|
4225
|
+
let commandsPromise;
|
|
4226
|
+
function loadCommands() {
|
|
4227
|
+
commandsPromise ??= import("@aws-sdk/client-bedrock-runtime").then((mod) => mod, (cause) => {
|
|
4228
|
+
commandsPromise = void 0;
|
|
4229
|
+
throw new LLMError("fromBedrock requires \"@aws-sdk/client-bedrock-runtime\" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.", "validation", { cause });
|
|
4230
|
+
});
|
|
4231
|
+
return commandsPromise;
|
|
4232
|
+
}
|
|
4233
|
+
return {
|
|
4234
|
+
converse: async (params, requestOptions) => {
|
|
4235
|
+
const { ConverseCommand } = await loadCommands();
|
|
4236
|
+
return client.send(new ConverseCommand(params), { abortSignal: requestOptions.signal });
|
|
4237
|
+
},
|
|
4238
|
+
converseStream: async (params, requestOptions) => {
|
|
4239
|
+
const { ConverseStreamCommand } = await loadCommands();
|
|
4240
|
+
const result = await client.send(new ConverseStreamCommand(params), { abortSignal: requestOptions.signal });
|
|
4241
|
+
if (!result.stream) throw new LLMError("Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.", "api", { code: "server_error" });
|
|
4242
|
+
return { stream: normalizeBedrockEventStream(result.stream) };
|
|
4243
|
+
}
|
|
4244
|
+
};
|
|
4245
|
+
}
|
|
4246
|
+
/**
|
|
3168
4247
|
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
3169
4248
|
* interface VernLLM uses for OpenAI/Groq. The Converse API is unified
|
|
3170
4249
|
* across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
|
|
@@ -3172,6 +4251,16 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3172
4251
|
* regardless of which underlying model `modelId` points at, as long as
|
|
3173
4252
|
* that model supports Converse (most current-generation ones do)
|
|
3174
4253
|
*
|
|
4254
|
+
* `bedrockClient` accepts either a hand-written `BedrockConverseClient`
|
|
4255
|
+
* (a `.converse()`/`.converseStream()` wrapper you provide) or a real AWS
|
|
4256
|
+
* SDK v3 client (anything with `.send()`, matching `BedrockRuntimeClient`)
|
|
4257
|
+
* directly, detected structurally. Passing a raw AWS client skips the
|
|
4258
|
+
* hand-written wrapper entirely, internally doing what it would
|
|
4259
|
+
* (`send(new ConverseCommand(...))`, `send(new
|
|
4260
|
+
* ConverseStreamCommand(...))`). See `wrapAwsSendClient` for how that path
|
|
4261
|
+
* is implemented, including why `@aws-sdk/client-bedrock-runtime` stays
|
|
4262
|
+
* out of this package's dependencies either way.
|
|
4263
|
+
*
|
|
3175
4264
|
* `response_format: json_schema`, on a model covered by
|
|
3176
4265
|
* `options.nativeStructuredOutputModels` (opt-in, unset by default), is
|
|
3177
4266
|
* sent as `outputConfig.textFormat`, its own request field, independent of
|
|
@@ -3193,12 +4282,15 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3193
4282
|
* `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
|
|
3194
4283
|
* unsupported model surfaces Bedrock's raw error unchanged.
|
|
3195
4284
|
*
|
|
3196
|
-
* `response_format: json_object` (
|
|
3197
|
-
*
|
|
3198
|
-
*
|
|
3199
|
-
*
|
|
3200
|
-
*
|
|
3201
|
-
*
|
|
4285
|
+
* `response_format: json_object` throws `LLMError('validation')`: Converse
|
|
4286
|
+
* has no field that mechanically guarantees JSON output, and the only way
|
|
4287
|
+
* to emulate it was an unenforced system-prompt instruction, a guarantee
|
|
4288
|
+
* this adapter no longer pretends to make. Use `jsonSchema` instead.
|
|
4289
|
+
* `reasoning_effort` (no Converse equivalent) is converted to a token
|
|
4290
|
+
* budget and forwarded via `additionalModelRequestFields` for Claude
|
|
4291
|
+
* models only; `budget_tokens` is forwarded the same way directly. Both
|
|
4292
|
+
* are silently dropped for non-Claude models, which have no equivalent
|
|
4293
|
+
* field to reach for. See `adapters/internal/reasoningBudget.utils.ts`.
|
|
3202
4294
|
*
|
|
3203
4295
|
* `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
|
|
3204
4296
|
* `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
|
|
@@ -3216,104 +4308,122 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3216
4308
|
* `create` branch above unwraps it.
|
|
3217
4309
|
*/
|
|
3218
4310
|
function fromBedrock(bedrockClient, options) {
|
|
4311
|
+
const client = isAwsSendClient(bedrockClient) ? wrapAwsSendClient(bedrockClient) : bedrockClient;
|
|
3219
4312
|
const toolUseSupportedModels = options?.toolUseSupportedModels;
|
|
3220
4313
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
const
|
|
3229
|
-
text
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
const
|
|
3236
|
-
|
|
3237
|
-
|
|
3238
|
-
|
|
3239
|
-
|
|
3240
|
-
|
|
3241
|
-
|
|
3242
|
-
|
|
3243
|
-
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
|
|
3249
|
-
|
|
3250
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3251
|
-
} }],
|
|
3252
|
-
usage: {
|
|
3253
|
-
prompt_tokens: response.usage?.inputTokens,
|
|
3254
|
-
completion_tokens: response.usage?.outputTokens,
|
|
3255
|
-
total_tokens: response.usage?.totalTokens
|
|
4314
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
4315
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
4316
|
+
return {
|
|
4317
|
+
supportsJsonObjectMode: false,
|
|
4318
|
+
chat: { completions: {
|
|
4319
|
+
async create(params, requestOptions) {
|
|
4320
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4321
|
+
const response = await client.converse(request, requestOptions);
|
|
4322
|
+
let text;
|
|
4323
|
+
let wireToolCalls;
|
|
4324
|
+
if (toolName) {
|
|
4325
|
+
const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
|
|
4326
|
+
text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
|
|
4327
|
+
} else {
|
|
4328
|
+
const blocks = response.output?.message?.content ?? [];
|
|
4329
|
+
text = blocks.map((c) => c.text ?? "").join("");
|
|
4330
|
+
const toolUses = blocks.filter((block) => Boolean(block.toolUse));
|
|
4331
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
|
|
4332
|
+
const toolUse = block.toolUse;
|
|
4333
|
+
if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
|
|
4334
|
+
return {
|
|
4335
|
+
id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
|
|
4336
|
+
type: "function",
|
|
4337
|
+
function: {
|
|
4338
|
+
name: toolUse.name,
|
|
4339
|
+
arguments: JSON.stringify(toolUse.input ?? {})
|
|
4340
|
+
}
|
|
4341
|
+
};
|
|
4342
|
+
});
|
|
3256
4343
|
}
|
|
3257
|
-
|
|
3258
|
-
|
|
3259
|
-
|
|
3260
|
-
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
3268
|
-
blockKinds.set(contentBlockIndex, kind);
|
|
3269
|
-
if (kind === "tool_use" && !toolName) yield {
|
|
3270
|
-
type: "tool_call_delta",
|
|
3271
|
-
index: contentBlockIndex,
|
|
3272
|
-
id: start.toolUse.toolUseId,
|
|
3273
|
-
name: start.toolUse.name
|
|
3274
|
-
};
|
|
3275
|
-
} else blockKinds.set(contentBlockIndex, "text");
|
|
3276
|
-
} else if ("contentBlockDelta" in event) {
|
|
3277
|
-
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
3278
|
-
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3279
|
-
type: "text-delta",
|
|
3280
|
-
delta: delta.text
|
|
4344
|
+
return {
|
|
4345
|
+
choices: [{ message: {
|
|
4346
|
+
content: text,
|
|
4347
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
4348
|
+
} }],
|
|
4349
|
+
usage: {
|
|
4350
|
+
prompt_tokens: response.usage?.inputTokens,
|
|
4351
|
+
completion_tokens: response.usage?.outputTokens,
|
|
4352
|
+
total_tokens: response.usage?.totalTokens
|
|
4353
|
+
}
|
|
3281
4354
|
};
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
4355
|
+
},
|
|
4356
|
+
async *createStream(params, requestOptions) {
|
|
4357
|
+
if (!client.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
|
|
4358
|
+
code: "unsupported_capability",
|
|
4359
|
+
issues: { capability: "converseStream" }
|
|
4360
|
+
});
|
|
4361
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4362
|
+
const { stream } = await client.converseStream(request, requestOptions);
|
|
4363
|
+
const blockKinds = new Map();
|
|
4364
|
+
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
4365
|
+
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
4366
|
+
if (start?.toolUse) {
|
|
4367
|
+
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
4368
|
+
blockKinds.set(contentBlockIndex, kind);
|
|
4369
|
+
if (kind === "tool_use" && !toolName) yield {
|
|
4370
|
+
type: "tool_call_delta",
|
|
4371
|
+
index: contentBlockIndex,
|
|
4372
|
+
id: start.toolUse.toolUseId,
|
|
4373
|
+
name: start.toolUse.name
|
|
4374
|
+
};
|
|
4375
|
+
} else blockKinds.set(contentBlockIndex, "text");
|
|
4376
|
+
} else if ("contentBlockDelta" in event) {
|
|
4377
|
+
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
4378
|
+
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3285
4379
|
type: "text-delta",
|
|
3286
|
-
delta: delta.
|
|
3287
|
-
};
|
|
3288
|
-
else if (!toolName) yield {
|
|
3289
|
-
type: "tool_call_delta",
|
|
3290
|
-
index: contentBlockIndex,
|
|
3291
|
-
argumentsDelta: delta.toolUse.input
|
|
4380
|
+
delta: delta.text
|
|
3292
4381
|
};
|
|
4382
|
+
else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
|
|
4383
|
+
const kind = blockKinds.get(contentBlockIndex);
|
|
4384
|
+
if (kind === "json-tool") yield {
|
|
4385
|
+
type: "text-delta",
|
|
4386
|
+
delta: delta.toolUse.input
|
|
4387
|
+
};
|
|
4388
|
+
else if (!toolName) yield {
|
|
4389
|
+
type: "tool_call_delta",
|
|
4390
|
+
index: contentBlockIndex,
|
|
4391
|
+
argumentsDelta: delta.toolUse.input
|
|
4392
|
+
};
|
|
4393
|
+
}
|
|
4394
|
+
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
4395
|
+
type: "usage",
|
|
4396
|
+
usage: {
|
|
4397
|
+
prompt_tokens: event.metadata.usage.inputTokens,
|
|
4398
|
+
completion_tokens: event.metadata.usage.outputTokens,
|
|
4399
|
+
total_tokens: event.metadata.usage.totalTokens
|
|
4400
|
+
}
|
|
4401
|
+
};
|
|
4402
|
+
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
|
|
4403
|
+
status: 429,
|
|
4404
|
+
code: "provider_rate_limited"
|
|
4405
|
+
});
|
|
4406
|
+
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
4407
|
+
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
4408
|
+
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
4409
|
+
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
4410
|
+
throw new LLMError(detail, "api", {
|
|
4411
|
+
status,
|
|
4412
|
+
code: status >= 500 ? "server_error" : void 0
|
|
4413
|
+
});
|
|
3293
4414
|
}
|
|
3294
|
-
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
3295
|
-
type: "usage",
|
|
3296
|
-
usage: {
|
|
3297
|
-
prompt_tokens: event.metadata.usage.inputTokens,
|
|
3298
|
-
completion_tokens: event.metadata.usage.outputTokens,
|
|
3299
|
-
total_tokens: event.metadata.usage.totalTokens
|
|
3300
|
-
}
|
|
3301
|
-
};
|
|
3302
|
-
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", 429);
|
|
3303
|
-
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
3304
|
-
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
3305
|
-
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
3306
|
-
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
3307
|
-
throw new LLMError(detail, "api", status);
|
|
3308
4415
|
}
|
|
3309
|
-
}
|
|
3310
|
-
}
|
|
4416
|
+
} }
|
|
4417
|
+
};
|
|
3311
4418
|
}
|
|
3312
4419
|
/** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
|
|
3313
4420
|
function toBedrockToolChoice(toolChoice) {
|
|
3314
4421
|
if (!toolChoice || toolChoice === "auto") return { auto: {} };
|
|
3315
4422
|
if (toolChoice === "required") return { any: {} };
|
|
3316
|
-
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "
|
|
4423
|
+
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "invalid_params", {
|
|
4424
|
+
code: "unsupported_capability",
|
|
4425
|
+
issues: { capability: "toolChoice: 'none'" }
|
|
4426
|
+
});
|
|
3317
4427
|
return { tool: { name: toolChoice.function.name } };
|
|
3318
4428
|
}
|
|
3319
4429
|
/**
|
|
@@ -3338,7 +4448,7 @@ function toBedrockMessage(m) {
|
|
|
3338
4448
|
else try {
|
|
3339
4449
|
input = JSON.parse(tc.function.arguments);
|
|
3340
4450
|
} catch (cause) {
|
|
3341
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation",
|
|
4451
|
+
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
3342
4452
|
}
|
|
3343
4453
|
blocks.push({ toolUse: {
|
|
3344
4454
|
toolUseId: tc.id,
|
|
@@ -3509,8 +4619,14 @@ function fromFetch(config) {
|
|
|
3509
4619
|
};
|
|
3510
4620
|
},
|
|
3511
4621
|
async *createStream(params, options) {
|
|
3512
|
-
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "
|
|
3513
|
-
|
|
4622
|
+
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "invalid_params", {
|
|
4623
|
+
code: "unsupported_capability",
|
|
4624
|
+
issues: { capability: "mapStreamEvent" }
|
|
4625
|
+
});
|
|
4626
|
+
if (config.request && !config.requestStream) throw new LLMError("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.", "invalid_params", {
|
|
4627
|
+
code: "unsupported_capability",
|
|
4628
|
+
issues: { capability: "requestStream" }
|
|
4629
|
+
});
|
|
3514
4630
|
const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
|
|
3515
4631
|
const requestStream = config.requestStream ?? defaultRequestStream;
|
|
3516
4632
|
const parseFrames = config.parseStreamFrames ?? parseSseStream;
|
|
@@ -3537,6 +4653,22 @@ function fromFetch(config) {
|
|
|
3537
4653
|
//#endregion
|
|
3538
4654
|
//#region src/adapters/openaiCompatible.ts
|
|
3539
4655
|
/**
|
|
4656
|
+
* OpenAI's wire format only understands `reasoning_effort`, not a raw
|
|
4657
|
+
* token budget. When the caller set `reasoningEffort`, it's already on
|
|
4658
|
+
* `params` and passed through unchanged, this function does nothing.
|
|
4659
|
+
* When only `budgetTokens` was set, it's converted to the nearest tier
|
|
4660
|
+
* and `budget_tokens` is dropped, since OpenAI's API would otherwise
|
|
4661
|
+
* silently ignore an unrecognized field.
|
|
4662
|
+
*/
|
|
4663
|
+
function applyReasoningBudget(params, effortTokenTable) {
|
|
4664
|
+
if (params.budget_tokens === void 0) return params;
|
|
4665
|
+
const { budget_tokens,...rest } = params;
|
|
4666
|
+
return rest.reasoning_effort !== void 0 ? rest : {
|
|
4667
|
+
...rest,
|
|
4668
|
+
reasoning_effort: budgetTokensToEffort(budget_tokens, effortTokenTable)
|
|
4669
|
+
};
|
|
4670
|
+
}
|
|
4671
|
+
/**
|
|
3540
4672
|
* Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
|
|
3541
4673
|
* array. Text blocks become `{ type: 'text', text }`; image blocks become
|
|
3542
4674
|
* `{ type: 'image_url', image_url: { url } }` with the base64 payload
|
|
@@ -3602,23 +4734,24 @@ function* toWireStreamChunks(chunk) {
|
|
|
3602
4734
|
function fromOpenAICompatible(client, options = {}) {
|
|
3603
4735
|
const raw = client;
|
|
3604
4736
|
const { supportsStreamUsage = true } = options;
|
|
4737
|
+
const effortTokenTable = resolveEffortTokenTable(options.reasoningEffortTokens);
|
|
3605
4738
|
const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
|
|
3606
4739
|
return { chat: { completions: {
|
|
3607
4740
|
async create(params, options$1) {
|
|
3608
4741
|
const messages = toOpenAIMessages(params);
|
|
3609
|
-
return raw.chat.completions.create({
|
|
4742
|
+
return raw.chat.completions.create(applyReasoningBudget({
|
|
3610
4743
|
...params,
|
|
3611
4744
|
messages
|
|
3612
|
-
}, options$1);
|
|
4745
|
+
}, effortTokenTable), options$1);
|
|
3613
4746
|
},
|
|
3614
4747
|
async *createStream(params, options$1) {
|
|
3615
4748
|
const messages = toOpenAIMessages(params);
|
|
3616
|
-
const stream = await rawCreate({
|
|
4749
|
+
const stream = await rawCreate(applyReasoningBudget({
|
|
3617
4750
|
...params,
|
|
3618
4751
|
messages,
|
|
3619
4752
|
stream: true,
|
|
3620
4753
|
...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
|
|
3621
|
-
}, options$1);
|
|
4754
|
+
}, effortTokenTable), options$1);
|
|
3622
4755
|
for await (const chunk of stream) yield* toWireStreamChunks(chunk);
|
|
3623
4756
|
}
|
|
3624
4757
|
} } };
|
|
@@ -3742,6 +4875,8 @@ exports.TieredCacheAdapter = TieredCacheAdapter
|
|
|
3742
4875
|
exports.VernLLM = VernLLM
|
|
3743
4876
|
exports.defaultEstimateTokens = defaultEstimateTokens
|
|
3744
4877
|
exports.defaultFallbackOn = defaultFallbackOn
|
|
4878
|
+
exports.defineCachedCallParams = defineCachedCallParams
|
|
4879
|
+
exports.defineCallParams = defineCallParams
|
|
3745
4880
|
exports.from01AI = from01AI
|
|
3746
4881
|
exports.fromAnthropic = fromAnthropic
|
|
3747
4882
|
exports.fromAnyscale = fromAnyscale
|
|
@@ -3785,6 +4920,8 @@ exports.fromVLLM = fromVLLM
|
|
|
3785
4920
|
exports.fromVercelAIGateway = fromVercelAIGateway
|
|
3786
4921
|
exports.fromXAI = fromXAI
|
|
3787
4922
|
exports.fromZhipu = fromZhipu
|
|
4923
|
+
exports.hasIssues = hasIssues
|
|
4924
|
+
exports.isFallbackExhaustedError = isFallbackExhaustedError
|
|
3788
4925
|
exports.isLLMError = isLLMError
|
|
3789
4926
|
exports.isToolCallResult = isToolCallResult
|
|
3790
4927
|
exports.parseSseStream = parseSseStream
|