vern-llm 2.1.1 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/index.cjs +629 -137
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +390 -132
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +390 -132
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +628 -136
- package/dist/index.js.map +1 -1
- package/package.json +16 -2
package/dist/index.cjs
CHANGED
|
@@ -25,23 +25,205 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
|
|
|
25
25
|
const crypto = __toESM(require("crypto"));
|
|
26
26
|
|
|
27
27
|
//#region src/types/errors.ts
|
|
28
|
+
/**
|
|
29
|
+
* Tool contract codes: a model or provider response defect, not a
|
|
30
|
+
* transient provider fault. Deterministic on the wire request, so
|
|
31
|
+
* retrying can't change the outcome and it shouldn't count toward the
|
|
32
|
+
* circuit breaker either. Shared by `LLMError.retryable` below and by
|
|
33
|
+
* `CallExecutor`'s own retry/breaker accounting, so the two can't drift
|
|
34
|
+
* apart.
|
|
35
|
+
*/
|
|
36
|
+
const NON_RETRYABLE_TOOL_CONTRACT_CODES = new Set([
|
|
37
|
+
"unknown_tool",
|
|
38
|
+
"duplicate_tool_call_id",
|
|
39
|
+
"tool_choice_none_violated",
|
|
40
|
+
"unexpected_tool_calls"
|
|
41
|
+
]);
|
|
42
|
+
/**
|
|
43
|
+
* Local rate-limit codes: the call never reached the provider, so it says
|
|
44
|
+
* nothing about the provider's health, and retrying either just requeues
|
|
45
|
+
* behind the same limit (the two queue codes) or can never succeed at all
|
|
46
|
+
* (`rate_limit_capacity_exceeded`). Shared for the same reason as
|
|
47
|
+
* {@link NON_RETRYABLE_TOOL_CONTRACT_CODES}.
|
|
48
|
+
*/
|
|
49
|
+
const LOCAL_RATE_LIMIT_CODES = new Set([
|
|
50
|
+
"rate_limit_queue_full",
|
|
51
|
+
"rate_limit_queue_timeout",
|
|
52
|
+
"rate_limit_capacity_exceeded"
|
|
53
|
+
]);
|
|
54
|
+
/**
|
|
55
|
+
* Types that are never worth retrying on their own: deterministic
|
|
56
|
+
* caller-input, model-response, or cancellation failures rather than a
|
|
57
|
+
* transient provider fault.
|
|
58
|
+
*/
|
|
59
|
+
const NON_RETRYABLE_TYPES = new Set([
|
|
60
|
+
"parse",
|
|
61
|
+
"validation",
|
|
62
|
+
"invalid_params",
|
|
63
|
+
"aborted"
|
|
64
|
+
]);
|
|
65
|
+
/**
|
|
66
|
+
* Shared retryability rule behind both `LLMError.retryable` and
|
|
67
|
+
* `LLMErrorSnapshot.retryable`. Pulled out so the two can't drift apart:
|
|
68
|
+
* a snapshot is a point-in-time copy of an error's fields, and this is
|
|
69
|
+
* one of them, so it has to be computed the same way in both places.
|
|
70
|
+
*/
|
|
71
|
+
function computeRetryable(type, code) {
|
|
72
|
+
if (NON_RETRYABLE_TYPES.has(type)) return false;
|
|
73
|
+
if (code && NON_RETRYABLE_TOOL_CONTRACT_CODES.has(code)) return false;
|
|
74
|
+
if (code && LOCAL_RATE_LIMIT_CODES.has(code)) return false;
|
|
75
|
+
return true;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Returns `issues` unchanged when it can survive `JSON.stringify`.
|
|
79
|
+
* Most `issues` values are VernLLM's own structured shapes (see
|
|
80
|
+
* `LLMErrorIssuesByCode`) and always safe. The one exception is a
|
|
81
|
+
* schema validation failure, where `issues` is a caller supplied
|
|
82
|
+
* `SchemaLike` validator's own `error: unknown`, not controlled by
|
|
83
|
+
* VernLLM and not guaranteed to be circular free. Rather than silently
|
|
84
|
+
* dropping it in that case, this returns a marker string so a reader
|
|
85
|
+
* of serialized output can tell "no issues data" apart from "issues
|
|
86
|
+
* existed but could not be shown".
|
|
87
|
+
*/
|
|
88
|
+
function safeIssues(issues) {
|
|
89
|
+
if (issues === void 0) return void 0;
|
|
90
|
+
try {
|
|
91
|
+
JSON.stringify(issues);
|
|
92
|
+
return issues;
|
|
93
|
+
} catch {
|
|
94
|
+
return "[Unserializable: issues contained a circular reference]";
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
/**
|
|
98
|
+
* Depth cap for `safeAttempts`, guarding against a pathological,
|
|
99
|
+
* self referential `attempts` array. `attempts` is a public
|
|
100
|
+
* `LLMErrorOptions` field, so a caller can construct one by hand; this
|
|
101
|
+
* keeps that path bounded the same way a circular `issues` value is
|
|
102
|
+
* bounded, rather than assuming well formed input.
|
|
103
|
+
*/
|
|
104
|
+
const MAX_ATTEMPTS_DEPTH = 20;
|
|
105
|
+
/**
|
|
106
|
+
* Returns a copy of `attempts` with every nested snapshot's `issues`
|
|
107
|
+
* re-checked through `safeIssues`, recursively through each snapshot's
|
|
108
|
+
* own `attempts`. Needed for two reasons: `safeIssues` returns a safe
|
|
109
|
+
* `issues` value by reference, so a shared object can be mutated into a
|
|
110
|
+
* circular one after the snapshot was created, and `attempts` is a
|
|
111
|
+
* public constructor option, so a caller can hand build a `RetryAttempt`
|
|
112
|
+
* (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
|
|
113
|
+
* in directly, never touching `toSnapshot()` at all. Extra fields on an
|
|
114
|
+
* attempt (e.g. `FallbackAttempt`'s `provider`/`model`) are preserved.
|
|
115
|
+
*/
|
|
116
|
+
function safeAttempts(attempts, depth = 0) {
|
|
117
|
+
if (attempts === void 0) return void 0;
|
|
118
|
+
if (depth >= MAX_ATTEMPTS_DEPTH) return [];
|
|
119
|
+
return attempts.map((attempt) => ({
|
|
120
|
+
...attempt,
|
|
121
|
+
error: {
|
|
122
|
+
...attempt.error,
|
|
123
|
+
issues: safeIssues(attempt.error.issues),
|
|
124
|
+
attempts: safeAttempts(attempt.error.attempts, depth + 1)
|
|
125
|
+
}
|
|
126
|
+
}));
|
|
127
|
+
}
|
|
28
128
|
var LLMError = class extends Error {
|
|
29
|
-
|
|
129
|
+
status;
|
|
130
|
+
issues;
|
|
131
|
+
cause;
|
|
132
|
+
retryAfterMs;
|
|
133
|
+
/** Stable discriminator within `type`. Absent on errors predating it. */
|
|
134
|
+
code;
|
|
135
|
+
/** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
|
|
136
|
+
attempts;
|
|
137
|
+
constructor(message, type, options = {}) {
|
|
30
138
|
super(message);
|
|
31
139
|
this.type = type;
|
|
32
|
-
this.status = status;
|
|
33
|
-
this.issues = issues;
|
|
34
|
-
this.cause = cause;
|
|
35
|
-
this.retryAfterMs = retryAfterMs;
|
|
36
|
-
this.code = code;
|
|
37
140
|
this.name = "LLMError";
|
|
141
|
+
this.status = options.status;
|
|
142
|
+
this.issues = options.issues;
|
|
143
|
+
this.cause = options.cause;
|
|
144
|
+
this.retryAfterMs = options.retryAfterMs;
|
|
145
|
+
this.code = options.code;
|
|
146
|
+
this.attempts = options.attempts;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Computed purely from `type`/`code`, independent of any specific call's
|
|
150
|
+
* `nonRetryableStatus` list. False for `parse`/`validation`/
|
|
151
|
+
* `invalid_params`/`aborted` types (the caller's own input, the model's
|
|
152
|
+
* own response, or intentional cancellation, none of which are the
|
|
153
|
+
* provider being unhealthy), the tool contract codes, and the local
|
|
154
|
+
* rate limit codes. Subclasses (see `FallbackExhaustedError`) may
|
|
155
|
+
* override this when `type` alone carries no retry signal.
|
|
156
|
+
*/
|
|
157
|
+
get retryable() {
|
|
158
|
+
return computeRetryable(this.type, this.code);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Copies this error's fields into an {@link LLMErrorSnapshot}, for
|
|
162
|
+
* recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
|
|
163
|
+
* captured here since a snapshot has no getter of its own. `cause` is
|
|
164
|
+
* not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
|
|
165
|
+
* nested `attempts` entry's own `issues` go through `safeAttempts`,
|
|
166
|
+
* since a schema validation failure's `issues` is a caller supplied
|
|
167
|
+
* value, not controlled by VernLLM, and `attempts` is itself a public
|
|
168
|
+
* constructor option a caller can hand build.
|
|
169
|
+
*/
|
|
170
|
+
toSnapshot() {
|
|
171
|
+
return {
|
|
172
|
+
message: this.message,
|
|
173
|
+
type: this.type,
|
|
174
|
+
status: this.status,
|
|
175
|
+
issues: safeIssues(this.issues),
|
|
176
|
+
retryAfterMs: this.retryAfterMs,
|
|
177
|
+
code: this.code,
|
|
178
|
+
retryable: this.retryable,
|
|
179
|
+
attempts: safeAttempts(this.attempts)
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Controls what `JSON.stringify(err)` produces. Omits `cause` for the
|
|
184
|
+
* same reason `toSnapshot()` does: `cause` is `unknown` and never
|
|
185
|
+
* validated by VernLLM, and some SDK errors carry circular structures
|
|
186
|
+
* `JSON.stringify` cannot serialize at all. Read `err.cause` directly
|
|
187
|
+
* instead. `issues`, including every nested `attempts` entry's own
|
|
188
|
+
* `issues`, goes through `safeAttempts` for the same reason: a schema
|
|
189
|
+
* validation failure's `issues` is caller supplied and not guaranteed
|
|
190
|
+
* circular free. Also includes `message` and `retryable`, which a
|
|
191
|
+
* plain property walk would otherwise miss: `message` is
|
|
192
|
+
* non-enumerable on `Error`, and `retryable` is a getter, not an own
|
|
193
|
+
* property.
|
|
194
|
+
*/
|
|
195
|
+
toJSON() {
|
|
196
|
+
return {
|
|
197
|
+
name: this.name,
|
|
198
|
+
message: this.message,
|
|
199
|
+
type: this.type,
|
|
200
|
+
status: this.status,
|
|
201
|
+
issues: safeIssues(this.issues),
|
|
202
|
+
retryAfterMs: this.retryAfterMs,
|
|
203
|
+
code: this.code,
|
|
204
|
+
retryable: this.retryable,
|
|
205
|
+
attempts: safeAttempts(this.attempts)
|
|
206
|
+
};
|
|
38
207
|
}
|
|
39
|
-
/** Every tool contract failure in one response, when there is more than one. */
|
|
40
|
-
toolIssues;
|
|
41
208
|
};
|
|
42
209
|
function isLLMError(err) {
|
|
43
210
|
return err instanceof LLMError;
|
|
44
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
|
|
214
|
+
* `code` to, for any code listed there. `code` stays the only discriminator
|
|
215
|
+
* VernLLM uses; this just gives that existing check a typed return instead
|
|
216
|
+
* of requiring a manual cast of `issues`:
|
|
217
|
+
*
|
|
218
|
+
* ```ts
|
|
219
|
+
* if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
|
|
220
|
+
* console.log(err.issues.names); // string[], no cast needed
|
|
221
|
+
* }
|
|
222
|
+
* ```
|
|
223
|
+
*/
|
|
224
|
+
function hasIssues(err, code) {
|
|
225
|
+
return err.code === code && err.issues !== void 0;
|
|
226
|
+
}
|
|
45
227
|
|
|
46
228
|
//#endregion
|
|
47
229
|
//#region src/types/cache.ts
|
|
@@ -171,7 +353,12 @@ function isToolCallResult(result) {
|
|
|
171
353
|
//#endregion
|
|
172
354
|
//#region src/types/fallback.ts
|
|
173
355
|
/** Tool contract failures are the model ignoring the request, not a sick provider: repeating it elsewhere can't help. */
|
|
174
|
-
const TOOL_CONTRACT_CODES = new Set([
|
|
356
|
+
const TOOL_CONTRACT_CODES = new Set([
|
|
357
|
+
"unknown_tool",
|
|
358
|
+
"duplicate_tool_call_id",
|
|
359
|
+
"tool_choice_none_violated",
|
|
360
|
+
"unexpected_tool_calls"
|
|
361
|
+
]);
|
|
175
362
|
/**
|
|
176
363
|
* The default `fallbackOn` policy. Exported so a caller can wrap rather
|
|
177
364
|
* than replace it, e.g. `fallbackOn: (e, ctx) => myCheck(e) ? 'stop' : defaultFallbackOn(e, ctx)`.
|
|
@@ -187,16 +374,37 @@ const defaultFallbackOn = (error) => {
|
|
|
187
374
|
* or `fallbackOn` chose to stop early. Carries each attempt in order so
|
|
188
375
|
* an outage across providers stays debuggable without reproducing it.
|
|
189
376
|
* Extends `LLMError` so `isLLMError` and any `instanceof LLMError` check
|
|
190
|
-
* still passes, inheriting the last failure's `type`
|
|
191
|
-
* type-based handling
|
|
377
|
+
* still passes, inheriting the last failure's `type`/`status`/`retryAfterMs`
|
|
378
|
+
* so existing type-based handling, including reading `retryAfterMs` on an
|
|
379
|
+
* `'api'`-typed error, keeps working on a fallback-exhausted error too.
|
|
192
380
|
*/
|
|
193
381
|
var FallbackExhaustedError = class extends LLMError {
|
|
194
382
|
constructor(attempts) {
|
|
195
383
|
const last = attempts[attempts.length - 1]?.error;
|
|
196
|
-
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`,
|
|
384
|
+
super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`, "fallback_exhausted", {
|
|
385
|
+
status: last?.status,
|
|
386
|
+
cause: last,
|
|
387
|
+
retryAfterMs: last?.retryAfterMs,
|
|
388
|
+
code: "fallback_exhausted",
|
|
389
|
+
attempts
|
|
390
|
+
});
|
|
197
391
|
this.attempts = attempts;
|
|
198
392
|
}
|
|
393
|
+
/**
|
|
394
|
+
* `type: 'fallback_exhausted'` by itself says nothing about whether
|
|
395
|
+
* retrying could help; the reason the last target failed does. Defers to
|
|
396
|
+
* that attempt's own `retryable` instead of anything about this class's
|
|
397
|
+
* own type.
|
|
398
|
+
*/
|
|
399
|
+
get retryable() {
|
|
400
|
+
const last = this.attempts[this.attempts.length - 1]?.error;
|
|
401
|
+
return last ? last.retryable : super.retryable;
|
|
402
|
+
}
|
|
199
403
|
};
|
|
404
|
+
/** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
|
|
405
|
+
function isFallbackExhaustedError(err) {
|
|
406
|
+
return err instanceof FallbackExhaustedError;
|
|
407
|
+
}
|
|
200
408
|
|
|
201
409
|
//#endregion
|
|
202
410
|
//#region src/internal/execution/usage.utils.ts
|
|
@@ -218,7 +426,7 @@ async function reserve(params, coalesced, signal) {
|
|
|
218
426
|
return true;
|
|
219
427
|
} catch (error) {
|
|
220
428
|
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
221
|
-
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded",
|
|
429
|
+
throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded", { cause: error });
|
|
222
430
|
}
|
|
223
431
|
}
|
|
224
432
|
/**
|
|
@@ -418,13 +626,30 @@ var CacheOrchestrator = class {
|
|
|
418
626
|
*/
|
|
419
627
|
async deleteCache(key) {
|
|
420
628
|
if (!this.cache.delete) return;
|
|
421
|
-
|
|
629
|
+
try {
|
|
630
|
+
await this.cache.delete(await this.resolveCacheKey(key));
|
|
631
|
+
} catch (error) {
|
|
632
|
+
this.logger.warn(`[VernLLM] cache delete failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
633
|
+
}
|
|
422
634
|
}
|
|
423
635
|
/** Logs a failed refundUsage attempt via the configured logger. */
|
|
424
636
|
logRefundError(logMessage, error) {
|
|
425
637
|
this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
|
|
426
638
|
}
|
|
427
639
|
/**
|
|
640
|
+
* Reads from the cache, treating a failed adapter read as a miss rather
|
|
641
|
+
* than letting it fail the call. The request still falls through to a
|
|
642
|
+
* real provider call, but that fallback is now logged instead of silent.
|
|
643
|
+
*/
|
|
644
|
+
async getCached(key) {
|
|
645
|
+
try {
|
|
646
|
+
return await this.cache.get(key);
|
|
647
|
+
} catch (error) {
|
|
648
|
+
this.logger.warn(`[VernLLM] cache read failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
649
|
+
return { hit: false };
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
428
653
|
* Internal cache primitive around caller-supplied logic. Concurrent misses
|
|
429
654
|
* for the same `cacheKey` share a single in-flight call, avoiding cache
|
|
430
655
|
* stampedes.
|
|
@@ -444,7 +669,7 @@ var CacheOrchestrator = class {
|
|
|
444
669
|
...params,
|
|
445
670
|
cacheKey: resolvedKey
|
|
446
671
|
};
|
|
447
|
-
const cached = await this.
|
|
672
|
+
const cached = await this.getCached(resolvedKey);
|
|
448
673
|
if (cached.hit) return cached.value;
|
|
449
674
|
const existing = this.inFlight.get(resolvedKey);
|
|
450
675
|
if (existing) return withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
|
|
@@ -465,7 +690,7 @@ var CacheOrchestrator = class {
|
|
|
465
690
|
try {
|
|
466
691
|
await this.cache.set(params.cacheKey, result, params.ttl);
|
|
467
692
|
} catch (error) {
|
|
468
|
-
this.logger.
|
|
693
|
+
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
469
694
|
}
|
|
470
695
|
return result;
|
|
471
696
|
}
|
|
@@ -494,7 +719,7 @@ var CacheOrchestrator = class {
|
|
|
494
719
|
...params,
|
|
495
720
|
cacheKey: resolvedKey
|
|
496
721
|
};
|
|
497
|
-
const cached = await this.
|
|
722
|
+
const cached = await this.getCached(resolvedKey);
|
|
498
723
|
if (cached.hit) {
|
|
499
724
|
const value = cached.value;
|
|
500
725
|
return {
|
|
@@ -543,7 +768,7 @@ var CacheOrchestrator = class {
|
|
|
543
768
|
try {
|
|
544
769
|
await this.cache.set(params.cacheKey, value, params.ttl);
|
|
545
770
|
} catch (error) {
|
|
546
|
-
this.logger.
|
|
771
|
+
this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
|
|
547
772
|
}
|
|
548
773
|
return value;
|
|
549
774
|
}, (error) => {
|
|
@@ -585,6 +810,7 @@ var CircuitBreaker = class {
|
|
|
585
810
|
threshold;
|
|
586
811
|
cooldownMs;
|
|
587
812
|
onStateChange;
|
|
813
|
+
/** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
|
|
588
814
|
isolateByModel;
|
|
589
815
|
sharedBucket = newBucket();
|
|
590
816
|
bucketsByModel = new Map();
|
|
@@ -630,12 +856,12 @@ var CircuitBreaker = class {
|
|
|
630
856
|
if (bucket.state === "closed") return;
|
|
631
857
|
if (bucket.state === "open") {
|
|
632
858
|
const elapsed = Date.now() - bucket.openedAt;
|
|
633
|
-
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open");
|
|
859
|
+
if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open", { code: "circuit_cooling_down" });
|
|
634
860
|
bucket.trialInFlight = true;
|
|
635
861
|
this.transition(bucket, "half-open", model);
|
|
636
862
|
return;
|
|
637
863
|
}
|
|
638
|
-
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open");
|
|
864
|
+
if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open", { code: "circuit_trial_in_flight" });
|
|
639
865
|
bucket.trialInFlight = true;
|
|
640
866
|
}
|
|
641
867
|
recordSuccess(model) {
|
|
@@ -670,6 +896,33 @@ var CircuitBreaker = class {
|
|
|
670
896
|
getState(model) {
|
|
671
897
|
return this.lookupBucket(model)?.state ?? "closed";
|
|
672
898
|
}
|
|
899
|
+
/**
|
|
900
|
+
* Manually opens the circuit, as if `threshold` consecutive failures had
|
|
901
|
+
* just happened, e.g. to pull a provider out of rotation ahead of known
|
|
902
|
+
* maintenance. Resets the cooldown window from now, same as a real
|
|
903
|
+
* threshold-crossing failure would, and clears any in-flight half-open
|
|
904
|
+
* trial since it no longer applies once the circuit is (re)opened.
|
|
905
|
+
*/
|
|
906
|
+
open(model) {
|
|
907
|
+
const bucket = this.ensureBucketFor(model);
|
|
908
|
+
bucket.openedAt = Date.now();
|
|
909
|
+
bucket.trialInFlight = false;
|
|
910
|
+
this.transition(bucket, "open", model);
|
|
911
|
+
}
|
|
912
|
+
/**
|
|
913
|
+
* Manually closes the circuit and resets its failure count, e.g. once a
|
|
914
|
+
* provider is confirmed healthy again without waiting out the cooldown.
|
|
915
|
+
* Mirrors `recordSuccess`'s bookkeeping (including dropping the
|
|
916
|
+
* per-model bucket under `isolateByModel`, once idle) but without
|
|
917
|
+
* requiring an actual successful call first.
|
|
918
|
+
*/
|
|
919
|
+
close(model) {
|
|
920
|
+
const bucket = this.ensureBucketFor(model);
|
|
921
|
+
bucket.consecutiveFailures = 0;
|
|
922
|
+
bucket.trialInFlight = false;
|
|
923
|
+
this.transition(bucket, "closed", model);
|
|
924
|
+
if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
|
|
925
|
+
}
|
|
673
926
|
};
|
|
674
927
|
|
|
675
928
|
//#endregion
|
|
@@ -788,7 +1041,7 @@ async function withTimeout(fn, timeoutMs, externalSignal) {
|
|
|
788
1041
|
try {
|
|
789
1042
|
return await fn(signal);
|
|
790
1043
|
} catch (err) {
|
|
791
|
-
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout");
|
|
1044
|
+
if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout", { code: "request_timeout" });
|
|
792
1045
|
throw err;
|
|
793
1046
|
} finally {
|
|
794
1047
|
clearTimeout(timer);
|
|
@@ -821,7 +1074,7 @@ function withChunkIdleTimeout(next, timeoutMs, onIdle, logger) {
|
|
|
821
1074
|
const timer = setTimeout(() => {
|
|
822
1075
|
settled = true;
|
|
823
1076
|
onIdle?.();
|
|
824
|
-
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout"));
|
|
1077
|
+
reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout", { code: "idle_timeout" }));
|
|
825
1078
|
}, clampTimeoutMs(activeTimeoutMs));
|
|
826
1079
|
next().then((result) => {
|
|
827
1080
|
clearTimeout(timer);
|
|
@@ -914,6 +1167,52 @@ function extractStatus(err) {
|
|
|
914
1167
|
if (typeof error.$metadata?.httpStatusCode === "number") return error.$metadata.httpStatusCode;
|
|
915
1168
|
return void 0;
|
|
916
1169
|
}
|
|
1170
|
+
/**
|
|
1171
|
+
* POSIX/libuv error codes libuv (and so Node's `fetch`/undici) attaches to
|
|
1172
|
+
* genuine transport-level failures: connection refused, DNS lookup
|
|
1173
|
+
* failure, connection reset mid-request, a connect that never completed,
|
|
1174
|
+
* DNS server unreachable, broken pipe, or host/network unreachable.
|
|
1175
|
+
* Deliberately narrow: only codes that can only mean "the connection
|
|
1176
|
+
* itself failed," not anything that could also indicate an application
|
|
1177
|
+
* error.
|
|
1178
|
+
*/
|
|
1179
|
+
const NETWORK_ERROR_CODES = new Set([
|
|
1180
|
+
"ECONNREFUSED",
|
|
1181
|
+
"ENOTFOUND",
|
|
1182
|
+
"ECONNRESET",
|
|
1183
|
+
"ETIMEDOUT",
|
|
1184
|
+
"EAI_AGAIN",
|
|
1185
|
+
"EPIPE",
|
|
1186
|
+
"ECONNABORTED",
|
|
1187
|
+
"EHOSTUNREACH",
|
|
1188
|
+
"ENETUNREACH"
|
|
1189
|
+
]);
|
|
1190
|
+
/** `fetch`'s own wording for a transport-level failure, across runtimes/browsers. */
|
|
1191
|
+
const NETWORK_ERROR_MESSAGES = new Set([
|
|
1192
|
+
"fetch failed",
|
|
1193
|
+
"failed to fetch",
|
|
1194
|
+
"load failed",
|
|
1195
|
+
"networkerror when attempting to fetch resource."
|
|
1196
|
+
]);
|
|
1197
|
+
/**
|
|
1198
|
+
* Whether `error` is, with reasonable confidence, a transport-level
|
|
1199
|
+
* failure (never reached the provider, as opposed to the provider itself
|
|
1200
|
+
* responding with an error) rather than some other unexpected exception.
|
|
1201
|
+
* Checked via explicit, well-known signals only, so a genuinely unknown
|
|
1202
|
+
* error never gets misclassified as a connection failure just because it
|
|
1203
|
+
* also lacked an HTTP status.
|
|
1204
|
+
*/
|
|
1205
|
+
function isNetworkError(error) {
|
|
1206
|
+
if (!error || typeof error !== "object") return false;
|
|
1207
|
+
const err = error;
|
|
1208
|
+
if (typeof err.code === "string" && NETWORK_ERROR_CODES.has(err.code)) return true;
|
|
1209
|
+
if (typeof err.message === "string" && NETWORK_ERROR_MESSAGES.has(err.message.toLowerCase())) return true;
|
|
1210
|
+
if (err.cause && typeof err.cause === "object") {
|
|
1211
|
+
const cause = err.cause;
|
|
1212
|
+
if (typeof cause.code === "string" && NETWORK_ERROR_CODES.has(cause.code)) return true;
|
|
1213
|
+
}
|
|
1214
|
+
return false;
|
|
1215
|
+
}
|
|
917
1216
|
function formatSafely(value) {
|
|
918
1217
|
try {
|
|
919
1218
|
return JSON.stringify(value, null, 2) ?? String(value);
|
|
@@ -940,17 +1239,58 @@ function describeError(err) {
|
|
|
940
1239
|
} catch {}
|
|
941
1240
|
return formatSafely(err);
|
|
942
1241
|
}
|
|
943
|
-
/**
|
|
944
|
-
|
|
945
|
-
|
|
1242
|
+
/**
|
|
1243
|
+
* Maps an HTTP status to its corresponding `LLMErrorCode`, derived purely
|
|
1244
|
+
* from the status itself so it applies the same way regardless of which
|
|
1245
|
+
* adapter or client raised the error. Used both when building a fresh
|
|
1246
|
+
* `LLMError` and when filling in a `code` on an already-normalized one
|
|
1247
|
+
* that doesn't have one yet, so the two paths can't drift apart.
|
|
1248
|
+
*/
|
|
1249
|
+
function codeForStatus(status) {
|
|
1250
|
+
switch (status) {
|
|
1251
|
+
case 429: return "provider_rate_limited";
|
|
1252
|
+
case 401: return "authentication";
|
|
1253
|
+
case 403: return "authorization";
|
|
1254
|
+
case 404: return "not_found";
|
|
1255
|
+
case 413: return "payload_too_large";
|
|
1256
|
+
default: return status >= 500 ? "server_error" : void 0;
|
|
1257
|
+
}
|
|
1258
|
+
}
|
|
1259
|
+
/**
|
|
1260
|
+
* Converts any thrown value into a well-typed LLMError. `attempts`, when
|
|
1261
|
+
* given, is the accumulated record of every attempt made before `error`
|
|
1262
|
+
* was thrown; it's passed straight into the constructed error's options
|
|
1263
|
+
* rather than assigned onto the error afterward, so `attempts` is always
|
|
1264
|
+
* settled once, through the constructor, like every other field on
|
|
1265
|
+
* `LLMError`.
|
|
1266
|
+
*/
|
|
1267
|
+
function normalizeError(error, signal, attempts) {
|
|
1268
|
+
if (signal?.aborted) return new LLMError("LLM request aborted", "aborted", { attempts });
|
|
946
1269
|
if (error instanceof LLMError) {
|
|
947
|
-
if (error.
|
|
1270
|
+
if (error.code === void 0 && error.status !== void 0) error.code = codeForStatus(error.status);
|
|
1271
|
+
if (error.attempts === void 0 && attempts !== void 0) error.attempts = attempts;
|
|
948
1272
|
return error;
|
|
949
1273
|
}
|
|
950
1274
|
const status = extractStatus(error);
|
|
951
1275
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
952
|
-
if (status !== void 0) return new LLMError("LLM request failed", "api",
|
|
953
|
-
|
|
1276
|
+
if (status !== void 0) return new LLMError("LLM request failed", "api", {
|
|
1277
|
+
status,
|
|
1278
|
+
cause: error,
|
|
1279
|
+
retryAfterMs,
|
|
1280
|
+
code: codeForStatus(status),
|
|
1281
|
+
attempts
|
|
1282
|
+
});
|
|
1283
|
+
if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
|
|
1284
|
+
cause: error,
|
|
1285
|
+
retryAfterMs,
|
|
1286
|
+
code: "connection_failed",
|
|
1287
|
+
attempts
|
|
1288
|
+
});
|
|
1289
|
+
return new LLMError("LLM request failed", "unknown", {
|
|
1290
|
+
cause: error,
|
|
1291
|
+
retryAfterMs,
|
|
1292
|
+
attempts
|
|
1293
|
+
});
|
|
954
1294
|
}
|
|
955
1295
|
|
|
956
1296
|
//#endregion
|
|
@@ -999,7 +1339,7 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
999
1339
|
try {
|
|
1000
1340
|
parsedArgs = wc.function.arguments.trim() ? JSON.parse(wc.function.arguments) : {};
|
|
1001
1341
|
} catch {
|
|
1002
|
-
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse");
|
|
1342
|
+
throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse", { code: "tool_arguments_parse_failed" });
|
|
1003
1343
|
}
|
|
1004
1344
|
return {
|
|
1005
1345
|
id: wc.id,
|
|
@@ -1014,9 +1354,13 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1014
1354
|
/**
|
|
1015
1355
|
* Builds the wire request object for one call, applying per-instance
|
|
1016
1356
|
* defaults (model, max tokens, temperature) and per-call overrides.
|
|
1017
|
-
* Owns every
|
|
1018
|
-
* history alternation, duplicate/empty tool lists,
|
|
1019
|
-
* real tool.
|
|
1357
|
+
* Owns every check that depends only on the caller's own input shape, not
|
|
1358
|
+
* on execution: history alternation, duplicate/empty tool lists,
|
|
1359
|
+
* `toolChoice` naming a real tool. All deterministic on the call site's
|
|
1360
|
+
* own input and never touch the network, so every throw here is
|
|
1361
|
+
* `type: 'invalid_params'`, not `'validation'` (which is reserved for the
|
|
1362
|
+
* model/provider's own response failing a contract check). Has no
|
|
1363
|
+
* knowledge of retry, timeouts, or the breaker, only
|
|
1020
1364
|
* the three defaults a `FallbackTarget` can override per-target (see the
|
|
1021
1365
|
* `defaultMaxTokens`/`defaultTemperature` overrides in the fallback
|
|
1022
1366
|
* design), which is what keeps it separable from `CallExecutor`.
|
|
@@ -1034,7 +1378,7 @@ var RequestBuilder = class {
|
|
|
1034
1378
|
build(params) {
|
|
1035
1379
|
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, reasoningEffort, jsonSchema, tools, toolChoice } = params;
|
|
1036
1380
|
const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
|
|
1037
|
-
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "
|
|
1381
|
+
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
|
|
1038
1382
|
if (tools) {
|
|
1039
1383
|
const seen = new Set();
|
|
1040
1384
|
const duplicates = new Set();
|
|
@@ -1042,13 +1386,22 @@ var RequestBuilder = class {
|
|
|
1042
1386
|
if (seen.has(tool.name)) duplicates.add(tool.name);
|
|
1043
1387
|
seen.add(tool.name);
|
|
1044
1388
|
}
|
|
1045
|
-
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "
|
|
1389
|
+
if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "invalid_params", {
|
|
1390
|
+
code: "duplicate_tool_names",
|
|
1391
|
+
issues: { names: [...duplicates] }
|
|
1392
|
+
});
|
|
1046
1393
|
}
|
|
1047
|
-
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "
|
|
1048
|
-
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "
|
|
1394
|
+
if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "invalid_params");
|
|
1395
|
+
if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "invalid_params", {
|
|
1396
|
+
code: "unknown_tool_choice",
|
|
1397
|
+
issues: {
|
|
1398
|
+
requested: toolChoice.name,
|
|
1399
|
+
available: tools.map((t) => t.name)
|
|
1400
|
+
}
|
|
1401
|
+
});
|
|
1049
1402
|
const jsonMode = params.jsonMode ?? (tools ? false : true);
|
|
1050
1403
|
const useJson = jsonMode || Boolean(jsonSchema);
|
|
1051
|
-
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "
|
|
1404
|
+
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
|
|
1052
1405
|
const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
|
|
1053
1406
|
this.validateHistory(history);
|
|
1054
1407
|
const request = {
|
|
@@ -1085,29 +1438,47 @@ var RequestBuilder = class {
|
|
|
1085
1438
|
let previousTurn;
|
|
1086
1439
|
for (const [index, turn] of history.entries()) {
|
|
1087
1440
|
if (turn.role === "tool") {
|
|
1088
|
-
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "
|
|
1089
|
-
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "
|
|
1441
|
+
if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "invalid_params");
|
|
1442
|
+
if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "invalid_params");
|
|
1090
1443
|
const requestedIds = new Set(previousTurn.toolCalls.map((tc) => tc.id));
|
|
1091
1444
|
const resultIds = turn.toolResults.map((tr) => tr.toolCallId);
|
|
1092
1445
|
const unknownIds = resultIds.filter((id) => !requestedIds.has(id));
|
|
1093
|
-
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "
|
|
1446
|
+
if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "invalid_params", {
|
|
1447
|
+
code: "unknown_tool_result_ids",
|
|
1448
|
+
issues: {
|
|
1449
|
+
historyIndex: index,
|
|
1450
|
+
ids: unknownIds
|
|
1451
|
+
}
|
|
1452
|
+
});
|
|
1094
1453
|
const seenIds = new Set();
|
|
1095
1454
|
const duplicateIds = new Set();
|
|
1096
1455
|
for (const id of resultIds) {
|
|
1097
1456
|
if (seenIds.has(id)) duplicateIds.add(id);
|
|
1098
1457
|
seenIds.add(id);
|
|
1099
1458
|
}
|
|
1100
|
-
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "
|
|
1459
|
+
if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "invalid_params", {
|
|
1460
|
+
code: "duplicate_tool_result_ids",
|
|
1461
|
+
issues: {
|
|
1462
|
+
historyIndex: index,
|
|
1463
|
+
ids: [...duplicateIds]
|
|
1464
|
+
}
|
|
1465
|
+
});
|
|
1101
1466
|
const missingIds = [...requestedIds].filter((id) => !resultIds.includes(id));
|
|
1102
|
-
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "
|
|
1467
|
+
if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "invalid_params", {
|
|
1468
|
+
code: "missing_tool_results",
|
|
1469
|
+
issues: {
|
|
1470
|
+
historyIndex: index,
|
|
1471
|
+
ids: missingIds
|
|
1472
|
+
}
|
|
1473
|
+
});
|
|
1103
1474
|
} else {
|
|
1104
|
-
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "
|
|
1105
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "
|
|
1475
|
+
if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "invalid_params");
|
|
1476
|
+
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "invalid_params");
|
|
1106
1477
|
}
|
|
1107
1478
|
previousTurn = turn;
|
|
1108
1479
|
}
|
|
1109
|
-
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "
|
|
1110
|
-
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "
|
|
1480
|
+
if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "invalid_params");
|
|
1481
|
+
if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "invalid_params");
|
|
1111
1482
|
}
|
|
1112
1483
|
/** Maps VernLLM's app-facing `ToolChoice` onto the OpenAI-shaped wire `tool_choice`. */
|
|
1113
1484
|
buildWireToolChoice(toolChoice) {
|
|
@@ -1375,6 +1746,18 @@ var CallExecutor = class {
|
|
|
1375
1746
|
getCircuitState(model) {
|
|
1376
1747
|
return this.breaker?.getState(model);
|
|
1377
1748
|
}
|
|
1749
|
+
/** Whether this target's breaker tracks failures per model. `false` if no breaker is configured. */
|
|
1750
|
+
get isolateByModel() {
|
|
1751
|
+
return this.breaker?.isolateByModel ?? false;
|
|
1752
|
+
}
|
|
1753
|
+
/** Manually opens this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1754
|
+
openCircuit(model) {
|
|
1755
|
+
this.breaker?.open(model);
|
|
1756
|
+
}
|
|
1757
|
+
/** Manually closes this target's circuit breaker, if one is configured. No-op otherwise. */
|
|
1758
|
+
closeCircuit(model) {
|
|
1759
|
+
this.breaker?.close(model);
|
|
1760
|
+
}
|
|
1378
1761
|
/**
|
|
1379
1762
|
* Throws if the breaker is open for this target/model, exactly like the
|
|
1380
1763
|
* check `run`/`runStream` used to make internally. Exposed so `VernLLM`
|
|
@@ -1396,10 +1779,11 @@ var CallExecutor = class {
|
|
|
1396
1779
|
*/
|
|
1397
1780
|
async run(params, requestId, onAttempt) {
|
|
1398
1781
|
const model = params.model ?? this.model;
|
|
1782
|
+
const attempts = [];
|
|
1399
1783
|
try {
|
|
1400
|
-
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt);
|
|
1784
|
+
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1401
1785
|
} catch (error) {
|
|
1402
|
-
const normalized = normalizeError(error, params.signal);
|
|
1786
|
+
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1403
1787
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1404
1788
|
this.logger.debug(`[VernLLM:${requestId}] error:\n${this.redactText(describeError(error))}`);
|
|
1405
1789
|
throw normalized;
|
|
@@ -1408,10 +1792,11 @@ var CallExecutor = class {
|
|
|
1408
1792
|
/** Streaming counterpart to `run`. Mirrors the old streaming branch of `VernLLM.call`. */
|
|
1409
1793
|
async runStream(params, requestId, onAttempt) {
|
|
1410
1794
|
const model = params.model ?? this.model;
|
|
1795
|
+
const attempts = [];
|
|
1411
1796
|
try {
|
|
1412
|
-
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt);
|
|
1797
|
+
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1413
1798
|
} catch (error) {
|
|
1414
|
-
const normalized = normalizeError(error, params.signal);
|
|
1799
|
+
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1415
1800
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
1416
1801
|
this.logger.debug(`[VernLLM:${requestId}] stream-open error:\n${this.redactText(describeError(error))}`);
|
|
1417
1802
|
throw normalized;
|
|
@@ -1479,10 +1864,11 @@ var CallExecutor = class {
|
|
|
1479
1864
|
finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt) {
|
|
1480
1865
|
try {
|
|
1481
1866
|
const content = rawContent?.trim();
|
|
1482
|
-
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api");
|
|
1867
|
+
if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api", { code: "empty_response" });
|
|
1483
1868
|
this.logger.debug(`[VernLLM:${requestId}] output:\n${this.redactedOutput(content, wireToolCalls).slice(0, 800)}`);
|
|
1484
1869
|
if (wireToolCalls?.length) {
|
|
1485
|
-
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "
|
|
1870
|
+
if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "validation", { code: "unexpected_tool_calls" });
|
|
1871
|
+
if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "validation", { code: "tool_choice_none_violated" });
|
|
1486
1872
|
const toolCalls = parseWireToolCalls(wireToolCalls);
|
|
1487
1873
|
this.validateToolCallArguments(toolCalls, params.tools);
|
|
1488
1874
|
this.breaker?.recordSuccess(model);
|
|
@@ -1535,7 +1921,10 @@ var CallExecutor = class {
|
|
|
1535
1921
|
async executeStreamCall(params, requestId, attempt) {
|
|
1536
1922
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1537
1923
|
const completions = this.client.chat.completions;
|
|
1538
|
-
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "
|
|
1924
|
+
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
|
|
1925
|
+
code: "unsupported_capability",
|
|
1926
|
+
issues: { capability: "createStream" }
|
|
1927
|
+
});
|
|
1539
1928
|
const createStream = completions.createStream.bind(completions);
|
|
1540
1929
|
let release;
|
|
1541
1930
|
if (this.limiter) {
|
|
@@ -1596,13 +1985,13 @@ var CallExecutor = class {
|
|
|
1596
1985
|
* `argumentsSchema`, if present.
|
|
1597
1986
|
*
|
|
1598
1987
|
* Contract failures (unknown name, duplicate id) are collected across
|
|
1599
|
-
* every call and thrown together
|
|
1600
|
-
*
|
|
1601
|
-
*
|
|
1602
|
-
*
|
|
1603
|
-
*
|
|
1604
|
-
*
|
|
1605
|
-
*
|
|
1988
|
+
* every call and thrown together as one `type: 'validation'` error with
|
|
1989
|
+
* `issues: ToolIssue[]`, since retrying a request that already has these
|
|
1990
|
+
* errors cannot help (excluded from retry by `type`) and a caller fixing
|
|
1991
|
+
* them wants to see every one, not just the first. Schema failures keep
|
|
1992
|
+
* the original single-error, `type: 'validation'` shape rather than being
|
|
1993
|
+
* folded into the aggregate, since they're a distinct failure kind from
|
|
1994
|
+
* the contract failures above.
|
|
1606
1995
|
*/
|
|
1607
1996
|
validateToolCallArguments(toolCalls, tools) {
|
|
1608
1997
|
const known = new Map(tools.map((t) => [t.name, t]));
|
|
@@ -1624,20 +2013,31 @@ var CallExecutor = class {
|
|
|
1624
2013
|
if (toolIssues.length > 0) {
|
|
1625
2014
|
const unknownTool = toolIssues.find((i) => i.code === "unknown_tool");
|
|
1626
2015
|
const primary = unknownTool ? `Model requested tool "${unknownTool.name}", which was not in the tools offered ([${[...known.keys()].join(", ")}]).` : `Duplicate tool call id "${toolIssues[0].toolCallId}" in the model's response.`;
|
|
1627
|
-
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
2016
|
+
const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see error.issues.)` : primary;
|
|
2017
|
+
throw new LLMError(message, "validation", {
|
|
2018
|
+
code: unknownTool ? "unknown_tool" : "duplicate_tool_call_id",
|
|
2019
|
+
issues: toolIssues
|
|
2020
|
+
});
|
|
1631
2021
|
}
|
|
1632
2022
|
for (const call of toolCalls) {
|
|
1633
2023
|
const definition = known.get(call.name);
|
|
1634
2024
|
if (!definition?.argumentsSchema) continue;
|
|
1635
2025
|
const result = definition.argumentsSchema.safeParse(call.arguments);
|
|
1636
|
-
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation",
|
|
2026
|
+
if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation", { issues: result.error });
|
|
1637
2027
|
}
|
|
1638
2028
|
}
|
|
1639
|
-
/**
|
|
1640
|
-
|
|
2029
|
+
/**
|
|
2030
|
+
* Runs `fn`, retrying with backoff according to `shouldRetry`. When
|
|
2031
|
+
* `attempts` is given, every failed attempt that is actually followed by
|
|
2032
|
+
* a retry is recorded, in order. This mirrors `LLMError.attempts`'s
|
|
2033
|
+
* contract: every attempt made before this error was thrown. The
|
|
2034
|
+
* terminal failure is never pushed since it isn't a prior attempt, it
|
|
2035
|
+
* is the error being thrown. `attempts` stays empty when nothing was
|
|
2036
|
+
* retried, so no separate bookkeeping is needed at the call sites.
|
|
2037
|
+
* Each failure is recorded as a snapshot (`LLMError.toSnapshot()`),
|
|
2038
|
+
* not the live `LLMError`, per `RetryAttempt`'s contract.
|
|
2039
|
+
*/
|
|
2040
|
+
async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
|
|
1641
2041
|
let lastError;
|
|
1642
2042
|
for (let attempt = 0; attempt <= this.maxRetries; attempt++) try {
|
|
1643
2043
|
if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
|
|
@@ -1645,7 +2045,12 @@ var CallExecutor = class {
|
|
|
1645
2045
|
return await fn(attempt);
|
|
1646
2046
|
} catch (error) {
|
|
1647
2047
|
lastError = error;
|
|
1648
|
-
|
|
2048
|
+
const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
|
|
2049
|
+
if (!willRetry) break;
|
|
2050
|
+
attempts?.push({
|
|
2051
|
+
index: attempt,
|
|
2052
|
+
error: normalizeError(error, signal).toSnapshot()
|
|
2053
|
+
});
|
|
1649
2054
|
}
|
|
1650
2055
|
throw lastError;
|
|
1651
2056
|
}
|
|
@@ -1715,7 +2120,7 @@ var CallExecutor = class {
|
|
|
1715
2120
|
if (parsed === null || parsed === void 0) throw new LLMError("Invalid JSON response", "parse");
|
|
1716
2121
|
if (!schema) return parsed;
|
|
1717
2122
|
const result = schema.safeParse(parsed);
|
|
1718
|
-
if (!result.success) throw new LLMError("Schema validation failed", "validation",
|
|
2123
|
+
if (!result.success) throw new LLMError("Schema validation failed", "validation", { issues: result.error });
|
|
1719
2124
|
return result.data;
|
|
1720
2125
|
}
|
|
1721
2126
|
/**
|
|
@@ -1728,7 +2133,7 @@ var CallExecutor = class {
|
|
|
1728
2133
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
1729
2134
|
const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
|
|
1730
2135
|
const retryAfterHonored = retryAfterMs !== void 0;
|
|
1731
|
-
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${delay}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
2136
|
+
this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${Math.ceil(delay)}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
|
|
1732
2137
|
this.reportEvent({
|
|
1733
2138
|
kind: "retry",
|
|
1734
2139
|
requestId,
|
|
@@ -1742,33 +2147,61 @@ var CallExecutor = class {
|
|
|
1742
2147
|
});
|
|
1743
2148
|
await waitForRetry(delay, signal);
|
|
1744
2149
|
}
|
|
1745
|
-
isNonRetryableToolContractError(error) {
|
|
1746
|
-
return error instanceof LLMError && (error.code === "unknown_tool" || error.code === "duplicate_tool_call_id");
|
|
1747
|
-
}
|
|
1748
2150
|
/** Decides whether a failed attempt is worth retrying. */
|
|
1749
2151
|
shouldRetry(error, signal) {
|
|
1750
2152
|
if (signal?.aborted) return false;
|
|
1751
|
-
if (error instanceof LLMError &&
|
|
1752
|
-
if (error instanceof LLMError && error.code === "local_rate_limit") return false;
|
|
1753
|
-
if (this.isNonRetryableToolContractError(error)) return false;
|
|
2153
|
+
if (error instanceof LLMError && !error.retryable) return false;
|
|
1754
2154
|
const status = extractStatus(error);
|
|
1755
2155
|
return !(status !== void 0 && this.nonRetryableStatus.includes(status));
|
|
1756
2156
|
}
|
|
1757
2157
|
/**
|
|
1758
2158
|
* Decides whether a failed attempt should count toward the circuit
|
|
1759
|
-
* breaker's failure threshold. A model hallucinating a tool name
|
|
1760
|
-
* reusing a call id
|
|
1761
|
-
*
|
|
1762
|
-
*
|
|
1763
|
-
*
|
|
1764
|
-
*
|
|
2159
|
+
* breaker's failure threshold. A model hallucinating a tool name,
|
|
2160
|
+
* reusing a call id, or a provider ignoring `toolChoice: 'none'` isn't
|
|
2161
|
+
* the provider being unhealthy, it's a model/provider response defect
|
|
2162
|
+
* that will very likely recur regardless of provider health, so it
|
|
2163
|
+
* shouldn't push a healthy provider's circuit toward opening. Same for
|
|
2164
|
+
* a caller-input bug or a local rate-limit rejection: neither ever
|
|
2165
|
+
* reached the provider at all. This is exactly what `LLMError.retryable`
|
|
2166
|
+
* already excludes, so this defers to it directly.
|
|
1765
2167
|
*/
|
|
1766
2168
|
countsTowardBreaker(error) {
|
|
1767
|
-
|
|
1768
|
-
return true;
|
|
2169
|
+
return error.retryable;
|
|
1769
2170
|
}
|
|
1770
2171
|
};
|
|
1771
2172
|
|
|
2173
|
+
//#endregion
|
|
2174
|
+
//#region src/internal/logger.utils.ts
|
|
2175
|
+
/**
|
|
2176
|
+
* Wraps a `Logger` so a throwing implementation can never break the call
|
|
2177
|
+
* it's trying to describe. `logger` is user-supplied (`VernLLMOptions.logger`),
|
|
2178
|
+
* so a custom logger that ships to a file, Datadog, etc. can throw for
|
|
2179
|
+
* reasons unrelated to VernLLM. Wrap once at construction so every
|
|
2180
|
+
* downstream `this.logger.warn(...)` call stays as-is and is safe by
|
|
2181
|
+
* construction, instead of guarding each call site individually.
|
|
2182
|
+
*/
|
|
2183
|
+
function createSafeLogger(logger) {
|
|
2184
|
+
return {
|
|
2185
|
+
debug: safe(logger, "debug"),
|
|
2186
|
+
warn: safe(logger, "warn"),
|
|
2187
|
+
error: safe(logger, "error")
|
|
2188
|
+
};
|
|
2189
|
+
}
|
|
2190
|
+
function safe(logger, method) {
|
|
2191
|
+
const fn = logger[method].bind(logger);
|
|
2192
|
+
return (...args) => {
|
|
2193
|
+
try {
|
|
2194
|
+
swallowRejection(fn(...args));
|
|
2195
|
+
} catch {}
|
|
2196
|
+
};
|
|
2197
|
+
}
|
|
2198
|
+
function isPromiseLike(value) {
|
|
2199
|
+
return typeof value?.then === "function";
|
|
2200
|
+
}
|
|
2201
|
+
function swallowRejection(result) {
|
|
2202
|
+
if (isPromiseLike(result)) Promise.resolve(result).catch(() => {});
|
|
2203
|
+
}
|
|
2204
|
+
|
|
1772
2205
|
//#endregion
|
|
1773
2206
|
//#region src/logger.ts
|
|
1774
2207
|
/**
|
|
@@ -1823,7 +2256,8 @@ var TokenBucket = class {
|
|
|
1823
2256
|
refill() {
|
|
1824
2257
|
if (this.refillPerMs === 0) return;
|
|
1825
2258
|
const now = Date.now();
|
|
1826
|
-
|
|
2259
|
+
const elapsedMs = now - this.lastRefill;
|
|
2260
|
+
this.available = Math.min(this.capacity, this.available + Math.max(0, elapsedMs) * this.refillPerMs);
|
|
1827
2261
|
this.lastRefill = now;
|
|
1828
2262
|
}
|
|
1829
2263
|
/** Refills, then takes `amount` if available. Leaves the bucket untouched if it can't. */
|
|
@@ -1908,22 +2342,21 @@ var RateLimiter = class {
|
|
|
1908
2342
|
*/
|
|
1909
2343
|
async acquire(estimatedTokens, signal) {
|
|
1910
2344
|
if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
1911
|
-
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "
|
|
1912
|
-
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "
|
|
2345
|
+
if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "invalid_params");
|
|
2346
|
+
if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "rate_limited", { code: "rate_limit_capacity_exceeded" });
|
|
1913
2347
|
if (this.queue.length === 0) {
|
|
1914
2348
|
const attempt = this.tryAcquireBuckets(estimatedTokens);
|
|
1915
2349
|
if (attempt.ok) return {
|
|
1916
2350
|
release: this.makeRelease(estimatedTokens),
|
|
1917
2351
|
waitedMs: 0
|
|
1918
2352
|
};
|
|
1919
|
-
if (this.maxQueueSize > 0 && this.queue.length >= this.maxQueueSize) throw this.queueFullError();
|
|
1920
2353
|
return this.enqueue(estimatedTokens, attempt.reason, signal);
|
|
1921
2354
|
}
|
|
1922
2355
|
if (this.maxQueueSize > 0 && this.queue.length >= this.maxQueueSize) throw this.queueFullError();
|
|
1923
2356
|
return this.enqueue(estimatedTokens, void 0, signal);
|
|
1924
2357
|
}
|
|
1925
2358
|
queueFullError() {
|
|
1926
|
-
return new LLMError("Rate limit queue is full", "
|
|
2359
|
+
return new LLMError("Rate limit queue is full", "rate_limited", { code: "rate_limit_queue_full" });
|
|
1927
2360
|
}
|
|
1928
2361
|
enqueue(estimatedTokens, initialReason, signal) {
|
|
1929
2362
|
return new Promise((resolvePromise, rejectPromise) => {
|
|
@@ -1956,7 +2389,7 @@ var RateLimiter = class {
|
|
|
1956
2389
|
if (index !== -1) this.queue.splice(index, 1);
|
|
1957
2390
|
};
|
|
1958
2391
|
if (this.maxQueueMs > 0) queueTimer = setTimeout(() => {
|
|
1959
|
-
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "
|
|
2392
|
+
waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "rate_limited", { code: "rate_limit_queue_timeout" }));
|
|
1960
2393
|
}, this.maxQueueMs);
|
|
1961
2394
|
signal?.addEventListener("abort", onAbort, { once: true });
|
|
1962
2395
|
this.queue.push(waiter);
|
|
@@ -2090,7 +2523,7 @@ var VernLLM = class {
|
|
|
2090
2523
|
* `nonRetryableStatus` `[400, 401, 403, 404, 422]`, `debug` false.
|
|
2091
2524
|
*/
|
|
2092
2525
|
constructor(options) {
|
|
2093
|
-
this.logger = options.logger ?? new ConsoleLogger(options.debug ?? false);
|
|
2526
|
+
this.logger = createSafeLogger(options.logger ?? new ConsoleLogger(options.debug ?? false));
|
|
2094
2527
|
const providerName = options.name ?? "primary";
|
|
2095
2528
|
this.cacheOrchestrator = new CacheOrchestrator(options.cache ?? new InMemoryCacheAdapter(), this.logger);
|
|
2096
2529
|
this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
|
|
@@ -2183,7 +2616,7 @@ var VernLLM = class {
|
|
|
2183
2616
|
index: i - 1,
|
|
2184
2617
|
provider: executor.providerName,
|
|
2185
2618
|
model: params.model ?? executor.model,
|
|
2186
|
-
error: normalized
|
|
2619
|
+
error: normalized.toSnapshot()
|
|
2187
2620
|
});
|
|
2188
2621
|
const isLast = i === this.executors.length - 1;
|
|
2189
2622
|
const policyDecision = this.fallbackOn(normalized, { isLastTarget: isLast });
|
|
@@ -2202,7 +2635,7 @@ var VernLLM = class {
|
|
|
2202
2635
|
});
|
|
2203
2636
|
}
|
|
2204
2637
|
}
|
|
2205
|
-
throw new LLMError("No provider targets configured", "
|
|
2638
|
+
throw new LLMError("No provider targets configured", "invalid_params");
|
|
2206
2639
|
}
|
|
2207
2640
|
async call(params) {
|
|
2208
2641
|
if (params.signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
|
|
@@ -2247,8 +2680,8 @@ var VernLLM = class {
|
|
|
2247
2680
|
}
|
|
2248
2681
|
async cachedCall(params) {
|
|
2249
2682
|
const { call: callParams,...cacheParams } = params;
|
|
2250
|
-
const
|
|
2251
|
-
if (reserveUsage || refundUsage)
|
|
2683
|
+
const restCallParams = callParams;
|
|
2684
|
+
if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "invalid_params");
|
|
2252
2685
|
if (restCallParams.stream) {
|
|
2253
2686
|
const streamParams = restCallParams;
|
|
2254
2687
|
return this.cacheOrchestrator.runCachedStream({
|
|
@@ -2262,35 +2695,67 @@ var VernLLM = class {
|
|
|
2262
2695
|
});
|
|
2263
2696
|
}
|
|
2264
2697
|
/**
|
|
2265
|
-
* @param
|
|
2266
|
-
* model
|
|
2267
|
-
*
|
|
2268
|
-
*
|
|
2269
|
-
*
|
|
2270
|
-
*
|
|
2698
|
+
* @param target.index Which target to read. Defaults to the primary.
|
|
2699
|
+
* @param target.model Which model bucket to read, if the target isolates by model.
|
|
2700
|
+
* @returns The breaker state, or `undefined` if that target has no breaker.
|
|
2701
|
+
* @throws {RangeError} If `target.index` names no target. Lets a real
|
|
2702
|
+
* target with no breaker (`undefined`) stay distinguishable from a
|
|
2703
|
+
* target that doesn't exist.
|
|
2271
2704
|
*/
|
|
2272
|
-
getCircuitState(
|
|
2273
|
-
|
|
2705
|
+
getCircuitState(target) {
|
|
2706
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "getCircuitState");
|
|
2707
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "getCircuitState");
|
|
2708
|
+
return executor.getCircuitState(target?.model ?? executor.model);
|
|
2274
2709
|
}
|
|
2275
2710
|
/**
|
|
2276
|
-
* @param model
|
|
2277
|
-
* target's
|
|
2278
|
-
* Ignored otherwise. Omit for the shared circuit (the default) or, under
|
|
2279
|
-
* isolation, the state of calls that didn't resolve a model.
|
|
2280
|
-
* @returns The current circuit state for every target in declaration
|
|
2281
|
-
* order, including the primary and all fallback targets. Each entry
|
|
2282
|
-
* includes the target's provider name, chain index, whether it is a
|
|
2283
|
-
* fallback, and its circuit state, or undefined if that target has no
|
|
2284
|
-
* circuit breaker configured.
|
|
2711
|
+
* @param model Which model bucket to read, for targets that isolate by model.
|
|
2712
|
+
* @returns Every target's state, in chain order.
|
|
2285
2713
|
*/
|
|
2286
2714
|
getCircuitStates(model) {
|
|
2287
2715
|
return this.executors.map((executor, index) => ({
|
|
2288
2716
|
provider: executor.providerName,
|
|
2289
2717
|
index,
|
|
2290
2718
|
isFallback: index > 0,
|
|
2291
|
-
|
|
2719
|
+
isolateByModel: executor.isolateByModel,
|
|
2720
|
+
state: executor.getCircuitState(model ?? executor.model)
|
|
2292
2721
|
}));
|
|
2293
2722
|
}
|
|
2723
|
+
/**
|
|
2724
|
+
* Manually opens a target's breaker, e.g. to pull a provider out of
|
|
2725
|
+
* rotation ahead of known maintenance instead of waiting for it to fail.
|
|
2726
|
+
*
|
|
2727
|
+
* @param target.index Which target to open. Defaults to the primary.
|
|
2728
|
+
* @param target.model Which model bucket to open, if the target isolates by model.
|
|
2729
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
2730
|
+
*/
|
|
2731
|
+
openCircuit(target) {
|
|
2732
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "openCircuit");
|
|
2733
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "openCircuit");
|
|
2734
|
+
executor.openCircuit(target?.model ?? executor.model);
|
|
2735
|
+
}
|
|
2736
|
+
/**
|
|
2737
|
+
* Manually closes a target's breaker, e.g. once a provider is confirmed
|
|
2738
|
+
* healthy again without waiting out the cooldown.
|
|
2739
|
+
*
|
|
2740
|
+
* @param target.index Which target to close. Defaults to the primary.
|
|
2741
|
+
* @param target.model Which model bucket to close, if the target isolates by model.
|
|
2742
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
2743
|
+
*/
|
|
2744
|
+
closeCircuit(target) {
|
|
2745
|
+
const executor = this.resolveExecutor(target?.index ?? 0, "closeCircuit");
|
|
2746
|
+
this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "closeCircuit");
|
|
2747
|
+
executor.closeCircuit(target?.model ?? executor.model);
|
|
2748
|
+
}
|
|
2749
|
+
/** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
|
|
2750
|
+
resolveExecutor(index, caller) {
|
|
2751
|
+
const executor = this.executors[index];
|
|
2752
|
+
if (!executor) throw new RangeError(`${caller}: no target at index ${index} (chain has ${this.executors.length} target${this.executors.length === 1 ? "" : "s"})`);
|
|
2753
|
+
return executor;
|
|
2754
|
+
}
|
|
2755
|
+
/** Warns when `model` can't do anything on this target, so it's never silently ignored. */
|
|
2756
|
+
warnIfModelUnsupported(isolateByModel, model, caller) {
|
|
2757
|
+
if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
|
|
2758
|
+
}
|
|
2294
2759
|
};
|
|
2295
2760
|
|
|
2296
2761
|
//#endregion
|
|
@@ -2330,7 +2795,7 @@ async function* parseSseStream(source) {
|
|
|
2330
2795
|
try {
|
|
2331
2796
|
text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
|
|
2332
2797
|
} catch (cause) {
|
|
2333
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse",
|
|
2798
|
+
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2334
2799
|
}
|
|
2335
2800
|
buffer = (buffer + text).replace(/\r\n/g, "\n").replace(/\r(?!$)/g, "\n");
|
|
2336
2801
|
let boundary$1 = buffer.indexOf("\n\n");
|
|
@@ -2346,7 +2811,7 @@ async function* parseSseStream(source) {
|
|
|
2346
2811
|
try {
|
|
2347
2812
|
buffer += decoder.decode();
|
|
2348
2813
|
} catch (cause) {
|
|
2349
|
-
throw new LLMError("Invalid UTF-8 in SSE stream", "parse",
|
|
2814
|
+
throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
|
|
2350
2815
|
}
|
|
2351
2816
|
buffer = buffer.replace(/\r$/, "\n");
|
|
2352
2817
|
let boundary = buffer.indexOf("\n\n");
|
|
@@ -2391,7 +2856,10 @@ function parseSseFrame(frame) {
|
|
|
2391
2856
|
try {
|
|
2392
2857
|
return JSON.parse(data);
|
|
2393
2858
|
} catch (cause) {
|
|
2394
|
-
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse",
|
|
2859
|
+
throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse", {
|
|
2860
|
+
cause,
|
|
2861
|
+
code: "stream_frame_invalid"
|
|
2862
|
+
});
|
|
2395
2863
|
}
|
|
2396
2864
|
}
|
|
2397
2865
|
|
|
@@ -2411,13 +2879,14 @@ const SUPPORTED_IMAGE_MIME_TYPES = [
|
|
|
2411
2879
|
];
|
|
2412
2880
|
/**
|
|
2413
2881
|
* Validates an `ImageBlock.mimeType` against the shared supported set.
|
|
2414
|
-
* Throws a non-retryable `LLMError('
|
|
2415
|
-
* mimeType is a
|
|
2416
|
-
* the same
|
|
2882
|
+
* Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
|
|
2883
|
+
* mimeType is a bug in the caller's own input, deterministic before any
|
|
2884
|
+
* request is built, the same class of failure as every other check in
|
|
2885
|
+
* `RequestBuilder`.
|
|
2417
2886
|
*/
|
|
2418
2887
|
function assertSupportedImageMimeType(mimeType) {
|
|
2419
2888
|
if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
|
|
2420
|
-
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "
|
|
2889
|
+
throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "invalid_params");
|
|
2421
2890
|
}
|
|
2422
2891
|
|
|
2423
2892
|
//#endregion
|
|
@@ -2730,7 +3199,7 @@ function toAnthropicMessage(m) {
|
|
|
2730
3199
|
try {
|
|
2731
3200
|
input = tc.function.arguments.trim() ? JSON.parse(tc.function.arguments) : {};
|
|
2732
3201
|
} catch (cause) {
|
|
2733
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation",
|
|
3202
|
+
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
2734
3203
|
}
|
|
2735
3204
|
if (input === null || Array.isArray(input) || typeof input !== "object") throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) arguments must be a JSON object.`, "validation");
|
|
2736
3205
|
blocks.push({
|
|
@@ -2811,7 +3280,10 @@ function parseToolArguments(text, toolName) {
|
|
|
2811
3280
|
try {
|
|
2812
3281
|
parsed = text.trim() ? JSON.parse(text) : {};
|
|
2813
3282
|
} catch (cause) {
|
|
2814
|
-
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "
|
|
3283
|
+
throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "parse", {
|
|
3284
|
+
cause,
|
|
3285
|
+
code: "tool_arguments_parse_failed"
|
|
3286
|
+
});
|
|
2815
3287
|
}
|
|
2816
3288
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
|
|
2817
3289
|
return parsed;
|
|
@@ -2945,7 +3417,10 @@ function fromGemini(geminiClient) {
|
|
|
2945
3417
|
};
|
|
2946
3418
|
},
|
|
2947
3419
|
async *createStream(params, options) {
|
|
2948
|
-
if (!geminiClient.generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "
|
|
3420
|
+
if (!geminiClient.generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
|
|
3421
|
+
code: "unsupported_capability",
|
|
3422
|
+
issues: { capability: "generateContentStream" }
|
|
3423
|
+
});
|
|
2949
3424
|
const request = buildGeminiRequest(params);
|
|
2950
3425
|
request.config = {
|
|
2951
3426
|
...request.config,
|
|
@@ -3093,7 +3568,10 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3093
3568
|
if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
|
|
3094
3569
|
if (jsonSchema && toolConfig && toolUseSupportedModels) {
|
|
3095
3570
|
const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
|
|
3096
|
-
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "
|
|
3571
|
+
if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "invalid_params", {
|
|
3572
|
+
code: "unsupported_capability",
|
|
3573
|
+
issues: { capability: "toolUseSupportedModels" }
|
|
3574
|
+
});
|
|
3097
3575
|
}
|
|
3098
3576
|
const systemParts = [systemMessage?.content, jsonInstruction].filter((s) => Boolean(s));
|
|
3099
3577
|
const request = {
|
|
@@ -3205,7 +3683,10 @@ function fromBedrock(bedrockClient, options) {
|
|
|
3205
3683
|
};
|
|
3206
3684
|
},
|
|
3207
3685
|
async *createStream(params, requestOptions) {
|
|
3208
|
-
if (!bedrockClient.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "
|
|
3686
|
+
if (!bedrockClient.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
|
|
3687
|
+
code: "unsupported_capability",
|
|
3688
|
+
issues: { capability: "converseStream" }
|
|
3689
|
+
});
|
|
3209
3690
|
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
|
|
3210
3691
|
const { stream } = await bedrockClient.converseStream(request, requestOptions);
|
|
3211
3692
|
const blockKinds = new Map();
|
|
@@ -3247,12 +3728,18 @@ function fromBedrock(bedrockClient, options) {
|
|
|
3247
3728
|
total_tokens: event.metadata.usage.totalTokens
|
|
3248
3729
|
}
|
|
3249
3730
|
};
|
|
3250
|
-
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api",
|
|
3731
|
+
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
|
|
3732
|
+
status: 429,
|
|
3733
|
+
code: "provider_rate_limited"
|
|
3734
|
+
});
|
|
3251
3735
|
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
3252
3736
|
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
3253
3737
|
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
3254
3738
|
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
3255
|
-
throw new LLMError(detail, "api",
|
|
3739
|
+
throw new LLMError(detail, "api", {
|
|
3740
|
+
status,
|
|
3741
|
+
code: status >= 500 ? "server_error" : void 0
|
|
3742
|
+
});
|
|
3256
3743
|
}
|
|
3257
3744
|
}
|
|
3258
3745
|
} } };
|
|
@@ -3261,7 +3748,10 @@ function fromBedrock(bedrockClient, options) {
|
|
|
3261
3748
|
function toBedrockToolChoice(toolChoice) {
|
|
3262
3749
|
if (!toolChoice || toolChoice === "auto") return { auto: {} };
|
|
3263
3750
|
if (toolChoice === "required") return { any: {} };
|
|
3264
|
-
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "
|
|
3751
|
+
if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "invalid_params", {
|
|
3752
|
+
code: "unsupported_capability",
|
|
3753
|
+
issues: { capability: "toolChoice: 'none'" }
|
|
3754
|
+
});
|
|
3265
3755
|
return { tool: { name: toolChoice.function.name } };
|
|
3266
3756
|
}
|
|
3267
3757
|
/**
|
|
@@ -3286,7 +3776,7 @@ function toBedrockMessage(m) {
|
|
|
3286
3776
|
else try {
|
|
3287
3777
|
input = JSON.parse(tc.function.arguments);
|
|
3288
3778
|
} catch (cause) {
|
|
3289
|
-
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation",
|
|
3779
|
+
throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
|
|
3290
3780
|
}
|
|
3291
3781
|
blocks.push({ toolUse: {
|
|
3292
3782
|
toolUseId: tc.id,
|
|
@@ -3457,8 +3947,14 @@ function fromFetch(config) {
|
|
|
3457
3947
|
};
|
|
3458
3948
|
},
|
|
3459
3949
|
async *createStream(params, options) {
|
|
3460
|
-
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "
|
|
3461
|
-
|
|
3950
|
+
if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "invalid_params", {
|
|
3951
|
+
code: "unsupported_capability",
|
|
3952
|
+
issues: { capability: "mapStreamEvent" }
|
|
3953
|
+
});
|
|
3954
|
+
if (config.request && !config.requestStream) throw new LLMError("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.", "invalid_params", {
|
|
3955
|
+
code: "unsupported_capability",
|
|
3956
|
+
issues: { capability: "requestStream" }
|
|
3957
|
+
});
|
|
3462
3958
|
const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
|
|
3463
3959
|
const requestStream = config.requestStream ?? defaultRequestStream;
|
|
3464
3960
|
const parseFrames = config.parseStreamFrames ?? parseSseStream;
|
|
@@ -3642,8 +4138,6 @@ const fromNvidiaNIM = fromOpenAICompatible;
|
|
|
3642
4138
|
const fromVercelAIGateway = fromOpenAICompatible;
|
|
3643
4139
|
/** Cloudflare Workers AI exposes an OpenAI-compatible endpoint */
|
|
3644
4140
|
const fromCloudflareWorkersAI = fromOpenAICompatible;
|
|
3645
|
-
/** GitHub Models is OpenAI-compatible */
|
|
3646
|
-
const fromGitHubModels = fromOpenAICompatible;
|
|
3647
4141
|
/** Nebius AI Studio is OpenAI-compatible */
|
|
3648
4142
|
const fromNebius = fromOpenAICompatible;
|
|
3649
4143
|
/** SambaNova Cloud's API is OpenAI-compatible */
|
|
@@ -3670,8 +4164,6 @@ const fromSnowflakeCortex = fromOpenAICompatible;
|
|
|
3670
4164
|
const fromAnyscale = fromOpenAICompatible;
|
|
3671
4165
|
/** Lepton AI's inference API is OpenAI-compatible */
|
|
3672
4166
|
const fromLepton = fromOpenAICompatible;
|
|
3673
|
-
/** kluster.ai's inference API is OpenAI-compatible */
|
|
3674
|
-
const fromKlusterAI = fromOpenAICompatible;
|
|
3675
4167
|
/** Inference.net's API is OpenAI-compatible */
|
|
3676
4168
|
const fromInferenceNet = fromOpenAICompatible;
|
|
3677
4169
|
/** Infermatic's API is OpenAI-compatible */
|
|
@@ -3709,12 +4201,10 @@ exports.fromFetch = fromFetch
|
|
|
3709
4201
|
exports.fromFireworks = fromFireworks
|
|
3710
4202
|
exports.fromFriendli = fromFriendli
|
|
3711
4203
|
exports.fromGemini = fromGemini
|
|
3712
|
-
exports.fromGitHubModels = fromGitHubModels
|
|
3713
4204
|
exports.fromGroq = fromGroq
|
|
3714
4205
|
exports.fromHyperbolic = fromHyperbolic
|
|
3715
4206
|
exports.fromInferenceNet = fromInferenceNet
|
|
3716
4207
|
exports.fromInfermatic = fromInfermatic
|
|
3717
|
-
exports.fromKlusterAI = fromKlusterAI
|
|
3718
4208
|
exports.fromLMStudio = fromLMStudio
|
|
3719
4209
|
exports.fromLambdaLabs = fromLambdaLabs
|
|
3720
4210
|
exports.fromLepton = fromLepton
|
|
@@ -3739,6 +4229,8 @@ exports.fromVLLM = fromVLLM
|
|
|
3739
4229
|
exports.fromVercelAIGateway = fromVercelAIGateway
|
|
3740
4230
|
exports.fromXAI = fromXAI
|
|
3741
4231
|
exports.fromZhipu = fromZhipu
|
|
4232
|
+
exports.hasIssues = hasIssues
|
|
4233
|
+
exports.isFallbackExhaustedError = isFallbackExhaustedError
|
|
3742
4234
|
exports.isLLMError = isLLMError
|
|
3743
4235
|
exports.isToolCallResult = isToolCallResult
|
|
3744
4236
|
exports.parseSseStream = parseSseStream
|