vern-llm 2.2.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/index.cjs +1512 -375
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +893 -222
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +893 -222
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1509 -376
- package/dist/index.js.map +1 -1
- package/package.json +16 -2
package/dist/index.d.cts
CHANGED
|
@@ -1,87 +1,22 @@
|
|
|
1
|
-
//#region src/circuitBreaker.d.ts
|
|
2
|
-
interface CircuitBreakerOptions {
|
|
3
|
-
/** Consecutive failures before the circuit opens, default 5 */
|
|
4
|
-
threshold?: number;
|
|
5
|
-
/** How long the circuit stays open before allowing a trial request, in ms. Default 30000 */
|
|
6
|
-
cooldownMs?: number;
|
|
7
|
-
/**
|
|
8
|
-
* Called after every real state change, never for a no-op transition
|
|
9
|
-
* (e.g. open to open). `model` is the resolved model of whichever call
|
|
10
|
-
* triggered this specific transition (the `model` passed to whichever
|
|
11
|
-
* of `assertClosed`/`recordSuccess`/`recordFailure` caused it).
|
|
12
|
-
*
|
|
13
|
-
* With `isolateByModel` off (the default), this is a label only: the
|
|
14
|
-
* breaker still counts failures across every model together, so a
|
|
15
|
-
* threshold crossing can be the sum of several different models'
|
|
16
|
-
* failures even though only the triggering call's `model` is reported
|
|
17
|
-
* here. With `isolateByModel` on, it's exact: each model has its own
|
|
18
|
-
* counter, so the transition really was caused solely by that model.
|
|
19
|
-
*/
|
|
20
|
-
onStateChange?: (from: CircuitState, to: CircuitState, consecutiveFailures: number, model?: string) => void;
|
|
21
|
-
/**
|
|
22
|
-
* Track a separate circuit per resolved model instead of one shared
|
|
23
|
-
* circuit for the whole instance. A failure on one model then never
|
|
24
|
-
* opens another model's circuit, at the cost of slower detection for
|
|
25
|
-
* an outage spread across many distinct models (each model's counter
|
|
26
|
-
* must independently cross `threshold`). Default false: one shared
|
|
27
|
-
* circuit, matching every version before this option existed.
|
|
28
|
-
*
|
|
29
|
-
* A call that omits `model` (only possible calling `CircuitBreaker`
|
|
30
|
-
* directly, `VernLLM` always passes one) falls into one shared bucket
|
|
31
|
-
* alongside every other call that also omits it.
|
|
32
|
-
*/
|
|
33
|
-
isolateByModel?: boolean;
|
|
34
|
-
}
|
|
35
|
-
type CircuitState = 'closed' | 'open' | 'half-open';
|
|
36
|
-
/**
|
|
37
|
-
* Per retry VernLLM-instance circuit breaker. Tracks consecutive failures across
|
|
38
|
-
* calls. Once the threshold is hit, short-circuits new calls with an
|
|
39
|
-
* LLMError('circuit_open') instead of hitting the provider, until the
|
|
40
|
-
* cooldown elapses and a single trial call is allowed through
|
|
41
|
-
*/
|
|
42
|
-
declare class CircuitBreaker {
|
|
43
|
-
private readonly threshold;
|
|
44
|
-
private readonly cooldownMs;
|
|
45
|
-
private readonly onStateChange?;
|
|
46
|
-
private readonly isolateByModel;
|
|
47
|
-
private readonly sharedBucket;
|
|
48
|
-
private readonly bucketsByModel;
|
|
49
|
-
constructor(options?: CircuitBreakerOptions);
|
|
50
|
-
/** Returns the bucket for a model if one already exists, without allocating. */
|
|
51
|
-
private lookupBucket;
|
|
52
|
-
/** Creates and stores a bucket for a model when the first mutation needs one. */
|
|
53
|
-
private ensureBucketFor;
|
|
54
|
-
/** Every state mutation routes through here, so `onStateChange` fires exactly once per real change. */
|
|
55
|
-
private transition;
|
|
56
|
-
/**
|
|
57
|
-
* Throws if the circuit is open and the cooldown hasn't elapsed, or if
|
|
58
|
-
* the circuit is half-open and a trial call is already in flight.
|
|
59
|
-
* Otherwise, if the circuit just became eligible for a trial (cooldown
|
|
60
|
-
* elapsed, or half-open with no trial currently running), this call
|
|
61
|
-
* becomes that trial
|
|
62
|
-
*/
|
|
63
|
-
assertClosed(model?: string): void;
|
|
64
|
-
recordSuccess(model?: string): void;
|
|
65
|
-
recordFailure(model?: string): void;
|
|
66
|
-
/**
|
|
67
|
-
* With `isolateByModel` off (the default), `model` is ignored and the
|
|
68
|
-
* one shared circuit's state is returned, unchanged from every version
|
|
69
|
-
* before this option existed. With `isolateByModel` on, returns that
|
|
70
|
-
* model's own state, `'closed'` for a model never seen yet, same as a
|
|
71
|
-
* fresh breaker.
|
|
72
|
-
*/
|
|
73
|
-
getState(model?: string): CircuitState;
|
|
74
|
-
} //#endregion
|
|
75
1
|
//#region src/types/errors.d.ts
|
|
76
|
-
|
|
77
|
-
//# sourceMappingURL=circuitBreaker.d.ts.map
|
|
78
|
-
type LLMErrorType = 'timeout' | 'api' | 'parse' | 'validation' | 'circuit_open' | 'quota_exceeded' | 'unknown' | 'aborted';
|
|
2
|
+
type LLMErrorType = 'timeout' | 'api' | 'network' | 'parse' | 'validation' | 'invalid_params' | 'rate_limited' | 'quota_exceeded' | 'circuit_open' | 'fallback_exhausted' | 'aborted' | 'unknown';
|
|
79
3
|
/**
|
|
80
4
|
* Machine readable discriminator within a `type`, for cases where `type`
|
|
81
5
|
* alone is too coarse to act on. Optional and additive: errors thrown
|
|
82
|
-
* before a given code existed simply omit it.
|
|
6
|
+
* before a given code existed simply omit it. Not owned by a single type;
|
|
7
|
+
* e.g. `authentication`/`authorization` apply the same way regardless of
|
|
8
|
+
* which type wraps them.
|
|
83
9
|
*/
|
|
84
|
-
type LLMErrorCode = 'unknown_tool' | 'duplicate_tool_call_id' | '
|
|
10
|
+
type LLMErrorCode = 'unknown_tool' | 'duplicate_tool_call_id' | 'tool_choice_none_violated' | 'unexpected_tool_calls' | 'unsupported_capability' | 'duplicate_tool_names' | 'unknown_tool_choice' | 'duplicate_tool_result_ids' | 'unknown_tool_result_ids' | 'missing_tool_results' | 'rate_limit_queue_full' | 'rate_limit_queue_timeout' | 'rate_limit_capacity_exceeded' | 'provider_rate_limited' | 'request_timeout' | 'idle_timeout' | 'authentication' | 'authorization' | 'not_found' | 'payload_too_large' | 'server_error' | 'empty_response' | 'connection_failed' | 'circuit_cooling_down' | 'circuit_trial_in_flight' | 'fallback_exhausted' | 'tool_arguments_parse_failed' | 'stream_frame_invalid';
|
|
11
|
+
/**
|
|
12
|
+
* Tool contract codes: a model or provider response defect, not a
|
|
13
|
+
* transient provider fault. Deterministic on the wire request, so
|
|
14
|
+
* retrying can't change the outcome and it shouldn't count toward the
|
|
15
|
+
* circuit breaker either. Shared by `LLMError.retryable` below and by
|
|
16
|
+
* `CallExecutor`'s own retry/breaker accounting, so the two can't drift
|
|
17
|
+
* apart.
|
|
18
|
+
*/
|
|
19
|
+
|
|
85
20
|
/** One tool call's contract failure, used to report every bad call in a response at once. */
|
|
86
21
|
interface ToolIssue {
|
|
87
22
|
name: string;
|
|
@@ -89,23 +24,219 @@ interface ToolIssue {
|
|
|
89
24
|
code: LLMErrorCode;
|
|
90
25
|
detail?: unknown;
|
|
91
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* The specific values behind a `duplicate_tool_names` failure: the
|
|
29
|
+
* offending call's `tools` array had more than one entry sharing a name.
|
|
30
|
+
*/
|
|
31
|
+
interface DuplicateToolNamesIssue {
|
|
32
|
+
names: string[];
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The specific values behind an `unknown_tool_choice` failure: `toolChoice`
|
|
36
|
+
* named a tool that wasn't in the call's own `tools` array.
|
|
37
|
+
*/
|
|
38
|
+
interface UnknownToolChoiceIssue {
|
|
39
|
+
requested: string;
|
|
40
|
+
available: string[];
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The specific values behind a `duplicate_tool_result_ids` /
|
|
44
|
+
* `unknown_tool_result_ids` / `missing_tool_results` failure: which
|
|
45
|
+
* `history` turn was affected, and which `toolCallId`s were the problem.
|
|
46
|
+
*/
|
|
47
|
+
interface HistoryToolResultIssue {
|
|
48
|
+
historyIndex: number;
|
|
49
|
+
ids: string[];
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The specific values behind an `unsupported_capability` failure: which
|
|
53
|
+
* capability the current adapter/client/model doesn't support.
|
|
54
|
+
*/
|
|
55
|
+
interface UnsupportedCapabilityIssue {
|
|
56
|
+
capability: string;
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Maps each `LLMErrorCode` that carries structured `issues` to that
|
|
60
|
+
* payload's exact shape. Not every code appears here: most `invalid_params`
|
|
61
|
+
* failures are a single deterministic fact the `message` already states in
|
|
62
|
+
* full, so adding a typed `issues` entry for them would only duplicate the
|
|
63
|
+
* message into a field, the same near-duplicate-code problem `code` itself
|
|
64
|
+
* avoids. Codes that repeat here are exactly the ones whose `message`
|
|
65
|
+
* already string-joins a list a caller might want to consume directly
|
|
66
|
+
* rather than re-parse out of prose, or that otherwise want a place to
|
|
67
|
+
* report the exact captured values of a failure.
|
|
68
|
+
*
|
|
69
|
+
* Deliberately not a mapped type over the whole `LLMErrorCode` union: a
|
|
70
|
+
* schema-validation failure's `issues` (the caller's own Zod-compatible
|
|
71
|
+
* validator's error object) has no code and no shape VernLLM could know in
|
|
72
|
+
* advance, so it stays untyped on `LLMError.issues` itself rather than
|
|
73
|
+
* forcing every code into this table.
|
|
74
|
+
*/
|
|
75
|
+
interface LLMErrorIssuesByCode {
|
|
76
|
+
unknown_tool: ToolIssue[];
|
|
77
|
+
duplicate_tool_call_id: ToolIssue[];
|
|
78
|
+
duplicate_tool_names: DuplicateToolNamesIssue;
|
|
79
|
+
unknown_tool_choice: UnknownToolChoiceIssue;
|
|
80
|
+
duplicate_tool_result_ids: HistoryToolResultIssue;
|
|
81
|
+
unknown_tool_result_ids: HistoryToolResultIssue;
|
|
82
|
+
missing_tool_results: HistoryToolResultIssue;
|
|
83
|
+
unsupported_capability: UnsupportedCapabilityIssue;
|
|
84
|
+
}
|
|
85
|
+
/**
|
|
86
|
+
* Point-in-time copy of an `LLMError`'s fields, produced by
|
|
87
|
+
* `LLMError.toSnapshot()`. This is what `RetryAttempt.error` holds
|
|
88
|
+
* instead of a live `LLMError`.
|
|
89
|
+
*
|
|
90
|
+
* A past attempt only needs to be describable (message, type, code,
|
|
91
|
+
* whether it was retryable), never thrown again. So it skips `Error`'s
|
|
92
|
+
* behavior, `instanceof` identity, and any live getter. Using the full
|
|
93
|
+
* `LLMError` class here would also make the type self referential
|
|
94
|
+
* through its own `attempts` field.
|
|
95
|
+
*
|
|
96
|
+
* Has no `cause`. `cause` is `unknown` and never validated by VernLLM,
|
|
97
|
+
* and it is meant to be read directly on the live error you just
|
|
98
|
+
* caught, not carried indefinitely inside history. `type`, `code`,
|
|
99
|
+
* `status`, and `issues` are the structured fields a snapshot carries
|
|
100
|
+
* instead.
|
|
101
|
+
*
|
|
102
|
+
* `attempts` is still present, since a recorded attempt can itself be
|
|
103
|
+
* the terminal failure of an inner retry loop with its own history (see
|
|
104
|
+
* `FallbackAttempt`). That's a tree of past data, not a cycle.
|
|
105
|
+
*/
|
|
106
|
+
interface LLMErrorSnapshot {
|
|
107
|
+
message: string;
|
|
108
|
+
type: LLMErrorType;
|
|
109
|
+
status?: number;
|
|
110
|
+
issues?: unknown;
|
|
111
|
+
retryAfterMs?: number;
|
|
112
|
+
code?: LLMErrorCode;
|
|
113
|
+
/** Computed once, at snapshot time, since a snapshot has no live getter. */
|
|
114
|
+
retryable: boolean;
|
|
115
|
+
/** This attempt's own prior attempts, if it was itself the terminal failure of a retry loop. */
|
|
116
|
+
attempts?: RetryAttempt[];
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Point-in-time copy of the request an attempt sent, produced by
|
|
120
|
+
* `toRequestSnapshot()`. This is what `RetryAttempt.request` holds.
|
|
121
|
+
* Mirrors `LLMErrorSnapshot`: plain data, never thrown or dispatched
|
|
122
|
+
* again, safe to serialize and store.
|
|
123
|
+
*/
|
|
124
|
+
interface LLMRequestSnapshot {
|
|
125
|
+
/** Provider id this attempt targeted, e.g. "openai". */
|
|
126
|
+
provider: string;
|
|
127
|
+
/** Model id this attempt targeted. */
|
|
128
|
+
model: string;
|
|
129
|
+
/** The payload as actually sent for this attempt, after any transform/repair. Passed through `safeBody`. */
|
|
130
|
+
body: unknown;
|
|
131
|
+
/** Non sensitive request headers. Auth headers are stripped before the snapshot is built, never included. */
|
|
132
|
+
headers?: Record<string, string>;
|
|
133
|
+
/** Wall clock time the attempt started, ms since epoch. */
|
|
134
|
+
startedAt: number;
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Builds a point-in-time, plain data copy of one attempt's outgoing
|
|
138
|
+
* request. Mirrors `LLMError.toSnapshot()`: never thrown or dispatched
|
|
139
|
+
* again, safe to serialize and store. A plain function rather than a
|
|
140
|
+
* method, since unlike `LLMError` a request has no throwable identity or
|
|
141
|
+
* derived state worth wrapping in a class.
|
|
142
|
+
*
|
|
143
|
+
* `startedAt` is optional so existing call sites (and tests) that don't
|
|
144
|
+
* care about exact timing keep working, but a caller that has a real
|
|
145
|
+
* capture time should always pass it: this function may run well after
|
|
146
|
+
* the request was actually dispatched (e.g. `callExecutor` only builds
|
|
147
|
+
* the snapshot once an attempt has failed), so defaulting to `Date.now()`
|
|
148
|
+
* here would record failure-handling time, not request-start time.
|
|
149
|
+
*/
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* One failed attempt on the way to a terminal error: which attempt index
|
|
153
|
+
* it was, and a snapshot of the error it failed with. The base shape
|
|
154
|
+
* every richer attempt record (e.g. `FallbackAttempt`) extends, rather
|
|
155
|
+
* than duplicates.
|
|
156
|
+
*/
|
|
157
|
+
interface RetryAttempt {
|
|
158
|
+
index: number;
|
|
159
|
+
error: LLMErrorSnapshot;
|
|
160
|
+
/** What was sent for this attempt. Optional: absent for attempts predating this field. */
|
|
161
|
+
request?: LLMRequestSnapshot;
|
|
162
|
+
}
|
|
163
|
+
/** Optional fields for constructing an {@link LLMError}. `message` and `type` stay positional since every throw site sets both. */
|
|
164
|
+
interface LLMErrorOptions {
|
|
165
|
+
status?: number;
|
|
166
|
+
issues?: unknown;
|
|
167
|
+
cause?: unknown;
|
|
168
|
+
retryAfterMs?: number;
|
|
169
|
+
/** Stable discriminator within `type`. Absent on errors predating it. */
|
|
170
|
+
code?: LLMErrorCode;
|
|
171
|
+
/** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
|
|
172
|
+
attempts?: RetryAttempt[];
|
|
173
|
+
}
|
|
92
174
|
declare class LLMError extends Error {
|
|
93
175
|
type: LLMErrorType;
|
|
94
|
-
status?: number
|
|
95
|
-
issues?: unknown
|
|
96
|
-
cause?: unknown
|
|
97
|
-
retryAfterMs?: number
|
|
176
|
+
status?: number;
|
|
177
|
+
issues?: unknown;
|
|
178
|
+
cause?: unknown;
|
|
179
|
+
retryAfterMs?: number;
|
|
98
180
|
/** Stable discriminator within `type`. Absent on errors predating it. */
|
|
99
|
-
code?: LLMErrorCode
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
181
|
+
code?: LLMErrorCode;
|
|
182
|
+
/** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
|
|
183
|
+
attempts?: RetryAttempt[];
|
|
184
|
+
constructor(message: string, type: LLMErrorType, options?: LLMErrorOptions);
|
|
185
|
+
/**
|
|
186
|
+
* Computed purely from `type`/`code`, independent of any specific call's
|
|
187
|
+
* `nonRetryableStatus` list. False for `parse`/`validation`/
|
|
188
|
+
* `invalid_params`/`aborted` types (the caller's own input, the model's
|
|
189
|
+
* own response, or intentional cancellation, none of which are the
|
|
190
|
+
* provider being unhealthy), the tool contract codes, and the local
|
|
191
|
+
* rate limit codes. Subclasses (see `FallbackExhaustedError`) may
|
|
192
|
+
* override this when `type` alone carries no retry signal.
|
|
193
|
+
*/
|
|
194
|
+
get retryable(): boolean;
|
|
195
|
+
/**
|
|
196
|
+
* Copies this error's fields into an {@link LLMErrorSnapshot}, for
|
|
197
|
+
* recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
|
|
198
|
+
* captured here since a snapshot has no getter of its own. `cause` is
|
|
199
|
+
* not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
|
|
200
|
+
* nested `attempts` entry's own `issues` go through `safeAttempts`,
|
|
201
|
+
* since a schema validation failure's `issues` is a caller supplied
|
|
202
|
+
* value, not controlled by VernLLM, and `attempts` is itself a public
|
|
203
|
+
* constructor option a caller can hand build.
|
|
204
|
+
*/
|
|
205
|
+
toSnapshot(): LLMErrorSnapshot;
|
|
206
|
+
/**
|
|
207
|
+
* Controls what `JSON.stringify(err)` produces. Omits `cause` for the
|
|
208
|
+
* same reason `toSnapshot()` does: `cause` is `unknown` and never
|
|
209
|
+
* validated by VernLLM, and some SDK errors carry circular structures
|
|
210
|
+
* `JSON.stringify` cannot serialize at all. Read `err.cause` directly
|
|
211
|
+
* instead. `issues`, including every nested `attempts` entry's own
|
|
212
|
+
* `issues`, goes through `safeAttempts` for the same reason: a schema
|
|
213
|
+
* validation failure's `issues` is caller supplied and not guaranteed
|
|
214
|
+
* circular free. Also includes `message` and `retryable`, which a
|
|
215
|
+
* plain property walk would otherwise miss: `message` is
|
|
216
|
+
* non-enumerable on `Error`, and `retryable` is a getter, not an own
|
|
217
|
+
* property.
|
|
218
|
+
*/
|
|
219
|
+
toJSON(): Record<string, unknown>;
|
|
104
220
|
}
|
|
105
221
|
declare function isLLMError(err: unknown): err is LLMError;
|
|
106
|
-
|
|
107
|
-
|
|
222
|
+
/**
|
|
223
|
+
* Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
|
|
224
|
+
* `code` to, for any code listed there. `code` stays the only discriminator
|
|
225
|
+
* VernLLM uses; this just gives that existing check a typed return instead
|
|
226
|
+
* of requiring a manual cast of `issues`:
|
|
227
|
+
*
|
|
228
|
+
* ```ts
|
|
229
|
+
* if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
|
|
230
|
+
* console.log(err.issues.names); // string[], no cast needed
|
|
231
|
+
* }
|
|
232
|
+
* ```
|
|
233
|
+
*/
|
|
234
|
+
declare function hasIssues<C extends keyof LLMErrorIssuesByCode>(err: LLMError, code: C): err is LLMError & {
|
|
235
|
+
code: C;
|
|
236
|
+
issues: LLMErrorIssuesByCode[C];
|
|
237
|
+
}; //#endregion
|
|
108
238
|
//#region src/types/cache.d.ts
|
|
239
|
+
|
|
109
240
|
//# sourceMappingURL=errors.d.ts.map
|
|
110
241
|
interface CacheAdapter<T = unknown> {
|
|
111
242
|
get(key: string): Promise<{
|
|
@@ -172,8 +303,102 @@ declare class TieredCacheAdapter<T = unknown> implements CacheAdapter<T> {
|
|
|
172
303
|
}
|
|
173
304
|
|
|
174
305
|
//#endregion
|
|
175
|
-
//#region src/
|
|
306
|
+
//#region src/circuitBreaker.d.ts
|
|
176
307
|
//# sourceMappingURL=cache.d.ts.map
|
|
308
|
+
interface CircuitBreakerOptions {
|
|
309
|
+
/** Consecutive failures before the circuit opens, default 5 */
|
|
310
|
+
threshold?: number;
|
|
311
|
+
/** How long the circuit stays open before allowing a trial request, in ms. Default 30000 */
|
|
312
|
+
cooldownMs?: number;
|
|
313
|
+
/**
|
|
314
|
+
* Called after every real state change, never for a no-op transition
|
|
315
|
+
* (e.g. open to open). `model` is the resolved model of whichever call
|
|
316
|
+
* triggered this specific transition (the `model` passed to whichever
|
|
317
|
+
* of `assertClosed`/`recordSuccess`/`recordFailure` caused it).
|
|
318
|
+
*
|
|
319
|
+
* With `isolateByModel` off (the default), this is a label only: the
|
|
320
|
+
* breaker still counts failures across every model together, so a
|
|
321
|
+
* threshold crossing can be the sum of several different models'
|
|
322
|
+
* failures even though only the triggering call's `model` is reported
|
|
323
|
+
* here. With `isolateByModel` on, it's exact: each model has its own
|
|
324
|
+
* counter, so the transition really was caused solely by that model.
|
|
325
|
+
*/
|
|
326
|
+
onStateChange?: (from: CircuitState, to: CircuitState, consecutiveFailures: number, model?: string) => void;
|
|
327
|
+
/**
|
|
328
|
+
* Track a separate circuit per resolved model instead of one shared
|
|
329
|
+
* circuit for the whole instance. A failure on one model then never
|
|
330
|
+
* opens another model's circuit, at the cost of slower detection for
|
|
331
|
+
* an outage spread across many distinct models (each model's counter
|
|
332
|
+
* must independently cross `threshold`). Default false: one shared
|
|
333
|
+
* circuit, matching every version before this option existed.
|
|
334
|
+
*
|
|
335
|
+
* A call that omits `model` (only possible calling `CircuitBreaker`
|
|
336
|
+
* directly, `VernLLM` always passes one) falls into one shared bucket
|
|
337
|
+
* alongside every other call that also omits it.
|
|
338
|
+
*/
|
|
339
|
+
isolateByModel?: boolean;
|
|
340
|
+
}
|
|
341
|
+
type CircuitState = 'closed' | 'open' | 'half-open';
|
|
342
|
+
/**
|
|
343
|
+
* Per retry VernLLM-instance circuit breaker. Tracks consecutive failures across
|
|
344
|
+
* calls. Once the threshold is hit, short-circuits new calls with an
|
|
345
|
+
* LLMError('circuit_open') instead of hitting the provider, until the
|
|
346
|
+
* cooldown elapses and a single trial call is allowed through
|
|
347
|
+
*/
|
|
348
|
+
declare class CircuitBreaker {
|
|
349
|
+
private readonly threshold;
|
|
350
|
+
private readonly cooldownMs;
|
|
351
|
+
private readonly onStateChange?;
|
|
352
|
+
/** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
|
|
353
|
+
readonly isolateByModel: boolean;
|
|
354
|
+
private readonly sharedBucket;
|
|
355
|
+
private readonly bucketsByModel;
|
|
356
|
+
constructor(options?: CircuitBreakerOptions);
|
|
357
|
+
/** Returns the bucket for a model if one already exists, without allocating. */
|
|
358
|
+
private lookupBucket;
|
|
359
|
+
/** Creates and stores a bucket for a model when the first mutation needs one. */
|
|
360
|
+
private ensureBucketFor;
|
|
361
|
+
/** Every state mutation routes through here, so `onStateChange` fires exactly once per real change. */
|
|
362
|
+
private transition;
|
|
363
|
+
/**
|
|
364
|
+
* Throws if the circuit is open and the cooldown hasn't elapsed, or if
|
|
365
|
+
* the circuit is half-open and a trial call is already in flight.
|
|
366
|
+
* Otherwise, if the circuit just became eligible for a trial (cooldown
|
|
367
|
+
* elapsed, or half-open with no trial currently running), this call
|
|
368
|
+
* becomes that trial
|
|
369
|
+
*/
|
|
370
|
+
assertClosed(model?: string): void;
|
|
371
|
+
recordSuccess(model?: string): void;
|
|
372
|
+
recordFailure(model?: string): void;
|
|
373
|
+
/**
|
|
374
|
+
* With `isolateByModel` off (the default), `model` is ignored and the
|
|
375
|
+
* one shared circuit's state is returned, unchanged from every version
|
|
376
|
+
* before this option existed. With `isolateByModel` on, returns that
|
|
377
|
+
* model's own state, `'closed'` for a model never seen yet, same as a
|
|
378
|
+
* fresh breaker.
|
|
379
|
+
*/
|
|
380
|
+
getState(model?: string): CircuitState;
|
|
381
|
+
/**
|
|
382
|
+
* Manually opens the circuit, as if `threshold` consecutive failures had
|
|
383
|
+
* just happened, e.g. to pull a provider out of rotation ahead of known
|
|
384
|
+
* maintenance. Resets the cooldown window from now, same as a real
|
|
385
|
+
* threshold-crossing failure would, and clears any in-flight half-open
|
|
386
|
+
* trial since it no longer applies once the circuit is (re)opened.
|
|
387
|
+
*/
|
|
388
|
+
open(model?: string): void;
|
|
389
|
+
/**
|
|
390
|
+
* Manually closes the circuit and resets its failure count, e.g. once a
|
|
391
|
+
* provider is confirmed healthy again without waiting out the cooldown.
|
|
392
|
+
* Mirrors `recordSuccess`'s bookkeeping (including dropping the
|
|
393
|
+
* per-model bucket under `isolateByModel`, once idle) but without
|
|
394
|
+
* requiring an actual successful call first.
|
|
395
|
+
*/
|
|
396
|
+
close(model?: string): void;
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
//#endregion
|
|
400
|
+
//#region src/rateLimit.d.ts
|
|
401
|
+
//# sourceMappingURL=circuitBreaker.d.ts.map
|
|
177
402
|
/** The request shape sent to `LLMClient['chat']['completions']['create']`, used for token estimation. */
|
|
178
403
|
type WireRequest = Parameters<LLMClient['chat']['completions']['create']>[0];
|
|
179
404
|
/** Which configured bucket is currently blocking a call. */
|
|
@@ -305,6 +530,8 @@ interface FallbackTarget {
|
|
|
305
530
|
baseDelayMs?: number;
|
|
306
531
|
defaultMaxTokens?: number;
|
|
307
532
|
defaultTemperature?: number | null;
|
|
533
|
+
defaultReasoningEffort?: 'minimal' | 'low' | 'medium' | 'high';
|
|
534
|
+
defaultBudgetTokens?: number;
|
|
308
535
|
nonRetryableStatus?: number[];
|
|
309
536
|
/** This target's own circuit breaker, independent of every other target's. Not inherited from the parent's `circuitBreaker`. */
|
|
310
537
|
circuitBreaker?: boolean | CircuitBreakerOptions;
|
|
@@ -331,16 +558,27 @@ interface TargetCircuitState {
|
|
|
331
558
|
/** Position in the chain: `0` for the primary, `1`+ for fallback targets. */
|
|
332
559
|
index: number;
|
|
333
560
|
isFallback: boolean;
|
|
561
|
+
/** Whether this target tracks failures per model. `false` means `model` on `getCircuitStates` had no effect on this entry. */
|
|
562
|
+
isolateByModel: boolean;
|
|
334
563
|
/** `undefined` if that target has no circuit breaker configured. */
|
|
335
564
|
state: CircuitState | undefined;
|
|
336
565
|
}
|
|
337
|
-
/**
|
|
338
|
-
interface
|
|
339
|
-
/**
|
|
340
|
-
index
|
|
566
|
+
/** Which target/model `VernLLM.getCircuitState`, `openCircuit`, and `closeCircuit` act on. */
|
|
567
|
+
interface CircuitTarget {
|
|
568
|
+
/** Which target to act on. `0` is the primary, `1`+ are fallbacks. Defaults to `0`. */
|
|
569
|
+
index?: number;
|
|
570
|
+
/** Which model bucket to act on, if the resolved target isolates by model. */
|
|
571
|
+
model?: string;
|
|
572
|
+
}
|
|
573
|
+
/**
|
|
574
|
+
* One target's failure, recorded on the way to either the next target or
|
|
575
|
+
* `FallbackExhaustedError`. Extends `RetryAttempt`: `index` is `-1` for
|
|
576
|
+
* the primary target here (rather than a plain retry count), and
|
|
577
|
+
* `provider`/`model` identify which target failed.
|
|
578
|
+
*/
|
|
579
|
+
interface FallbackAttempt extends RetryAttempt {
|
|
341
580
|
provider: string;
|
|
342
581
|
model: string;
|
|
343
|
-
error: LLMError;
|
|
344
582
|
}
|
|
345
583
|
/**
|
|
346
584
|
* Decides what happens after a target's own retries are exhausted or
|
|
@@ -368,7 +606,16 @@ declare const defaultFallbackOn: FallbackOn;
|
|
|
368
606
|
declare class FallbackExhaustedError extends LLMError {
|
|
369
607
|
readonly attempts: FallbackAttempt[];
|
|
370
608
|
constructor(attempts: FallbackAttempt[]);
|
|
609
|
+
/**
|
|
610
|
+
* `type: 'fallback_exhausted'` by itself says nothing about whether
|
|
611
|
+
* retrying could help; the reason the last target failed does. Defers to
|
|
612
|
+
* that attempt's own `retryable` instead of anything about this class's
|
|
613
|
+
* own type.
|
|
614
|
+
*/
|
|
615
|
+
get retryable(): boolean;
|
|
371
616
|
}
|
|
617
|
+
/** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
|
|
618
|
+
declare function isFallbackExhaustedError(err: unknown): err is FallbackExhaustedError;
|
|
372
619
|
|
|
373
620
|
//#endregion
|
|
374
621
|
//#region src/types/schema.d.ts
|
|
@@ -505,6 +752,13 @@ interface TokenUsage {
|
|
|
505
752
|
promptTokens: number;
|
|
506
753
|
completionTokens: number;
|
|
507
754
|
totalTokens: number;
|
|
755
|
+
/**
|
|
756
|
+
* Tokens spent on internal reasoning, a subset of `completionTokens`,
|
|
757
|
+
* never added on top of it. Undefined when the provider's response
|
|
758
|
+
* doesn't report a separate reasoning figure, e.g. Bedrock Converse
|
|
759
|
+
* without an explicit `additionalModelResponseFieldPaths` request.
|
|
760
|
+
*/
|
|
761
|
+
reasoningTokens?: number;
|
|
508
762
|
requestId: string;
|
|
509
763
|
model: string;
|
|
510
764
|
/**
|
|
@@ -537,6 +791,20 @@ type OnUsageFailure = (usage: TokenUsage, error: LLMError) => void;
|
|
|
537
791
|
//#endregion
|
|
538
792
|
//#region src/types/call.d.ts
|
|
539
793
|
//# sourceMappingURL=usage.d.ts.map
|
|
794
|
+
/**
|
|
795
|
+
* Any valid JSON value: a primitive, `null`, or a JSON array/object made
|
|
796
|
+
* of the same. This is what `call()` returns when `jsonMode: true`.
|
|
797
|
+
*/
|
|
798
|
+
type JsonValue = string | number | boolean | null | JsonValue[] | {
|
|
799
|
+
[key: string]: JsonValue;
|
|
800
|
+
};
|
|
801
|
+
/**
|
|
802
|
+
* Content for an `assistant` turn in `history`. Accepts a string or a
|
|
803
|
+
* parsed `JsonValue`, so a prior `jsonMode: true` response can be pushed
|
|
804
|
+
* straight back into history. Request construction stringifies non-string
|
|
805
|
+
* content before it's sent to the provider.
|
|
806
|
+
*/
|
|
807
|
+
type AssistantContent = string | JsonValue;
|
|
540
808
|
/**
|
|
541
809
|
* A single prior turn in a multi-turn conversation, passed via `history`.
|
|
542
810
|
*
|
|
@@ -550,7 +818,7 @@ type ConversationTurn = {
|
|
|
550
818
|
content: string;
|
|
551
819
|
} | {
|
|
552
820
|
role: 'assistant';
|
|
553
|
-
content?:
|
|
821
|
+
content?: AssistantContent;
|
|
554
822
|
toolCalls?: ToolCall[];
|
|
555
823
|
} | {
|
|
556
824
|
role: 'tool';
|
|
@@ -607,8 +875,29 @@ interface CallParams<T = unknown> extends UsageHooks {
|
|
|
607
875
|
chunkIdleTimeoutMs?: number;
|
|
608
876
|
/** Overrides the instance model for this call. */
|
|
609
877
|
model?: string;
|
|
610
|
-
/**
|
|
611
|
-
|
|
878
|
+
/**
|
|
879
|
+
* Reasoning effort for supported reasoning models. Pass `null` to
|
|
880
|
+
* explicitly skip an instance-level `defaultReasoningEffort` for this
|
|
881
|
+
* one call (e.g. a call using a forced `toolChoice`, which Anthropic
|
|
882
|
+
* rejects alongside any reasoning at all), the same way `temperature:
|
|
883
|
+
* null` opts a call out of `defaultTemperature`. Omitting the field
|
|
884
|
+
* entirely (`undefined`) defers to the instance default instead.
|
|
885
|
+
*/
|
|
886
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high' | null;
|
|
887
|
+
/**
|
|
888
|
+
* Token budget for internal reasoning, for models with a native numeric
|
|
889
|
+
* budget (Anthropic's `budget_tokens`, Gemini's `thinkingBudget`). On a
|
|
890
|
+
* provider that only understands `reasoningEffort` tiers (OpenAI-
|
|
891
|
+
* compatible), this is converted to the nearest tier instead of sent as
|
|
892
|
+
* a raw number. When both `budgetTokens` and `reasoningEffort` are set,
|
|
893
|
+
* each adapter prefers whichever field it natively understands and
|
|
894
|
+
* ignores the other. See the reasoning budget docs for the conversion
|
|
895
|
+
* table used in each direction. Pass `null` to explicitly skip an
|
|
896
|
+
* instance-level `defaultBudgetTokens` for this one call, mirroring
|
|
897
|
+
* `reasoningEffort: null` above; omitting the field entirely defers to
|
|
898
|
+
* the instance default.
|
|
899
|
+
*/
|
|
900
|
+
budgetTokens?: number | null;
|
|
612
901
|
/**
|
|
613
902
|
* Provider-native JSON Schema output constraint. Implies jsonMode: true.
|
|
614
903
|
*/
|
|
@@ -619,23 +908,9 @@ interface CallParams<T = unknown> extends UsageHooks {
|
|
|
619
908
|
*/
|
|
620
909
|
schema?: SchemaLike<T>;
|
|
621
910
|
/**
|
|
622
|
-
* Tools the model may call. When set, `call()`
|
|
623
|
-
* `CallWithToolsResult<T>`
|
|
624
|
-
*
|
|
625
|
-
* keeps `call()`'s old `Promise<T>` behavior exactly.
|
|
626
|
-
*
|
|
627
|
-
* Can be combined with `jsonSchema` on Gemini and OpenAI-compatible
|
|
628
|
-
* clients unconditionally (neither ever restricted the combination:
|
|
629
|
-
* Gemini builds `responseSchema`/`tools` as independent fields, OpenAI-
|
|
630
|
-
* compatible clients pass both straight through). On Anthropic and
|
|
631
|
-
* Bedrock, combining the two is opt-in per call site, via each
|
|
632
|
-
* adapter's `nativeStructuredOutputModels` option: models not covered
|
|
633
|
-
* by it still throw `LLMError('validation')`, since `jsonSchema` falls
|
|
634
|
-
* back to a forced single-tool call there, which would collide with
|
|
635
|
-
* real tools. See `fromAnthropic`/`fromBedrock`.
|
|
636
|
-
*
|
|
637
|
-
* `schema` (client-side validation, distinct from `jsonSchema`) was
|
|
638
|
-
* never restricted from combining with `tools` on any provider.
|
|
911
|
+
* Tools the model may call. When set, `call()` returns a
|
|
912
|
+
* `CallWithToolsResult<T>` union instead of `T` directly. Combining with
|
|
913
|
+
* `jsonSchema` is provider-dependent; see the Tool Calling docs.
|
|
639
914
|
*/
|
|
640
915
|
tools?: ToolDefinition[];
|
|
641
916
|
/** Defaults to `'auto'` when `tools` is set. */
|
|
@@ -677,6 +952,20 @@ interface CallParams<T = unknown> extends UsageHooks {
|
|
|
677
952
|
type ToolEnabledCallParams<T> = CallParams<T> & {
|
|
678
953
|
tools: NonNullable<CallParams<T>['tools']>;
|
|
679
954
|
};
|
|
955
|
+
/**
|
|
956
|
+
* A `CallParams` variant for tools set conditionally, e.g. `tools:
|
|
957
|
+
* someCondition ? [myTool] : undefined`. Selects the `call()` overload
|
|
958
|
+
* returning the honest union `T | CallWithToolsResult<T>` instead of
|
|
959
|
+
* falling through to plain `T` (which is what happened before this type
|
|
960
|
+
* existed, since `ToolDefinition[] | undefined` matched neither
|
|
961
|
+
* `ToolEnabledCallParams` nor `ToolsDisabledCallParams`). Forces an
|
|
962
|
+
* `isToolCallResult()` check before treating the result as plain
|
|
963
|
+
* content. Omitting `tools` entirely still resolves to plain `T`, since
|
|
964
|
+
* tools genuinely cannot have run there.
|
|
965
|
+
*/
|
|
966
|
+
type ConditionalToolCallParams<T> = CallParams<T> & {
|
|
967
|
+
tools: ToolDefinition[] | undefined;
|
|
968
|
+
};
|
|
680
969
|
/**
|
|
681
970
|
* A `CallParams` variant where tools are offered but the model is barred
|
|
682
971
|
* from calling one. `toolChoice: 'none'` guarantees the response can never
|
|
@@ -691,6 +980,35 @@ type ToolsDisabledCallParams<T> = CallParams<T> & {
|
|
|
691
980
|
tools: NonNullable<CallParams<T>['tools']>;
|
|
692
981
|
toolChoice: 'none';
|
|
693
982
|
};
|
|
983
|
+
/**
|
|
984
|
+
* `CallParams` with `jsonMode: false`. Selects the `call()` overload
|
|
985
|
+
* that returns a plain `string`. `jsonSchema` is typed `never` here: a
|
|
986
|
+
* truthy `jsonSchema` forces JSON parsing at runtime regardless of
|
|
987
|
+
* `jsonMode` (see `RequestBuilder.build()`), so `jsonMode: false` +
|
|
988
|
+
* `jsonSchema` together would otherwise still match this overload and
|
|
989
|
+
* falsely promise a `string`.
|
|
990
|
+
*/
|
|
991
|
+
type JsonModeDisabledCallParams = Omit<CallParams<unknown>, 'jsonSchema'> & {
|
|
992
|
+
jsonMode: false;
|
|
993
|
+
jsonSchema?: never;
|
|
994
|
+
};
|
|
995
|
+
/**
|
|
996
|
+
* `CallParams` with `jsonMode: true` and no `schema`. Selects the
|
|
997
|
+
* `call()` overload that returns a `JsonValue`.
|
|
998
|
+
*
|
|
999
|
+
* `schema` is explicitly typed `never` here, not just omitted: `CallParams<JsonValue>['schema']`
|
|
1000
|
+
* would be `SchemaLike<JsonValue> | undefined`, and a schema whose inferred result type is
|
|
1001
|
+
* itself structurally assignable to `JsonValue` (e.g. a schema for `string[]` or
|
|
1002
|
+
* `Record<string, string>`) would still satisfy that shape, incorrectly selecting this
|
|
1003
|
+
* overload over the schema-aware generic one and widening the result to `JsonValue`. Forcing
|
|
1004
|
+
* `schema?: never` makes any call that sets `schema` fail this overload's structural check
|
|
1005
|
+
* regardless of the schema's result type, so it always falls through to the generic
|
|
1006
|
+
* `CallParams<T>` overload and infers `T` from the schema instead.
|
|
1007
|
+
*/
|
|
1008
|
+
type JsonModeEnabledCallParams = Omit<CallParams<JsonValue>, 'schema'> & {
|
|
1009
|
+
jsonMode: true;
|
|
1010
|
+
schema?: never;
|
|
1011
|
+
};
|
|
694
1012
|
/** Shared cache-configuration fields, minus the internal `fn` primitive. */
|
|
695
1013
|
interface CachedCallInput extends UsageHooks {
|
|
696
1014
|
cacheKey: string;
|
|
@@ -698,22 +1016,10 @@ interface CachedCallInput extends UsageHooks {
|
|
|
698
1016
|
signal?: AbortSignal;
|
|
699
1017
|
}
|
|
700
1018
|
/**
|
|
701
|
-
* Parameters for a cached LLM call without tool calling
|
|
702
|
-
*
|
|
703
|
-
*
|
|
704
|
-
*
|
|
705
|
-
*
|
|
706
|
-
* `reserveUsage`/`refundUsage` are omitted from `call`'s type on purpose:
|
|
707
|
-
* `CachedCallInput` already extends `UsageHooks`, so those two hooks
|
|
708
|
-
* belong at the top level, alongside `cacheKey`/`ttl`, not nested inside
|
|
709
|
-
* `call`. Both positions used to typecheck, which meant `cachedCall`
|
|
710
|
-
* could only catch the mistake at runtime with a warning, after silently
|
|
711
|
-
* ignoring the caller's usage hooks. Putting them inside `call` as an
|
|
712
|
-
* inline object literal is now a compile error instead; TypeScript's
|
|
713
|
-
* excess-property check only applies to object literals though, so a
|
|
714
|
-
* preconstructed value carrying `reserveUsage`/`refundUsage` can still be
|
|
715
|
-
* structurally assignable, which is why `cachedCall` also checks for and
|
|
716
|
-
* rejects both hooks at runtime.
|
|
1019
|
+
* Parameters for a cached LLM call without tool calling: cache config
|
|
1020
|
+
* plus the `CallParams` passed to `call()`. `reserveUsage`/`refundUsage`
|
|
1021
|
+
* belong at the top level (`CachedCallInput`), not nested in `call`; see
|
|
1022
|
+
* the caching docs for why.
|
|
717
1023
|
*/
|
|
718
1024
|
type CachedCallParams<T> = CachedCallInput & {
|
|
719
1025
|
call: Omit<CallParams<T>, 'reserveUsage' | 'refundUsage'>;
|
|
@@ -731,6 +1037,29 @@ type CachedCallParams<T> = CachedCallInput & {
|
|
|
731
1037
|
type CachedToolCallParams<T> = CachedCallInput & {
|
|
732
1038
|
call: Omit<ToolEnabledCallParams<T>, 'reserveUsage' | 'refundUsage'>;
|
|
733
1039
|
};
|
|
1040
|
+
/**
|
|
1041
|
+
* Parameters for a cached LLM call with `call.tools` set conditionally.
|
|
1042
|
+
* Selects the `cachedCall()` overload that returns the honest union
|
|
1043
|
+
* `T | CallWithToolsResult<T>` instead of narrowing to plain `T`. See
|
|
1044
|
+
* `ConditionalToolCallParams` for why this overload exists.
|
|
1045
|
+
*/
|
|
1046
|
+
type CachedConditionalToolCallParams<T> = CachedCallInput & {
|
|
1047
|
+
call: Omit<ConditionalToolCallParams<T>, 'reserveUsage' | 'refundUsage'>;
|
|
1048
|
+
};
|
|
1049
|
+
/**
|
|
1050
|
+
* Parameters for a cached LLM call with `jsonMode: false`. Selects the
|
|
1051
|
+
* `cachedCall()` overload that returns a plain `string`.
|
|
1052
|
+
*/
|
|
1053
|
+
type CachedJsonModeDisabledCallParams = CachedCallInput & {
|
|
1054
|
+
call: Omit<JsonModeDisabledCallParams, 'reserveUsage' | 'refundUsage'>;
|
|
1055
|
+
};
|
|
1056
|
+
/**
|
|
1057
|
+
* Parameters for a cached LLM call with `jsonMode: true` and no `schema`.
|
|
1058
|
+
* Selects the `cachedCall()` overload that returns a `JsonValue`.
|
|
1059
|
+
*/
|
|
1060
|
+
type CachedJsonModeEnabledCallParams = CachedCallInput & {
|
|
1061
|
+
call: Omit<JsonModeEnabledCallParams, 'reserveUsage' | 'refundUsage'>;
|
|
1062
|
+
};
|
|
734
1063
|
|
|
735
1064
|
//#endregion
|
|
736
1065
|
//#region src/types/stream.d.ts
|
|
@@ -759,32 +1088,10 @@ type StreamChunk = {
|
|
|
759
1088
|
usage: TokenUsage;
|
|
760
1089
|
};
|
|
761
1090
|
/**
|
|
762
|
-
* What `call()` returns when `stream: true`. `
|
|
763
|
-
* `
|
|
764
|
-
*
|
|
765
|
-
*
|
|
766
|
-
*
|
|
767
|
-
* `chunks` is single-use and supports only one consumer: iterating it more
|
|
768
|
-
* than once, or from more than one place concurrently, shares the same
|
|
769
|
-
* underlying buffered stream rather than replaying or forking it, which can
|
|
770
|
-
* split chunks unpredictably between consumers. Stopping iteration early
|
|
771
|
-
* (e.g. `break`ing out of a `for await`) does not cancel or otherwise
|
|
772
|
-
* signal the underlying stream, the background pump keeps running to
|
|
773
|
-
* completion regardless, buffering any chunks emitted after that point, so
|
|
774
|
-
* `finalResult` still settles normally even if `chunks` is abandoned or
|
|
775
|
-
* never read at all.
|
|
776
|
-
*
|
|
777
|
-
* Unread chunks are buffered internally for the duration of one stream,
|
|
778
|
-
* this is what lets a caller start iterating `chunks` after the stream has
|
|
779
|
-
* already progressed (or finished) and still see everything. That backlog
|
|
780
|
-
* is capped: an unusually large stream whose `chunks` is never read at all
|
|
781
|
-
* has its oldest buffered chunks dropped once the backlog grows past
|
|
782
|
-
* roughly twice a fixed internal limit, trimmed back down to that limit in
|
|
783
|
-
* one batch rather than one-at-a-time, bounding both peak memory and the
|
|
784
|
-
* eviction work itself for that pathological case instead of the array
|
|
785
|
-
* growing (or being trimmed) proportional to the whole stream's output.
|
|
786
|
-
* Ordinary consumption, even started somewhat late, stays far under the
|
|
787
|
-
* limit and is unaffected.
|
|
1091
|
+
* What `call()` returns when `stream: true`. `finalResult` resolves to
|
|
1092
|
+
* the same shape `call()` would have returned with `stream` omitted.
|
|
1093
|
+
* `chunks` is single-use and buffered; see the streaming docs for the
|
|
1094
|
+
* full consumption/backpressure semantics.
|
|
788
1095
|
*/
|
|
789
1096
|
interface StreamCallResult<R> {
|
|
790
1097
|
chunks: AsyncIterable<StreamChunk>;
|
|
@@ -800,6 +1107,31 @@ interface StreamCallResult<R> {
|
|
|
800
1107
|
type StreamEnabledCallParams<T> = CallParams<T> & {
|
|
801
1108
|
stream: true;
|
|
802
1109
|
};
|
|
1110
|
+
/**
|
|
1111
|
+
* `StreamEnabledCallParams` with `jsonMode: false`. Selects the streaming
|
|
1112
|
+
* `call()` overload whose `finalResult` resolves to a plain `string`.
|
|
1113
|
+
* `jsonSchema` is typed `never` for the same reason as
|
|
1114
|
+
* `JsonModeDisabledCallParams`.
|
|
1115
|
+
*/
|
|
1116
|
+
type StreamJsonModeDisabledCallParams = Omit<StreamEnabledCallParams<unknown>, 'jsonSchema'> & {
|
|
1117
|
+
jsonMode: false;
|
|
1118
|
+
jsonSchema?: never;
|
|
1119
|
+
};
|
|
1120
|
+
/**
|
|
1121
|
+
* `StreamEnabledCallParams` with `jsonMode: true` and no `schema`. Selects
|
|
1122
|
+
* the streaming `call()` overload whose `finalResult` resolves to a
|
|
1123
|
+
* `JsonValue`.
|
|
1124
|
+
*
|
|
1125
|
+
* `schema` is explicitly `never` here for the same reason as
|
|
1126
|
+
* `JsonModeEnabledCallParams`: a schema whose result type is itself
|
|
1127
|
+
* structurally assignable to `JsonValue` would otherwise still satisfy this
|
|
1128
|
+
* overload's shape and incorrectly widen the result to `JsonValue` instead
|
|
1129
|
+
* of the schema's real type.
|
|
1130
|
+
*/
|
|
1131
|
+
type StreamJsonModeEnabledCallParams = Omit<StreamEnabledCallParams<JsonValue>, 'schema'> & {
|
|
1132
|
+
jsonMode: true;
|
|
1133
|
+
schema?: never;
|
|
1134
|
+
};
|
|
803
1135
|
/**
|
|
804
1136
|
* The adapter-facing, pre-normalization shape a `createStream` client
|
|
805
1137
|
* implementation emits, analogous to how `WireMessage`/`WireToolCall`
|
|
@@ -822,6 +1154,9 @@ type WireStreamChunk = {
|
|
|
822
1154
|
prompt_tokens?: number;
|
|
823
1155
|
completion_tokens?: number;
|
|
824
1156
|
total_tokens?: number;
|
|
1157
|
+
completion_tokens_details?: {
|
|
1158
|
+
reasoning_tokens?: number;
|
|
1159
|
+
};
|
|
825
1160
|
};
|
|
826
1161
|
} | {
|
|
827
1162
|
/**
|
|
@@ -857,6 +1192,32 @@ type CachedStreamCallParams<T> = CachedCallInput & {
|
|
|
857
1192
|
type CachedStreamToolCallParams<T> = CachedCallInput & {
|
|
858
1193
|
call: Omit<StreamEnabledCallParams<T> & ToolEnabledCallParams<T>, 'reserveUsage' | 'refundUsage'>;
|
|
859
1194
|
};
|
|
1195
|
+
/**
|
|
1196
|
+
* Parameters for a cached, streaming LLM call with `call.tools` set
|
|
1197
|
+
* conditionally. Selects the `cachedCall()` overload whose `finalResult`
|
|
1198
|
+
* (on a miss) or cached value (on a hit) is the honest union
|
|
1199
|
+
* `T | CallWithToolsResult<T>` instead of narrowing to plain `T`. See
|
|
1200
|
+
* `ConditionalToolCallParams` for why this overload exists.
|
|
1201
|
+
*/
|
|
1202
|
+
type CachedStreamConditionalToolCallParams<T> = CachedCallInput & {
|
|
1203
|
+
call: Omit<StreamEnabledCallParams<T> & ConditionalToolCallParams<T>, 'reserveUsage' | 'refundUsage'>;
|
|
1204
|
+
};
|
|
1205
|
+
/**
|
|
1206
|
+
* Parameters for a cached, streaming LLM call with `jsonMode: false`.
|
|
1207
|
+
* Selects the `cachedCall()` overload whose `finalResult` (on a miss) or
|
|
1208
|
+
* cached value (on a hit) is a plain `string`.
|
|
1209
|
+
*/
|
|
1210
|
+
type CachedStreamJsonModeDisabledCallParams = CachedCallInput & {
|
|
1211
|
+
call: Omit<StreamJsonModeDisabledCallParams, 'reserveUsage' | 'refundUsage'>;
|
|
1212
|
+
};
|
|
1213
|
+
/**
|
|
1214
|
+
* Parameters for a cached, streaming LLM call with `jsonMode: true` and no
|
|
1215
|
+
* `schema`. Selects the `cachedCall()` overload whose `finalResult` (on a
|
|
1216
|
+
* miss) or cached value (on a hit) is a `JsonValue`.
|
|
1217
|
+
*/
|
|
1218
|
+
type CachedStreamJsonModeEnabledCallParams = CachedCallInput & {
|
|
1219
|
+
call: Omit<StreamJsonModeEnabledCallParams, 'reserveUsage' | 'refundUsage'>;
|
|
1220
|
+
};
|
|
860
1221
|
|
|
861
1222
|
//#endregion
|
|
862
1223
|
//#region src/types/client.d.ts
|
|
@@ -905,6 +1266,19 @@ type WireToolChoice = 'auto' | 'none' | 'required' | {
|
|
|
905
1266
|
* rather than importing the SDKs own params type
|
|
906
1267
|
*/
|
|
907
1268
|
interface LLMClient {
|
|
1269
|
+
/**
|
|
1270
|
+
* Whether this client supports OpenAI's `response_format: { type:
|
|
1271
|
+
* 'json_object' }` as a real, API-level constraint. Defaults to `true`
|
|
1272
|
+
* when omitted (every OpenAI-compatible client and `fromGemini` map it to
|
|
1273
|
+
* a real field). `fromAnthropic` and `fromBedrock` set this to `false`:
|
|
1274
|
+
* neither provider has a field that mechanically guarantees JSON output
|
|
1275
|
+
* for this mode, so `RequestBuilder` downgrades a *default* (unset)
|
|
1276
|
+
* `jsonMode` to plain text for these clients instead of requesting
|
|
1277
|
+
* `json_object` and getting an unenforced, provider-side no-op back. An
|
|
1278
|
+
* *explicit* `jsonMode: true` still throws for such clients, since that's
|
|
1279
|
+
* a caller deliberately asking for a guarantee the client can't provide.
|
|
1280
|
+
*/
|
|
1281
|
+
supportsJsonObjectMode?: boolean;
|
|
908
1282
|
chat: {
|
|
909
1283
|
completions: {
|
|
910
1284
|
create(params: {
|
|
@@ -924,6 +1298,13 @@ interface LLMClient {
|
|
|
924
1298
|
};
|
|
925
1299
|
/** OpenAI reasoning-model param (o-series, gpt-5), ignored by providers that don't support it */
|
|
926
1300
|
reasoning_effort?: 'minimal' | 'low' | 'medium' | 'high';
|
|
1301
|
+
/**
|
|
1302
|
+
* Numeric reasoning token budget, for providers with a native
|
|
1303
|
+
* budget field (Anthropic, Gemini). Ignored by clients that only
|
|
1304
|
+
* understand `reasoning_effort` tiers, use that field instead for
|
|
1305
|
+
* those.
|
|
1306
|
+
*/
|
|
1307
|
+
budget_tokens?: number;
|
|
927
1308
|
/** Tools the model may call, OpenAI's `function`-wrapped shape. */
|
|
928
1309
|
tools?: Array<{
|
|
929
1310
|
type: 'function';
|
|
@@ -953,6 +1334,9 @@ interface LLMClient {
|
|
|
953
1334
|
prompt_tokens?: number;
|
|
954
1335
|
completion_tokens?: number;
|
|
955
1336
|
total_tokens?: number;
|
|
1337
|
+
completion_tokens_details?: {
|
|
1338
|
+
reasoning_tokens?: number;
|
|
1339
|
+
};
|
|
956
1340
|
};
|
|
957
1341
|
}>;
|
|
958
1342
|
/**
|
|
@@ -1086,6 +1470,20 @@ interface VernLLMOptions {
|
|
|
1086
1470
|
* request entirely, so the provider applies its own default instead.
|
|
1087
1471
|
*/
|
|
1088
1472
|
defaultTemperature?: number | null;
|
|
1473
|
+
/**
|
|
1474
|
+
* Default reasoning effort for calls that don't override it. Not sent
|
|
1475
|
+
* when omitted, same as leaving `reasoningEffort` unset on a call. See
|
|
1476
|
+
* `budgetTokens`/`reasoningEffort` on `CallParams` for how the two
|
|
1477
|
+
* relate and how each adapter converts between them.
|
|
1478
|
+
*/
|
|
1479
|
+
defaultReasoningEffort?: 'minimal' | 'low' | 'medium' | 'high';
|
|
1480
|
+
/**
|
|
1481
|
+
* Default reasoning token budget for calls that don't override it. Not
|
|
1482
|
+
* sent when omitted. If both this and `defaultReasoningEffort` are set,
|
|
1483
|
+
* each adapter still prefers whichever field it natively understands,
|
|
1484
|
+
* same as at the per-call level.
|
|
1485
|
+
*/
|
|
1486
|
+
defaultBudgetTokens?: number;
|
|
1089
1487
|
/**
|
|
1090
1488
|
* Enables debug logging of raw model output (logs up to 800 chars of each
|
|
1091
1489
|
* response) and provider errors. Off by default. Only controls the
|
|
@@ -1177,7 +1575,7 @@ interface VernLLMOptions {
|
|
|
1177
1575
|
//#region src/vernLLM.d.ts
|
|
1178
1576
|
//# sourceMappingURL=options.d.ts.map
|
|
1179
1577
|
/**
|
|
1180
|
-
* A
|
|
1578
|
+
* A LLM call framework for resilience, observability and control. This is VernLLM!
|
|
1181
1579
|
*
|
|
1182
1580
|
* Adds retry with backoff and jitter, per-attempt timeouts, an optional
|
|
1183
1581
|
* circuit breaker, JSON parsing with optional schema validation, usage
|
|
@@ -1239,10 +1637,16 @@ declare class VernLLM {
|
|
|
1239
1637
|
* continue via `history` (see `ConversationTurn`). Mutually exclusive
|
|
1240
1638
|
* with `jsonSchema`/`schema`.
|
|
1241
1639
|
*
|
|
1242
|
-
* TypeScript
|
|
1243
|
-
*
|
|
1244
|
-
* `
|
|
1245
|
-
*
|
|
1640
|
+
* TypeScript picks the tools-aware overload (`CallWithToolsResult<T>`)
|
|
1641
|
+
* when `tools` is a literal array on `params`, and the conditional-tools
|
|
1642
|
+
* overload (`T | CallWithToolsResult<T>`, see `ConditionalToolCallParams`)
|
|
1643
|
+
* when `tools` is present but statically `ToolDefinition[] | undefined`,
|
|
1644
|
+
* e.g. `const tools = condition ? [myTool] : undefined`. Either way, use
|
|
1645
|
+
* `isToolCallResult()` to narrow the result once `tools` isn't a literal
|
|
1646
|
+
* array: TypeScript's static type can't know from the `ConditionalToolCallParams`
|
|
1647
|
+
* shape alone whether tools actually ran on a given call. Only omitting
|
|
1648
|
+
* `tools` entirely resolves to the plain `T` overload, since then tools
|
|
1649
|
+
* genuinely cannot have run. See the Tool Calling docs for details.
|
|
1246
1650
|
*
|
|
1247
1651
|
* The same static-vs-dynamic caveat applies to `stream`: TypeScript only
|
|
1248
1652
|
* selects the streaming overload (returning `StreamCallResult<...>`) when
|
|
@@ -1264,9 +1668,15 @@ declare class VernLLM {
|
|
|
1264
1668
|
*/
|
|
1265
1669
|
call<T = unknown>(params: StreamEnabledCallParams<T> & ToolsDisabledCallParams<T>): Promise<StreamCallResult<ContentResult<T>>>;
|
|
1266
1670
|
call<T = unknown>(params: StreamEnabledCallParams<T> & ToolEnabledCallParams<T>): Promise<StreamCallResult<CallWithToolsResult<T>>>;
|
|
1671
|
+
call<T = unknown>(params: StreamEnabledCallParams<T> & ConditionalToolCallParams<T>): Promise<StreamCallResult<T | CallWithToolsResult<T>>>;
|
|
1672
|
+
call(params: StreamJsonModeDisabledCallParams): Promise<StreamCallResult<string>>;
|
|
1673
|
+
call(params: StreamJsonModeEnabledCallParams): Promise<StreamCallResult<JsonValue>>;
|
|
1267
1674
|
call<T = unknown>(params: StreamEnabledCallParams<T>): Promise<StreamCallResult<T>>;
|
|
1268
1675
|
call<T = unknown>(params: ToolsDisabledCallParams<T>): Promise<ContentResult<T>>;
|
|
1269
1676
|
call<T = unknown>(params: ToolEnabledCallParams<T>): Promise<CallWithToolsResult<T>>;
|
|
1677
|
+
call<T = unknown>(params: ConditionalToolCallParams<T>): Promise<T | CallWithToolsResult<T>>;
|
|
1678
|
+
call(params: JsonModeDisabledCallParams): Promise<string>;
|
|
1679
|
+
call(params: JsonModeEnabledCallParams): Promise<JsonValue>;
|
|
1270
1680
|
call<T = unknown>(params: CallParams<T>): Promise<T>;
|
|
1271
1681
|
/**
|
|
1272
1682
|
* Thin delegator kept private on `VernLLM` (rather than only existing on
|
|
@@ -1313,31 +1723,87 @@ declare class VernLLM {
|
|
|
1313
1723
|
* @returns The cached value on a hit, or the freshly-called result on a miss.
|
|
1314
1724
|
*/
|
|
1315
1725
|
cachedCall<T>(params: CachedStreamToolCallParams<T>): Promise<StreamCallResult<CallWithToolsResult<T>>>;
|
|
1726
|
+
cachedCall<T>(params: CachedStreamConditionalToolCallParams<T>): Promise<StreamCallResult<T | CallWithToolsResult<T>>>;
|
|
1727
|
+
cachedCall(params: CachedStreamJsonModeDisabledCallParams): Promise<StreamCallResult<string>>;
|
|
1728
|
+
cachedCall(params: CachedStreamJsonModeEnabledCallParams): Promise<StreamCallResult<JsonValue>>;
|
|
1316
1729
|
cachedCall<T>(params: CachedStreamCallParams<T>): Promise<StreamCallResult<T>>;
|
|
1317
1730
|
cachedCall<T>(params: CachedToolCallParams<T>): Promise<CallWithToolsResult<T>>;
|
|
1731
|
+
cachedCall<T>(params: CachedConditionalToolCallParams<T>): Promise<T | CallWithToolsResult<T>>;
|
|
1732
|
+
cachedCall(params: CachedJsonModeDisabledCallParams): Promise<string>;
|
|
1733
|
+
cachedCall(params: CachedJsonModeEnabledCallParams): Promise<JsonValue>;
|
|
1318
1734
|
cachedCall<T>(params: CachedCallParams<T>): Promise<T>;
|
|
1319
1735
|
/**
|
|
1320
|
-
* @param
|
|
1321
|
-
* model
|
|
1322
|
-
*
|
|
1323
|
-
*
|
|
1324
|
-
*
|
|
1325
|
-
*
|
|
1326
|
-
*/
|
|
1327
|
-
getCircuitState(
|
|
1328
|
-
/**
|
|
1329
|
-
* @param model
|
|
1330
|
-
* target's
|
|
1331
|
-
* Ignored otherwise. Omit for the shared circuit (the default) or, under
|
|
1332
|
-
* isolation, the state of calls that didn't resolve a model.
|
|
1333
|
-
* @returns The current circuit state for every target in declaration
|
|
1334
|
-
* order, including the primary and all fallback targets. Each entry
|
|
1335
|
-
* includes the target's provider name, chain index, whether it is a
|
|
1336
|
-
* fallback, and its circuit state, or undefined if that target has no
|
|
1337
|
-
* circuit breaker configured.
|
|
1736
|
+
* @param target.index Which target to read. Defaults to the primary.
|
|
1737
|
+
* @param target.model Which model bucket to read, if the target isolates by model.
|
|
1738
|
+
* @returns The breaker state, or `undefined` if that target has no breaker.
|
|
1739
|
+
* @throws {RangeError} If `target.index` names no target. Lets a real
|
|
1740
|
+
* target with no breaker (`undefined`) stay distinguishable from a
|
|
1741
|
+
* target that doesn't exist.
|
|
1742
|
+
*/
|
|
1743
|
+
getCircuitState(target?: CircuitTarget): CircuitState | undefined;
|
|
1744
|
+
/**
|
|
1745
|
+
* @param model Which model bucket to read, for targets that isolate by model.
|
|
1746
|
+
* @returns Every target's state, in chain order.
|
|
1338
1747
|
*/
|
|
1339
1748
|
getCircuitStates(model?: string): TargetCircuitState[];
|
|
1749
|
+
/**
|
|
1750
|
+
* Manually opens a target's breaker, e.g. to pull a provider out of
|
|
1751
|
+
* rotation ahead of known maintenance instead of waiting for it to fail.
|
|
1752
|
+
*
|
|
1753
|
+
* @param target.index Which target to open. Defaults to the primary.
|
|
1754
|
+
* @param target.model Which model bucket to open, if the target isolates by model.
|
|
1755
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
1756
|
+
*/
|
|
1757
|
+
openCircuit(target?: CircuitTarget): void;
|
|
1758
|
+
/**
|
|
1759
|
+
* Manually closes a target's breaker, e.g. once a provider is confirmed
|
|
1760
|
+
* healthy again without waiting out the cooldown.
|
|
1761
|
+
*
|
|
1762
|
+
* @param target.index Which target to close. Defaults to the primary.
|
|
1763
|
+
* @param target.model Which model bucket to close, if the target isolates by model.
|
|
1764
|
+
* @throws {RangeError} If `target.index` names no target.
|
|
1765
|
+
*/
|
|
1766
|
+
closeCircuit(target?: CircuitTarget): void;
|
|
1767
|
+
/** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
|
|
1768
|
+
private resolveExecutor;
|
|
1769
|
+
/** Warns when `model` can't do anything on this target, so it's never silently ignored. */
|
|
1770
|
+
private warnIfModelUnsupported;
|
|
1340
1771
|
}
|
|
1772
|
+
/**
|
|
1773
|
+
* Identity function preserving `params`'s own precise type, unlike a `:
|
|
1774
|
+
* CallParams<T>` annotation, which would widen `tools` away and break the
|
|
1775
|
+
* `ConditionalToolCallParams<T>` overload for `tools: someCondition ?
|
|
1776
|
+
* [tool] : undefined`. Use it when you need `call()` params in a named,
|
|
1777
|
+
* reusable variable; skip it when you can pass the object inline.
|
|
1778
|
+
*
|
|
1779
|
+
* ```ts
|
|
1780
|
+
* const params = defineCallParams({
|
|
1781
|
+
* userContent: 'What is the weather?',
|
|
1782
|
+
* tools: someCondition ? [weatherTool] : undefined,
|
|
1783
|
+
* });
|
|
1784
|
+
* const result = await llm.call(params);
|
|
1785
|
+
* // result: unknown | CallWithToolsResult<unknown>, same as inline
|
|
1786
|
+
* ```
|
|
1787
|
+
*
|
|
1788
|
+
* `T` isn't a parameter here; pin it via `llm.call<T>(params)` as usual.
|
|
1789
|
+
* `defineCachedCallParams` is the `cachedCall()` counterpart.
|
|
1790
|
+
*/
|
|
1791
|
+
declare function defineCallParams<P extends CallParams<unknown>>(params: P): P;
|
|
1792
|
+
/**
|
|
1793
|
+
* The `cachedCall()` counterpart to `defineCallParams`: preserves the
|
|
1794
|
+
* whole `{ cacheKey, ttl, call }` object, `call.tools` included, in one
|
|
1795
|
+
* named variable.
|
|
1796
|
+
*
|
|
1797
|
+
* ```ts
|
|
1798
|
+
* const params = defineCachedCallParams({
|
|
1799
|
+
* cacheKey: 'weather-ny',
|
|
1800
|
+
* ttl: 60,
|
|
1801
|
+
* call: { userContent: 'What is the weather?', tools: someCondition ? [weatherTool] : undefined },
|
|
1802
|
+
* });
|
|
1803
|
+
* const result = await llm.cachedCall(params);
|
|
1804
|
+
* ```
|
|
1805
|
+
*/
|
|
1806
|
+
declare function defineCachedCallParams<P extends CachedCallParams<unknown>>(params: P): P;
|
|
1341
1807
|
|
|
1342
1808
|
//#endregion
|
|
1343
1809
|
//#region src/adapters/internal/sse.d.ts
|
|
@@ -1394,9 +1860,10 @@ type SupportedImageMimeType = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number];
|
|
|
1394
1860
|
//#region src/adapters/internal/nativeStructuredOutput.d.ts
|
|
1395
1861
|
/**
|
|
1396
1862
|
* Validates an `ImageBlock.mimeType` against the shared supported set.
|
|
1397
|
-
* Throws a non-retryable `LLMError('
|
|
1398
|
-
* mimeType is a
|
|
1399
|
-
* the same
|
|
1863
|
+
* Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
|
|
1864
|
+
* mimeType is a bug in the caller's own input, deterministic before any
|
|
1865
|
+
* request is built, the same class of failure as every other check in
|
|
1866
|
+
* `RequestBuilder`.
|
|
1400
1867
|
*/
|
|
1401
1868
|
|
|
1402
1869
|
/**
|
|
@@ -1421,8 +1888,31 @@ type SupportedImageMimeType = (typeof SUPPORTED_IMAGE_MIME_TYPES)[number];
|
|
|
1421
1888
|
type ModelCapabilityOverride = string[] | ((model: string) => boolean);
|
|
1422
1889
|
|
|
1423
1890
|
//#endregion
|
|
1424
|
-
//#region src/adapters/
|
|
1891
|
+
//#region src/adapters/internal/reasoningBudget.utils.d.ts
|
|
1425
1892
|
/** Resolves whether `model` is covered by a caller-supplied allow-list/predicate. */
|
|
1893
|
+
/**
|
|
1894
|
+
* Shared conversion between the two reasoning controls VernLLM exposes:
|
|
1895
|
+
* `reasoningEffort` (a tier string, OpenAI's native shape) and
|
|
1896
|
+
* `budgetTokens` (a raw integer, Anthropic's and Gemini's native shape).
|
|
1897
|
+
*
|
|
1898
|
+
* Every adapter prefers its own native field when the caller set it, and
|
|
1899
|
+
* only calls into this table when the caller set the other one instead.
|
|
1900
|
+
* The numbers here are a guess, not a provider guarantee, callers who
|
|
1901
|
+
* need a precise budget on a specific model should set `budgetTokens`
|
|
1902
|
+
* directly rather than relying on this table's `reasoningEffort` mapping.
|
|
1903
|
+
*
|
|
1904
|
+
* The table itself is overridable per adapter instance, via
|
|
1905
|
+
* `reasoningEffortTokens` on each `from*` adapter's options (see
|
|
1906
|
+
* `AnthropicAdapterOptions`, `GeminiAdapterOptions`,
|
|
1907
|
+
* `OpenAICompatibleAdapterOptions`, `BedrockAdapterOptions`), for callers
|
|
1908
|
+
* who want `reasoningEffort` tiers to map onto different token counts
|
|
1909
|
+
* than the defaults below, e.g. a model whose useful reasoning range
|
|
1910
|
+
* doesn't match these numbers.
|
|
1911
|
+
*/
|
|
1912
|
+
type EffortTokenTable = Record<'minimal' | 'low' | 'medium' | 'high', number>;
|
|
1913
|
+
|
|
1914
|
+
//#endregion
|
|
1915
|
+
//#region src/adapters/anthropic.d.ts
|
|
1426
1916
|
/** Anthropic's native per-block content shape for a message. */
|
|
1427
1917
|
type AnthropicContentBlock = {
|
|
1428
1918
|
type: 'text';
|
|
@@ -1493,10 +1983,33 @@ interface AnthropicClient {
|
|
|
1493
1983
|
* output endpoint has no equivalent for any of them.
|
|
1494
1984
|
*/
|
|
1495
1985
|
output_config?: {
|
|
1496
|
-
format
|
|
1986
|
+
format?: {
|
|
1497
1987
|
type: 'json_schema';
|
|
1498
1988
|
schema: Record<string, unknown>;
|
|
1499
1989
|
};
|
|
1990
|
+
/**
|
|
1991
|
+
* Effort control for adaptive thinking, on models where manual
|
|
1992
|
+
* `budget_tokens` thinking is no longer accepted (see
|
|
1993
|
+
* `supportsManualThinkingBudget` in
|
|
1994
|
+
* `adapters/internal/reasoningBudget.utils.ts`). Sibling to
|
|
1995
|
+
* `format`, either or both may be present independently.
|
|
1996
|
+
*/
|
|
1997
|
+
effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
1998
|
+
};
|
|
1999
|
+
/**
|
|
2000
|
+
* Native reasoning control. `{ type: 'enabled', budget_tokens }`
|
|
2001
|
+
* is built directly from `CallParams.budgetTokens`, or converted
|
|
2002
|
+
* from `reasoningEffort`, on models that still accept a manual
|
|
2003
|
+
* token budget. `{ type: 'adaptive' }` is sent instead, paired
|
|
2004
|
+
* with `output_config.effort`, on models that only support
|
|
2005
|
+
* adaptive thinking. See
|
|
2006
|
+
* `adapters/internal/reasoningBudget.utils.ts`.
|
|
2007
|
+
*/
|
|
2008
|
+
thinking?: {
|
|
2009
|
+
type: 'enabled';
|
|
2010
|
+
budget_tokens: number;
|
|
2011
|
+
} | {
|
|
2012
|
+
type: 'adaptive';
|
|
1500
2013
|
};
|
|
1501
2014
|
}, options: {
|
|
1502
2015
|
signal: AbortSignal;
|
|
@@ -1511,6 +2024,9 @@ interface AnthropicClient {
|
|
|
1511
2024
|
usage?: {
|
|
1512
2025
|
input_tokens?: number;
|
|
1513
2026
|
output_tokens?: number;
|
|
2027
|
+
output_tokens_details?: {
|
|
2028
|
+
thinking_tokens?: number;
|
|
2029
|
+
} | null;
|
|
1514
2030
|
};
|
|
1515
2031
|
}>;
|
|
1516
2032
|
};
|
|
@@ -1529,6 +2045,25 @@ interface AnthropicAdapterOptions {
|
|
|
1529
2045
|
* exactly this adapter's behavior before native support was added.
|
|
1530
2046
|
*/
|
|
1531
2047
|
nativeStructuredOutputModels?: ModelCapabilityOverride;
|
|
2048
|
+
/**
|
|
2049
|
+
* Overrides the token count `reasoningEffort` tiers map onto when the
|
|
2050
|
+
* caller sets `reasoningEffort` but not `budgetTokens` (Claude has no
|
|
2051
|
+
* tier concept of its own, see `adapters/internal/reasoningBudget.utils.ts`).
|
|
2052
|
+
* Only the tiers listed are changed; any omitted tier keeps the
|
|
2053
|
+
* built-in default. Has no effect when `budgetTokens` is set directly.
|
|
2054
|
+
*/
|
|
2055
|
+
reasoningEffortTokens?: Partial<EffortTokenTable>;
|
|
2056
|
+
/**
|
|
2057
|
+
* Marks additional models as adaptive-only, on top of this package's
|
|
2058
|
+
* own built-in rule (Claude Opus 4.7 and later, every Claude 5 tier
|
|
2059
|
+
* model, see `isAdaptiveOnlyModel` in
|
|
2060
|
+
* `adapters/internal/reasoningBudget.utils.ts`). Additive, not a
|
|
2061
|
+
* replacement: it can correct a false negative (a newer model this
|
|
2062
|
+
* package doesn't know about yet), it can't un-mark a model the
|
|
2063
|
+
* built-in rule already caught. Pass a static list of model IDs or a
|
|
2064
|
+
* predicate.
|
|
2065
|
+
*/
|
|
2066
|
+
adaptiveOnlyModels?: ModelCapabilityOverride;
|
|
1532
2067
|
}
|
|
1533
2068
|
/**
|
|
1534
2069
|
* Wraps an Anthropic SDK client so it satisfies the same `LLMClient`
|
|
@@ -1552,17 +2087,25 @@ interface AnthropicAdapterOptions {
|
|
|
1552
2087
|
* schema matching applies only when `strict: true` is forwarded and
|
|
1553
2088
|
* supported.
|
|
1554
2089
|
*
|
|
1555
|
-
* `response_format: json_object` (
|
|
1556
|
-
*
|
|
1557
|
-
*
|
|
1558
|
-
*
|
|
1559
|
-
*
|
|
2090
|
+
* `response_format: json_object` throws `LLMError('validation')`. Anthropic
|
|
2091
|
+
* has no API-level field that mechanically guarantees JSON output the way
|
|
2092
|
+
* OpenAI's `json_object` mode does; the only way to emulate it was a
|
|
2093
|
+
* system-prompt instruction with no actual enforcement behind it, a
|
|
2094
|
+
* guarantee this adapter no longer pretends to make. Use `jsonSchema`
|
|
2095
|
+
* instead, which maps to a real constraint either way (native
|
|
2096
|
+
* `output_config.format` or a forced tool call).
|
|
1560
2097
|
*/
|
|
1561
2098
|
declare function fromAnthropic(anthropicClient: AnthropicClient, options?: AnthropicAdapterOptions): LLMClient;
|
|
1562
2099
|
|
|
1563
2100
|
//#endregion
|
|
1564
2101
|
//#region src/adapters/gemini.d.ts
|
|
1565
|
-
/**
|
|
2102
|
+
/**
|
|
2103
|
+
* Gemini's native per-part content shape for a `contents` entry.
|
|
2104
|
+
* `functionCall.args` and `functionResponse.response` are typed as
|
|
2105
|
+
* `Record<string, unknown>` (not `unknown`) to match the real SDK's
|
|
2106
|
+
* `FunctionCall.args` / `FunctionResponse.response`, see the doc comment
|
|
2107
|
+
* on {@link GeminiClient}.
|
|
2108
|
+
*/
|
|
1566
2109
|
type GeminiPart = {
|
|
1567
2110
|
text: string;
|
|
1568
2111
|
} | {
|
|
@@ -1573,31 +2116,44 @@ type GeminiPart = {
|
|
|
1573
2116
|
} | {
|
|
1574
2117
|
functionCall: {
|
|
1575
2118
|
name: string;
|
|
1576
|
-
args: unknown
|
|
2119
|
+
args: Record<string, unknown>;
|
|
1577
2120
|
};
|
|
1578
2121
|
} | {
|
|
1579
2122
|
functionResponse: {
|
|
1580
2123
|
name: string;
|
|
1581
|
-
response: unknown
|
|
2124
|
+
response: Record<string, unknown>;
|
|
1582
2125
|
};
|
|
1583
2126
|
};
|
|
1584
2127
|
/**
|
|
1585
|
-
* Structural type matching the real `@google/genai` SDK
|
|
1586
|
-
*
|
|
1587
|
-
*
|
|
1588
|
-
*
|
|
1589
|
-
* `abortSignal` all together), matching the real SDK closely enough that
|
|
1590
|
-
* `fromGemini(ai.models)` works directly, e.g:
|
|
2128
|
+
* Structural type matching the real `@google/genai` SDK, in either shape
|
|
2129
|
+
* it's commonly held in: the callable model methods directly (`ai.models`),
|
|
2130
|
+
* or the complete top-level client (`ai`, via the optional `models` field
|
|
2131
|
+
* below). Both work with `fromGemini` directly, with no cast:
|
|
1591
2132
|
*
|
|
1592
2133
|
* ```ts
|
|
1593
2134
|
* import { GoogleGenAI } from '@google/genai';
|
|
1594
2135
|
* const ai = new GoogleGenAI({ apiKey: '...' });
|
|
1595
|
-
* const llm = new VernLLM({ client: fromGemini(ai
|
|
2136
|
+
* const llm = new VernLLM({ client: fromGemini(ai), model: 'gemini-2.5-flash' });
|
|
1596
2137
|
* ```
|
|
2138
|
+
*
|
|
2139
|
+
* `generateContent` is optional so a `{ models: ... }`-shaped value is
|
|
2140
|
+
* still a structural `GeminiClient`; `fromGemini` resolves `models` at
|
|
2141
|
+
* runtime and throws if nothing callable results.
|
|
2142
|
+
*
|
|
2143
|
+
* Every field is shaped to be structurally assignable from the real SDK's
|
|
2144
|
+
* generated types without importing them, so provider SDKs stay optional:
|
|
2145
|
+
* `model` is required (the real SDK requires it), `functionCall.args` /
|
|
2146
|
+
* `functionResponse.response` are `Record<string, unknown>` (matching the
|
|
2147
|
+
* real SDK, not `unknown`), `toolConfig...mode` is `any` (TypeScript never
|
|
2148
|
+
* treats a string-literal union as assignable to the real SDK's string
|
|
2149
|
+
* enum), and response-side `functionCall.name` is optional (matching the
|
|
2150
|
+
* real SDK).
|
|
1597
2151
|
*/
|
|
1598
2152
|
interface GeminiClient {
|
|
1599
|
-
|
|
1600
|
-
|
|
2153
|
+
/** Present when this is the whole top-level SDK client, not `ai.models`. `fromGemini` unwraps it at runtime. */
|
|
2154
|
+
models?: GeminiClient;
|
|
2155
|
+
generateContent?(params: {
|
|
2156
|
+
model: string;
|
|
1601
2157
|
contents: Array<{
|
|
1602
2158
|
role: 'user' | 'model';
|
|
1603
2159
|
parts: GeminiPart[];
|
|
@@ -1621,10 +2177,25 @@ interface GeminiClient {
|
|
|
1621
2177
|
}>;
|
|
1622
2178
|
toolConfig?: {
|
|
1623
2179
|
functionCallingConfig: {
|
|
1624
|
-
mode:
|
|
2180
|
+
mode: any;
|
|
1625
2181
|
allowedFunctionNames?: string[];
|
|
1626
2182
|
};
|
|
1627
2183
|
};
|
|
2184
|
+
/**
|
|
2185
|
+
* Native reasoning control. `thinkingBudget` is built from
|
|
2186
|
+
* `CallParams.budgetTokens` directly when set (0 disables thinking,
|
|
2187
|
+
* -1 requests automatic budgeting, both passed through unchanged),
|
|
2188
|
+
* or converted from `reasoningEffort`, on Gemini 2.5 and earlier
|
|
2189
|
+
* models. `thinkingLevel` is used instead on Gemini 3 and later,
|
|
2190
|
+
* which use a level-based control rather than a numeric budget.
|
|
2191
|
+
* `any`, same reason as `toolConfig...mode` above, see class doc
|
|
2192
|
+
* comment. See `usesGeminiThinkingLevel` in
|
|
2193
|
+
* `adapters/internal/reasoningBudget.utils.ts`.
|
|
2194
|
+
*/
|
|
2195
|
+
thinkingConfig?: {
|
|
2196
|
+
thinkingBudget?: number;
|
|
2197
|
+
thinkingLevel?: any;
|
|
2198
|
+
};
|
|
1628
2199
|
abortSignal?: AbortSignal;
|
|
1629
2200
|
};
|
|
1630
2201
|
}): Promise<{
|
|
@@ -1633,8 +2204,8 @@ interface GeminiClient {
|
|
|
1633
2204
|
parts?: Array<{
|
|
1634
2205
|
text?: string;
|
|
1635
2206
|
functionCall?: {
|
|
1636
|
-
name
|
|
1637
|
-
args
|
|
2207
|
+
name?: string;
|
|
2208
|
+
args?: unknown;
|
|
1638
2209
|
};
|
|
1639
2210
|
}>;
|
|
1640
2211
|
};
|
|
@@ -1643,6 +2214,7 @@ interface GeminiClient {
|
|
|
1643
2214
|
promptTokenCount?: number;
|
|
1644
2215
|
candidatesTokenCount?: number;
|
|
1645
2216
|
totalTokenCount?: number;
|
|
2217
|
+
thoughtsTokenCount?: number;
|
|
1646
2218
|
};
|
|
1647
2219
|
}>;
|
|
1648
2220
|
/**
|
|
@@ -1653,14 +2225,14 @@ interface GeminiClient {
|
|
|
1653
2225
|
* holding the same `candidates[].content.parts[]` structure as
|
|
1654
2226
|
* `generateContent`'s response, just incremental.
|
|
1655
2227
|
*/
|
|
1656
|
-
generateContentStream?(params: Parameters<GeminiClient['generateContent']
|
|
2228
|
+
generateContentStream?(params: Parameters<NonNullable<GeminiClient['generateContent']>>[0]): Promise<AsyncIterable<{
|
|
1657
2229
|
candidates?: Array<{
|
|
1658
2230
|
content?: {
|
|
1659
2231
|
parts?: Array<{
|
|
1660
2232
|
text?: string;
|
|
1661
2233
|
functionCall?: {
|
|
1662
|
-
name
|
|
1663
|
-
args
|
|
2234
|
+
name?: string;
|
|
2235
|
+
args?: unknown;
|
|
1664
2236
|
};
|
|
1665
2237
|
}>;
|
|
1666
2238
|
};
|
|
@@ -1669,6 +2241,7 @@ interface GeminiClient {
|
|
|
1669
2241
|
promptTokenCount?: number;
|
|
1670
2242
|
candidatesTokenCount?: number;
|
|
1671
2243
|
totalTokenCount?: number;
|
|
2244
|
+
thoughtsTokenCount?: number;
|
|
1672
2245
|
};
|
|
1673
2246
|
}>>;
|
|
1674
2247
|
}
|
|
@@ -1679,9 +2252,10 @@ interface GeminiClient {
|
|
|
1679
2252
|
* `systemInstruction` field instead of a `system` role message,
|
|
1680
2253
|
* `generationConfig` instead of top-level `temperature`/`max_tokens`, and
|
|
1681
2254
|
* native JSON Schema support via `responseMimeType: 'application/json'` +
|
|
1682
|
-
* `responseSchema`. `reasoning_effort` has no
|
|
1683
|
-
*
|
|
1684
|
-
*
|
|
2255
|
+
* `responseSchema`. `reasoning_effort` has no native Gemini equivalent, so
|
|
2256
|
+
* it's converted to a `thinkingConfig.thinkingBudget` token count; `budget_tokens`
|
|
2257
|
+
* maps to `thinkingBudget` directly, Gemini's native reasoning control. See
|
|
2258
|
+
* `adapters/internal/reasoningBudget.utils.ts`.
|
|
1685
2259
|
*
|
|
1686
2260
|
* `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
|
|
1687
2261
|
* `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
|
|
@@ -1701,8 +2275,36 @@ interface GeminiClient {
|
|
|
1701
2275
|
* own behavior) only reliably present on the last chunk, so the `usage`
|
|
1702
2276
|
* `WireStreamChunk` is emitted once, after the stream completes, from
|
|
1703
2277
|
* whichever chunk's `usageMetadata` was seen last.
|
|
2278
|
+
*
|
|
2279
|
+
* Accepts a `GeminiClient` in either shape it structurally covers: the
|
|
2280
|
+
* callable model methods directly (`ai.models`), or the complete
|
|
2281
|
+
* top-level client (`ai`), unwrapping `.models` internally when present.
|
|
2282
|
+
* Both work with no cast: `fromGemini(ai.models)` and `fromGemini(ai)`.
|
|
2283
|
+
* Throws `LLMError('invalid_params')` up front if nothing callable
|
|
2284
|
+
* results.
|
|
1704
2285
|
*/
|
|
1705
|
-
|
|
2286
|
+
interface GeminiAdapterOptions {
|
|
2287
|
+
/**
|
|
2288
|
+
* Overrides the token count `reasoningEffort` tiers map onto when the
|
|
2289
|
+
* caller sets `reasoningEffort` but not `budgetTokens` (Gemini has no
|
|
2290
|
+
* tier string of its own, see `adapters/internal/reasoningBudget.utils.ts`).
|
|
2291
|
+
* Only the tiers listed are changed; any omitted tier keeps the
|
|
2292
|
+
* built-in default. Has no effect when `budgetTokens` is set directly.
|
|
2293
|
+
*/
|
|
2294
|
+
reasoningEffortTokens?: Partial<EffortTokenTable>;
|
|
2295
|
+
/**
|
|
2296
|
+
* Marks additional models as using `thinkingLevel` instead of
|
|
2297
|
+
* `thinkingBudget`, on top of this package's own built-in rule (every
|
|
2298
|
+
* Gemini 3 series model and later, see `usesGeminiThinkingLevel` in
|
|
2299
|
+
* `adapters/internal/reasoningBudget.utils.ts`). Additive, not a
|
|
2300
|
+
* replacement: it can correct a false negative (a newer model this
|
|
2301
|
+
* package doesn't know about yet), it can't un-mark a model the
|
|
2302
|
+
* built-in rule already caught. Pass a static list of model IDs or a
|
|
2303
|
+
* predicate.
|
|
2304
|
+
*/
|
|
2305
|
+
thinkingLevelModels?: ModelCapabilityOverride;
|
|
2306
|
+
}
|
|
2307
|
+
declare function fromGemini(client: GeminiClient, options?: GeminiAdapterOptions): LLMClient;
|
|
1706
2308
|
|
|
1707
2309
|
//#endregion
|
|
1708
2310
|
//#region src/adapters/bedrock.d.ts
|
|
@@ -1801,7 +2403,7 @@ interface BedrockConverseClient {
|
|
|
1801
2403
|
* included). There is no `strict` field here, unlike `toolSpec`.
|
|
1802
2404
|
*/
|
|
1803
2405
|
outputConfig?: {
|
|
1804
|
-
textFormat
|
|
2406
|
+
textFormat?: {
|
|
1805
2407
|
type: 'json_schema';
|
|
1806
2408
|
structure: {
|
|
1807
2409
|
jsonSchema: {
|
|
@@ -1811,7 +2413,23 @@ interface BedrockConverseClient {
|
|
|
1811
2413
|
};
|
|
1812
2414
|
};
|
|
1813
2415
|
};
|
|
2416
|
+
/**
|
|
2417
|
+
* Effort control for adaptive thinking, on Claude models where
|
|
2418
|
+
* manual `budget_tokens` thinking is no longer accepted (see
|
|
2419
|
+
* `supportsManualThinkingBudget` in
|
|
2420
|
+
* `adapters/internal/reasoningBudget.utils.ts`). Sibling to
|
|
2421
|
+
* `textFormat`, either or both may be present independently.
|
|
2422
|
+
*/
|
|
2423
|
+
effort?: 'low' | 'medium' | 'high' | 'xhigh' | 'max';
|
|
1814
2424
|
};
|
|
2425
|
+
/**
|
|
2426
|
+
* Model-specific passthrough. Converse has no reasoning-budget field
|
|
2427
|
+
* of its own, so a token budget for a Claude model on Bedrock is
|
|
2428
|
+
* forwarded here under Anthropic's own key, `{ thinking: { type:
|
|
2429
|
+
* 'enabled', budget_tokens } }`. Non-Claude models get nothing here,
|
|
2430
|
+
* there is no equivalent field to reach for.
|
|
2431
|
+
*/
|
|
2432
|
+
additionalModelRequestFields?: Record<string, unknown>;
|
|
1815
2433
|
}, options: {
|
|
1816
2434
|
signal: AbortSignal;
|
|
1817
2435
|
}): Promise<{
|
|
@@ -1949,6 +2567,37 @@ interface BedrockAdapterOptions {
|
|
|
1949
2567
|
* added.
|
|
1950
2568
|
*/
|
|
1951
2569
|
nativeStructuredOutputModels?: ModelCapabilityOverride;
|
|
2570
|
+
/**
|
|
2571
|
+
* Overrides the token count `reasoningEffort` tiers map onto when the
|
|
2572
|
+
* caller sets `reasoningEffort` but not `budgetTokens` (Converse has no
|
|
2573
|
+
* tier string of its own, see `adapters/internal/reasoningBudget.utils.ts`).
|
|
2574
|
+
* Only the tiers listed are changed; any omitted tier keeps the
|
|
2575
|
+
* built-in default. Has no effect when `budgetTokens` is set directly,
|
|
2576
|
+
* or when the target model isn't a Claude model.
|
|
2577
|
+
*/
|
|
2578
|
+
reasoningEffortTokens?: Partial<EffortTokenTable>;
|
|
2579
|
+
/**
|
|
2580
|
+
* Marks additional models as adaptive-only, on top of this package's
|
|
2581
|
+
* own built-in rule (Claude Opus 4.7 and later, every Claude 5 tier
|
|
2582
|
+
* model, see `isAdaptiveOnlyModel` in
|
|
2583
|
+
* `adapters/internal/reasoningBudget.utils.ts`). Additive, not a
|
|
2584
|
+
* replacement: it can correct a false negative (a newer model this
|
|
2585
|
+
* package doesn't know about yet), it can't un-mark a model the
|
|
2586
|
+
* built-in rule already caught. Pass a static list of model IDs or a
|
|
2587
|
+
* predicate.
|
|
2588
|
+
*/
|
|
2589
|
+
adaptiveOnlyModels?: ModelCapabilityOverride;
|
|
2590
|
+
}
|
|
2591
|
+
/**
|
|
2592
|
+
* Minimal structural shape of an AWS SDK v3 client that exposes `.send()`,
|
|
2593
|
+
* matching `BedrockRuntimeClient` (and its abort-signal-aware call
|
|
2594
|
+
* convention). Avoids importing `@aws-sdk/client-bedrock-runtime` for the
|
|
2595
|
+
* type.
|
|
2596
|
+
*/
|
|
2597
|
+
interface AwsSendClient {
|
|
2598
|
+
send(command: unknown, options?: {
|
|
2599
|
+
abortSignal?: AbortSignal;
|
|
2600
|
+
}): Promise<unknown>;
|
|
1952
2601
|
}
|
|
1953
2602
|
/**
|
|
1954
2603
|
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
@@ -1958,6 +2607,16 @@ interface BedrockAdapterOptions {
|
|
|
1958
2607
|
* regardless of which underlying model `modelId` points at, as long as
|
|
1959
2608
|
* that model supports Converse (most current-generation ones do)
|
|
1960
2609
|
*
|
|
2610
|
+
* `bedrockClient` accepts either a hand-written `BedrockConverseClient`
|
|
2611
|
+
* (a `.converse()`/`.converseStream()` wrapper you provide) or a real AWS
|
|
2612
|
+
* SDK v3 client (anything with `.send()`, matching `BedrockRuntimeClient`)
|
|
2613
|
+
* directly, detected structurally. Passing a raw AWS client skips the
|
|
2614
|
+
* hand-written wrapper entirely, internally doing what it would
|
|
2615
|
+
* (`send(new ConverseCommand(...))`, `send(new
|
|
2616
|
+
* ConverseStreamCommand(...))`). See `wrapAwsSendClient` for how that path
|
|
2617
|
+
* is implemented, including why `@aws-sdk/client-bedrock-runtime` stays
|
|
2618
|
+
* out of this package's dependencies either way.
|
|
2619
|
+
*
|
|
1961
2620
|
* `response_format: json_schema`, on a model covered by
|
|
1962
2621
|
* `options.nativeStructuredOutputModels` (opt-in, unset by default), is
|
|
1963
2622
|
* sent as `outputConfig.textFormat`, its own request field, independent of
|
|
@@ -1979,12 +2638,15 @@ interface BedrockAdapterOptions {
|
|
|
1979
2638
|
* `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
|
|
1980
2639
|
* unsupported model surfaces Bedrock's raw error unchanged.
|
|
1981
2640
|
*
|
|
1982
|
-
* `response_format: json_object` (
|
|
1983
|
-
*
|
|
1984
|
-
*
|
|
1985
|
-
*
|
|
1986
|
-
*
|
|
1987
|
-
*
|
|
2641
|
+
* `response_format: json_object` throws `LLMError('validation')`: Converse
|
|
2642
|
+
* has no field that mechanically guarantees JSON output, and the only way
|
|
2643
|
+
* to emulate it was an unenforced system-prompt instruction, a guarantee
|
|
2644
|
+
* this adapter no longer pretends to make. Use `jsonSchema` instead.
|
|
2645
|
+
* `reasoning_effort` (no Converse equivalent) is converted to a token
|
|
2646
|
+
* budget and forwarded via `additionalModelRequestFields` for Claude
|
|
2647
|
+
* models only; `budget_tokens` is forwarded the same way directly. Both
|
|
2648
|
+
* are silently dropped for non-Claude models, which have no equivalent
|
|
2649
|
+
* field to reach for. See `adapters/internal/reasoningBudget.utils.ts`.
|
|
1988
2650
|
*
|
|
1989
2651
|
* `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
|
|
1990
2652
|
* `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
|
|
@@ -2001,7 +2663,7 @@ interface BedrockAdapterOptions {
|
|
|
2001
2663
|
* `finalizeResponse`'s `content` path exactly like the non-streaming
|
|
2002
2664
|
* `create` branch above unwraps it.
|
|
2003
2665
|
*/
|
|
2004
|
-
declare function fromBedrock(bedrockClient: BedrockConverseClient, options?: BedrockAdapterOptions): LLMClient;
|
|
2666
|
+
declare function fromBedrock(bedrockClient: BedrockConverseClient | AwsSendClient, options?: BedrockAdapterOptions): LLMClient;
|
|
2005
2667
|
|
|
2006
2668
|
//#endregion
|
|
2007
2669
|
//#region src/adapters/fetch.d.ts
|
|
@@ -2212,6 +2874,15 @@ interface OpenAICompatibleAdapterOptions {
|
|
|
2212
2874
|
* with such a provider won't get one.
|
|
2213
2875
|
*/
|
|
2214
2876
|
supportsStreamUsage?: boolean;
|
|
2877
|
+
/**
|
|
2878
|
+
* Overrides the token count `budgetTokens` buckets into when the caller
|
|
2879
|
+
* sets `budgetTokens` but not `reasoningEffort` (OpenAI-compatible
|
|
2880
|
+
* clients have no numeric budget field of their own, see
|
|
2881
|
+
* `adapters/internal/reasoningBudget.utils.ts`). Only the tiers listed
|
|
2882
|
+
* are changed; any omitted tier keeps the built-in default. Has no
|
|
2883
|
+
* effect when `reasoningEffort` is set directly.
|
|
2884
|
+
*/
|
|
2885
|
+
reasoningEffortTokens?: Partial<EffortTokenTable>;
|
|
2215
2886
|
}
|
|
2216
2887
|
declare function fromOpenAICompatible(client: unknown, options?: OpenAICompatibleAdapterOptions): LLMClient;
|
|
2217
2888
|
/**
|
|
@@ -2323,5 +2994,5 @@ declare const from01AI: typeof fromOpenAICompatible;
|
|
|
2323
2994
|
//#endregion
|
|
2324
2995
|
//# sourceMappingURL=openaiCompatible.d.ts.map
|
|
2325
2996
|
|
|
2326
|
-
export { AnthropicClient, BedrockConverseClient, CacheAdapter, CachedCallParams, CachedStreamCallParams, CachedStreamToolCallParams, CachedToolCallParams, CallMeta, CallParams, CallWithToolsResult, CircuitBreaker, CircuitBreakerOptions, CircuitState, ConsoleLogger, ContentBlock, ContentResult, ConversationTurn, FallbackAttempt, FallbackExhaustedError, FallbackOn, FallbackTarget, FetchAdapterConfig, GeminiClient, ImageBlock, InMemoryCacheAdapter, JsonSchemaSpec, LLMClient, LLMError, LLMErrorCode, LLMErrorType, Logger, NormalizedCacheAdapter, OnEvent, OnUsage, RateLimitAcquireResult, RateLimitOptions, RateLimitReason, RateLimiter, RefundUsage, ReserveUsage, SSE_PING, SchemaLike, StreamCallResult, StreamChunk, StreamEnabledCallParams, TargetCircuitState, TextBlock, TieredCacheAdapter, TokenUsage, ToolCall, ToolCallResult, ToolChoice, ToolDefinition, ToolEnabledCallParams, ToolIssue, ToolResult, ToolsDisabledCallParams, VernLLM, VernLLMEvent, VernLLMOptions, WireMessage, WireRequest, WireStreamChunk, WireToolCall, WireToolChoice, defaultEstimateTokens, defaultFallbackOn, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, isLLMError, isToolCallResult, parseSseStream };
|
|
2997
|
+
export { AnthropicClient, AssistantContent, BedrockConverseClient, CacheAdapter, CachedCallParams, CachedConditionalToolCallParams, CachedJsonModeDisabledCallParams, CachedJsonModeEnabledCallParams, CachedStreamCallParams, CachedStreamConditionalToolCallParams, CachedStreamJsonModeDisabledCallParams, CachedStreamJsonModeEnabledCallParams, CachedStreamToolCallParams, CachedToolCallParams, CallMeta, CallParams, CallWithToolsResult, CircuitBreaker, CircuitBreakerOptions, CircuitState, CircuitTarget, ConditionalToolCallParams, ConsoleLogger, ContentBlock, ContentResult, ConversationTurn, DuplicateToolNamesIssue, FallbackAttempt, FallbackExhaustedError, FallbackOn, FallbackTarget, FetchAdapterConfig, GeminiClient, HistoryToolResultIssue, ImageBlock, InMemoryCacheAdapter, JsonModeDisabledCallParams, JsonModeEnabledCallParams, JsonSchemaSpec, JsonValue, LLMClient, LLMError, LLMErrorCode, LLMErrorIssuesByCode, LLMErrorSnapshot, LLMErrorType, LLMRequestSnapshot, Logger, NormalizedCacheAdapter, OnEvent, OnUsage, RateLimitAcquireResult, RateLimitOptions, RateLimitReason, RateLimiter, RefundUsage, ReserveUsage, RetryAttempt, SSE_PING, SchemaLike, StreamCallResult, StreamChunk, StreamEnabledCallParams, StreamJsonModeDisabledCallParams, StreamJsonModeEnabledCallParams, TargetCircuitState, TextBlock, TieredCacheAdapter, TokenUsage, ToolCall, ToolCallResult, ToolChoice, ToolDefinition, ToolEnabledCallParams, ToolIssue, ToolResult, ToolsDisabledCallParams, UnknownToolChoiceIssue, UnsupportedCapabilityIssue, VernLLM, VernLLMEvent, VernLLMOptions, WireMessage, WireRequest, WireStreamChunk, WireToolCall, WireToolChoice, defaultEstimateTokens, defaultFallbackOn, defineCachedCallParams, defineCallParams, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, hasIssues, isFallbackExhaustedError, isLLMError, isToolCallResult, parseSseStream };
|
|
2327
2998
|
//# sourceMappingURL=index.d.cts.map
|