vern-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +7 -0
- package/README.md +315 -0
- package/dist/index.cjs +742 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +608 -0
- package/dist/index.d.cts.map +1 -0
- package/dist/index.d.ts +608 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +701 -0
- package/dist/index.js.map +1 -0
- package/package.json +59 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,608 @@
|
|
|
1
|
+
//#region src/logger.d.ts
|
|
2
|
+
interface Logger {
|
|
3
|
+
debug(message: string): void;
|
|
4
|
+
warn(message: string): void;
|
|
5
|
+
error(message: string, meta?: Record<string, unknown>): void;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Default logger. `debug` is gated by the `debug` option on VernLLM
|
|
9
|
+
* warn/error always fire since they indicate real problems (retries, cache failures)
|
|
10
|
+
*/
|
|
11
|
+
declare class ConsoleLogger implements Logger {
|
|
12
|
+
private debugEnabled;
|
|
13
|
+
constructor(debugEnabled: boolean);
|
|
14
|
+
debug(message: string): void;
|
|
15
|
+
warn(message: string): void;
|
|
16
|
+
error(message: string, meta?: Record<string, unknown>): void;
|
|
17
|
+
} //#endregion
|
|
18
|
+
//#region src/circuitBreaker.d.ts
|
|
19
|
+
interface CircuitBreakerOptions {
|
|
20
|
+
/** Consecutive failures before the circuit opens, default 5 */
|
|
21
|
+
threshold?: number;
|
|
22
|
+
/** How long the circuit stays open before allowing a trial request, in ms. Default 30000 */
|
|
23
|
+
cooldownMs?: number;
|
|
24
|
+
}
|
|
25
|
+
type CircuitState = 'closed' | 'open' | 'half-open';
|
|
26
|
+
/**
|
|
27
|
+
* Per retry VernLLM-instance circuit breaker. Tracks consecutive failures across
|
|
28
|
+
* calls. Once the threshold is hit, short-circuits new calls with an
|
|
29
|
+
* LLMError('circuit_open') instead of hitting the provider, until the
|
|
30
|
+
* cooldown elapses and a single trial call is allowed through
|
|
31
|
+
*/
|
|
32
|
+
declare class CircuitBreaker {
|
|
33
|
+
private state;
|
|
34
|
+
private consecutiveFailures;
|
|
35
|
+
private openedAt;
|
|
36
|
+
private threshold;
|
|
37
|
+
private cooldownMs;
|
|
38
|
+
constructor(options?: CircuitBreakerOptions);
|
|
39
|
+
/** Throws if the circuit is open and the cooldown hasnt elapsed */
|
|
40
|
+
assertClosed(): void;
|
|
41
|
+
recordSuccess(): void;
|
|
42
|
+
recordFailure(): void;
|
|
43
|
+
getState(): CircuitState;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
//#endregion
|
|
47
|
+
//#region src/types.d.ts
|
|
48
|
+
type LLMErrorType = 'timeout' | 'api' | 'parse' | 'validation' | 'circuit_open' | 'unknown' | 'aborted';
|
|
49
|
+
declare class LLMError extends Error {
|
|
50
|
+
type: LLMErrorType;
|
|
51
|
+
status?: number | undefined;
|
|
52
|
+
issues?: unknown | undefined;
|
|
53
|
+
constructor(message: string, type: LLMErrorType, status?: number | undefined, issues?: unknown | undefined);
|
|
54
|
+
}
|
|
55
|
+
declare function isLLMError(err: unknown): err is LLMError;
|
|
56
|
+
interface CacheAdapter<T = unknown> {
|
|
57
|
+
get(key: string): Promise<T | null>;
|
|
58
|
+
set(key: string, value: T, ttl: number): Promise<void>;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Trivial default so the package works out of the box with no external deps
|
|
62
|
+
* Not shared across processes, swap in Redis/Upstash/etc for production
|
|
63
|
+
*/
|
|
64
|
+
declare class InMemoryCacheAdapter<T = unknown> implements CacheAdapter<T> {
|
|
65
|
+
private store;
|
|
66
|
+
get(key: string): Promise<T | null>;
|
|
67
|
+
set(key: string, value: T, ttl: number): Promise<void>;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Minimal shape compatible with the OpenAI SDKs chat.completions.create,
|
|
71
|
+
* so consumers can pass an OpenAI client directly
|
|
72
|
+
* `response_format.json_schema` and `reasoning_effort` are optional on the wire
|
|
73
|
+
* providers that don't support them will just ignore fields they don't recognize,
|
|
74
|
+
* but not every SDKs TS types accept them, hence this being a structural type
|
|
75
|
+
* rather than importing the SDKs own params type
|
|
76
|
+
*/
|
|
77
|
+
interface LLMClient {
|
|
78
|
+
chat: {
|
|
79
|
+
completions: {
|
|
80
|
+
create(params: {
|
|
81
|
+
model: string;
|
|
82
|
+
temperature: number;
|
|
83
|
+
max_tokens: number;
|
|
84
|
+
response_format?: {
|
|
85
|
+
type: 'json_object';
|
|
86
|
+
} | {
|
|
87
|
+
type: 'json_schema';
|
|
88
|
+
json_schema: {
|
|
89
|
+
name: string;
|
|
90
|
+
schema: Record<string, unknown>;
|
|
91
|
+
strict?: boolean;
|
|
92
|
+
description?: string;
|
|
93
|
+
};
|
|
94
|
+
};
|
|
95
|
+
/** OpenAI reasoning-model param (o-series, gpt-5), ignored by providers that don't support it */
|
|
96
|
+
reasoning_effort?: 'minimal' | 'low' | 'medium' | 'high';
|
|
97
|
+
messages: Array<{
|
|
98
|
+
role: 'system' | 'user';
|
|
99
|
+
content: string;
|
|
100
|
+
}>;
|
|
101
|
+
}, options: {
|
|
102
|
+
signal: AbortSignal;
|
|
103
|
+
}): Promise<{
|
|
104
|
+
choices?: Array<{
|
|
105
|
+
message?: {
|
|
106
|
+
content?: string | null;
|
|
107
|
+
};
|
|
108
|
+
}>;
|
|
109
|
+
usage?: {
|
|
110
|
+
prompt_tokens?: number;
|
|
111
|
+
completion_tokens?: number;
|
|
112
|
+
total_tokens?: number;
|
|
113
|
+
};
|
|
114
|
+
}>;
|
|
115
|
+
};
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
type ReserveUsage = () => Promise<void>;
|
|
119
|
+
type RefundUsage = () => Promise<void>;
|
|
120
|
+
interface TokenUsage {
|
|
121
|
+
promptTokens: number;
|
|
122
|
+
completionTokens: number;
|
|
123
|
+
totalTokens: number;
|
|
124
|
+
requestId: string;
|
|
125
|
+
model: string;
|
|
126
|
+
}
|
|
127
|
+
type OnUsage = (usage: TokenUsage) => void;
|
|
128
|
+
/**
|
|
129
|
+
* Minimal structural type for a Zod-like schema, so this package doesnt need
|
|
130
|
+
* a hard dependency on a specific Zod major version. Any object exposing
|
|
131
|
+
* `safeParse` (Zod v3/v4, and most Zod-compatible validators) should satisfy this
|
|
132
|
+
*/
|
|
133
|
+
interface SchemaLike<T> {
|
|
134
|
+
safeParse(data: unknown): {
|
|
135
|
+
success: true;
|
|
136
|
+
data: T;
|
|
137
|
+
} | {
|
|
138
|
+
success: false;
|
|
139
|
+
error: unknown;
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
interface VernLLMOptions {
|
|
143
|
+
client: LLMClient;
|
|
144
|
+
model: string;
|
|
145
|
+
/** Max retries after the first attempt. Default 1 (2 attempts total) */
|
|
146
|
+
maxRetries?: number;
|
|
147
|
+
/** Per-attempt timeout in ms. Default 25000 */
|
|
148
|
+
timeoutMs?: number;
|
|
149
|
+
/** Base delay for exponential backoff in ms. Default 500 */
|
|
150
|
+
baseDelayMs?: number;
|
|
151
|
+
/** Default max_tokens for calls that don't override it. Default 1000 */
|
|
152
|
+
defaultMaxTokens?: number;
|
|
153
|
+
/** Enables debug logging of raw model output. Default: NODE_ENV !== 'production' */
|
|
154
|
+
debug?: boolean;
|
|
155
|
+
/** Cache adapter for cachedCall. Defaults to an in-memory adapter */
|
|
156
|
+
cache?: CacheAdapter;
|
|
157
|
+
/** HTTP status codes that should fail fast without retrying. Default [400, 401, 403] */
|
|
158
|
+
nonRetryableStatus?: number[];
|
|
159
|
+
/** Custom JSON parser. Must return undefined/null on failure. Default: JSON.parse wrapped in try/catch */
|
|
160
|
+
parseJson?: (content: string) => unknown;
|
|
161
|
+
/** Called after every successful call with token usage, if the provider reports it */
|
|
162
|
+
onUsage?: OnUsage;
|
|
163
|
+
/** Injectable logger. Defaults to a console-based logger gated by `debug` */
|
|
164
|
+
logger?: Logger;
|
|
165
|
+
/**
|
|
166
|
+
* Enables a circuit breaker that short-circuits calls after repeated
|
|
167
|
+
* consecutive failures, instead of continuing to hammer a down provider
|
|
168
|
+
* Pass `true` for defaults, or an options object to tune threshold/cooldown
|
|
169
|
+
*/
|
|
170
|
+
circuitBreaker?: boolean | CircuitBreakerOptions;
|
|
171
|
+
}
|
|
172
|
+
/**
|
|
173
|
+
* A provider-native JSON Schema for structured outputs (OpenAI/Groq
|
|
174
|
+
* `response_format: { type: 'json_schema' }`) This is the wire-format
|
|
175
|
+
* schema the model is constrained to generate against, distinct from
|
|
176
|
+
* `schema`, which is a client-side Zod validator run on the parsed result
|
|
177
|
+
* You can use one, both, or neither; using both gets you provider-level
|
|
178
|
+
* constraint plus client-side type inference/validation as a safety net
|
|
179
|
+
*/
|
|
180
|
+
interface JsonSchemaSpec {
|
|
181
|
+
name: string;
|
|
182
|
+
schema: Record<string, unknown>;
|
|
183
|
+
/** Enforces the schema strictly (OpenAI-specific), default true when supported */
|
|
184
|
+
strict?: boolean;
|
|
185
|
+
description?: string;
|
|
186
|
+
}
|
|
187
|
+
interface CallParams<T = unknown> {
|
|
188
|
+
systemPrompt: string;
|
|
189
|
+
userContent: string;
|
|
190
|
+
temperature?: number;
|
|
191
|
+
jsonMode?: boolean;
|
|
192
|
+
maxTokens?: number;
|
|
193
|
+
requestId?: string;
|
|
194
|
+
signal?: AbortSignal;
|
|
195
|
+
/** Overrides the model set on the VernLLM instance for this call only */
|
|
196
|
+
model?: string;
|
|
197
|
+
/**
|
|
198
|
+
* OpenAI-style reasoning effort for reasoning models (o-series, gpt-5, etc).
|
|
199
|
+
* Passed through as-is, providers/models that don't support it ignore it
|
|
200
|
+
*/
|
|
201
|
+
reasoningEffort?: 'minimal' | 'low' | 'medium' | 'high';
|
|
202
|
+
/**
|
|
203
|
+
* Provider-native JSON Schema structured-output mode. When set, this is sent
|
|
204
|
+
* as `response_format: { type: 'json_schema', json_schema: ... }` instead of
|
|
205
|
+
* the looser `json_object` mode, constraining the models output shape at
|
|
206
|
+
* generation time (not just validating it after the fact). Implies jsonMode: true.
|
|
207
|
+
*/
|
|
208
|
+
jsonSchema?: JsonSchemaSpec;
|
|
209
|
+
/**
|
|
210
|
+
* Optional Zod (or Zod-compatible) schema. When provided, the parsed JSON
|
|
211
|
+
* is validated against it; on failure an LLMError('validation') is thrown
|
|
212
|
+
* with `.issues` set to the schema's error object. Implies jsonMode: true.
|
|
213
|
+
* Can be combined with `jsonSchema` for provider-level constraint + client-side typing.
|
|
214
|
+
*/
|
|
215
|
+
schema?: SchemaLike<T>;
|
|
216
|
+
}
|
|
217
|
+
interface CachedCallParams<T> {
|
|
218
|
+
cacheKey: string;
|
|
219
|
+
ttl: number;
|
|
220
|
+
fn: () => Promise<T>;
|
|
221
|
+
reserveUsage?: ReserveUsage;
|
|
222
|
+
refundUsage?: RefundUsage;
|
|
223
|
+
} //#endregion
|
|
224
|
+
//#region src/vernLLM.d.ts
|
|
225
|
+
/**
|
|
226
|
+
* A resilient wrapper around an LLM chat completions client, this is VernLLM!
|
|
227
|
+
*
|
|
228
|
+
* Adds retry with exponential backoff and jitter, per-attempt timeouts,
|
|
229
|
+
* an optional circuit breaker, JSON parsing with optional schema
|
|
230
|
+
* validation, usage tracking, and an optional response cache, all
|
|
231
|
+
* configurable, all opt-in beyond sensible defaults
|
|
232
|
+
*/
|
|
233
|
+
declare class VernLLM {
|
|
234
|
+
private readonly client;
|
|
235
|
+
private readonly model;
|
|
236
|
+
private readonly maxRetries;
|
|
237
|
+
private readonly timeoutMs;
|
|
238
|
+
private readonly baseDelayMs;
|
|
239
|
+
private readonly defaultMaxTokens;
|
|
240
|
+
private readonly cache;
|
|
241
|
+
private readonly nonRetryableStatus;
|
|
242
|
+
private readonly parseJson;
|
|
243
|
+
private readonly onUsage?;
|
|
244
|
+
private readonly logger;
|
|
245
|
+
private readonly breaker?;
|
|
246
|
+
/**
|
|
247
|
+
* @param options: Client, model, and all tunables (retries, timeout,
|
|
248
|
+
* backoff, cache, circuit breaker, logger, etc). See VernLLMOptions in `types.ts`
|
|
249
|
+
* for individual defaults
|
|
250
|
+
*/
|
|
251
|
+
constructor(options: VernLLMOptions);
|
|
252
|
+
/**
|
|
253
|
+
* Resolves retry/timeout/token defaults from the given options,
|
|
254
|
+
* falling back to the librarys built-in defaults for anything unset
|
|
255
|
+
*/
|
|
256
|
+
private resolveRetryConfig;
|
|
257
|
+
/**
|
|
258
|
+
* Returns the caller supplied logger, or a console-based logger whose
|
|
259
|
+
* debug output is gated by the `debug` option (defaulting to on
|
|
260
|
+
* outside production)
|
|
261
|
+
*/
|
|
262
|
+
private resolveLogger;
|
|
263
|
+
/**
|
|
264
|
+
* Builds a circuit breaker if `circuitBreaker` is truthy on the
|
|
265
|
+
* options. Passing `true` uses default thresholds, passing an options
|
|
266
|
+
* object tunes them. Returns undefined when the breaker is disabled
|
|
267
|
+
*/
|
|
268
|
+
private resolveCircuitBreaker;
|
|
269
|
+
/**
|
|
270
|
+
* Makes a single logical LLM call, transparently retrying on failure
|
|
271
|
+
* according to the configured retry policy
|
|
272
|
+
*
|
|
273
|
+
* Fails fast if the circuit breaker is open or the signal is already
|
|
274
|
+
* aborted, before any request is dispatched. On exhausting all
|
|
275
|
+
* retries, records a circuit breaker failure and rejects with a
|
|
276
|
+
* normalized LLMError
|
|
277
|
+
*
|
|
278
|
+
* @param params : System/user content plus per call overrides
|
|
279
|
+
* (model, temperature, jsonMode, schema, signal, etc)
|
|
280
|
+
* @returns The parsed and optionally schema-validated response, or
|
|
281
|
+
* the raw string content when jsonMode is disabled
|
|
282
|
+
*/
|
|
283
|
+
call<T = unknown>(params: CallParams<T>): Promise<T>;
|
|
284
|
+
/**
|
|
285
|
+
* Runs `fn`, retrying with backoff according to `shouldRetry` policy
|
|
286
|
+
* Purely mechanical: knows nothing about LLM specifics beyond the retry
|
|
287
|
+
* predicate, so its testable independent of request/response shaping
|
|
288
|
+
*/
|
|
289
|
+
private retryWithBackoff;
|
|
290
|
+
/**
|
|
291
|
+
* Converts any thrown value into a well-typed LLMError for the public
|
|
292
|
+
* API surface. Preserves an existing LLMError as is, reports aborted
|
|
293
|
+
* signals as such, classifies errors carrying an http status as
|
|
294
|
+
* type api, and otherwise falls back to a generic unknown error.
|
|
295
|
+
*/
|
|
296
|
+
private normalizeError;
|
|
297
|
+
/**
|
|
298
|
+
* Performs a single attempt: builds the request, dispatches it with a
|
|
299
|
+
* timeout, and shapes the response. Throws on an empty response so
|
|
300
|
+
* the retry loop treats it like any other transient failure. Records
|
|
301
|
+
* usage and a circuit breaker success before returning
|
|
302
|
+
*/
|
|
303
|
+
private executeCall;
|
|
304
|
+
/**
|
|
305
|
+
* Applies per call defaults and shapes the params into the request
|
|
306
|
+
* object expected by the underlying client, including the resolved
|
|
307
|
+
* response format. Also returns whether JSON parsing should be
|
|
308
|
+
* applied to the response and which model was ultimately used
|
|
309
|
+
*/
|
|
310
|
+
private buildRequestPayload;
|
|
311
|
+
/**
|
|
312
|
+
* Chooses the response format to send to the provider. A provider
|
|
313
|
+
* native json schema takes priority when supplied, constraining
|
|
314
|
+
* generation directly, otherwise falls back to the looser json
|
|
315
|
+
* object mode when JSON output is requested, or no format at all
|
|
316
|
+
* for plain text responses
|
|
317
|
+
*/
|
|
318
|
+
private buildResponseFormat;
|
|
319
|
+
/**
|
|
320
|
+
* Reports token usage to the caller supplied onUsage callback, when
|
|
321
|
+
* both a callback was configured and the provider actually returned
|
|
322
|
+
* usage data on this response. A no op otherwise.
|
|
323
|
+
*/
|
|
324
|
+
private recordUsage;
|
|
325
|
+
/**
|
|
326
|
+
* Parses the raw response content as JSON and, when a schema is
|
|
327
|
+
* supplied, validates the parsed value against it. Throws a parse
|
|
328
|
+
* type LLMError on malformed JSON and a validation type LLMError,
|
|
329
|
+
* carrying the schemas issues, on a failed validation
|
|
330
|
+
*/
|
|
331
|
+
private parseAndValidate;
|
|
332
|
+
/**
|
|
333
|
+
* Waits out the backoff delay for a given retry attempt, logging the
|
|
334
|
+
* attempt for observability before the wait begins. Rejects early if
|
|
335
|
+
* the signal aborts during the wait
|
|
336
|
+
*/
|
|
337
|
+
private recoverDelay;
|
|
338
|
+
/**
|
|
339
|
+
* Decides whether a failed attempt is worth retrying. Never retries
|
|
340
|
+
* once the signal has aborted, never retries a parse or validation
|
|
341
|
+
* failure since those stem from the response content rather than a
|
|
342
|
+
* transient fault, and never retries a status code the caller has
|
|
343
|
+
* marked as non retryable. Retries everything else
|
|
344
|
+
*/
|
|
345
|
+
private shouldRetry;
|
|
346
|
+
/**
|
|
347
|
+
* Thin cache wrapper around caller supplied logic. `params.fn` is expected
|
|
348
|
+
* to be a call that itself invokes `this.call(...)` (see `cachedLLMCall`
|
|
349
|
+
* below for a convenience wrapper that wires this up automatically),
|
|
350
|
+
* `cachedCall` does not itself apply retry/timeout policy.
|
|
351
|
+
*/
|
|
352
|
+
cachedCall<T>(params: CachedCallParams<T>): Promise<T>;
|
|
353
|
+
/**
|
|
354
|
+
* Convenience wrapper composing `call` + `cachedCall`, so cached LLM calls
|
|
355
|
+
* automatically get retry/timeout/circuit-breaker behavior without callers
|
|
356
|
+
* having to remember to wire `fn: () => this.call(...)` themselves
|
|
357
|
+
*/
|
|
358
|
+
cachedLLMCall<T>(params: Omit<CachedCallParams<T>, 'fn'> & {
|
|
359
|
+
call: CallParams<T>;
|
|
360
|
+
}): Promise<T>;
|
|
361
|
+
/**
|
|
362
|
+
* Returns the current circuit breaker state, or undefined when no
|
|
363
|
+
* circuit breaker was configured on this instance
|
|
364
|
+
*/
|
|
365
|
+
getCircuitState(): ("closed" | "open" | "half-open") | undefined;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
//#endregion
|
|
369
|
+
//#region src/adapters/anthropic.d.ts
|
|
370
|
+
/** Minimal structural type for the Anthropic SDKs `messages.create` */
|
|
371
|
+
interface AnthropicClient {
|
|
372
|
+
messages: {
|
|
373
|
+
create(params: {
|
|
374
|
+
model: string;
|
|
375
|
+
max_tokens: number;
|
|
376
|
+
temperature?: number;
|
|
377
|
+
system?: string;
|
|
378
|
+
messages: Array<{
|
|
379
|
+
role: 'user' | 'assistant';
|
|
380
|
+
content: string;
|
|
381
|
+
}>;
|
|
382
|
+
}, options: {
|
|
383
|
+
signal: AbortSignal;
|
|
384
|
+
}): Promise<{
|
|
385
|
+
content: Array<{
|
|
386
|
+
type: string;
|
|
387
|
+
text?: string;
|
|
388
|
+
}>;
|
|
389
|
+
usage?: {
|
|
390
|
+
input_tokens?: number;
|
|
391
|
+
output_tokens?: number;
|
|
392
|
+
};
|
|
393
|
+
}>;
|
|
394
|
+
};
|
|
395
|
+
}
|
|
396
|
+
/**
|
|
397
|
+
* Wraps an Anthropic SDK client so it satisfies the same `LLMClient`
|
|
398
|
+
* interface VernLLM uses for OpenAI/Groq. Anthropics Messages API has no
|
|
399
|
+
* `response_format: json_object` equivalent, so when the caller requests
|
|
400
|
+
* JSON mode, this adapter appends an instruction to the system prompt
|
|
401
|
+
* asking the model to respond with JSON only
|
|
402
|
+
*/
|
|
403
|
+
declare function fromAnthropic(anthropicClient: AnthropicClient): LLMClient;
|
|
404
|
+
|
|
405
|
+
//#endregion
|
|
406
|
+
//#region src/adapters/gemini.d.ts
|
|
407
|
+
/**
|
|
408
|
+
* Minimal structural type for Geminis `generateContent`, matching both the
|
|
409
|
+
* legacy `@google/generative-ai` SDKs `model.generateContent(...)` and the
|
|
410
|
+
* newer `@google/genai` SDKs `ai.models.generateContent({ model, ... })`
|
|
411
|
+
* closely enough to adapt either — pass whichever `.generateContent` you have.
|
|
412
|
+
*/
|
|
413
|
+
interface GeminiClient {
|
|
414
|
+
generateContent(params: {
|
|
415
|
+
model?: string;
|
|
416
|
+
contents: Array<{
|
|
417
|
+
role: "user";
|
|
418
|
+
parts: Array<{
|
|
419
|
+
text: string;
|
|
420
|
+
}>;
|
|
421
|
+
}>;
|
|
422
|
+
systemInstruction?: {
|
|
423
|
+
parts: Array<{
|
|
424
|
+
text: string;
|
|
425
|
+
}>;
|
|
426
|
+
};
|
|
427
|
+
generationConfig?: {
|
|
428
|
+
temperature?: number;
|
|
429
|
+
maxOutputTokens?: number;
|
|
430
|
+
responseMimeType?: string;
|
|
431
|
+
responseSchema?: Record<string, unknown>;
|
|
432
|
+
};
|
|
433
|
+
}, options: {
|
|
434
|
+
signal: AbortSignal;
|
|
435
|
+
}): Promise<{
|
|
436
|
+
candidates?: Array<{
|
|
437
|
+
content?: {
|
|
438
|
+
parts?: Array<{
|
|
439
|
+
text?: string;
|
|
440
|
+
}>;
|
|
441
|
+
};
|
|
442
|
+
}>;
|
|
443
|
+
usageMetadata?: {
|
|
444
|
+
promptTokenCount?: number;
|
|
445
|
+
candidatesTokenCount?: number;
|
|
446
|
+
totalTokenCount?: number;
|
|
447
|
+
};
|
|
448
|
+
}>;
|
|
449
|
+
}
|
|
450
|
+
/**
|
|
451
|
+
* Wraps a Gemini client so it satisfies the `LLMClient` interface VernLLM
|
|
452
|
+
* uses for OpenAI/Groq. Geminis shape differs on nearly every axis: a
|
|
453
|
+
* `contents` array instead of `messages`, a separate `systemInstruction`
|
|
454
|
+
* field instead of a `system` role message, `generationConfig` instead of
|
|
455
|
+
* top-level `temperature`/`max_tokens`, and native JSON Schema support via
|
|
456
|
+
* `responseMimeType: 'application/json'` + `responseSchema` (so `jsonSchema`
|
|
457
|
+
* is provider-enforced here, unlike the Anthropic adapters prompt-embedding
|
|
458
|
+
* fallback). `reasoning_effort` has no equivalent. Geminis thinking models
|
|
459
|
+
* use a token budget, not an effort tier, so its dropped, same as Anthropic.
|
|
460
|
+
*/
|
|
461
|
+
declare function fromGemini(geminiClient: GeminiClient): LLMClient;
|
|
462
|
+
|
|
463
|
+
//#endregion
|
|
464
|
+
//#region src/adapters/bedrock.d.ts
|
|
465
|
+
/**
|
|
466
|
+
* Minimal structural type matching AWS Bedrocks Converse API. This is
|
|
467
|
+
* intentionally NOT `BedrockRuntimeClient` itself, the AWS SDK v3 client
|
|
468
|
+
* exposes `.send(command)`, not a direct `.converse()` method, and pulling
|
|
469
|
+
* in `@aws-sdk/client-bedrock-runtime` as a dependency just for its types
|
|
470
|
+
* isn't worth it for a structural adapter. Wrap your client, e.g:
|
|
471
|
+
*
|
|
472
|
+
* ```ts
|
|
473
|
+
* import { BedrockRuntimeClient, ConverseCommand } from '@aws-sdk/client-bedrock-runtime';
|
|
474
|
+
* const client = new BedrockRuntimeClient({ region: 'us-east-1' });
|
|
475
|
+
* const converseClient = {
|
|
476
|
+
* converse: (params, options) =>
|
|
477
|
+
* client.send(new ConverseCommand(params), { abortSignal: options.signal }),
|
|
478
|
+
* };
|
|
479
|
+
* ```
|
|
480
|
+
*/
|
|
481
|
+
interface BedrockConverseClient {
|
|
482
|
+
converse(params: {
|
|
483
|
+
modelId: string;
|
|
484
|
+
messages: Array<{
|
|
485
|
+
role: 'user';
|
|
486
|
+
content: Array<{
|
|
487
|
+
text: string;
|
|
488
|
+
}>;
|
|
489
|
+
}>;
|
|
490
|
+
system?: Array<{
|
|
491
|
+
text: string;
|
|
492
|
+
}>;
|
|
493
|
+
inferenceConfig?: {
|
|
494
|
+
temperature?: number;
|
|
495
|
+
maxTokens?: number;
|
|
496
|
+
};
|
|
497
|
+
}, options: {
|
|
498
|
+
signal: AbortSignal;
|
|
499
|
+
}): Promise<{
|
|
500
|
+
output?: {
|
|
501
|
+
message?: {
|
|
502
|
+
content?: Array<{
|
|
503
|
+
text?: string;
|
|
504
|
+
}>;
|
|
505
|
+
};
|
|
506
|
+
};
|
|
507
|
+
usage?: {
|
|
508
|
+
inputTokens?: number;
|
|
509
|
+
outputTokens?: number;
|
|
510
|
+
totalTokens?: number;
|
|
511
|
+
};
|
|
512
|
+
}>;
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
516
|
+
* interface VernLLM uses for OpenAI/Groq. The Converse API is unified
|
|
517
|
+
* across Bedrocks model families (Anthropic, Titan, Llama, Mistral, etc.),
|
|
518
|
+
* so unlike raw per-model Bedrock invocation, this one adapter works
|
|
519
|
+
* regardless of which underlying model `modelId` points at, as long as
|
|
520
|
+
* that model supports Converse (most current-generation ones do)
|
|
521
|
+
*
|
|
522
|
+
* Theres no uniform native JSON Schema enforcement across families here
|
|
523
|
+
* (some support it via forced tool-use, which varies per model), so
|
|
524
|
+
* `jsonSchema`/`jsonMode` are emulated via a system-prompt instruction, same
|
|
525
|
+
* approach as the Anthropic adapter. `reasoning_effort` has no Converse
|
|
526
|
+
* equivalent and is dropped
|
|
527
|
+
*/
|
|
528
|
+
declare function fromBedrock(bedrockClient: BedrockConverseClient): LLMClient;
|
|
529
|
+
|
|
530
|
+
//#endregion
|
|
531
|
+
//#region src/adapters/fetch.d.ts
|
|
532
|
+
/** The chat-completion-shaped request VernLLM builds internally */
|
|
533
|
+
type ChatRequest = Parameters<LLMClient['chat']['completions']['create']>[0];
|
|
534
|
+
interface FetchAdapterConfig {
|
|
535
|
+
/** Endpoint URL, or a function of the request in case it depends on model/params */
|
|
536
|
+
url: string | ((params: ChatRequest) => string);
|
|
537
|
+
/** Static headers, or a function (sync or async) for things like refreshed auth tokens */
|
|
538
|
+
headers?: Record<string, string> | (() => Record<string, string> | Promise<Record<string, string>>);
|
|
539
|
+
/** HTTP method. Default 'POST' */
|
|
540
|
+
method?: string;
|
|
541
|
+
/** Maps VernLLMs internal chat-completion request into the providers raw request body */
|
|
542
|
+
mapRequest: (params: ChatRequest) => unknown;
|
|
543
|
+
/**
|
|
544
|
+
* Maps the providers raw JSON response into `{ content, usage? }`
|
|
545
|
+
* `content` is the assistants text (JSON string when JSON mode was requested)
|
|
546
|
+
*/
|
|
547
|
+
mapResponse: (json: unknown) => {
|
|
548
|
+
content: string;
|
|
549
|
+
usage?: {
|
|
550
|
+
promptTokens?: number;
|
|
551
|
+
completionTokens?: number;
|
|
552
|
+
totalTokens?: number;
|
|
553
|
+
};
|
|
554
|
+
};
|
|
555
|
+
}
|
|
556
|
+
/**
|
|
557
|
+
* A fetch-based escape hatch for providers with no SDK, or where pulling one
|
|
558
|
+
* in isnt worth it. You supply the URL, headers, and two small mapping
|
|
559
|
+
* functions; this handles the HTTP call and slots the result into the same
|
|
560
|
+
* `LLMClient` shape every other adapter produces, so retries, timeouts,
|
|
561
|
+
* the circuit breaker, and JSON/schema handling all still work unmodified
|
|
562
|
+
*
|
|
563
|
+
* Non-2xx responses throw an error with `.status` set to the HTTP status
|
|
564
|
+
* code, so VernLLMs `nonRetryableStatus` handling (e.g. failing fast on
|
|
565
|
+
* 401/403) applies here too
|
|
566
|
+
*/
|
|
567
|
+
declare function fromFetch(config: FetchAdapterConfig): LLMClient;
|
|
568
|
+
|
|
569
|
+
//#endregion
|
|
570
|
+
//#region src/adapters/openaiCompatible.d.ts
|
|
571
|
+
/**
|
|
572
|
+
* Passthrough adapter for any SDK/client whose `chat.completions.create`
|
|
573
|
+
* already matches the OpenAI wire format 1:1 : this covers most hosted
|
|
574
|
+
* inference providers, since "OpenAI-compatible" is a de facto standard for
|
|
575
|
+
* chat completion APIs. No transformation happens here, this exists purely
|
|
576
|
+
* so call sites read clearly (`fromMistral(client)` vs handing a Mistral
|
|
577
|
+
* client to something typed for OpenAI) and so a real transformation could
|
|
578
|
+
* be added later, per-provider, without a breaking change.
|
|
579
|
+
*
|
|
580
|
+
* Not every SDKs own TypeScript types line up exactly with `LLMClient`
|
|
581
|
+
* (extra fields, stricter unions, etc.), so this takes `unknown` and casts:
|
|
582
|
+
* the actual compatibility contract is the JSON each provider sends and
|
|
583
|
+
* receives over the wire, not the SDKs TS types.
|
|
584
|
+
*/
|
|
585
|
+
declare function fromOpenAICompatible(client: unknown): LLMClient;
|
|
586
|
+
/** Groqs SDK matches the OpenAI wire format */
|
|
587
|
+
declare const fromGroq: typeof fromOpenAICompatible;
|
|
588
|
+
/** Mistrals `chat.completions`-shaped client (or their OpenAI-compat endpoint) */
|
|
589
|
+
declare const fromMistral: typeof fromOpenAICompatible;
|
|
590
|
+
/** DeepSeeks API is OpenAI-compatible */
|
|
591
|
+
declare const fromDeepSeek: typeof fromOpenAICompatible;
|
|
592
|
+
/** Cerebras inference API is OpenAI-compatible */
|
|
593
|
+
declare const fromCerebras: typeof fromOpenAICompatible;
|
|
594
|
+
/** Together AIs API is OpenAI-compatible */
|
|
595
|
+
declare const fromTogether: typeof fromOpenAICompatible;
|
|
596
|
+
/** Fireworks AIs API is OpenAI-compatible */
|
|
597
|
+
declare const fromFireworks: typeof fromOpenAICompatible;
|
|
598
|
+
/**
|
|
599
|
+
* Ollama exposes an OpenAI-compatible endpoint at `/v1/chat/completions`
|
|
600
|
+
* (as opposed to its native `/api/chat` format, which differs). Point an
|
|
601
|
+
* OpenAI SDK instances `baseURL` at your Ollama server and pass it here:
|
|
602
|
+
* this does not talk to Ollamas native API directly.
|
|
603
|
+
*/
|
|
604
|
+
declare const fromOllama: typeof fromOpenAICompatible;
|
|
605
|
+
|
|
606
|
+
//#endregion
|
|
607
|
+
export { CacheAdapter, CachedCallParams, CallParams, CircuitBreaker, CircuitBreakerOptions, ConsoleLogger, FetchAdapterConfig, InMemoryCacheAdapter, JsonSchemaSpec, LLMClient, LLMError, LLMErrorType, Logger, OnUsage, RefundUsage, ReserveUsage, SchemaLike, TokenUsage, VernLLM, VernLLMOptions, fromAnthropic, fromBedrock, fromCerebras, fromDeepSeek, fromFetch, fromFireworks, fromGemini, fromGroq, fromMistral, fromOllama, fromOpenAICompatible, fromTogether, isLLMError };
|
|
608
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","names":[],"sources":["../src/logger.d.ts","../src/circuitBreaker.d.ts","../src/types.d.ts","../src/vernLLM.d.ts","../src/adapters/anthropic.d.ts","../src/adapters/gemini.d.ts","../src/adapters/bedrock.d.ts","../src/adapters/fetch.d.ts","../src/adapters/openaiCompatible.d.ts"],"sourcesContent":null,"mappings":";AAAA,IAAW,SAAS,CAAC,IAAG,MAAA,MAAA;;;;;AAKxB,IAAE,gBAAA;CAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;ACLF,IAAW,wBAAwB,CAAC,EAAG;AACvC,IAAI,eAAe,CAAC,EAAG;;;;;;;AAOvB,IAAW,iBAAiB;CAAC;CAAI,MAAM;CAAuB,MAAM;AAAY;;;;ACNhF,IAAW,eAAW,CAAA,EAAA;AACtB,IAAW,WAAW;CAAC;CAAI,MAAI;CAAA,MAAA;CAAA,MAAA;AAAA;AAC/B,IAAW,aAAa,CAAC,IAAI,MAAG,QAAA;AAChC,IAAW,eAAe;CAAC;CAAI,MAAM;CAAG,MAAM;CAAS,MAAM;CAAG,MAAM;AAAQ;;;;;AAK9E,IAAW,uBAAuB;CAAC;CAAI,MAAM;CAAG,MAAM;CAAG,MAAC;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;AAS1D,IAAW,YAAY;CAAC;CAAI,MAAM;CAAQ,MAAM;CAAO,MAAG;CAAA,MAAA;CAAA,MAAA;AAAA;AAC1D,IAAA,eAAA,CAAA,IAAA,MAAA,OAAA;AACA,IAAE,cAAA,CAAA,IAAA,MAAA,OAAA;AACF,IAAW,aAAa,CAAC,EAAG;AAC5B,IAAW,UAAU,CAAC,IAAI,MAAM,UAAW;;;;;;AAM3C,IAAW,aAAa,CAAC,IAAE,MAAA,CAAA;AAC3B,IAAU,iBAAA;CAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA;CAAA;AAAA;;;;;;;;;AASV,IAAW,iBAAiB,CAAC,IAAI,MAAM,MAAA;AACvC,IAAW,aAAa;CAAC;CAAI,MAAI;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;AACjC,IAAW,mBAAmB;CAAC;CAAI,MAAC;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;;;;AChCpC,IAAW,UAAU;CAAC;CAAI,MAAG;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;ACP7B,IAAI,kBAAkB;CAAC;CAAG,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;AAQ1B,IAAW,gBAAgB;CAAC;CAAI,MAAM;CAAI,MAAA;AAAA;;;;;;;;;;ACH1C,IAAI,eAAe;CAAC;CAAG,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;;;;AAYvB,IAAW,aAAG;CAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;;;;;;;;;;;;ACFd,IAAI,wBAAwB;CAAC;CAAG,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;;;;;;;AAehC,IAAS,cAAA;CAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;AC9BT,IAAI,cAAc;CAAC;CAAG,MAAM;CAAW,MAAM;AAAW;AACxD,IAAW,qBAAqB;CAAC;CAAG,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;;;;AAYpC,IAAM,YAAA;CAAA;CAAA,MAAA;CAAA,MAAA;AAAA;;;;;;;;;;;;;;;;;;ACAN,IAAW,uBAAuB,CAAC,GAAG,MAAM,SAAU;;AAEtD,IAAW,WAAW,CAAC,GAAG,MAAM,oBAAqB;;AAErD,IAAW,cAAc,CAAC,GAAG,MAAM,oBAAqB;;AAExD,IAAW,eAAe,CAAC,IAAI,MAAM,oBAAqB;;AAE1D,IAAW,eAAe,CAAC,IAAI,MAAM,oBAAqB;;AAE1D,IAAW,eAAe,CAAC,IAAI,MAAM,oBAAqB;;AAE1D,IAAW,gBAAgB,CAAC,IAAI,MAAM,oBAAqB;;;;;;;AAO3D,IAAW,aAAa,CAAC,IAAI,MAAM,oBAAqB"}
|