vern-llm 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.cjs +969 -278
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +550 -105
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +550 -105
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +968 -279
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -95,6 +95,42 @@ function safeIssues(issues) {
|
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
/**
|
|
98
|
+
* Returns a JSON safe, independent copy of `body`, or a marker string if
|
|
99
|
+
* `body` can't survive `JSON.stringify` (e.g. a circular reference). A
|
|
100
|
+
* request body built from adapter-transformed messages is normally
|
|
101
|
+
* always plain data, but tool call arguments or a caller supplied
|
|
102
|
+
* `cause`-adjacent value could in principle carry a circular reference,
|
|
103
|
+
* so this guards the same way `safeIssues` does rather than assuming it
|
|
104
|
+
* can't happen. Unlike `safeIssues`, this clones rather than returning
|
|
105
|
+
* the same reference: the object backing a request body can still be
|
|
106
|
+
* mutated by adapter code between when a request is dispatched and when
|
|
107
|
+
* an attempt is later recorded as failed (e.g. `fromGemini` sets
|
|
108
|
+
* `request.config` in place), so returning the same reference here could
|
|
109
|
+
* make a stored snapshot silently reflect a later, different state than
|
|
110
|
+
* what was actually sent.
|
|
111
|
+
*/
|
|
112
|
+
function safeBody(body) {
|
|
113
|
+
if (body === void 0) return void 0;
|
|
114
|
+
try {
|
|
115
|
+
return JSON.parse(JSON.stringify(body));
|
|
116
|
+
} catch {
|
|
117
|
+
return "[Unserializable: request body contained a circular reference]";
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
const AUTH_HEADER_NAMES = new Set([
|
|
121
|
+
"authorization",
|
|
122
|
+
"x-api-key",
|
|
123
|
+
"x-goog-api-key",
|
|
124
|
+
"api-key"
|
|
125
|
+
]);
|
|
126
|
+
/** Removes auth headers before a request snapshot is built. Case insensitive on header names. */
|
|
127
|
+
function stripAuthHeaders(headers) {
|
|
128
|
+
if (headers === void 0) return void 0;
|
|
129
|
+
const out = {};
|
|
130
|
+
for (const [key, value] of Object.entries(headers)) if (!AUTH_HEADER_NAMES.has(key.toLowerCase())) out[key] = value;
|
|
131
|
+
return out;
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
98
134
|
* Depth cap for `safeAttempts`, guarding against a pathological,
|
|
99
135
|
* self referential `attempts` array. `attempts` is a public
|
|
100
136
|
* `LLMErrorOptions` field, so a caller can construct one by hand; this
|
|
@@ -110,8 +146,12 @@ const MAX_ATTEMPTS_DEPTH = 20;
|
|
|
110
146
|
* circular one after the snapshot was created, and `attempts` is a
|
|
111
147
|
* public constructor option, so a caller can hand build a `RetryAttempt`
|
|
112
148
|
* (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
|
|
113
|
-
* in directly, never touching `toSnapshot()` at all.
|
|
114
|
-
*
|
|
149
|
+
* in directly, never touching `toSnapshot()` at all. The same applies to
|
|
150
|
+
* `request`: its `body` is re-checked through `safeBody`, and its
|
|
151
|
+
* `headers` are re-stripped through `stripAuthHeaders`, so a hand built
|
|
152
|
+
* `RetryAttempt.request` can't smuggle an auth header past `toSnapshot()`
|
|
153
|
+
* either. Extra fields on an attempt (e.g. `FallbackAttempt`'s
|
|
154
|
+
* `provider`/`model`) are preserved.
|
|
115
155
|
*/
|
|
116
156
|
function safeAttempts(attempts, depth = 0) {
|
|
117
157
|
if (attempts === void 0) return void 0;
|
|
@@ -122,9 +162,37 @@ function safeAttempts(attempts, depth = 0) {
|
|
|
122
162
|
...attempt.error,
|
|
123
163
|
issues: safeIssues(attempt.error.issues),
|
|
124
164
|
attempts: safeAttempts(attempt.error.attempts, depth + 1)
|
|
165
|
+
},
|
|
166
|
+
request: attempt.request && {
|
|
167
|
+
...attempt.request,
|
|
168
|
+
body: safeBody(attempt.request.body),
|
|
169
|
+
headers: stripAuthHeaders(attempt.request.headers)
|
|
125
170
|
}
|
|
126
171
|
}));
|
|
127
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Builds a point-in-time, plain data copy of one attempt's outgoing
|
|
175
|
+
* request. Mirrors `LLMError.toSnapshot()`: never thrown or dispatched
|
|
176
|
+
* again, safe to serialize and store. A plain function rather than a
|
|
177
|
+
* method, since unlike `LLMError` a request has no throwable identity or
|
|
178
|
+
* derived state worth wrapping in a class.
|
|
179
|
+
*
|
|
180
|
+
* `startedAt` is optional so existing call sites (and tests) that don't
|
|
181
|
+
* care about exact timing keep working, but a caller that has a real
|
|
182
|
+
* capture time should always pass it: this function may run well after
|
|
183
|
+
* the request was actually dispatched (e.g. `callExecutor` only builds
|
|
184
|
+
* the snapshot once an attempt has failed), so defaulting to `Date.now()`
|
|
185
|
+
* here would record failure-handling time, not request-start time.
|
|
186
|
+
*/
|
|
187
|
+
function toRequestSnapshot(provider, model, body, headers, startedAt = Date.now()) {
|
|
188
|
+
return {
|
|
189
|
+
provider,
|
|
190
|
+
model,
|
|
191
|
+
body: safeBody(body),
|
|
192
|
+
headers: stripAuthHeaders(headers),
|
|
193
|
+
startedAt
|
|
194
|
+
};
|
|
195
|
+
}
|
|
128
196
|
var LLMError = class extends Error {
|
|
129
197
|
status;
|
|
130
198
|
issues;
|
|
@@ -1257,6 +1325,41 @@ function codeForStatus(status) {
|
|
|
1257
1325
|
}
|
|
1258
1326
|
}
|
|
1259
1327
|
/**
|
|
1328
|
+
* Whether a provider's error response actually contains anything a person
|
|
1329
|
+
* could act on. Some providers return a non-2xx status with **no body at
|
|
1330
|
+
* all** for certain field-validation failures (Mistral's OpenAI-compatible
|
|
1331
|
+
* endpoint does this, for example, when a request includes a field the
|
|
1332
|
+
* target model doesn't support). SDKs built on top of `openai` render that
|
|
1333
|
+
* specific case as a message like `"400 status code (no body)"`.
|
|
1334
|
+
*
|
|
1335
|
+
* Derived from the object's own `error`/`message` fields directly, rather
|
|
1336
|
+
* than from whatever `describeError` rendered, because `describeError`
|
|
1337
|
+
* falls back to serializing the *whole* thrown value when neither field is
|
|
1338
|
+
* present or meaningful. That fallback is local echo (e.g. just the
|
|
1339
|
+
* `status` a caller passed in), not provider diagnostic content, and
|
|
1340
|
+
* treating it as "detail" defeats the whole point of this check.
|
|
1341
|
+
*/
|
|
1342
|
+
const NO_BODY_MESSAGE_PATTERN = /\(no body\)/i;
|
|
1343
|
+
function isEmptyObject(value) {
|
|
1344
|
+
return Object.keys(value).length === 0;
|
|
1345
|
+
}
|
|
1346
|
+
function hasNoDiagnosticDetail(error) {
|
|
1347
|
+
if (error && typeof error === "object") {
|
|
1348
|
+
const { error: errorField, message } = error;
|
|
1349
|
+
if (errorField !== void 0 && errorField !== null) {
|
|
1350
|
+
const isEmptyString = typeof errorField === "string" && errorField.trim().length === 0;
|
|
1351
|
+
const isEmptyStruct = typeof errorField === "object" && isEmptyObject(errorField);
|
|
1352
|
+
if (!isEmptyString && !isEmptyStruct) return false;
|
|
1353
|
+
}
|
|
1354
|
+
if (typeof message === "string") {
|
|
1355
|
+
const trimmed = message.trim();
|
|
1356
|
+
return trimmed.length === 0 || NO_BODY_MESSAGE_PATTERN.test(trimmed);
|
|
1357
|
+
}
|
|
1358
|
+
return true;
|
|
1359
|
+
}
|
|
1360
|
+
return true;
|
|
1361
|
+
}
|
|
1362
|
+
/**
|
|
1260
1363
|
* Converts any thrown value into a well-typed LLMError. `attempts`, when
|
|
1261
1364
|
* given, is the accumulated record of every attempt made before `error`
|
|
1262
1365
|
* was thrown; it's passed straight into the constructed error's options
|
|
@@ -1273,13 +1376,19 @@ function normalizeError(error, signal, attempts) {
|
|
|
1273
1376
|
}
|
|
1274
1377
|
const status = extractStatus(error);
|
|
1275
1378
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
1276
|
-
if (status !== void 0)
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
|
|
1379
|
+
if (status !== void 0) {
|
|
1380
|
+
const description = describeError(error);
|
|
1381
|
+
const code = codeForStatus(status);
|
|
1382
|
+
const isRequestValidationStatus = code === void 0;
|
|
1383
|
+
const message = hasNoDiagnosticDetail(error) ? isRequestValidationStatus ? `LLM request failed with status ${status} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.` : `LLM request failed with status ${status} and no error detail from the provider.` : `LLM request failed: ${description}`;
|
|
1384
|
+
return new LLMError(message, "api", {
|
|
1385
|
+
status,
|
|
1386
|
+
cause: error,
|
|
1387
|
+
retryAfterMs,
|
|
1388
|
+
code,
|
|
1389
|
+
attempts
|
|
1390
|
+
});
|
|
1391
|
+
}
|
|
1283
1392
|
if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
|
|
1284
1393
|
cause: error,
|
|
1285
1394
|
retryAfterMs,
|
|
@@ -1352,6 +1461,13 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1352
1461
|
//#endregion
|
|
1353
1462
|
//#region src/internal/execution/requestBuilder.ts
|
|
1354
1463
|
/**
|
|
1464
|
+
* Serializes `ConversationTurn` assistant content for the wire. Strings
|
|
1465
|
+
* pass through unchanged. Parsed JSON values are `JSON.stringify`'d.
|
|
1466
|
+
*/
|
|
1467
|
+
function serializeAssistantContent(content) {
|
|
1468
|
+
return typeof content === "string" ? content : JSON.stringify(content);
|
|
1469
|
+
}
|
|
1470
|
+
/**
|
|
1355
1471
|
* Builds the wire request object for one call, applying per-instance
|
|
1356
1472
|
* defaults (model, max tokens, temperature) and per-call overrides.
|
|
1357
1473
|
* Owns every check that depends only on the caller's own input shape, not
|
|
@@ -1369,15 +1485,23 @@ var RequestBuilder = class {
|
|
|
1369
1485
|
model;
|
|
1370
1486
|
defaultMaxTokens;
|
|
1371
1487
|
defaultTemperature;
|
|
1488
|
+
defaultReasoningEffort;
|
|
1489
|
+
defaultBudgetTokens;
|
|
1490
|
+
supportsJsonObjectMode;
|
|
1372
1491
|
constructor(options) {
|
|
1373
1492
|
this.model = options.model;
|
|
1374
1493
|
this.defaultMaxTokens = options.defaultMaxTokens;
|
|
1375
1494
|
this.defaultTemperature = options.defaultTemperature;
|
|
1495
|
+
this.defaultReasoningEffort = options.defaultReasoningEffort;
|
|
1496
|
+
this.defaultBudgetTokens = options.defaultBudgetTokens;
|
|
1497
|
+
this.supportsJsonObjectMode = options.supportsJsonObjectMode;
|
|
1376
1498
|
}
|
|
1377
1499
|
/** Applies per-call defaults and shapes params into the client's request object. */
|
|
1378
1500
|
build(params) {
|
|
1379
|
-
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model,
|
|
1501
|
+
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, jsonSchema, tools, toolChoice } = params;
|
|
1380
1502
|
const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
|
|
1503
|
+
const reasoningEffort = params.reasoningEffort === void 0 ? this.defaultReasoningEffort : params.reasoningEffort;
|
|
1504
|
+
const budgetTokens = params.budgetTokens === void 0 ? this.defaultBudgetTokens : params.budgetTokens;
|
|
1381
1505
|
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
|
|
1382
1506
|
if (tools) {
|
|
1383
1507
|
const seen = new Set();
|
|
@@ -1399,8 +1523,12 @@ var RequestBuilder = class {
|
|
|
1399
1523
|
available: tools.map((t) => t.name)
|
|
1400
1524
|
}
|
|
1401
1525
|
});
|
|
1402
|
-
const
|
|
1403
|
-
const
|
|
1526
|
+
const jsonModeExplicit = params.jsonMode;
|
|
1527
|
+
const jsonMode = jsonModeExplicit ?? (tools ? false : true);
|
|
1528
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === true) throw new LLMError("jsonMode: true was set explicitly, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.", "invalid_params");
|
|
1529
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 && params.schema) throw new LLMError("`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.", "invalid_params");
|
|
1530
|
+
const jsonModeEffective = !this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 ? false : jsonMode;
|
|
1531
|
+
const useJson = jsonModeEffective || Boolean(jsonSchema);
|
|
1404
1532
|
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
|
|
1405
1533
|
const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
|
|
1406
1534
|
this.validateHistory(history);
|
|
@@ -1410,6 +1538,7 @@ var RequestBuilder = class {
|
|
|
1410
1538
|
max_tokens: maxTokens,
|
|
1411
1539
|
...responseFormat ? { response_format: responseFormat } : {},
|
|
1412
1540
|
...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
|
|
1541
|
+
...budgetTokens !== void 0 && budgetTokens !== null ? { budget_tokens: budgetTokens } : {},
|
|
1413
1542
|
...tools ? { tools: toWireTools(tools) } : {},
|
|
1414
1543
|
...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
|
|
1415
1544
|
messages: [
|
|
@@ -1505,9 +1634,13 @@ var RequestBuilder = class {
|
|
|
1505
1634
|
}));
|
|
1506
1635
|
if (turn.role === "assistant" && turn.toolCalls?.length) return [{
|
|
1507
1636
|
role: "assistant",
|
|
1508
|
-
...turn.content ? { content: turn.content } : {},
|
|
1637
|
+
...turn.content !== void 0 ? { content: serializeAssistantContent(turn.content) } : {},
|
|
1509
1638
|
tool_calls: toWireToolCalls(turn.toolCalls)
|
|
1510
1639
|
}];
|
|
1640
|
+
if (turn.role === "assistant") return [{
|
|
1641
|
+
role: "assistant",
|
|
1642
|
+
content: serializeAssistantContent(turn.content === void 0 ? "" : turn.content)
|
|
1643
|
+
}];
|
|
1511
1644
|
return [{
|
|
1512
1645
|
role: turn.role,
|
|
1513
1646
|
content: turn.content ?? ""
|
|
@@ -1641,10 +1774,12 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1641
1774
|
complete: wireChunk.complete
|
|
1642
1775
|
});
|
|
1643
1776
|
} else if (wireChunk.type === "usage") {
|
|
1777
|
+
const reasoningTokens = wireChunk.usage.completion_tokens_details?.reasoning_tokens;
|
|
1644
1778
|
usage = {
|
|
1645
1779
|
promptTokens: wireChunk.usage.prompt_tokens ?? 0,
|
|
1646
1780
|
completionTokens: wireChunk.usage.completion_tokens ?? 0,
|
|
1647
1781
|
totalTokens: wireChunk.usage.total_tokens ?? 0,
|
|
1782
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
1648
1783
|
requestId,
|
|
1649
1784
|
model,
|
|
1650
1785
|
provider: providerName,
|
|
@@ -1698,6 +1833,16 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1698
1833
|
//#endregion
|
|
1699
1834
|
//#region src/internal/execution/callExecutor.ts
|
|
1700
1835
|
/**
|
|
1836
|
+
* Identity function with its own parameter, used only to sidestep a TS
|
|
1837
|
+
* quirk: a `let` reassigned solely inside a nested closure (like
|
|
1838
|
+
* `retryWithBackoff`'s `onRequest`) gets narrowed to `undefined` at the
|
|
1839
|
+
* point it was last synchronously assigned, which would otherwise make
|
|
1840
|
+
* `lastRequestForAttempt` read as `never` at the point it's used below.
|
|
1841
|
+
*/
|
|
1842
|
+
function passThroughRequestSnapshot(snapshot) {
|
|
1843
|
+
return snapshot;
|
|
1844
|
+
}
|
|
1845
|
+
/**
|
|
1701
1846
|
* Everything one provider target needs to attempt a call: request
|
|
1702
1847
|
* building, retry with backoff, the per-target breaker, the per-target
|
|
1703
1848
|
* limiter. Never exported publicly. `VernLLM` holds one per target and
|
|
@@ -1740,7 +1885,10 @@ var CallExecutor = class {
|
|
|
1740
1885
|
this.requestBuilder = new RequestBuilder({
|
|
1741
1886
|
model,
|
|
1742
1887
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
1743
|
-
defaultTemperature: options.defaultTemperature
|
|
1888
|
+
defaultTemperature: options.defaultTemperature,
|
|
1889
|
+
defaultReasoningEffort: options.defaultReasoningEffort,
|
|
1890
|
+
defaultBudgetTokens: options.defaultBudgetTokens,
|
|
1891
|
+
supportsJsonObjectMode: client.supportsJsonObjectMode ?? true
|
|
1744
1892
|
});
|
|
1745
1893
|
}
|
|
1746
1894
|
getCircuitState(model) {
|
|
@@ -1781,7 +1929,7 @@ var CallExecutor = class {
|
|
|
1781
1929
|
const model = params.model ?? this.model;
|
|
1782
1930
|
const attempts = [];
|
|
1783
1931
|
try {
|
|
1784
|
-
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1932
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1785
1933
|
} catch (error) {
|
|
1786
1934
|
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1787
1935
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
@@ -1794,7 +1942,7 @@ var CallExecutor = class {
|
|
|
1794
1942
|
const model = params.model ?? this.model;
|
|
1795
1943
|
const attempts = [];
|
|
1796
1944
|
try {
|
|
1797
|
-
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1945
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeStreamCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1798
1946
|
} catch (error) {
|
|
1799
1947
|
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1800
1948
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
@@ -1809,8 +1957,9 @@ var CallExecutor = class {
|
|
|
1809
1957
|
* set. Throws on an empty response (no text and no tool_calls) so the
|
|
1810
1958
|
* retry loop treats it like any other transient failure.
|
|
1811
1959
|
*/
|
|
1812
|
-
async executeCall(params, requestId, attempt) {
|
|
1960
|
+
async executeCall(params, requestId, attempt, onRequest) {
|
|
1813
1961
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1962
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1814
1963
|
let release;
|
|
1815
1964
|
if (this.limiter) {
|
|
1816
1965
|
const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
|
|
@@ -1918,8 +2067,9 @@ var CallExecutor = class {
|
|
|
1918
2067
|
* not on the first chunk arriving, so a connection that opens but then
|
|
1919
2068
|
* dies mid-stream isn't masked as a success (see `buildStreamResult`).
|
|
1920
2069
|
*/
|
|
1921
|
-
async executeStreamCall(params, requestId, attempt) {
|
|
2070
|
+
async executeStreamCall(params, requestId, attempt, onRequest) {
|
|
1922
2071
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
2072
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1923
2073
|
const completions = this.client.chat.completions;
|
|
1924
2074
|
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
|
|
1925
2075
|
code: "unsupported_capability",
|
|
@@ -2039,18 +2189,25 @@ var CallExecutor = class {
|
|
|
2039
2189
|
*/
|
|
2040
2190
|
async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
|
|
2041
2191
|
let lastError;
|
|
2042
|
-
|
|
2043
|
-
|
|
2044
|
-
|
|
2045
|
-
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
|
|
2049
|
-
|
|
2050
|
-
|
|
2051
|
-
|
|
2052
|
-
|
|
2053
|
-
|
|
2192
|
+
let lastRequestForAttempt;
|
|
2193
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
2194
|
+
lastRequestForAttempt = void 0;
|
|
2195
|
+
try {
|
|
2196
|
+
if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
|
|
2197
|
+
onAttempt?.();
|
|
2198
|
+
return await fn(attempt, (req) => {
|
|
2199
|
+
lastRequestForAttempt = req;
|
|
2200
|
+
});
|
|
2201
|
+
} catch (error) {
|
|
2202
|
+
lastError = error;
|
|
2203
|
+
const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
|
|
2204
|
+
if (!willRetry) break;
|
|
2205
|
+
attempts?.push({
|
|
2206
|
+
index: attempt,
|
|
2207
|
+
error: normalizeError(error, signal).toSnapshot(),
|
|
2208
|
+
request: passThroughRequestSnapshot(lastRequestForAttempt)
|
|
2209
|
+
});
|
|
2210
|
+
}
|
|
2054
2211
|
}
|
|
2055
2212
|
throw lastError;
|
|
2056
2213
|
}
|
|
@@ -2062,10 +2219,12 @@ var CallExecutor = class {
|
|
|
2062
2219
|
*/
|
|
2063
2220
|
extractUsage(response, requestId, model) {
|
|
2064
2221
|
if (!response.usage) return void 0;
|
|
2222
|
+
const reasoningTokens = response.usage.completion_tokens_details?.reasoning_tokens;
|
|
2065
2223
|
return {
|
|
2066
2224
|
promptTokens: response.usage.prompt_tokens ?? 0,
|
|
2067
2225
|
completionTokens: response.usage.completion_tokens ?? 0,
|
|
2068
2226
|
totalTokens: response.usage.total_tokens ?? 0,
|
|
2227
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
2069
2228
|
requestId,
|
|
2070
2229
|
model,
|
|
2071
2230
|
provider: this.providerName,
|
|
@@ -2489,7 +2648,7 @@ var RateLimiter = class {
|
|
|
2489
2648
|
//#endregion
|
|
2490
2649
|
//#region src/vernLLM.ts
|
|
2491
2650
|
/**
|
|
2492
|
-
* A
|
|
2651
|
+
* A LLM call framework for resilience, observability and control. This is VernLLM!
|
|
2493
2652
|
*
|
|
2494
2653
|
* Adds retry with backoff and jitter, per-attempt timeouts, an optional
|
|
2495
2654
|
* circuit breaker, JSON parsing with optional schema validation, usage
|
|
@@ -2529,6 +2688,8 @@ var VernLLM = class {
|
|
|
2529
2688
|
this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
|
|
2530
2689
|
this.reportEvent = makeEventReporter(options.onEvent, this.logger);
|
|
2531
2690
|
const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
|
|
2691
|
+
const primaryDefaultReasoningEffort = options.defaultReasoningEffort;
|
|
2692
|
+
const primaryDefaultBudgetTokens = options.defaultBudgetTokens;
|
|
2532
2693
|
const primaryTarget = {
|
|
2533
2694
|
client: options.client,
|
|
2534
2695
|
model: options.model,
|
|
@@ -2539,6 +2700,8 @@ var VernLLM = class {
|
|
|
2539
2700
|
baseDelayMs: options.baseDelayMs,
|
|
2540
2701
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
2541
2702
|
defaultTemperature: primaryDefaultTemperature,
|
|
2703
|
+
defaultReasoningEffort: primaryDefaultReasoningEffort,
|
|
2704
|
+
defaultBudgetTokens: primaryDefaultBudgetTokens,
|
|
2542
2705
|
nonRetryableStatus: options.nonRetryableStatus,
|
|
2543
2706
|
circuitBreaker: options.circuitBreaker,
|
|
2544
2707
|
rateLimit: options.rateLimit
|
|
@@ -2556,6 +2719,8 @@ var VernLLM = class {
|
|
|
2556
2719
|
baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
|
|
2557
2720
|
defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
|
|
2558
2721
|
defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
|
|
2722
|
+
defaultReasoningEffort: target.defaultReasoningEffort === void 0 ? primaryDefaultReasoningEffort : target.defaultReasoningEffort,
|
|
2723
|
+
defaultBudgetTokens: target.defaultBudgetTokens === void 0 ? primaryDefaultBudgetTokens : target.defaultBudgetTokens,
|
|
2559
2724
|
nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
|
|
2560
2725
|
400,
|
|
2561
2726
|
401,
|
|
@@ -2757,6 +2922,45 @@ var VernLLM = class {
|
|
|
2757
2922
|
if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
|
|
2758
2923
|
}
|
|
2759
2924
|
};
|
|
2925
|
+
/**
|
|
2926
|
+
* Identity function preserving `params`'s own precise type, unlike a `:
|
|
2927
|
+
* CallParams<T>` annotation, which would widen `tools` away and break the
|
|
2928
|
+
* `ConditionalToolCallParams<T>` overload for `tools: someCondition ?
|
|
2929
|
+
* [tool] : undefined`. Use it when you need `call()` params in a named,
|
|
2930
|
+
* reusable variable; skip it when you can pass the object inline.
|
|
2931
|
+
*
|
|
2932
|
+
* ```ts
|
|
2933
|
+
* const params = defineCallParams({
|
|
2934
|
+
* userContent: 'What is the weather?',
|
|
2935
|
+
* tools: someCondition ? [weatherTool] : undefined,
|
|
2936
|
+
* });
|
|
2937
|
+
* const result = await llm.call(params);
|
|
2938
|
+
* // result: unknown | CallWithToolsResult<unknown>, same as inline
|
|
2939
|
+
* ```
|
|
2940
|
+
*
|
|
2941
|
+
* `T` isn't a parameter here; pin it via `llm.call<T>(params)` as usual.
|
|
2942
|
+
* `defineCachedCallParams` is the `cachedCall()` counterpart.
|
|
2943
|
+
*/
|
|
2944
|
+
function defineCallParams(params) {
|
|
2945
|
+
return params;
|
|
2946
|
+
}
|
|
2947
|
+
/**
|
|
2948
|
+
* The `cachedCall()` counterpart to `defineCallParams`: preserves the
|
|
2949
|
+
* whole `{ cacheKey, ttl, call }` object, `call.tools` included, in one
|
|
2950
|
+
* named variable.
|
|
2951
|
+
*
|
|
2952
|
+
* ```ts
|
|
2953
|
+
* const params = defineCachedCallParams({
|
|
2954
|
+
* cacheKey: 'weather-ny',
|
|
2955
|
+
* ttl: 60,
|
|
2956
|
+
* call: { userContent: 'What is the weather?', tools: someCondition ? [weatherTool] : undefined },
|
|
2957
|
+
* });
|
|
2958
|
+
* const result = await llm.cachedCall(params);
|
|
2959
|
+
* ```
|
|
2960
|
+
*/
|
|
2961
|
+
function defineCachedCallParams(params) {
|
|
2962
|
+
return params;
|
|
2963
|
+
}
|
|
2760
2964
|
|
|
2761
2965
|
//#endregion
|
|
2762
2966
|
//#region src/adapters/internal/sse.ts
|
|
@@ -2897,6 +3101,275 @@ function supportsNativeStructuredOutput(model, override) {
|
|
|
2897
3101
|
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
2898
3102
|
}
|
|
2899
3103
|
|
|
3104
|
+
//#endregion
|
|
3105
|
+
//#region src/adapters/internal/reasoningBudget.utils.ts
|
|
3106
|
+
const DEFAULT_EFFORT_TOKENS = {
|
|
3107
|
+
minimal: 1024,
|
|
3108
|
+
low: 4096,
|
|
3109
|
+
medium: 16e3,
|
|
3110
|
+
high: 32e3
|
|
3111
|
+
};
|
|
3112
|
+
/**
|
|
3113
|
+
* Merges a caller-supplied partial override over `DEFAULT_EFFORT_TOKENS`.
|
|
3114
|
+
* Called once per adapter instance (not per request), so a per-instance
|
|
3115
|
+
* override only needs to specify the tiers it actually wants to change.
|
|
3116
|
+
*
|
|
3117
|
+
* Throws `LLMError('invalid_params')` if the override doesn't keep the
|
|
3118
|
+
* tiers in strictly ascending order (`minimal < low < medium < high`).
|
|
3119
|
+
* `budgetTokensToEffort` buckets by walking the tiers low to high and
|
|
3120
|
+
* returning on the first one a value is `<=`, so an unordered table (e.g.
|
|
3121
|
+
* `low` above `medium`) wouldn't just produce a "wrong" bucket, it would
|
|
3122
|
+
* make some tiers unreachable outright, silently, with no signal to the
|
|
3123
|
+
* caller that their override doesn't do what they think it does.
|
|
3124
|
+
*/
|
|
3125
|
+
function resolveEffortTokenTable(override) {
|
|
3126
|
+
if (!override) return DEFAULT_EFFORT_TOKENS;
|
|
3127
|
+
const table = {
|
|
3128
|
+
...DEFAULT_EFFORT_TOKENS,
|
|
3129
|
+
...override
|
|
3130
|
+
};
|
|
3131
|
+
if (!(table.minimal < table.low && table.low < table.medium && table.medium < table.high)) throw new LLMError(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(table)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`, "invalid_params");
|
|
3132
|
+
return table;
|
|
3133
|
+
}
|
|
3134
|
+
/** Converts a `reasoningEffort` tier into the nearest `budgetTokens` value. */
|
|
3135
|
+
function effortToBudgetTokens(effort, table = DEFAULT_EFFORT_TOKENS) {
|
|
3136
|
+
return table[effort];
|
|
3137
|
+
}
|
|
3138
|
+
/**
|
|
3139
|
+
* Converts a raw `budgetTokens` value into the nearest `reasoningEffort`
|
|
3140
|
+
* tier, for providers that only understand tiers. Buckets by the same
|
|
3141
|
+
* `table` `effortToBudgetTokens` produces its values from, so the two
|
|
3142
|
+
* functions agree with each other at the boundary values, as long as the
|
|
3143
|
+
* same (possibly overridden) table is passed to both. A value strictly
|
|
3144
|
+
* between two tiers (e.g. 4097, one above the default `low`) rounds up to
|
|
3145
|
+
* the next tier it's still `<=`, i.e. `medium` here, not down to `low`.
|
|
3146
|
+
*/
|
|
3147
|
+
function budgetTokensToEffort(budgetTokens, table = DEFAULT_EFFORT_TOKENS) {
|
|
3148
|
+
if (budgetTokens <= table.minimal) return "minimal";
|
|
3149
|
+
if (budgetTokens <= table.low) return "low";
|
|
3150
|
+
if (budgetTokens <= table.medium) return "medium";
|
|
3151
|
+
return "high";
|
|
3152
|
+
}
|
|
3153
|
+
/**
|
|
3154
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3155
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3156
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3157
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3158
|
+
* non-Opus model id.
|
|
3159
|
+
*/
|
|
3160
|
+
/**
|
|
3161
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3162
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3163
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3164
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3165
|
+
* non-Opus model id.
|
|
3166
|
+
*
|
|
3167
|
+
* Anthropic model ids sometimes carry a trailing snapshot date instead of
|
|
3168
|
+
* (or in addition to) an explicit minor version, e.g. the real, still-
|
|
3169
|
+
* supported base `"claude-opus-4-20250514"` (no `.7`-style minor at all,
|
|
3170
|
+
* just a date suffix directly after the major version). Read naively,
|
|
3171
|
+
* `20250514` looks like a minor version far above any real threshold and
|
|
3172
|
+
* would misclassify this pre-4.6 model as adaptive-only. Snapshot dates
|
|
3173
|
+
* are always 8 digits (`YYYYMMDD`); a real minor version never is, so an
|
|
3174
|
+
* 8+ digit second segment is treated as a date, not a minor version.
|
|
3175
|
+
*/
|
|
3176
|
+
function parseOpusVersion(model) {
|
|
3177
|
+
const match = /opus-(\d+)(?:-(\d+))?/.exec(model);
|
|
3178
|
+
if (!match) return null;
|
|
3179
|
+
const minorStr = match[2];
|
|
3180
|
+
const minor = minorStr === void 0 || minorStr.length >= 8 ? 0 : Number(minorStr);
|
|
3181
|
+
return [Number(match[1]), minor];
|
|
3182
|
+
}
|
|
3183
|
+
/**
|
|
3184
|
+
* Default rule for whether `model` only supports adaptive thinking
|
|
3185
|
+
* (`thinking: { type: 'adaptive' }`) and returns a 400 for manual,
|
|
3186
|
+
* budget-based thinking (`thinking: { type: 'enabled', budget_tokens }`):
|
|
3187
|
+
* Claude Opus 4.7 and later (matched as a version threshold, so 4.8, 4.9,
|
|
3188
|
+
* 5, and every future Opus point release are covered automatically,
|
|
3189
|
+
* without a new list entry per release), and every Claude 5 tier model
|
|
3190
|
+
* outside the Opus family (Sonnet 5, Fable 5, Mythos 5, Mythos Preview).
|
|
3191
|
+
* `mythos` alone is enough to catch both Mythos names without listing
|
|
3192
|
+
* each separately.
|
|
3193
|
+
*
|
|
3194
|
+
* Necessarily best-effort: a new model family with its own name (not
|
|
3195
|
+
* `opus-*`, not `sonnet-5`/`fable-5`/`mythos-*`) still needs a code
|
|
3196
|
+
* update here, or a caller-supplied `adaptiveOnlyModels` override (see
|
|
3197
|
+
* `isAdaptiveOnlyModel`) covering it in the meantime.
|
|
3198
|
+
*/
|
|
3199
|
+
function isDefaultAdaptiveOnly(model) {
|
|
3200
|
+
const opusVersion = parseOpusVersion(model);
|
|
3201
|
+
if (opusVersion) {
|
|
3202
|
+
const [major, minor] = opusVersion;
|
|
3203
|
+
return major > 4 || major === 4 && minor >= 7;
|
|
3204
|
+
}
|
|
3205
|
+
return [
|
|
3206
|
+
"sonnet-5",
|
|
3207
|
+
"fable-5",
|
|
3208
|
+
"mythos"
|
|
3209
|
+
].some((s) => model.includes(s));
|
|
3210
|
+
}
|
|
3211
|
+
/**
|
|
3212
|
+
* Whether `model` is adaptive-only, per the built-in rule above, or per a
|
|
3213
|
+
* caller-supplied `adaptiveOnlyModels` override. The override is
|
|
3214
|
+
* additive, not a replacement: it can mark an *additional* model as
|
|
3215
|
+
* adaptive-only (useful for a model family this package doesn't know
|
|
3216
|
+
* about yet), but it can't un-mark one the built-in rule already caught,
|
|
3217
|
+
* since a caller correcting a false negative is the only direction that
|
|
3218
|
+
* needs covering, a false positive here would mean this package is
|
|
3219
|
+
* simply wrong and needs its own fix, not a per-caller workaround.
|
|
3220
|
+
*/
|
|
3221
|
+
function isAdaptiveOnlyModel(model, override) {
|
|
3222
|
+
if (isDefaultAdaptiveOnly(model)) return true;
|
|
3223
|
+
if (!override) return false;
|
|
3224
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3225
|
+
}
|
|
3226
|
+
/** Whether `model` is known to support manual, budget-based thinking. */
|
|
3227
|
+
function supportsManualThinkingBudget(model, override) {
|
|
3228
|
+
return !isAdaptiveOnlyModel(model, override);
|
|
3229
|
+
}
|
|
3230
|
+
/**
|
|
3231
|
+
* Anthropic (and Claude models on Bedrock) require `budget_tokens` to be
|
|
3232
|
+
* at least 1024 and strictly less than `max_tokens`, since the thinking
|
|
3233
|
+
* budget and the reply share the same `max_tokens` ceiling. VernLLM's own
|
|
3234
|
+
* default `maxTokens` is 1000 (see `RequestBuilder`'s `defaultMaxTokens`),
|
|
3235
|
+
* below the 1024 floor, so the *default* `minimal` tier (1024 tokens) is
|
|
3236
|
+
* silently invalid against the *default* `max_tokens` unless a caller
|
|
3237
|
+
* happens to raise one or the other. Checked here, once, right before a
|
|
3238
|
+
* `thinking` block would be built, rather than left for Anthropic's own
|
|
3239
|
+
* 400 to explain after a real network round trip.
|
|
3240
|
+
*/
|
|
3241
|
+
function assertValidClaudeBudgetTokens(budgetTokens, maxTokens) {
|
|
3242
|
+
if (budgetTokens < 1024) throw new LLMError(`budgetTokens (${budgetTokens}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`, "invalid_params");
|
|
3243
|
+
if (budgetTokens >= maxTokens) throw new LLMError(`budgetTokens (${budgetTokens}) must be less than maxTokens (${maxTokens}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`, "invalid_params");
|
|
3244
|
+
}
|
|
3245
|
+
/**
|
|
3246
|
+
* Anthropic rejects any form of `thinking` (manual `budget_tokens` or
|
|
3247
|
+
* adaptive) combined with a `tool_choice` that forces tool use, a forced
|
|
3248
|
+
* single tool or "must call some tool", with a 400: `"Thinking may not be
|
|
3249
|
+
* enabled when tool_choice forces tool use."` Auto/none (or no tools at
|
|
3250
|
+
* all) are unaffected, thinking only conflicts with a choice that removes
|
|
3251
|
+
* the model's ability to just reply with text. This is a Claude-model
|
|
3252
|
+
* constraint, not specific to the Anthropic API's own wire shape, so it
|
|
3253
|
+
* applies identically to Claude models called through Bedrock's Converse
|
|
3254
|
+
* API, which forwards `thinking` under `additionalModelRequestFields` but
|
|
3255
|
+
* is still talking to the same underlying model.
|
|
3256
|
+
*
|
|
3257
|
+
* This combination can arise two ways: a caller explicitly sets both
|
|
3258
|
+
* `budgetTokens`/`reasoningEffort` and a forced `toolChoice`, or, more
|
|
3259
|
+
* subtly (Anthropic adapter only), a caller sets `jsonSchema` on a model
|
|
3260
|
+
* without native structured output support, which silently forces a
|
|
3261
|
+
* single synthetic tool call to emulate it, with no `tool_choice` of the
|
|
3262
|
+
* caller's own in sight. Both end up resolving to a forced tool choice by
|
|
3263
|
+
* the time each adapter calls this, so checking the adapter's own
|
|
3264
|
+
* already-resolved choice (rather than the caller's raw
|
|
3265
|
+
* `params.tool_choice`) catches both, right before a `thinking` block
|
|
3266
|
+
* would be built, rather than left for Anthropic's own 400 to explain
|
|
3267
|
+
* after a real network round trip.
|
|
3268
|
+
*
|
|
3269
|
+
* Takes a plain description of the forced choice rather than either
|
|
3270
|
+
* adapter's own wire shape (Anthropic SDK's `{ type: 'tool' | 'any', ... }`
|
|
3271
|
+
* vs Converse's `{ tool: {...} } | { any: {} }`), so both adapters can
|
|
3272
|
+
* share one check without either shape leaking into this file. Pass
|
|
3273
|
+
* `undefined` when the resolved choice is `auto`/`none`/unset, forcing
|
|
3274
|
+
* nothing.
|
|
3275
|
+
*/
|
|
3276
|
+
function assertNoForcedToolChoiceWithThinking(forcedChoiceDescription) {
|
|
3277
|
+
if (!forcedChoiceDescription) return;
|
|
3278
|
+
throw new LLMError(`budgetTokens/reasoningEffort was set alongside ${forcedChoiceDescription}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`, "invalid_params");
|
|
3279
|
+
}
|
|
3280
|
+
/**
|
|
3281
|
+
* Maps VernLLM's four-tier `reasoningEffort` onto Anthropic's five-tier
|
|
3282
|
+
* adaptive effort. `xhigh` and `max` have no VernLLM-side equivalent and
|
|
3283
|
+
* are unreachable through this mapping; a caller who wants either has to
|
|
3284
|
+
* target Anthropic/Bedrock-specific behavior already, so there's no gap
|
|
3285
|
+
* the shared `CallParams` surface needs to cover for a first pass.
|
|
3286
|
+
*/
|
|
3287
|
+
function toClaudeAdaptiveEffort(effort) {
|
|
3288
|
+
return effort === "minimal" ? "low" : effort;
|
|
3289
|
+
}
|
|
3290
|
+
/** Converts VernLLM's `reasoningEffort` directly into Gemini's `ThinkingLevel` enum value. */
|
|
3291
|
+
function toGeminiThinkingLevel(effort, model) {
|
|
3292
|
+
return clampGeminiThinkingLevel(model, effort.toUpperCase());
|
|
3293
|
+
}
|
|
3294
|
+
/**
|
|
3295
|
+
* Parses a Gemini model id's minor version, e.g. `"gemini-3.1-pro"` -> `1`,
|
|
3296
|
+
* `"gemini-3-pro"` -> `0` (no explicit minor). Only meaningful alongside
|
|
3297
|
+
* `parseGeminiMajorVersion`.
|
|
3298
|
+
*/
|
|
3299
|
+
function parseGeminiMinorVersion(model) {
|
|
3300
|
+
const match = /gemini-\d+\.(\d+)/.exec(model);
|
|
3301
|
+
return match ? Number(match[1]) : 0;
|
|
3302
|
+
}
|
|
3303
|
+
/**
|
|
3304
|
+
* Some Gemini 3 "Pro" tier models accept a narrower set of `thinkingLevel`
|
|
3305
|
+
* values than VernLLM's four tiers map onto, confirmed against real API
|
|
3306
|
+
* 400s and Google's own migration guidance, not assumed:
|
|
3307
|
+
* - Gemini 3 Pro (major 3, minor 0, e.g. `"gemini-3-pro-preview"`): only
|
|
3308
|
+
* `LOW` and `HIGH`; `MEDIUM` returns a 400 ("Thinking level MEDIUM is
|
|
3309
|
+
* not supported for this model").
|
|
3310
|
+
* - Gemini 3.1 Pro (major 3, minor >= 1): `LOW`/`MEDIUM`/`HIGH`, no
|
|
3311
|
+
* `MINIMAL`, Google's own docs point users toward a Flash-tier model
|
|
3312
|
+
* instead for the lowest setting.
|
|
3313
|
+
* - Every Flash-tier Gemini 3+ model accepts the full four levels, no
|
|
3314
|
+
* clamping needed, matched by this function simply not applying to
|
|
3315
|
+
* anything without `"pro"` in the model id.
|
|
3316
|
+
*
|
|
3317
|
+
* Clamped automatically rather than left to error, since `reasoningEffort`
|
|
3318
|
+
* is a per-call value, a caller hitting this isn't misconfiguring an
|
|
3319
|
+
* instance once, they're getting an intermittent-looking failure on
|
|
3320
|
+
* whichever specific call happened to pick an unsupported tier. Necessarily
|
|
3321
|
+
* best-effort: a future Pro-tier release could add back a level this rule
|
|
3322
|
+
* still clamps, or clamp one this rule doesn't yet know to touch.
|
|
3323
|
+
*/
|
|
3324
|
+
function clampGeminiThinkingLevel(model, level) {
|
|
3325
|
+
if (!model.includes("pro")) return level;
|
|
3326
|
+
const major = parseGeminiMajorVersion(model);
|
|
3327
|
+
if (major === null || major < 3) return level;
|
|
3328
|
+
const minor = parseGeminiMinorVersion(model);
|
|
3329
|
+
if (minor === 0) return level === "HIGH" ? "HIGH" : "LOW";
|
|
3330
|
+
return level === "MINIMAL" ? "LOW" : level;
|
|
3331
|
+
}
|
|
3332
|
+
/**
|
|
3333
|
+
* Parses a Gemini model id's major generation number, e.g.
|
|
3334
|
+
* `"gemini-3.1-flash-lite"` -> `3`, `"gemini-2.5-flash"` -> `2`. Not
|
|
3335
|
+
* anchored, so a Vertex-prefixed or otherwise decorated id still matches.
|
|
3336
|
+
* Returns `null` for a non-Gemini model id.
|
|
3337
|
+
*/
|
|
3338
|
+
function parseGeminiMajorVersion(model) {
|
|
3339
|
+
const match = /gemini-(\d+)/.exec(model);
|
|
3340
|
+
return match ? Number(match[1]) : null;
|
|
3341
|
+
}
|
|
3342
|
+
/**
|
|
3343
|
+
* Default rule for whether `model` uses `thinkingLevel` instead of
|
|
3344
|
+
* `thinkingBudget`: every Gemini 3 series model and later, matched as a
|
|
3345
|
+
* version threshold so 3.1, 3.5, 3.6, and every future Gemini 3.x or
|
|
3346
|
+
* later release are covered automatically, without a new entry per
|
|
3347
|
+
* release, same reasoning as `isDefaultAdaptiveOnly`'s Opus threshold.
|
|
3348
|
+
* Gemini 2.5 and earlier still use `thinkingBudget`.
|
|
3349
|
+
*
|
|
3350
|
+
* `thinkingBudget` is still *accepted* on Gemini 3 for backward
|
|
3351
|
+
* compatibility, per Google's own docs, but "may result in unexpected
|
|
3352
|
+
* performance" there, so this rule switches VernLLM's own default
|
|
3353
|
+
* behavior over rather than leaving it on the old field indefinitely.
|
|
3354
|
+
*/
|
|
3355
|
+
function isDefaultThinkingLevelModel(model) {
|
|
3356
|
+
const major = parseGeminiMajorVersion(model);
|
|
3357
|
+
return major !== null && major >= 3;
|
|
3358
|
+
}
|
|
3359
|
+
/**
|
|
3360
|
+
* Whether `model` uses `thinkingLevel`, per the built-in version
|
|
3361
|
+
* threshold above, or per a caller-supplied `thinkingLevelModels`
|
|
3362
|
+
* override. Additive, not a replacement, same reasoning as
|
|
3363
|
+
* `isAdaptiveOnlyModel`: an override can mark an *additional* model as
|
|
3364
|
+
* using `thinkingLevel` (a model family this package doesn't recognize
|
|
3365
|
+
* yet), it can't un-mark one the built-in threshold already caught.
|
|
3366
|
+
*/
|
|
3367
|
+
function usesGeminiThinkingLevel(model, override) {
|
|
3368
|
+
if (isDefaultThinkingLevelModel(model)) return true;
|
|
3369
|
+
if (!override) return false;
|
|
3370
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3371
|
+
}
|
|
3372
|
+
|
|
2900
3373
|
//#endregion
|
|
2901
3374
|
//#region src/adapters/anthropic.ts
|
|
2902
3375
|
/**
|
|
@@ -2981,7 +3454,7 @@ function buildAnthropicTools(tools, toolChoiceParam) {
|
|
|
2981
3454
|
* `params.tools` are left for the normal, non-forced tool-call handling
|
|
2982
3455
|
* both `create` and `createStream` already do when `toolName` is unset.
|
|
2983
3456
|
*/
|
|
2984
|
-
function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
3457
|
+
function buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
2985
3458
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2986
3459
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2987
3460
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -2989,8 +3462,8 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2989
3462
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
2990
3463
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
2991
3464
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
|
|
3465
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic's native output_config.format on covered models, or a forced single tool call otherwise).", "validation");
|
|
2992
3466
|
let toolName;
|
|
2993
|
-
let jsonInstruction;
|
|
2994
3467
|
let outputFormat;
|
|
2995
3468
|
let tools;
|
|
2996
3469
|
let toolChoice;
|
|
@@ -3013,20 +3486,42 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
3013
3486
|
type: "tool",
|
|
3014
3487
|
name: toolName
|
|
3015
3488
|
};
|
|
3016
|
-
}
|
|
3489
|
+
}
|
|
3017
3490
|
if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
|
|
3018
|
-
|
|
3491
|
+
let thinking;
|
|
3492
|
+
let effort;
|
|
3493
|
+
if (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0) {
|
|
3494
|
+
assertNoForcedToolChoiceWithThinking(toolChoice?.type === "tool" ? `toolChoice forcing the "${toolChoice.name}" tool` : toolChoice?.type === "any" ? "toolChoice: 'required' (Anthropic's \"any\" tool_choice)" : void 0);
|
|
3495
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
3496
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
3497
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
3498
|
+
thinking = {
|
|
3499
|
+
type: "enabled",
|
|
3500
|
+
budget_tokens: budgetTokens
|
|
3501
|
+
};
|
|
3502
|
+
} else {
|
|
3503
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
3504
|
+
thinking = { type: "adaptive" };
|
|
3505
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
3506
|
+
}
|
|
3507
|
+
}
|
|
3508
|
+
const system = systemMessage?.content;
|
|
3509
|
+
const temperature = thinking ? void 0 : params.temperature;
|
|
3019
3510
|
const body = {
|
|
3020
3511
|
model: params.model,
|
|
3021
3512
|
max_tokens: params.max_tokens,
|
|
3022
|
-
...
|
|
3513
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
3023
3514
|
system: system || void 0,
|
|
3024
3515
|
messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
|
|
3025
3516
|
...tools ? {
|
|
3026
3517
|
tools,
|
|
3027
3518
|
tool_choice: toolChoice
|
|
3028
3519
|
} : {},
|
|
3029
|
-
...outputFormat ? { output_config: {
|
|
3520
|
+
...outputFormat || effort ? { output_config: {
|
|
3521
|
+
...outputFormat ? { format: outputFormat } : {},
|
|
3522
|
+
...effort ? { effort } : {}
|
|
3523
|
+
} } : {},
|
|
3524
|
+
...thinking ? { thinking } : {}
|
|
3030
3525
|
};
|
|
3031
3526
|
return {
|
|
3032
3527
|
body,
|
|
@@ -3055,103 +3550,113 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
3055
3550
|
* schema matching applies only when `strict: true` is forwarded and
|
|
3056
3551
|
* supported.
|
|
3057
3552
|
*
|
|
3058
|
-
* `response_format: json_object` (
|
|
3059
|
-
*
|
|
3060
|
-
*
|
|
3061
|
-
*
|
|
3062
|
-
*
|
|
3553
|
+
* `response_format: json_object` throws `LLMError('validation')`. Anthropic
|
|
3554
|
+
* has no API-level field that mechanically guarantees JSON output the way
|
|
3555
|
+
* OpenAI's `json_object` mode does; the only way to emulate it was a
|
|
3556
|
+
* system-prompt instruction with no actual enforcement behind it, a
|
|
3557
|
+
* guarantee this adapter no longer pretends to make. Use `jsonSchema`
|
|
3558
|
+
* instead, which maps to a real constraint either way (native
|
|
3559
|
+
* `output_config.format` or a forced tool call).
|
|
3063
3560
|
*/
|
|
3064
3561
|
function fromAnthropic(anthropicClient, options) {
|
|
3065
3562
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3563
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3564
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
3066
3565
|
const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
|
|
3067
|
-
return {
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
if (
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
choices: [{ message: {
|
|
3092
|
-
content: text,
|
|
3093
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3094
|
-
} }],
|
|
3095
|
-
usage: {
|
|
3096
|
-
prompt_tokens: response.usage?.input_tokens,
|
|
3097
|
-
completion_tokens: response.usage?.output_tokens,
|
|
3098
|
-
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
|
|
3566
|
+
return {
|
|
3567
|
+
supportsJsonObjectMode: false,
|
|
3568
|
+
chat: { completions: {
|
|
3569
|
+
async create(params, options$1) {
|
|
3570
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3571
|
+
const response = await anthropicClient.messages.create(body, options$1);
|
|
3572
|
+
let text;
|
|
3573
|
+
let wireToolCalls;
|
|
3574
|
+
if (toolName) {
|
|
3575
|
+
const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
|
|
3576
|
+
if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3577
|
+
if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
|
|
3578
|
+
text = JSON.stringify(toolUse.input);
|
|
3579
|
+
} else {
|
|
3580
|
+
text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
|
|
3581
|
+
const toolUses = response.content.filter((block) => block.type === "tool_use");
|
|
3582
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
|
|
3583
|
+
id: block.id,
|
|
3584
|
+
type: "function",
|
|
3585
|
+
function: {
|
|
3586
|
+
name: block.name,
|
|
3587
|
+
arguments: JSON.stringify(block.input ?? {})
|
|
3588
|
+
}
|
|
3589
|
+
}));
|
|
3099
3590
|
}
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3111
|
-
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3112
|
-
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3113
|
-
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3114
|
-
blockKinds.set(event.index, kind);
|
|
3115
|
-
if (kind === "json-tool") sawJsonTool = true;
|
|
3116
|
-
else if (!toolName) yield {
|
|
3117
|
-
type: "tool_call_delta",
|
|
3118
|
-
index: event.index,
|
|
3119
|
-
id: event.content_block.id,
|
|
3120
|
-
name: event.content_block.name
|
|
3591
|
+
return {
|
|
3592
|
+
choices: [{ message: {
|
|
3593
|
+
content: text,
|
|
3594
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3595
|
+
} }],
|
|
3596
|
+
usage: {
|
|
3597
|
+
prompt_tokens: response.usage?.input_tokens,
|
|
3598
|
+
completion_tokens: response.usage?.output_tokens,
|
|
3599
|
+
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0),
|
|
3600
|
+
...response.usage?.output_tokens_details?.thinking_tokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usage.output_tokens_details.thinking_tokens } } : {}
|
|
3601
|
+
}
|
|
3121
3602
|
};
|
|
3122
|
-
}
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3134
|
-
|
|
3603
|
+
},
|
|
3604
|
+
async *createStream(params, options$1) {
|
|
3605
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3606
|
+
const stream = await rawMessagesCreate({
|
|
3607
|
+
...body,
|
|
3608
|
+
stream: true
|
|
3609
|
+
}, options$1);
|
|
3610
|
+
const blockKinds = new Map();
|
|
3611
|
+
let inputTokens = 0;
|
|
3612
|
+
let sawJsonTool = false;
|
|
3613
|
+
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3614
|
+
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3615
|
+
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3616
|
+
blockKinds.set(event.index, kind);
|
|
3617
|
+
if (kind === "json-tool") sawJsonTool = true;
|
|
3135
3618
|
else if (!toolName) yield {
|
|
3136
3619
|
type: "tool_call_delta",
|
|
3137
3620
|
index: event.index,
|
|
3138
|
-
|
|
3621
|
+
id: event.content_block.id,
|
|
3622
|
+
name: event.content_block.name
|
|
3139
3623
|
};
|
|
3140
|
-
}
|
|
3141
|
-
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3148
|
-
|
|
3624
|
+
} else blockKinds.set(event.index, "text");
|
|
3625
|
+
else if (event.type === "content_block_delta") {
|
|
3626
|
+
if (event.delta.type === "text_delta") {
|
|
3627
|
+
if (!toolName) yield {
|
|
3628
|
+
type: "text-delta",
|
|
3629
|
+
delta: event.delta.text
|
|
3630
|
+
};
|
|
3631
|
+
} else if (event.delta.type === "input_json_delta") {
|
|
3632
|
+
const kind = blockKinds.get(event.index);
|
|
3633
|
+
if (kind === "json-tool") yield {
|
|
3634
|
+
type: "text-delta",
|
|
3635
|
+
delta: event.delta.partial_json
|
|
3636
|
+
};
|
|
3637
|
+
else if (!toolName) yield {
|
|
3638
|
+
type: "tool_call_delta",
|
|
3639
|
+
index: event.index,
|
|
3640
|
+
argumentsDelta: event.delta.partial_json
|
|
3641
|
+
};
|
|
3149
3642
|
}
|
|
3150
|
-
}
|
|
3151
|
-
|
|
3152
|
-
|
|
3153
|
-
|
|
3154
|
-
|
|
3643
|
+
} else if (event.type === "message_delta") {
|
|
3644
|
+
const outputTokens = event.usage?.output_tokens ?? 0;
|
|
3645
|
+
const thinkingTokens = event.usage?.output_tokens_details?.thinking_tokens;
|
|
3646
|
+
yield {
|
|
3647
|
+
type: "usage",
|
|
3648
|
+
usage: {
|
|
3649
|
+
prompt_tokens: inputTokens,
|
|
3650
|
+
completion_tokens: outputTokens,
|
|
3651
|
+
total_tokens: inputTokens + outputTokens,
|
|
3652
|
+
...thinkingTokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: thinkingTokens } } : {}
|
|
3653
|
+
}
|
|
3654
|
+
};
|
|
3655
|
+
} else if (event.type === "ping") yield { type: "ping" };
|
|
3656
|
+
if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3657
|
+
}
|
|
3658
|
+
} }
|
|
3659
|
+
};
|
|
3155
3660
|
}
|
|
3156
3661
|
/**
|
|
3157
3662
|
* Anthropic requires strict role alternation, so the per-wire-message
|
|
@@ -3288,12 +3793,23 @@ function parseToolArguments(text, toolName) {
|
|
|
3288
3793
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
|
|
3289
3794
|
return parsed;
|
|
3290
3795
|
}
|
|
3796
|
+
/**
|
|
3797
|
+
* Parses a wire tool message's `content` into the object Gemini's
|
|
3798
|
+
* `functionResponse.response` expects. Gemini (and the real SDK's
|
|
3799
|
+
* `FunctionResponse.response` type) requires an object, so a result that
|
|
3800
|
+
* parses to something other than a plain JSON object (a string, number,
|
|
3801
|
+
* array, or unparseable text) is wrapped under an `output` key, mirroring
|
|
3802
|
+
* Gemini's own documented convention for non-object function results.
|
|
3803
|
+
*/
|
|
3291
3804
|
function parseToolResult(text) {
|
|
3805
|
+
let parsed;
|
|
3292
3806
|
try {
|
|
3293
|
-
|
|
3807
|
+
parsed = text.trim() ? JSON.parse(text) : "";
|
|
3294
3808
|
} catch {
|
|
3295
|
-
|
|
3809
|
+
parsed = text;
|
|
3296
3810
|
}
|
|
3811
|
+
if (parsed && !Array.isArray(parsed) && typeof parsed === "object") return parsed;
|
|
3812
|
+
return { output: parsed };
|
|
3297
3813
|
}
|
|
3298
3814
|
/**
|
|
3299
3815
|
* Gemini expects the results of everything the model asked for in one turn
|
|
@@ -3322,7 +3838,7 @@ function mergeConsecutiveFunctionResponses(contents) {
|
|
|
3322
3838
|
* `abortSignal` is folded into `config` by the caller (`create`/
|
|
3323
3839
|
* `createStream`), once the request options are available.
|
|
3324
3840
|
*/
|
|
3325
|
-
function buildGeminiRequest(params) {
|
|
3841
|
+
function buildGeminiRequest(params, effortTokenTable, thinkingLevelModels) {
|
|
3326
3842
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3327
3843
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3328
3844
|
const wantsJson = Boolean(params.response_format);
|
|
@@ -3347,51 +3863,37 @@ function buildGeminiRequest(params) {
|
|
|
3347
3863
|
})) }];
|
|
3348
3864
|
config.toolConfig = toGeminiToolConfig(params.tool_choice);
|
|
3349
3865
|
}
|
|
3866
|
+
if (usesGeminiThinkingLevel(params.model, thinkingLevelModels)) {
|
|
3867
|
+
const effortTier = params.reasoning_effort ?? (params.budget_tokens !== void 0 ? budgetTokensToEffort(params.budget_tokens, effortTokenTable) : void 0);
|
|
3868
|
+
if (effortTier !== void 0) config.thinkingConfig = { thinkingLevel: toGeminiThinkingLevel(effortTier, params.model) };
|
|
3869
|
+
} else {
|
|
3870
|
+
const thinkingBudget = params.budget_tokens ?? (params.reasoning_effort ? effortToBudgetTokens(params.reasoning_effort, effortTokenTable) : void 0);
|
|
3871
|
+
if (thinkingBudget !== void 0) config.thinkingConfig = { thinkingBudget };
|
|
3872
|
+
}
|
|
3350
3873
|
return {
|
|
3351
3874
|
model: params.model,
|
|
3352
3875
|
contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
|
|
3353
3876
|
config
|
|
3354
3877
|
};
|
|
3355
3878
|
}
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
*
|
|
3367
|
-
* `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
|
|
3368
|
-
* `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
|
|
3369
|
-
* `responseSchema` and `tools` in the same request natively, so both are
|
|
3370
|
-
* set independently here and no special-casing is needed for the
|
|
3371
|
-
* combination, unlike `fromAnthropic`/`fromBedrock`.
|
|
3372
|
-
*
|
|
3373
|
-
* `createStream` calls `generateContentStream` (optional on `GeminiClient`
|
|
3374
|
-
*, required only if the caller sets `stream: true`) and translates each
|
|
3375
|
-
* partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
|
|
3376
|
-
* Gemini's own function-calling API doesn't stream tool-call arguments
|
|
3377
|
-
* incrementally: a `functionCall` part always arrives whole in one chunk,
|
|
3378
|
-
* so each one is emitted as a single, complete `tool_call_delta` (a
|
|
3379
|
-
* one-shot "delta" containing the full arguments) rather than accumulated
|
|
3380
|
-
* fragments, that's a real difference in the underlying API, not
|
|
3381
|
-
* something this adapter can smooth over. `usageMetadata` is (per Gemini's
|
|
3382
|
-
* own behavior) only reliably present on the last chunk, so the `usage`
|
|
3383
|
-
* `WireStreamChunk` is emitted once, after the stream completes, from
|
|
3384
|
-
* whichever chunk's `usageMetadata` was seen last.
|
|
3385
|
-
*/
|
|
3386
|
-
function fromGemini(geminiClient) {
|
|
3879
|
+
function fromGemini(client, options) {
|
|
3880
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3881
|
+
const thinkingLevelModels = options?.thinkingLevelModels;
|
|
3882
|
+
const resolved = client.models ?? client;
|
|
3883
|
+
if (typeof resolved.generateContent !== "function") throw new LLMError("fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).", "invalid_params", {
|
|
3884
|
+
code: "unsupported_capability",
|
|
3885
|
+
issues: { capability: "generateContent" }
|
|
3886
|
+
});
|
|
3887
|
+
const generateContent = resolved.generateContent.bind(resolved);
|
|
3888
|
+
const generateContentStream = typeof resolved.generateContentStream === "function" ? resolved.generateContentStream.bind(resolved) : void 0;
|
|
3387
3889
|
return { chat: { completions: {
|
|
3388
|
-
async create(params, options) {
|
|
3389
|
-
const request = buildGeminiRequest(params);
|
|
3890
|
+
async create(params, options$1) {
|
|
3891
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
3390
3892
|
request.config = {
|
|
3391
3893
|
...request.config,
|
|
3392
|
-
abortSignal: options.signal
|
|
3894
|
+
abortSignal: options$1.signal
|
|
3393
3895
|
};
|
|
3394
|
-
const response = await
|
|
3896
|
+
const response = await generateContent(request);
|
|
3395
3897
|
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
3396
3898
|
const text = parts.map((p) => p.text ?? "").join("");
|
|
3397
3899
|
const functionCalls = parts.filter((p) => p.functionCall);
|
|
@@ -3412,21 +3914,22 @@ function fromGemini(geminiClient) {
|
|
|
3412
3914
|
usage: {
|
|
3413
3915
|
prompt_tokens: response.usageMetadata?.promptTokenCount,
|
|
3414
3916
|
completion_tokens: response.usageMetadata?.candidatesTokenCount,
|
|
3415
|
-
total_tokens: response.usageMetadata?.totalTokenCount
|
|
3917
|
+
total_tokens: response.usageMetadata?.totalTokenCount,
|
|
3918
|
+
...response.usageMetadata?.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usageMetadata.thoughtsTokenCount } } : {}
|
|
3416
3919
|
}
|
|
3417
3920
|
};
|
|
3418
3921
|
},
|
|
3419
|
-
async *createStream(params, options) {
|
|
3420
|
-
if (!
|
|
3922
|
+
async *createStream(params, options$1) {
|
|
3923
|
+
if (!generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
|
|
3421
3924
|
code: "unsupported_capability",
|
|
3422
3925
|
issues: { capability: "generateContentStream" }
|
|
3423
3926
|
});
|
|
3424
|
-
const request = buildGeminiRequest(params);
|
|
3927
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
3425
3928
|
request.config = {
|
|
3426
3929
|
...request.config,
|
|
3427
|
-
abortSignal: options.signal
|
|
3930
|
+
abortSignal: options$1.signal
|
|
3428
3931
|
};
|
|
3429
|
-
const stream = await
|
|
3932
|
+
const stream = await generateContentStream(request);
|
|
3430
3933
|
let toolCallIndex = 0;
|
|
3431
3934
|
let lastUsage;
|
|
3432
3935
|
for await (const chunk of stream) {
|
|
@@ -3455,7 +3958,8 @@ function fromGemini(geminiClient) {
|
|
|
3455
3958
|
usage: {
|
|
3456
3959
|
prompt_tokens: lastUsage.promptTokenCount,
|
|
3457
3960
|
completion_tokens: lastUsage.candidatesTokenCount,
|
|
3458
|
-
total_tokens: lastUsage.totalTokenCount
|
|
3961
|
+
total_tokens: lastUsage.totalTokenCount,
|
|
3962
|
+
...lastUsage.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: lastUsage.thoughtsTokenCount } } : {}
|
|
3459
3963
|
}
|
|
3460
3964
|
};
|
|
3461
3965
|
}
|
|
@@ -3464,6 +3968,19 @@ function fromGemini(geminiClient) {
|
|
|
3464
3968
|
|
|
3465
3969
|
//#endregion
|
|
3466
3970
|
//#region src/adapters/bedrock.ts
|
|
3971
|
+
/**
|
|
3972
|
+
* Default heuristic for whether a Bedrock model id is a Claude model,
|
|
3973
|
+
* matching AWS's own `anthropic.claude-*`/`us.anthropic.claude-*` naming.
|
|
3974
|
+
* Only used to decide whether a reasoning token budget is worth forwarding
|
|
3975
|
+
* through `additionalModelRequestFields`, not a general capability check,
|
|
3976
|
+
* so a plain substring match is enough, no override hook needed the way
|
|
3977
|
+
* `nativeStructuredOutputModels`/`toolUseSupportedModels` have one: a
|
|
3978
|
+
* false positive here just sends an inert extra field, not a request that
|
|
3979
|
+
* fails outright.
|
|
3980
|
+
*/
|
|
3981
|
+
function isClaudeModel(model) {
|
|
3982
|
+
return model.includes("claude");
|
|
3983
|
+
}
|
|
3467
3984
|
/** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
|
|
3468
3985
|
function toBedrockImageFormat(mimeType) {
|
|
3469
3986
|
switch (assertSupportedImageMimeType(mimeType)) {
|
|
@@ -3530,7 +4047,7 @@ function buildBedrockToolConfig(tools, toolChoiceParam) {
|
|
|
3530
4047
|
* normal, non-forced tool-call handling both `create` and `createStream`
|
|
3531
4048
|
* already do when `toolName` is unset.
|
|
3532
4049
|
*/
|
|
3533
|
-
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
|
|
4050
|
+
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
3534
4051
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3535
4052
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3536
4053
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -3538,8 +4055,8 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3538
4055
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
3539
4056
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
3540
4057
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
|
|
4058
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse's native outputConfig.textFormat on covered models, or a forced tool call otherwise).", "validation");
|
|
3541
4059
|
let toolName;
|
|
3542
|
-
let jsonInstruction;
|
|
3543
4060
|
let toolConfig;
|
|
3544
4061
|
let outputConfig;
|
|
3545
4062
|
if (jsonSchema && isNative) {
|
|
@@ -3564,7 +4081,7 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3564
4081
|
} }],
|
|
3565
4082
|
toolChoice: { tool: { name: toolName } }
|
|
3566
4083
|
};
|
|
3567
|
-
}
|
|
4084
|
+
}
|
|
3568
4085
|
if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
|
|
3569
4086
|
if (jsonSchema && toolConfig && toolUseSupportedModels) {
|
|
3570
4087
|
const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
|
|
@@ -3573,17 +4090,39 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3573
4090
|
issues: { capability: "toolUseSupportedModels" }
|
|
3574
4091
|
});
|
|
3575
4092
|
}
|
|
3576
|
-
|
|
4093
|
+
let additionalModelRequestFields;
|
|
4094
|
+
let effort;
|
|
4095
|
+
if (isClaudeModel(params.model) && (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0)) {
|
|
4096
|
+
const forcedToolChoice = toolConfig?.toolChoice;
|
|
4097
|
+
assertNoForcedToolChoiceWithThinking(forcedToolChoice && "tool" in forcedToolChoice ? `toolChoice forcing the "${forcedToolChoice.tool?.name}" tool` : forcedToolChoice && "any" in forcedToolChoice ? "toolChoice: 'required' (Converse's \"any\" tool_choice)" : void 0);
|
|
4098
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
4099
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
4100
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
4101
|
+
additionalModelRequestFields = { thinking: {
|
|
4102
|
+
type: "enabled",
|
|
4103
|
+
budget_tokens: budgetTokens
|
|
4104
|
+
} };
|
|
4105
|
+
} else {
|
|
4106
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
4107
|
+
additionalModelRequestFields = { thinking: { type: "adaptive" } };
|
|
4108
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
4109
|
+
}
|
|
4110
|
+
}
|
|
4111
|
+
const temperature = additionalModelRequestFields ? void 0 : params.temperature;
|
|
3577
4112
|
const request = {
|
|
3578
4113
|
modelId: params.model,
|
|
3579
4114
|
messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
|
|
3580
|
-
system:
|
|
4115
|
+
system: systemMessage?.content ? [{ text: systemMessage.content }] : void 0,
|
|
3581
4116
|
inferenceConfig: {
|
|
3582
|
-
...
|
|
4117
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
3583
4118
|
maxTokens: params.max_tokens
|
|
3584
4119
|
},
|
|
3585
4120
|
...toolConfig ? { toolConfig } : {},
|
|
3586
|
-
...outputConfig ? { outputConfig
|
|
4121
|
+
...outputConfig || effort ? { outputConfig: {
|
|
4122
|
+
...outputConfig ?? {},
|
|
4123
|
+
...effort ? { effort } : {}
|
|
4124
|
+
} } : {},
|
|
4125
|
+
...additionalModelRequestFields ? { additionalModelRequestFields } : {}
|
|
3587
4126
|
};
|
|
3588
4127
|
return {
|
|
3589
4128
|
request,
|
|
@@ -3591,6 +4130,120 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3591
4130
|
};
|
|
3592
4131
|
}
|
|
3593
4132
|
/**
|
|
4133
|
+
* Distinguishes a real AWS SDK v3 client (`.send(command)`) from a
|
|
4134
|
+
* hand-written `BedrockConverseClient` (`.converse(params)`) purely
|
|
4135
|
+
* structurally, so `fromBedrock` can accept either without the caller
|
|
4136
|
+
* saying which one they're passing. The two shapes don't overlap: nothing
|
|
4137
|
+
* implementing `.converse()` would also need `.send()`.
|
|
4138
|
+
*/
|
|
4139
|
+
function isAwsSendClient(client) {
|
|
4140
|
+
return typeof client.send === "function";
|
|
4141
|
+
}
|
|
4142
|
+
/**
|
|
4143
|
+
* Narrows one raw AWS stream event down to VernLLM's intentionally minimal
|
|
4144
|
+
* `BedrockConverseStreamEvent` union. Returns `undefined` if the event
|
|
4145
|
+
* isn't one of the kinds this adapter models.
|
|
4146
|
+
*
|
|
4147
|
+
* AWS's real `ConverseStreamOutput` type is a strictly larger union than
|
|
4148
|
+
* `BedrockConverseStreamEvent`. On top of every member modeled here, it
|
|
4149
|
+
* also includes a generated `$unknown` member, AWS's forward-compatibility
|
|
4150
|
+
* escape hatch for event kinds added to the service after this SDK version
|
|
4151
|
+
* was generated. A blind type assertion from one union to the other would
|
|
4152
|
+
* compile, but would let `$unknown` (or any other future member) reach
|
|
4153
|
+
* `fromBedrock`'s event-handling loop unnarrowed, as if it were one of the
|
|
4154
|
+
* kinds actually handled there.
|
|
4155
|
+
*
|
|
4156
|
+
* Returning `undefined` for anything unrecognized, filtered out by
|
|
4157
|
+
* `normalizeBedrockEventStream` below, keeps two guarantees. AWS SDK
|
|
4158
|
+
* generated types never leak into `fromBedrock`'s application code, only
|
|
4159
|
+
* this module's own `BedrockConverseStreamEvent` shape does. An event kind
|
|
4160
|
+
* this adapter doesn't yet know about is silently skipped, the same
|
|
4161
|
+
* forward-compatible behavior AWS's own `$unknown` convention implies,
|
|
4162
|
+
* rather than crashing the stream or being misrouted into a handler that
|
|
4163
|
+
* doesn't actually match its shape.
|
|
4164
|
+
*/
|
|
4165
|
+
function normalizeBedrockStreamEvent(raw) {
|
|
4166
|
+
if ("messageStart" in raw) return { messageStart: raw.messageStart };
|
|
4167
|
+
if ("contentBlockStart" in raw) return { contentBlockStart: raw.contentBlockStart };
|
|
4168
|
+
if ("contentBlockDelta" in raw) return { contentBlockDelta: raw.contentBlockDelta };
|
|
4169
|
+
if ("contentBlockStop" in raw) return { contentBlockStop: raw.contentBlockStop };
|
|
4170
|
+
if ("messageStop" in raw) return { messageStop: raw.messageStop };
|
|
4171
|
+
if ("metadata" in raw) return { metadata: raw.metadata };
|
|
4172
|
+
if ("internalServerException" in raw) return { internalServerException: raw.internalServerException };
|
|
4173
|
+
if ("modelStreamErrorException" in raw) return { modelStreamErrorException: raw.modelStreamErrorException };
|
|
4174
|
+
if ("validationException" in raw) return { validationException: raw.validationException };
|
|
4175
|
+
if ("throttlingException" in raw) return { throttlingException: raw.throttlingException };
|
|
4176
|
+
if ("serviceUnavailableException" in raw) return { serviceUnavailableException: raw.serviceUnavailableException };
|
|
4177
|
+
return void 0;
|
|
4178
|
+
}
|
|
4179
|
+
/**
|
|
4180
|
+
* Wraps a raw AWS event stream, narrowing each event through
|
|
4181
|
+
* `normalizeBedrockStreamEvent` and filtering out anything that doesn't
|
|
4182
|
+
* map onto `BedrockConverseStreamEvent`. `fromBedrock`'s event loop only
|
|
4183
|
+
* ever sees the shapes it actually models.
|
|
4184
|
+
*/
|
|
4185
|
+
async function* normalizeBedrockEventStream(rawStream) {
|
|
4186
|
+
for await (const raw of rawStream) {
|
|
4187
|
+
const event = normalizeBedrockStreamEvent(raw);
|
|
4188
|
+
if (event) yield event;
|
|
4189
|
+
}
|
|
4190
|
+
}
|
|
4191
|
+
/**
|
|
4192
|
+
* Adapts a real AWS SDK v3 client (anything with `.send()`, matching
|
|
4193
|
+
* `BedrockRuntimeClient`) into a `BedrockConverseClient`, so `fromBedrock`
|
|
4194
|
+
* can accept either without a hand-written `.converse()`/`.converseStream()`
|
|
4195
|
+
* wrapper. Internally does what that wrapper would: `client.send(new
|
|
4196
|
+
* ConverseCommand(params))`, `client.send(new
|
|
4197
|
+
* ConverseStreamCommand(params))`.
|
|
4198
|
+
*
|
|
4199
|
+
* `@aws-sdk/client-bedrock-runtime` is intentionally not a dependency (not
|
|
4200
|
+
* even a peer dependency) of this package. `vern-llm` otherwise has zero
|
|
4201
|
+
* runtime dependencies, and every other adapter works the same way:
|
|
4202
|
+
* structural typing over whatever client the caller already has. Instead,
|
|
4203
|
+
* `ConverseCommand`/`ConverseStreamCommand` are pulled in with a dynamic
|
|
4204
|
+
* `import()` the first time either method actually runs, and memoized
|
|
4205
|
+
* after that. Nothing is added to `package.json`, static or peer.
|
|
4206
|
+
* Bundlers only pull the AWS SDK in for code paths that actually pass a
|
|
4207
|
+
* raw AWS client to `fromBedrock`; a hand-written `BedrockConverseClient`
|
|
4208
|
+
* stays unaffected. If `@aws-sdk/client-bedrock-runtime` isn't installed,
|
|
4209
|
+
* the failure is a clear `LLMError` naming exactly what's missing, at the
|
|
4210
|
+
* moment it's needed, rather than a silent peer-dependency warning at
|
|
4211
|
+
* install time or a raw "Cannot find module" a caller has to trace back
|
|
4212
|
+
* themselves.
|
|
4213
|
+
*
|
|
4214
|
+
* Also closes two structural gaps between AWS's generated types and
|
|
4215
|
+
* `BedrockConverseClient`. AWS's `ConverseStreamCommandOutput.stream` is
|
|
4216
|
+
* optional, a response may not include it. This throws a clear `LLMError`
|
|
4217
|
+
* instead of letting `undefined` reach `fromBedrock`'s `for await` loop.
|
|
4218
|
+
* AWS's `ConverseStreamOutput` union is larger than
|
|
4219
|
+
* `BedrockConverseStreamEvent`, it includes a generated `$unknown` member.
|
|
4220
|
+
* Every event is narrowed through `normalizeBedrockStreamEvent` before it
|
|
4221
|
+
* reaches application code, instead of being asserted wholesale from one
|
|
4222
|
+
* type to the other.
|
|
4223
|
+
*/
|
|
4224
|
+
function wrapAwsSendClient(client) {
|
|
4225
|
+
let commandsPromise;
|
|
4226
|
+
function loadCommands() {
|
|
4227
|
+
commandsPromise ??= import("@aws-sdk/client-bedrock-runtime").then((mod) => mod, (cause) => {
|
|
4228
|
+
commandsPromise = void 0;
|
|
4229
|
+
throw new LLMError("fromBedrock requires \"@aws-sdk/client-bedrock-runtime\" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.", "validation", { cause });
|
|
4230
|
+
});
|
|
4231
|
+
return commandsPromise;
|
|
4232
|
+
}
|
|
4233
|
+
return {
|
|
4234
|
+
converse: async (params, requestOptions) => {
|
|
4235
|
+
const { ConverseCommand } = await loadCommands();
|
|
4236
|
+
return client.send(new ConverseCommand(params), { abortSignal: requestOptions.signal });
|
|
4237
|
+
},
|
|
4238
|
+
converseStream: async (params, requestOptions) => {
|
|
4239
|
+
const { ConverseStreamCommand } = await loadCommands();
|
|
4240
|
+
const result = await client.send(new ConverseStreamCommand(params), { abortSignal: requestOptions.signal });
|
|
4241
|
+
if (!result.stream) throw new LLMError("Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.", "api", { code: "server_error" });
|
|
4242
|
+
return { stream: normalizeBedrockEventStream(result.stream) };
|
|
4243
|
+
}
|
|
4244
|
+
};
|
|
4245
|
+
}
|
|
4246
|
+
/**
|
|
3594
4247
|
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
3595
4248
|
* interface VernLLM uses for OpenAI/Groq. The Converse API is unified
|
|
3596
4249
|
* across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
|
|
@@ -3598,6 +4251,16 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3598
4251
|
* regardless of which underlying model `modelId` points at, as long as
|
|
3599
4252
|
* that model supports Converse (most current-generation ones do)
|
|
3600
4253
|
*
|
|
4254
|
+
* `bedrockClient` accepts either a hand-written `BedrockConverseClient`
|
|
4255
|
+
* (a `.converse()`/`.converseStream()` wrapper you provide) or a real AWS
|
|
4256
|
+
* SDK v3 client (anything with `.send()`, matching `BedrockRuntimeClient`)
|
|
4257
|
+
* directly, detected structurally. Passing a raw AWS client skips the
|
|
4258
|
+
* hand-written wrapper entirely, internally doing what it would
|
|
4259
|
+
* (`send(new ConverseCommand(...))`, `send(new
|
|
4260
|
+
* ConverseStreamCommand(...))`). See `wrapAwsSendClient` for how that path
|
|
4261
|
+
* is implemented, including why `@aws-sdk/client-bedrock-runtime` stays
|
|
4262
|
+
* out of this package's dependencies either way.
|
|
4263
|
+
*
|
|
3601
4264
|
* `response_format: json_schema`, on a model covered by
|
|
3602
4265
|
* `options.nativeStructuredOutputModels` (opt-in, unset by default), is
|
|
3603
4266
|
* sent as `outputConfig.textFormat`, its own request field, independent of
|
|
@@ -3619,12 +4282,15 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3619
4282
|
* `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
|
|
3620
4283
|
* unsupported model surfaces Bedrock's raw error unchanged.
|
|
3621
4284
|
*
|
|
3622
|
-
* `response_format: json_object` (
|
|
3623
|
-
*
|
|
3624
|
-
*
|
|
3625
|
-
*
|
|
3626
|
-
*
|
|
3627
|
-
*
|
|
4285
|
+
* `response_format: json_object` throws `LLMError('validation')`: Converse
|
|
4286
|
+
* has no field that mechanically guarantees JSON output, and the only way
|
|
4287
|
+
* to emulate it was an unenforced system-prompt instruction, a guarantee
|
|
4288
|
+
* this adapter no longer pretends to make. Use `jsonSchema` instead.
|
|
4289
|
+
* `reasoning_effort` (no Converse equivalent) is converted to a token
|
|
4290
|
+
* budget and forwarded via `additionalModelRequestFields` for Claude
|
|
4291
|
+
* models only; `budget_tokens` is forwarded the same way directly. Both
|
|
4292
|
+
* are silently dropped for non-Claude models, which have no equivalent
|
|
4293
|
+
* field to reach for. See `adapters/internal/reasoningBudget.utils.ts`.
|
|
3628
4294
|
*
|
|
3629
4295
|
* `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
|
|
3630
4296
|
* `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
|
|
@@ -3642,107 +4308,113 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3642
4308
|
* `create` branch above unwraps it.
|
|
3643
4309
|
*/
|
|
3644
4310
|
function fromBedrock(bedrockClient, options) {
|
|
4311
|
+
const client = isAwsSendClient(bedrockClient) ? wrapAwsSendClient(bedrockClient) : bedrockClient;
|
|
3645
4312
|
const toolUseSupportedModels = options?.toolUseSupportedModels;
|
|
3646
4313
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
|
|
3653
|
-
|
|
3654
|
-
const
|
|
3655
|
-
text
|
|
3656
|
-
|
|
3657
|
-
|
|
3658
|
-
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
const
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
|
|
3670
|
-
|
|
3671
|
-
|
|
3672
|
-
|
|
3673
|
-
|
|
3674
|
-
|
|
3675
|
-
|
|
3676
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3677
|
-
} }],
|
|
3678
|
-
usage: {
|
|
3679
|
-
prompt_tokens: response.usage?.inputTokens,
|
|
3680
|
-
completion_tokens: response.usage?.outputTokens,
|
|
3681
|
-
total_tokens: response.usage?.totalTokens
|
|
4314
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
4315
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
4316
|
+
return {
|
|
4317
|
+
supportsJsonObjectMode: false,
|
|
4318
|
+
chat: { completions: {
|
|
4319
|
+
async create(params, requestOptions) {
|
|
4320
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4321
|
+
const response = await client.converse(request, requestOptions);
|
|
4322
|
+
let text;
|
|
4323
|
+
let wireToolCalls;
|
|
4324
|
+
if (toolName) {
|
|
4325
|
+
const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
|
|
4326
|
+
text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
|
|
4327
|
+
} else {
|
|
4328
|
+
const blocks = response.output?.message?.content ?? [];
|
|
4329
|
+
text = blocks.map((c) => c.text ?? "").join("");
|
|
4330
|
+
const toolUses = blocks.filter((block) => Boolean(block.toolUse));
|
|
4331
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
|
|
4332
|
+
const toolUse = block.toolUse;
|
|
4333
|
+
if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
|
|
4334
|
+
return {
|
|
4335
|
+
id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
|
|
4336
|
+
type: "function",
|
|
4337
|
+
function: {
|
|
4338
|
+
name: toolUse.name,
|
|
4339
|
+
arguments: JSON.stringify(toolUse.input ?? {})
|
|
4340
|
+
}
|
|
4341
|
+
};
|
|
4342
|
+
});
|
|
3682
4343
|
}
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
3694
|
-
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
3695
|
-
if (start?.toolUse) {
|
|
3696
|
-
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
3697
|
-
blockKinds.set(contentBlockIndex, kind);
|
|
3698
|
-
if (kind === "tool_use" && !toolName) yield {
|
|
3699
|
-
type: "tool_call_delta",
|
|
3700
|
-
index: contentBlockIndex,
|
|
3701
|
-
id: start.toolUse.toolUseId,
|
|
3702
|
-
name: start.toolUse.name
|
|
3703
|
-
};
|
|
3704
|
-
} else blockKinds.set(contentBlockIndex, "text");
|
|
3705
|
-
} else if ("contentBlockDelta" in event) {
|
|
3706
|
-
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
3707
|
-
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3708
|
-
type: "text-delta",
|
|
3709
|
-
delta: delta.text
|
|
4344
|
+
return {
|
|
4345
|
+
choices: [{ message: {
|
|
4346
|
+
content: text,
|
|
4347
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
4348
|
+
} }],
|
|
4349
|
+
usage: {
|
|
4350
|
+
prompt_tokens: response.usage?.inputTokens,
|
|
4351
|
+
completion_tokens: response.usage?.outputTokens,
|
|
4352
|
+
total_tokens: response.usage?.totalTokens
|
|
4353
|
+
}
|
|
3710
4354
|
};
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
4355
|
+
},
|
|
4356
|
+
async *createStream(params, requestOptions) {
|
|
4357
|
+
if (!client.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
|
|
4358
|
+
code: "unsupported_capability",
|
|
4359
|
+
issues: { capability: "converseStream" }
|
|
4360
|
+
});
|
|
4361
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4362
|
+
const { stream } = await client.converseStream(request, requestOptions);
|
|
4363
|
+
const blockKinds = new Map();
|
|
4364
|
+
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
4365
|
+
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
4366
|
+
if (start?.toolUse) {
|
|
4367
|
+
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
4368
|
+
blockKinds.set(contentBlockIndex, kind);
|
|
4369
|
+
if (kind === "tool_use" && !toolName) yield {
|
|
4370
|
+
type: "tool_call_delta",
|
|
4371
|
+
index: contentBlockIndex,
|
|
4372
|
+
id: start.toolUse.toolUseId,
|
|
4373
|
+
name: start.toolUse.name
|
|
4374
|
+
};
|
|
4375
|
+
} else blockKinds.set(contentBlockIndex, "text");
|
|
4376
|
+
} else if ("contentBlockDelta" in event) {
|
|
4377
|
+
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
4378
|
+
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3714
4379
|
type: "text-delta",
|
|
3715
|
-
delta: delta.
|
|
4380
|
+
delta: delta.text
|
|
3716
4381
|
};
|
|
3717
|
-
else if (
|
|
3718
|
-
|
|
3719
|
-
|
|
3720
|
-
|
|
3721
|
-
|
|
3722
|
-
|
|
3723
|
-
|
|
3724
|
-
|
|
3725
|
-
|
|
3726
|
-
|
|
3727
|
-
|
|
3728
|
-
|
|
3729
|
-
}
|
|
3730
|
-
|
|
3731
|
-
|
|
3732
|
-
|
|
3733
|
-
|
|
3734
|
-
|
|
3735
|
-
|
|
3736
|
-
|
|
3737
|
-
|
|
3738
|
-
|
|
3739
|
-
|
|
3740
|
-
status,
|
|
3741
|
-
code: status >= 500 ? "server_error" : void 0
|
|
4382
|
+
else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
|
|
4383
|
+
const kind = blockKinds.get(contentBlockIndex);
|
|
4384
|
+
if (kind === "json-tool") yield {
|
|
4385
|
+
type: "text-delta",
|
|
4386
|
+
delta: delta.toolUse.input
|
|
4387
|
+
};
|
|
4388
|
+
else if (!toolName) yield {
|
|
4389
|
+
type: "tool_call_delta",
|
|
4390
|
+
index: contentBlockIndex,
|
|
4391
|
+
argumentsDelta: delta.toolUse.input
|
|
4392
|
+
};
|
|
4393
|
+
}
|
|
4394
|
+
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
4395
|
+
type: "usage",
|
|
4396
|
+
usage: {
|
|
4397
|
+
prompt_tokens: event.metadata.usage.inputTokens,
|
|
4398
|
+
completion_tokens: event.metadata.usage.outputTokens,
|
|
4399
|
+
total_tokens: event.metadata.usage.totalTokens
|
|
4400
|
+
}
|
|
4401
|
+
};
|
|
4402
|
+
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
|
|
4403
|
+
status: 429,
|
|
4404
|
+
code: "provider_rate_limited"
|
|
3742
4405
|
});
|
|
4406
|
+
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
4407
|
+
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
4408
|
+
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
4409
|
+
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
4410
|
+
throw new LLMError(detail, "api", {
|
|
4411
|
+
status,
|
|
4412
|
+
code: status >= 500 ? "server_error" : void 0
|
|
4413
|
+
});
|
|
4414
|
+
}
|
|
3743
4415
|
}
|
|
3744
|
-
}
|
|
3745
|
-
}
|
|
4416
|
+
} }
|
|
4417
|
+
};
|
|
3746
4418
|
}
|
|
3747
4419
|
/** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
|
|
3748
4420
|
function toBedrockToolChoice(toolChoice) {
|
|
@@ -3981,6 +4653,22 @@ function fromFetch(config) {
|
|
|
3981
4653
|
//#endregion
|
|
3982
4654
|
//#region src/adapters/openaiCompatible.ts
|
|
3983
4655
|
/**
|
|
4656
|
+
* OpenAI's wire format only understands `reasoning_effort`, not a raw
|
|
4657
|
+
* token budget. When the caller set `reasoningEffort`, it's already on
|
|
4658
|
+
* `params` and passed through unchanged, this function does nothing.
|
|
4659
|
+
* When only `budgetTokens` was set, it's converted to the nearest tier
|
|
4660
|
+
* and `budget_tokens` is dropped, since OpenAI's API would otherwise
|
|
4661
|
+
* silently ignore an unrecognized field.
|
|
4662
|
+
*/
|
|
4663
|
+
function applyReasoningBudget(params, effortTokenTable) {
|
|
4664
|
+
if (params.budget_tokens === void 0) return params;
|
|
4665
|
+
const { budget_tokens,...rest } = params;
|
|
4666
|
+
return rest.reasoning_effort !== void 0 ? rest : {
|
|
4667
|
+
...rest,
|
|
4668
|
+
reasoning_effort: budgetTokensToEffort(budget_tokens, effortTokenTable)
|
|
4669
|
+
};
|
|
4670
|
+
}
|
|
4671
|
+
/**
|
|
3984
4672
|
* Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
|
|
3985
4673
|
* array. Text blocks become `{ type: 'text', text }`; image blocks become
|
|
3986
4674
|
* `{ type: 'image_url', image_url: { url } }` with the base64 payload
|
|
@@ -4046,23 +4734,24 @@ function* toWireStreamChunks(chunk) {
|
|
|
4046
4734
|
function fromOpenAICompatible(client, options = {}) {
|
|
4047
4735
|
const raw = client;
|
|
4048
4736
|
const { supportsStreamUsage = true } = options;
|
|
4737
|
+
const effortTokenTable = resolveEffortTokenTable(options.reasoningEffortTokens);
|
|
4049
4738
|
const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
|
|
4050
4739
|
return { chat: { completions: {
|
|
4051
4740
|
async create(params, options$1) {
|
|
4052
4741
|
const messages = toOpenAIMessages(params);
|
|
4053
|
-
return raw.chat.completions.create({
|
|
4742
|
+
return raw.chat.completions.create(applyReasoningBudget({
|
|
4054
4743
|
...params,
|
|
4055
4744
|
messages
|
|
4056
|
-
}, options$1);
|
|
4745
|
+
}, effortTokenTable), options$1);
|
|
4057
4746
|
},
|
|
4058
4747
|
async *createStream(params, options$1) {
|
|
4059
4748
|
const messages = toOpenAIMessages(params);
|
|
4060
|
-
const stream = await rawCreate({
|
|
4749
|
+
const stream = await rawCreate(applyReasoningBudget({
|
|
4061
4750
|
...params,
|
|
4062
4751
|
messages,
|
|
4063
4752
|
stream: true,
|
|
4064
4753
|
...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
|
|
4065
|
-
}, options$1);
|
|
4754
|
+
}, effortTokenTable), options$1);
|
|
4066
4755
|
for await (const chunk of stream) yield* toWireStreamChunks(chunk);
|
|
4067
4756
|
}
|
|
4068
4757
|
} } };
|
|
@@ -4186,6 +4875,8 @@ exports.TieredCacheAdapter = TieredCacheAdapter
|
|
|
4186
4875
|
exports.VernLLM = VernLLM
|
|
4187
4876
|
exports.defaultEstimateTokens = defaultEstimateTokens
|
|
4188
4877
|
exports.defaultFallbackOn = defaultFallbackOn
|
|
4878
|
+
exports.defineCachedCallParams = defineCachedCallParams
|
|
4879
|
+
exports.defineCallParams = defineCallParams
|
|
4189
4880
|
exports.from01AI = from01AI
|
|
4190
4881
|
exports.fromAnthropic = fromAnthropic
|
|
4191
4882
|
exports.fromAnyscale = fromAnyscale
|