vern-llm 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.cjs +969 -278
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +550 -105
- package/dist/index.d.cts.map +1 -1
- package/dist/index.d.ts +550 -105
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +968 -279
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.js
CHANGED
|
@@ -71,6 +71,42 @@ function safeIssues(issues) {
|
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
73
|
/**
|
|
74
|
+
* Returns a JSON safe, independent copy of `body`, or a marker string if
|
|
75
|
+
* `body` can't survive `JSON.stringify` (e.g. a circular reference). A
|
|
76
|
+
* request body built from adapter-transformed messages is normally
|
|
77
|
+
* always plain data, but tool call arguments or a caller supplied
|
|
78
|
+
* `cause`-adjacent value could in principle carry a circular reference,
|
|
79
|
+
* so this guards the same way `safeIssues` does rather than assuming it
|
|
80
|
+
* can't happen. Unlike `safeIssues`, this clones rather than returning
|
|
81
|
+
* the same reference: the object backing a request body can still be
|
|
82
|
+
* mutated by adapter code between when a request is dispatched and when
|
|
83
|
+
* an attempt is later recorded as failed (e.g. `fromGemini` sets
|
|
84
|
+
* `request.config` in place), so returning the same reference here could
|
|
85
|
+
* make a stored snapshot silently reflect a later, different state than
|
|
86
|
+
* what was actually sent.
|
|
87
|
+
*/
|
|
88
|
+
function safeBody(body) {
|
|
89
|
+
if (body === void 0) return void 0;
|
|
90
|
+
try {
|
|
91
|
+
return JSON.parse(JSON.stringify(body));
|
|
92
|
+
} catch {
|
|
93
|
+
return "[Unserializable: request body contained a circular reference]";
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const AUTH_HEADER_NAMES = new Set([
|
|
97
|
+
"authorization",
|
|
98
|
+
"x-api-key",
|
|
99
|
+
"x-goog-api-key",
|
|
100
|
+
"api-key"
|
|
101
|
+
]);
|
|
102
|
+
/** Removes auth headers before a request snapshot is built. Case insensitive on header names. */
|
|
103
|
+
function stripAuthHeaders(headers) {
|
|
104
|
+
if (headers === void 0) return void 0;
|
|
105
|
+
const out = {};
|
|
106
|
+
for (const [key, value] of Object.entries(headers)) if (!AUTH_HEADER_NAMES.has(key.toLowerCase())) out[key] = value;
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
74
110
|
* Depth cap for `safeAttempts`, guarding against a pathological,
|
|
75
111
|
* self referential `attempts` array. `attempts` is a public
|
|
76
112
|
* `LLMErrorOptions` field, so a caller can construct one by hand; this
|
|
@@ -86,8 +122,12 @@ const MAX_ATTEMPTS_DEPTH = 20;
|
|
|
86
122
|
* circular one after the snapshot was created, and `attempts` is a
|
|
87
123
|
* public constructor option, so a caller can hand build a `RetryAttempt`
|
|
88
124
|
* (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
|
|
89
|
-
* in directly, never touching `toSnapshot()` at all.
|
|
90
|
-
*
|
|
125
|
+
* in directly, never touching `toSnapshot()` at all. The same applies to
|
|
126
|
+
* `request`: its `body` is re-checked through `safeBody`, and its
|
|
127
|
+
* `headers` are re-stripped through `stripAuthHeaders`, so a hand built
|
|
128
|
+
* `RetryAttempt.request` can't smuggle an auth header past `toSnapshot()`
|
|
129
|
+
* either. Extra fields on an attempt (e.g. `FallbackAttempt`'s
|
|
130
|
+
* `provider`/`model`) are preserved.
|
|
91
131
|
*/
|
|
92
132
|
function safeAttempts(attempts, depth = 0) {
|
|
93
133
|
if (attempts === void 0) return void 0;
|
|
@@ -98,9 +138,37 @@ function safeAttempts(attempts, depth = 0) {
|
|
|
98
138
|
...attempt.error,
|
|
99
139
|
issues: safeIssues(attempt.error.issues),
|
|
100
140
|
attempts: safeAttempts(attempt.error.attempts, depth + 1)
|
|
141
|
+
},
|
|
142
|
+
request: attempt.request && {
|
|
143
|
+
...attempt.request,
|
|
144
|
+
body: safeBody(attempt.request.body),
|
|
145
|
+
headers: stripAuthHeaders(attempt.request.headers)
|
|
101
146
|
}
|
|
102
147
|
}));
|
|
103
148
|
}
|
|
149
|
+
/**
|
|
150
|
+
* Builds a point-in-time, plain data copy of one attempt's outgoing
|
|
151
|
+
* request. Mirrors `LLMError.toSnapshot()`: never thrown or dispatched
|
|
152
|
+
* again, safe to serialize and store. A plain function rather than a
|
|
153
|
+
* method, since unlike `LLMError` a request has no throwable identity or
|
|
154
|
+
* derived state worth wrapping in a class.
|
|
155
|
+
*
|
|
156
|
+
* `startedAt` is optional so existing call sites (and tests) that don't
|
|
157
|
+
* care about exact timing keep working, but a caller that has a real
|
|
158
|
+
* capture time should always pass it: this function may run well after
|
|
159
|
+
* the request was actually dispatched (e.g. `callExecutor` only builds
|
|
160
|
+
* the snapshot once an attempt has failed), so defaulting to `Date.now()`
|
|
161
|
+
* here would record failure-handling time, not request-start time.
|
|
162
|
+
*/
|
|
163
|
+
function toRequestSnapshot(provider, model, body, headers, startedAt = Date.now()) {
|
|
164
|
+
return {
|
|
165
|
+
provider,
|
|
166
|
+
model,
|
|
167
|
+
body: safeBody(body),
|
|
168
|
+
headers: stripAuthHeaders(headers),
|
|
169
|
+
startedAt
|
|
170
|
+
};
|
|
171
|
+
}
|
|
104
172
|
var LLMError = class extends Error {
|
|
105
173
|
status;
|
|
106
174
|
issues;
|
|
@@ -1233,6 +1301,41 @@ function codeForStatus(status) {
|
|
|
1233
1301
|
}
|
|
1234
1302
|
}
|
|
1235
1303
|
/**
|
|
1304
|
+
* Whether a provider's error response actually contains anything a person
|
|
1305
|
+
* could act on. Some providers return a non-2xx status with **no body at
|
|
1306
|
+
* all** for certain field-validation failures (Mistral's OpenAI-compatible
|
|
1307
|
+
* endpoint does this, for example, when a request includes a field the
|
|
1308
|
+
* target model doesn't support). SDKs built on top of `openai` render that
|
|
1309
|
+
* specific case as a message like `"400 status code (no body)"`.
|
|
1310
|
+
*
|
|
1311
|
+
* Derived from the object's own `error`/`message` fields directly, rather
|
|
1312
|
+
* than from whatever `describeError` rendered, because `describeError`
|
|
1313
|
+
* falls back to serializing the *whole* thrown value when neither field is
|
|
1314
|
+
* present or meaningful. That fallback is local echo (e.g. just the
|
|
1315
|
+
* `status` a caller passed in), not provider diagnostic content, and
|
|
1316
|
+
* treating it as "detail" defeats the whole point of this check.
|
|
1317
|
+
*/
|
|
1318
|
+
const NO_BODY_MESSAGE_PATTERN = /\(no body\)/i;
|
|
1319
|
+
function isEmptyObject(value) {
|
|
1320
|
+
return Object.keys(value).length === 0;
|
|
1321
|
+
}
|
|
1322
|
+
function hasNoDiagnosticDetail(error) {
|
|
1323
|
+
if (error && typeof error === "object") {
|
|
1324
|
+
const { error: errorField, message } = error;
|
|
1325
|
+
if (errorField !== void 0 && errorField !== null) {
|
|
1326
|
+
const isEmptyString = typeof errorField === "string" && errorField.trim().length === 0;
|
|
1327
|
+
const isEmptyStruct = typeof errorField === "object" && isEmptyObject(errorField);
|
|
1328
|
+
if (!isEmptyString && !isEmptyStruct) return false;
|
|
1329
|
+
}
|
|
1330
|
+
if (typeof message === "string") {
|
|
1331
|
+
const trimmed = message.trim();
|
|
1332
|
+
return trimmed.length === 0 || NO_BODY_MESSAGE_PATTERN.test(trimmed);
|
|
1333
|
+
}
|
|
1334
|
+
return true;
|
|
1335
|
+
}
|
|
1336
|
+
return true;
|
|
1337
|
+
}
|
|
1338
|
+
/**
|
|
1236
1339
|
* Converts any thrown value into a well-typed LLMError. `attempts`, when
|
|
1237
1340
|
* given, is the accumulated record of every attempt made before `error`
|
|
1238
1341
|
* was thrown; it's passed straight into the constructed error's options
|
|
@@ -1249,13 +1352,19 @@ function normalizeError(error, signal, attempts) {
|
|
|
1249
1352
|
}
|
|
1250
1353
|
const status = extractStatus(error);
|
|
1251
1354
|
const retryAfterMs = extractRetryAfterMs(error);
|
|
1252
|
-
if (status !== void 0)
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1355
|
+
if (status !== void 0) {
|
|
1356
|
+
const description = describeError(error);
|
|
1357
|
+
const code = codeForStatus(status);
|
|
1358
|
+
const isRequestValidationStatus = code === void 0;
|
|
1359
|
+
const message = hasNoDiagnosticDetail(error) ? isRequestValidationStatus ? `LLM request failed with status ${status} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.` : `LLM request failed with status ${status} and no error detail from the provider.` : `LLM request failed: ${description}`;
|
|
1360
|
+
return new LLMError(message, "api", {
|
|
1361
|
+
status,
|
|
1362
|
+
cause: error,
|
|
1363
|
+
retryAfterMs,
|
|
1364
|
+
code,
|
|
1365
|
+
attempts
|
|
1366
|
+
});
|
|
1367
|
+
}
|
|
1259
1368
|
if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
|
|
1260
1369
|
cause: error,
|
|
1261
1370
|
retryAfterMs,
|
|
@@ -1328,6 +1437,13 @@ function parseWireToolCalls(wireToolCalls) {
|
|
|
1328
1437
|
//#endregion
|
|
1329
1438
|
//#region src/internal/execution/requestBuilder.ts
|
|
1330
1439
|
/**
|
|
1440
|
+
* Serializes `ConversationTurn` assistant content for the wire. Strings
|
|
1441
|
+
* pass through unchanged. Parsed JSON values are `JSON.stringify`'d.
|
|
1442
|
+
*/
|
|
1443
|
+
function serializeAssistantContent(content) {
|
|
1444
|
+
return typeof content === "string" ? content : JSON.stringify(content);
|
|
1445
|
+
}
|
|
1446
|
+
/**
|
|
1331
1447
|
* Builds the wire request object for one call, applying per-instance
|
|
1332
1448
|
* defaults (model, max tokens, temperature) and per-call overrides.
|
|
1333
1449
|
* Owns every check that depends only on the caller's own input shape, not
|
|
@@ -1345,15 +1461,23 @@ var RequestBuilder = class {
|
|
|
1345
1461
|
model;
|
|
1346
1462
|
defaultMaxTokens;
|
|
1347
1463
|
defaultTemperature;
|
|
1464
|
+
defaultReasoningEffort;
|
|
1465
|
+
defaultBudgetTokens;
|
|
1466
|
+
supportsJsonObjectMode;
|
|
1348
1467
|
constructor(options) {
|
|
1349
1468
|
this.model = options.model;
|
|
1350
1469
|
this.defaultMaxTokens = options.defaultMaxTokens;
|
|
1351
1470
|
this.defaultTemperature = options.defaultTemperature;
|
|
1471
|
+
this.defaultReasoningEffort = options.defaultReasoningEffort;
|
|
1472
|
+
this.defaultBudgetTokens = options.defaultBudgetTokens;
|
|
1473
|
+
this.supportsJsonObjectMode = options.supportsJsonObjectMode;
|
|
1352
1474
|
}
|
|
1353
1475
|
/** Applies per-call defaults and shapes params into the client's request object. */
|
|
1354
1476
|
build(params) {
|
|
1355
|
-
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model,
|
|
1477
|
+
const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, jsonSchema, tools, toolChoice } = params;
|
|
1356
1478
|
const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
|
|
1479
|
+
const reasoningEffort = params.reasoningEffort === void 0 ? this.defaultReasoningEffort : params.reasoningEffort;
|
|
1480
|
+
const budgetTokens = params.budgetTokens === void 0 ? this.defaultBudgetTokens : params.budgetTokens;
|
|
1357
1481
|
if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
|
|
1358
1482
|
if (tools) {
|
|
1359
1483
|
const seen = new Set();
|
|
@@ -1375,8 +1499,12 @@ var RequestBuilder = class {
|
|
|
1375
1499
|
available: tools.map((t) => t.name)
|
|
1376
1500
|
}
|
|
1377
1501
|
});
|
|
1378
|
-
const
|
|
1379
|
-
const
|
|
1502
|
+
const jsonModeExplicit = params.jsonMode;
|
|
1503
|
+
const jsonMode = jsonModeExplicit ?? (tools ? false : true);
|
|
1504
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === true) throw new LLMError("jsonMode: true was set explicitly, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.", "invalid_params");
|
|
1505
|
+
if (!this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 && params.schema) throw new LLMError("`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: \"json_object\"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.", "invalid_params");
|
|
1506
|
+
const jsonModeEffective = !this.supportsJsonObjectMode && !jsonSchema && jsonModeExplicit === void 0 ? false : jsonMode;
|
|
1507
|
+
const useJson = jsonModeEffective || Boolean(jsonSchema);
|
|
1380
1508
|
if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
|
|
1381
1509
|
const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
|
|
1382
1510
|
this.validateHistory(history);
|
|
@@ -1386,6 +1514,7 @@ var RequestBuilder = class {
|
|
|
1386
1514
|
max_tokens: maxTokens,
|
|
1387
1515
|
...responseFormat ? { response_format: responseFormat } : {},
|
|
1388
1516
|
...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
|
|
1517
|
+
...budgetTokens !== void 0 && budgetTokens !== null ? { budget_tokens: budgetTokens } : {},
|
|
1389
1518
|
...tools ? { tools: toWireTools(tools) } : {},
|
|
1390
1519
|
...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
|
|
1391
1520
|
messages: [
|
|
@@ -1481,9 +1610,13 @@ var RequestBuilder = class {
|
|
|
1481
1610
|
}));
|
|
1482
1611
|
if (turn.role === "assistant" && turn.toolCalls?.length) return [{
|
|
1483
1612
|
role: "assistant",
|
|
1484
|
-
...turn.content ? { content: turn.content } : {},
|
|
1613
|
+
...turn.content !== void 0 ? { content: serializeAssistantContent(turn.content) } : {},
|
|
1485
1614
|
tool_calls: toWireToolCalls(turn.toolCalls)
|
|
1486
1615
|
}];
|
|
1616
|
+
if (turn.role === "assistant") return [{
|
|
1617
|
+
role: "assistant",
|
|
1618
|
+
content: serializeAssistantContent(turn.content === void 0 ? "" : turn.content)
|
|
1619
|
+
}];
|
|
1487
1620
|
return [{
|
|
1488
1621
|
role: turn.role,
|
|
1489
1622
|
content: turn.content ?? ""
|
|
@@ -1617,10 +1750,12 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1617
1750
|
complete: wireChunk.complete
|
|
1618
1751
|
});
|
|
1619
1752
|
} else if (wireChunk.type === "usage") {
|
|
1753
|
+
const reasoningTokens = wireChunk.usage.completion_tokens_details?.reasoning_tokens;
|
|
1620
1754
|
usage = {
|
|
1621
1755
|
promptTokens: wireChunk.usage.prompt_tokens ?? 0,
|
|
1622
1756
|
completionTokens: wireChunk.usage.completion_tokens ?? 0,
|
|
1623
1757
|
totalTokens: wireChunk.usage.total_tokens ?? 0,
|
|
1758
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
1624
1759
|
requestId,
|
|
1625
1760
|
model,
|
|
1626
1761
|
provider: providerName,
|
|
@@ -1674,6 +1809,16 @@ function buildStreamResult(iterator, first, options) {
|
|
|
1674
1809
|
//#endregion
|
|
1675
1810
|
//#region src/internal/execution/callExecutor.ts
|
|
1676
1811
|
/**
|
|
1812
|
+
* Identity function with its own parameter, used only to sidestep a TS
|
|
1813
|
+
* quirk: a `let` reassigned solely inside a nested closure (like
|
|
1814
|
+
* `retryWithBackoff`'s `onRequest`) gets narrowed to `undefined` at the
|
|
1815
|
+
* point it was last synchronously assigned, which would otherwise make
|
|
1816
|
+
* `lastRequestForAttempt` read as `never` at the point it's used below.
|
|
1817
|
+
*/
|
|
1818
|
+
function passThroughRequestSnapshot(snapshot) {
|
|
1819
|
+
return snapshot;
|
|
1820
|
+
}
|
|
1821
|
+
/**
|
|
1677
1822
|
* Everything one provider target needs to attempt a call: request
|
|
1678
1823
|
* building, retry with backoff, the per-target breaker, the per-target
|
|
1679
1824
|
* limiter. Never exported publicly. `VernLLM` holds one per target and
|
|
@@ -1716,7 +1861,10 @@ var CallExecutor = class {
|
|
|
1716
1861
|
this.requestBuilder = new RequestBuilder({
|
|
1717
1862
|
model,
|
|
1718
1863
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
1719
|
-
defaultTemperature: options.defaultTemperature
|
|
1864
|
+
defaultTemperature: options.defaultTemperature,
|
|
1865
|
+
defaultReasoningEffort: options.defaultReasoningEffort,
|
|
1866
|
+
defaultBudgetTokens: options.defaultBudgetTokens,
|
|
1867
|
+
supportsJsonObjectMode: client.supportsJsonObjectMode ?? true
|
|
1720
1868
|
});
|
|
1721
1869
|
}
|
|
1722
1870
|
getCircuitState(model) {
|
|
@@ -1757,7 +1905,7 @@ var CallExecutor = class {
|
|
|
1757
1905
|
const model = params.model ?? this.model;
|
|
1758
1906
|
const attempts = [];
|
|
1759
1907
|
try {
|
|
1760
|
-
return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1908
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1761
1909
|
} catch (error) {
|
|
1762
1910
|
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1763
1911
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
@@ -1770,7 +1918,7 @@ var CallExecutor = class {
|
|
|
1770
1918
|
const model = params.model ?? this.model;
|
|
1771
1919
|
const attempts = [];
|
|
1772
1920
|
try {
|
|
1773
|
-
return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
|
|
1921
|
+
return await this.retryWithBackoff((attempt, onRequest) => this.executeStreamCall(params, requestId, attempt, onRequest), requestId, model, params.signal, onAttempt, attempts);
|
|
1774
1922
|
} catch (error) {
|
|
1775
1923
|
const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
|
|
1776
1924
|
if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
|
|
@@ -1785,8 +1933,9 @@ var CallExecutor = class {
|
|
|
1785
1933
|
* set. Throws on an empty response (no text and no tool_calls) so the
|
|
1786
1934
|
* retry loop treats it like any other transient failure.
|
|
1787
1935
|
*/
|
|
1788
|
-
async executeCall(params, requestId, attempt) {
|
|
1936
|
+
async executeCall(params, requestId, attempt, onRequest) {
|
|
1789
1937
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
1938
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1790
1939
|
let release;
|
|
1791
1940
|
if (this.limiter) {
|
|
1792
1941
|
const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
|
|
@@ -1894,8 +2043,9 @@ var CallExecutor = class {
|
|
|
1894
2043
|
* not on the first chunk arriving, so a connection that opens but then
|
|
1895
2044
|
* dies mid-stream isn't masked as a success (see `buildStreamResult`).
|
|
1896
2045
|
*/
|
|
1897
|
-
async executeStreamCall(params, requestId, attempt) {
|
|
2046
|
+
async executeStreamCall(params, requestId, attempt, onRequest) {
|
|
1898
2047
|
const { useJson, model, request } = this.requestBuilder.build(params);
|
|
2048
|
+
onRequest?.(toRequestSnapshot(this.providerName, model, request, void 0, Date.now()));
|
|
1899
2049
|
const completions = this.client.chat.completions;
|
|
1900
2050
|
if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
|
|
1901
2051
|
code: "unsupported_capability",
|
|
@@ -2015,18 +2165,25 @@ var CallExecutor = class {
|
|
|
2015
2165
|
*/
|
|
2016
2166
|
async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
|
|
2017
2167
|
let lastError;
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
|
|
2029
|
-
|
|
2168
|
+
let lastRequestForAttempt;
|
|
2169
|
+
for (let attempt = 0; attempt <= this.maxRetries; attempt++) {
|
|
2170
|
+
lastRequestForAttempt = void 0;
|
|
2171
|
+
try {
|
|
2172
|
+
if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
|
|
2173
|
+
onAttempt?.();
|
|
2174
|
+
return await fn(attempt, (req) => {
|
|
2175
|
+
lastRequestForAttempt = req;
|
|
2176
|
+
});
|
|
2177
|
+
} catch (error) {
|
|
2178
|
+
lastError = error;
|
|
2179
|
+
const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
|
|
2180
|
+
if (!willRetry) break;
|
|
2181
|
+
attempts?.push({
|
|
2182
|
+
index: attempt,
|
|
2183
|
+
error: normalizeError(error, signal).toSnapshot(),
|
|
2184
|
+
request: passThroughRequestSnapshot(lastRequestForAttempt)
|
|
2185
|
+
});
|
|
2186
|
+
}
|
|
2030
2187
|
}
|
|
2031
2188
|
throw lastError;
|
|
2032
2189
|
}
|
|
@@ -2038,10 +2195,12 @@ var CallExecutor = class {
|
|
|
2038
2195
|
*/
|
|
2039
2196
|
extractUsage(response, requestId, model) {
|
|
2040
2197
|
if (!response.usage) return void 0;
|
|
2198
|
+
const reasoningTokens = response.usage.completion_tokens_details?.reasoning_tokens;
|
|
2041
2199
|
return {
|
|
2042
2200
|
promptTokens: response.usage.prompt_tokens ?? 0,
|
|
2043
2201
|
completionTokens: response.usage.completion_tokens ?? 0,
|
|
2044
2202
|
totalTokens: response.usage.total_tokens ?? 0,
|
|
2203
|
+
...reasoningTokens !== void 0 ? { reasoningTokens } : {},
|
|
2045
2204
|
requestId,
|
|
2046
2205
|
model,
|
|
2047
2206
|
provider: this.providerName,
|
|
@@ -2465,7 +2624,7 @@ var RateLimiter = class {
|
|
|
2465
2624
|
//#endregion
|
|
2466
2625
|
//#region src/vernLLM.ts
|
|
2467
2626
|
/**
|
|
2468
|
-
* A
|
|
2627
|
+
* A LLM call framework for resilience, observability and control. This is VernLLM!
|
|
2469
2628
|
*
|
|
2470
2629
|
* Adds retry with backoff and jitter, per-attempt timeouts, an optional
|
|
2471
2630
|
* circuit breaker, JSON parsing with optional schema validation, usage
|
|
@@ -2505,6 +2664,8 @@ var VernLLM = class {
|
|
|
2505
2664
|
this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
|
|
2506
2665
|
this.reportEvent = makeEventReporter(options.onEvent, this.logger);
|
|
2507
2666
|
const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
|
|
2667
|
+
const primaryDefaultReasoningEffort = options.defaultReasoningEffort;
|
|
2668
|
+
const primaryDefaultBudgetTokens = options.defaultBudgetTokens;
|
|
2508
2669
|
const primaryTarget = {
|
|
2509
2670
|
client: options.client,
|
|
2510
2671
|
model: options.model,
|
|
@@ -2515,6 +2676,8 @@ var VernLLM = class {
|
|
|
2515
2676
|
baseDelayMs: options.baseDelayMs,
|
|
2516
2677
|
defaultMaxTokens: options.defaultMaxTokens,
|
|
2517
2678
|
defaultTemperature: primaryDefaultTemperature,
|
|
2679
|
+
defaultReasoningEffort: primaryDefaultReasoningEffort,
|
|
2680
|
+
defaultBudgetTokens: primaryDefaultBudgetTokens,
|
|
2518
2681
|
nonRetryableStatus: options.nonRetryableStatus,
|
|
2519
2682
|
circuitBreaker: options.circuitBreaker,
|
|
2520
2683
|
rateLimit: options.rateLimit
|
|
@@ -2532,6 +2695,8 @@ var VernLLM = class {
|
|
|
2532
2695
|
baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
|
|
2533
2696
|
defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
|
|
2534
2697
|
defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
|
|
2698
|
+
defaultReasoningEffort: target.defaultReasoningEffort === void 0 ? primaryDefaultReasoningEffort : target.defaultReasoningEffort,
|
|
2699
|
+
defaultBudgetTokens: target.defaultBudgetTokens === void 0 ? primaryDefaultBudgetTokens : target.defaultBudgetTokens,
|
|
2535
2700
|
nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
|
|
2536
2701
|
400,
|
|
2537
2702
|
401,
|
|
@@ -2733,6 +2898,45 @@ var VernLLM = class {
|
|
|
2733
2898
|
if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
|
|
2734
2899
|
}
|
|
2735
2900
|
};
|
|
2901
|
+
/**
|
|
2902
|
+
* Identity function preserving `params`'s own precise type, unlike a `:
|
|
2903
|
+
* CallParams<T>` annotation, which would widen `tools` away and break the
|
|
2904
|
+
* `ConditionalToolCallParams<T>` overload for `tools: someCondition ?
|
|
2905
|
+
* [tool] : undefined`. Use it when you need `call()` params in a named,
|
|
2906
|
+
* reusable variable; skip it when you can pass the object inline.
|
|
2907
|
+
*
|
|
2908
|
+
* ```ts
|
|
2909
|
+
* const params = defineCallParams({
|
|
2910
|
+
* userContent: 'What is the weather?',
|
|
2911
|
+
* tools: someCondition ? [weatherTool] : undefined,
|
|
2912
|
+
* });
|
|
2913
|
+
* const result = await llm.call(params);
|
|
2914
|
+
* // result: unknown | CallWithToolsResult<unknown>, same as inline
|
|
2915
|
+
* ```
|
|
2916
|
+
*
|
|
2917
|
+
* `T` isn't a parameter here; pin it via `llm.call<T>(params)` as usual.
|
|
2918
|
+
* `defineCachedCallParams` is the `cachedCall()` counterpart.
|
|
2919
|
+
*/
|
|
2920
|
+
function defineCallParams(params) {
|
|
2921
|
+
return params;
|
|
2922
|
+
}
|
|
2923
|
+
/**
|
|
2924
|
+
* The `cachedCall()` counterpart to `defineCallParams`: preserves the
|
|
2925
|
+
* whole `{ cacheKey, ttl, call }` object, `call.tools` included, in one
|
|
2926
|
+
* named variable.
|
|
2927
|
+
*
|
|
2928
|
+
* ```ts
|
|
2929
|
+
* const params = defineCachedCallParams({
|
|
2930
|
+
* cacheKey: 'weather-ny',
|
|
2931
|
+
* ttl: 60,
|
|
2932
|
+
* call: { userContent: 'What is the weather?', tools: someCondition ? [weatherTool] : undefined },
|
|
2933
|
+
* });
|
|
2934
|
+
* const result = await llm.cachedCall(params);
|
|
2935
|
+
* ```
|
|
2936
|
+
*/
|
|
2937
|
+
function defineCachedCallParams(params) {
|
|
2938
|
+
return params;
|
|
2939
|
+
}
|
|
2736
2940
|
|
|
2737
2941
|
//#endregion
|
|
2738
2942
|
//#region src/adapters/internal/sse.ts
|
|
@@ -2873,6 +3077,275 @@ function supportsNativeStructuredOutput(model, override) {
|
|
|
2873
3077
|
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
2874
3078
|
}
|
|
2875
3079
|
|
|
3080
|
+
//#endregion
|
|
3081
|
+
//#region src/adapters/internal/reasoningBudget.utils.ts
|
|
3082
|
+
const DEFAULT_EFFORT_TOKENS = {
|
|
3083
|
+
minimal: 1024,
|
|
3084
|
+
low: 4096,
|
|
3085
|
+
medium: 16e3,
|
|
3086
|
+
high: 32e3
|
|
3087
|
+
};
|
|
3088
|
+
/**
|
|
3089
|
+
* Merges a caller-supplied partial override over `DEFAULT_EFFORT_TOKENS`.
|
|
3090
|
+
* Called once per adapter instance (not per request), so a per-instance
|
|
3091
|
+
* override only needs to specify the tiers it actually wants to change.
|
|
3092
|
+
*
|
|
3093
|
+
* Throws `LLMError('invalid_params')` if the override doesn't keep the
|
|
3094
|
+
* tiers in strictly ascending order (`minimal < low < medium < high`).
|
|
3095
|
+
* `budgetTokensToEffort` buckets by walking the tiers low to high and
|
|
3096
|
+
* returning on the first one a value is `<=`, so an unordered table (e.g.
|
|
3097
|
+
* `low` above `medium`) wouldn't just produce a "wrong" bucket, it would
|
|
3098
|
+
* make some tiers unreachable outright, silently, with no signal to the
|
|
3099
|
+
* caller that their override doesn't do what they think it does.
|
|
3100
|
+
*/
|
|
3101
|
+
function resolveEffortTokenTable(override) {
|
|
3102
|
+
if (!override) return DEFAULT_EFFORT_TOKENS;
|
|
3103
|
+
const table = {
|
|
3104
|
+
...DEFAULT_EFFORT_TOKENS,
|
|
3105
|
+
...override
|
|
3106
|
+
};
|
|
3107
|
+
if (!(table.minimal < table.low && table.low < table.medium && table.medium < table.high)) throw new LLMError(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(table)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`, "invalid_params");
|
|
3108
|
+
return table;
|
|
3109
|
+
}
|
|
3110
|
+
/** Converts a `reasoningEffort` tier into the nearest `budgetTokens` value. */
|
|
3111
|
+
function effortToBudgetTokens(effort, table = DEFAULT_EFFORT_TOKENS) {
|
|
3112
|
+
return table[effort];
|
|
3113
|
+
}
|
|
3114
|
+
/**
|
|
3115
|
+
* Converts a raw `budgetTokens` value into the nearest `reasoningEffort`
|
|
3116
|
+
* tier, for providers that only understand tiers. Buckets by the same
|
|
3117
|
+
* `table` `effortToBudgetTokens` produces its values from, so the two
|
|
3118
|
+
* functions agree with each other at the boundary values, as long as the
|
|
3119
|
+
* same (possibly overridden) table is passed to both. A value strictly
|
|
3120
|
+
* between two tiers (e.g. 4097, one above the default `low`) rounds up to
|
|
3121
|
+
* the next tier it's still `<=`, i.e. `medium` here, not down to `low`.
|
|
3122
|
+
*/
|
|
3123
|
+
function budgetTokensToEffort(budgetTokens, table = DEFAULT_EFFORT_TOKENS) {
|
|
3124
|
+
if (budgetTokens <= table.minimal) return "minimal";
|
|
3125
|
+
if (budgetTokens <= table.low) return "low";
|
|
3126
|
+
if (budgetTokens <= table.medium) return "medium";
|
|
3127
|
+
return "high";
|
|
3128
|
+
}
|
|
3129
|
+
/**
|
|
3130
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3131
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3132
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3133
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3134
|
+
* non-Opus model id.
|
|
3135
|
+
*/
|
|
3136
|
+
/**
|
|
3137
|
+
* Parses an Opus model id's generation and minor version, e.g.
|
|
3138
|
+
* `"claude-opus-4-7-20260101"` -> `[4, 7]`, `"anthropic.claude-opus-5-x"` ->
|
|
3139
|
+
* `[5, 0]`. Not anchored, so it matches equally inside a bare Anthropic id
|
|
3140
|
+
* or a Bedrock id carrying a provider prefix. Returns `null` for a
|
|
3141
|
+
* non-Opus model id.
|
|
3142
|
+
*
|
|
3143
|
+
* Anthropic model ids sometimes carry a trailing snapshot date instead of
|
|
3144
|
+
* (or in addition to) an explicit minor version, e.g. the real, still-
|
|
3145
|
+
* supported base `"claude-opus-4-20250514"` (no `.7`-style minor at all,
|
|
3146
|
+
* just a date suffix directly after the major version). Read naively,
|
|
3147
|
+
* `20250514` looks like a minor version far above any real threshold and
|
|
3148
|
+
* would misclassify this pre-4.6 model as adaptive-only. Snapshot dates
|
|
3149
|
+
* are always 8 digits (`YYYYMMDD`); a real minor version never is, so an
|
|
3150
|
+
* 8+ digit second segment is treated as a date, not a minor version.
|
|
3151
|
+
*/
|
|
3152
|
+
function parseOpusVersion(model) {
|
|
3153
|
+
const match = /opus-(\d+)(?:-(\d+))?/.exec(model);
|
|
3154
|
+
if (!match) return null;
|
|
3155
|
+
const minorStr = match[2];
|
|
3156
|
+
const minor = minorStr === void 0 || minorStr.length >= 8 ? 0 : Number(minorStr);
|
|
3157
|
+
return [Number(match[1]), minor];
|
|
3158
|
+
}
|
|
3159
|
+
/**
|
|
3160
|
+
* Default rule for whether `model` only supports adaptive thinking
|
|
3161
|
+
* (`thinking: { type: 'adaptive' }`) and returns a 400 for manual,
|
|
3162
|
+
* budget-based thinking (`thinking: { type: 'enabled', budget_tokens }`):
|
|
3163
|
+
* Claude Opus 4.7 and later (matched as a version threshold, so 4.8, 4.9,
|
|
3164
|
+
* 5, and every future Opus point release are covered automatically,
|
|
3165
|
+
* without a new list entry per release), and every Claude 5 tier model
|
|
3166
|
+
* outside the Opus family (Sonnet 5, Fable 5, Mythos 5, Mythos Preview).
|
|
3167
|
+
* `mythos` alone is enough to catch both Mythos names without listing
|
|
3168
|
+
* each separately.
|
|
3169
|
+
*
|
|
3170
|
+
* Necessarily best-effort: a new model family with its own name (not
|
|
3171
|
+
* `opus-*`, not `sonnet-5`/`fable-5`/`mythos-*`) still needs a code
|
|
3172
|
+
* update here, or a caller-supplied `adaptiveOnlyModels` override (see
|
|
3173
|
+
* `isAdaptiveOnlyModel`) covering it in the meantime.
|
|
3174
|
+
*/
|
|
3175
|
+
function isDefaultAdaptiveOnly(model) {
|
|
3176
|
+
const opusVersion = parseOpusVersion(model);
|
|
3177
|
+
if (opusVersion) {
|
|
3178
|
+
const [major, minor] = opusVersion;
|
|
3179
|
+
return major > 4 || major === 4 && minor >= 7;
|
|
3180
|
+
}
|
|
3181
|
+
return [
|
|
3182
|
+
"sonnet-5",
|
|
3183
|
+
"fable-5",
|
|
3184
|
+
"mythos"
|
|
3185
|
+
].some((s) => model.includes(s));
|
|
3186
|
+
}
|
|
3187
|
+
/**
|
|
3188
|
+
* Whether `model` is adaptive-only, per the built-in rule above, or per a
|
|
3189
|
+
* caller-supplied `adaptiveOnlyModels` override. The override is
|
|
3190
|
+
* additive, not a replacement: it can mark an *additional* model as
|
|
3191
|
+
* adaptive-only (useful for a model family this package doesn't know
|
|
3192
|
+
* about yet), but it can't un-mark one the built-in rule already caught,
|
|
3193
|
+
* since a caller correcting a false negative is the only direction that
|
|
3194
|
+
* needs covering, a false positive here would mean this package is
|
|
3195
|
+
* simply wrong and needs its own fix, not a per-caller workaround.
|
|
3196
|
+
*/
|
|
3197
|
+
function isAdaptiveOnlyModel(model, override) {
|
|
3198
|
+
if (isDefaultAdaptiveOnly(model)) return true;
|
|
3199
|
+
if (!override) return false;
|
|
3200
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3201
|
+
}
|
|
3202
|
+
/** Whether `model` is known to support manual, budget-based thinking. */
|
|
3203
|
+
function supportsManualThinkingBudget(model, override) {
|
|
3204
|
+
return !isAdaptiveOnlyModel(model, override);
|
|
3205
|
+
}
|
|
3206
|
+
/**
|
|
3207
|
+
* Anthropic (and Claude models on Bedrock) require `budget_tokens` to be
|
|
3208
|
+
* at least 1024 and strictly less than `max_tokens`, since the thinking
|
|
3209
|
+
* budget and the reply share the same `max_tokens` ceiling. VernLLM's own
|
|
3210
|
+
* default `maxTokens` is 1000 (see `RequestBuilder`'s `defaultMaxTokens`),
|
|
3211
|
+
* below the 1024 floor, so the *default* `minimal` tier (1024 tokens) is
|
|
3212
|
+
* silently invalid against the *default* `max_tokens` unless a caller
|
|
3213
|
+
* happens to raise one or the other. Checked here, once, right before a
|
|
3214
|
+
* `thinking` block would be built, rather than left for Anthropic's own
|
|
3215
|
+
* 400 to explain after a real network round trip.
|
|
3216
|
+
*/
|
|
3217
|
+
function assertValidClaudeBudgetTokens(budgetTokens, maxTokens) {
|
|
3218
|
+
if (budgetTokens < 1024) throw new LLMError(`budgetTokens (${budgetTokens}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`, "invalid_params");
|
|
3219
|
+
if (budgetTokens >= maxTokens) throw new LLMError(`budgetTokens (${budgetTokens}) must be less than maxTokens (${maxTokens}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`, "invalid_params");
|
|
3220
|
+
}
|
|
3221
|
+
/**
|
|
3222
|
+
* Anthropic rejects any form of `thinking` (manual `budget_tokens` or
|
|
3223
|
+
* adaptive) combined with a `tool_choice` that forces tool use, a forced
|
|
3224
|
+
* single tool or "must call some tool", with a 400: `"Thinking may not be
|
|
3225
|
+
* enabled when tool_choice forces tool use."` Auto/none (or no tools at
|
|
3226
|
+
* all) are unaffected, thinking only conflicts with a choice that removes
|
|
3227
|
+
* the model's ability to just reply with text. This is a Claude-model
|
|
3228
|
+
* constraint, not specific to the Anthropic API's own wire shape, so it
|
|
3229
|
+
* applies identically to Claude models called through Bedrock's Converse
|
|
3230
|
+
* API, which forwards `thinking` under `additionalModelRequestFields` but
|
|
3231
|
+
* is still talking to the same underlying model.
|
|
3232
|
+
*
|
|
3233
|
+
* This combination can arise two ways: a caller explicitly sets both
|
|
3234
|
+
* `budgetTokens`/`reasoningEffort` and a forced `toolChoice`, or, more
|
|
3235
|
+
* subtly (Anthropic adapter only), a caller sets `jsonSchema` on a model
|
|
3236
|
+
* without native structured output support, which silently forces a
|
|
3237
|
+
* single synthetic tool call to emulate it, with no `tool_choice` of the
|
|
3238
|
+
* caller's own in sight. Both end up resolving to a forced tool choice by
|
|
3239
|
+
* the time each adapter calls this, so checking the adapter's own
|
|
3240
|
+
* already-resolved choice (rather than the caller's raw
|
|
3241
|
+
* `params.tool_choice`) catches both, right before a `thinking` block
|
|
3242
|
+
* would be built, rather than left for Anthropic's own 400 to explain
|
|
3243
|
+
* after a real network round trip.
|
|
3244
|
+
*
|
|
3245
|
+
* Takes a plain description of the forced choice rather than either
|
|
3246
|
+
* adapter's own wire shape (Anthropic SDK's `{ type: 'tool' | 'any', ... }`
|
|
3247
|
+
* vs Converse's `{ tool: {...} } | { any: {} }`), so both adapters can
|
|
3248
|
+
* share one check without either shape leaking into this file. Pass
|
|
3249
|
+
* `undefined` when the resolved choice is `auto`/`none`/unset, forcing
|
|
3250
|
+
* nothing.
|
|
3251
|
+
*/
|
|
3252
|
+
function assertNoForcedToolChoiceWithThinking(forcedChoiceDescription) {
|
|
3253
|
+
if (!forcedChoiceDescription) return;
|
|
3254
|
+
throw new LLMError(`budgetTokens/reasoningEffort was set alongside ${forcedChoiceDescription}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`, "invalid_params");
|
|
3255
|
+
}
|
|
3256
|
+
/**
|
|
3257
|
+
* Maps VernLLM's four-tier `reasoningEffort` onto Anthropic's five-tier
|
|
3258
|
+
* adaptive effort. `xhigh` and `max` have no VernLLM-side equivalent and
|
|
3259
|
+
* are unreachable through this mapping; a caller who wants either has to
|
|
3260
|
+
* target Anthropic/Bedrock-specific behavior already, so there's no gap
|
|
3261
|
+
* the shared `CallParams` surface needs to cover for a first pass.
|
|
3262
|
+
*/
|
|
3263
|
+
function toClaudeAdaptiveEffort(effort) {
|
|
3264
|
+
return effort === "minimal" ? "low" : effort;
|
|
3265
|
+
}
|
|
3266
|
+
/** Converts VernLLM's `reasoningEffort` directly into Gemini's `ThinkingLevel` enum value. */
|
|
3267
|
+
function toGeminiThinkingLevel(effort, model) {
|
|
3268
|
+
return clampGeminiThinkingLevel(model, effort.toUpperCase());
|
|
3269
|
+
}
|
|
3270
|
+
/**
|
|
3271
|
+
* Parses a Gemini model id's minor version, e.g. `"gemini-3.1-pro"` -> `1`,
|
|
3272
|
+
* `"gemini-3-pro"` -> `0` (no explicit minor). Only meaningful alongside
|
|
3273
|
+
* `parseGeminiMajorVersion`.
|
|
3274
|
+
*/
|
|
3275
|
+
function parseGeminiMinorVersion(model) {
|
|
3276
|
+
const match = /gemini-\d+\.(\d+)/.exec(model);
|
|
3277
|
+
return match ? Number(match[1]) : 0;
|
|
3278
|
+
}
|
|
3279
|
+
/**
|
|
3280
|
+
* Some Gemini 3 "Pro" tier models accept a narrower set of `thinkingLevel`
|
|
3281
|
+
* values than VernLLM's four tiers map onto, confirmed against real API
|
|
3282
|
+
* 400s and Google's own migration guidance, not assumed:
|
|
3283
|
+
* - Gemini 3 Pro (major 3, minor 0, e.g. `"gemini-3-pro-preview"`): only
|
|
3284
|
+
* `LOW` and `HIGH`; `MEDIUM` returns a 400 ("Thinking level MEDIUM is
|
|
3285
|
+
* not supported for this model").
|
|
3286
|
+
* - Gemini 3.1 Pro (major 3, minor >= 1): `LOW`/`MEDIUM`/`HIGH`, no
|
|
3287
|
+
* `MINIMAL`, Google's own docs point users toward a Flash-tier model
|
|
3288
|
+
* instead for the lowest setting.
|
|
3289
|
+
* - Every Flash-tier Gemini 3+ model accepts the full four levels, no
|
|
3290
|
+
* clamping needed, matched by this function simply not applying to
|
|
3291
|
+
* anything without `"pro"` in the model id.
|
|
3292
|
+
*
|
|
3293
|
+
* Clamped automatically rather than left to error, since `reasoningEffort`
|
|
3294
|
+
* is a per-call value, a caller hitting this isn't misconfiguring an
|
|
3295
|
+
* instance once, they're getting an intermittent-looking failure on
|
|
3296
|
+
* whichever specific call happened to pick an unsupported tier. Necessarily
|
|
3297
|
+
* best-effort: a future Pro-tier release could add back a level this rule
|
|
3298
|
+
* still clamps, or clamp one this rule doesn't yet know to touch.
|
|
3299
|
+
*/
|
|
3300
|
+
function clampGeminiThinkingLevel(model, level) {
|
|
3301
|
+
if (!model.includes("pro")) return level;
|
|
3302
|
+
const major = parseGeminiMajorVersion(model);
|
|
3303
|
+
if (major === null || major < 3) return level;
|
|
3304
|
+
const minor = parseGeminiMinorVersion(model);
|
|
3305
|
+
if (minor === 0) return level === "HIGH" ? "HIGH" : "LOW";
|
|
3306
|
+
return level === "MINIMAL" ? "LOW" : level;
|
|
3307
|
+
}
|
|
3308
|
+
/**
|
|
3309
|
+
* Parses a Gemini model id's major generation number, e.g.
|
|
3310
|
+
* `"gemini-3.1-flash-lite"` -> `3`, `"gemini-2.5-flash"` -> `2`. Not
|
|
3311
|
+
* anchored, so a Vertex-prefixed or otherwise decorated id still matches.
|
|
3312
|
+
* Returns `null` for a non-Gemini model id.
|
|
3313
|
+
*/
|
|
3314
|
+
function parseGeminiMajorVersion(model) {
|
|
3315
|
+
const match = /gemini-(\d+)/.exec(model);
|
|
3316
|
+
return match ? Number(match[1]) : null;
|
|
3317
|
+
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Default rule for whether `model` uses `thinkingLevel` instead of
|
|
3320
|
+
* `thinkingBudget`: every Gemini 3 series model and later, matched as a
|
|
3321
|
+
* version threshold so 3.1, 3.5, 3.6, and every future Gemini 3.x or
|
|
3322
|
+
* later release are covered automatically, without a new entry per
|
|
3323
|
+
* release, same reasoning as `isDefaultAdaptiveOnly`'s Opus threshold.
|
|
3324
|
+
* Gemini 2.5 and earlier still use `thinkingBudget`.
|
|
3325
|
+
*
|
|
3326
|
+
* `thinkingBudget` is still *accepted* on Gemini 3 for backward
|
|
3327
|
+
* compatibility, per Google's own docs, but "may result in unexpected
|
|
3328
|
+
* performance" there, so this rule switches VernLLM's own default
|
|
3329
|
+
* behavior over rather than leaving it on the old field indefinitely.
|
|
3330
|
+
*/
|
|
3331
|
+
function isDefaultThinkingLevelModel(model) {
|
|
3332
|
+
const major = parseGeminiMajorVersion(model);
|
|
3333
|
+
return major !== null && major >= 3;
|
|
3334
|
+
}
|
|
3335
|
+
/**
|
|
3336
|
+
* Whether `model` uses `thinkingLevel`, per the built-in version
|
|
3337
|
+
* threshold above, or per a caller-supplied `thinkingLevelModels`
|
|
3338
|
+
* override. Additive, not a replacement, same reasoning as
|
|
3339
|
+
* `isAdaptiveOnlyModel`: an override can mark an *additional* model as
|
|
3340
|
+
* using `thinkingLevel` (a model family this package doesn't recognize
|
|
3341
|
+
* yet), it can't un-mark one the built-in threshold already caught.
|
|
3342
|
+
*/
|
|
3343
|
+
function usesGeminiThinkingLevel(model, override) {
|
|
3344
|
+
if (isDefaultThinkingLevelModel(model)) return true;
|
|
3345
|
+
if (!override) return false;
|
|
3346
|
+
return Array.isArray(override) ? override.includes(model) : override(model);
|
|
3347
|
+
}
|
|
3348
|
+
|
|
2876
3349
|
//#endregion
|
|
2877
3350
|
//#region src/adapters/anthropic.ts
|
|
2878
3351
|
/**
|
|
@@ -2957,7 +3430,7 @@ function buildAnthropicTools(tools, toolChoiceParam) {
|
|
|
2957
3430
|
* `params.tools` are left for the normal, non-forced tool-call handling
|
|
2958
3431
|
* both `create` and `createStream` already do when `toolName` is unset.
|
|
2959
3432
|
*/
|
|
2960
|
-
function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
3433
|
+
function buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
2961
3434
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
2962
3435
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
2963
3436
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -2965,8 +3438,8 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2965
3438
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
2966
3439
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
2967
3440
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
|
|
3441
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic's native output_config.format on covered models, or a forced single tool call otherwise).", "validation");
|
|
2968
3442
|
let toolName;
|
|
2969
|
-
let jsonInstruction;
|
|
2970
3443
|
let outputFormat;
|
|
2971
3444
|
let tools;
|
|
2972
3445
|
let toolChoice;
|
|
@@ -2989,20 +3462,42 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
2989
3462
|
type: "tool",
|
|
2990
3463
|
name: toolName
|
|
2991
3464
|
};
|
|
2992
|
-
}
|
|
3465
|
+
}
|
|
2993
3466
|
if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
|
|
2994
|
-
|
|
3467
|
+
let thinking;
|
|
3468
|
+
let effort;
|
|
3469
|
+
if (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0) {
|
|
3470
|
+
assertNoForcedToolChoiceWithThinking(toolChoice?.type === "tool" ? `toolChoice forcing the "${toolChoice.name}" tool` : toolChoice?.type === "any" ? "toolChoice: 'required' (Anthropic's \"any\" tool_choice)" : void 0);
|
|
3471
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
3472
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
3473
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
3474
|
+
thinking = {
|
|
3475
|
+
type: "enabled",
|
|
3476
|
+
budget_tokens: budgetTokens
|
|
3477
|
+
};
|
|
3478
|
+
} else {
|
|
3479
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
3480
|
+
thinking = { type: "adaptive" };
|
|
3481
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
3482
|
+
}
|
|
3483
|
+
}
|
|
3484
|
+
const system = systemMessage?.content;
|
|
3485
|
+
const temperature = thinking ? void 0 : params.temperature;
|
|
2995
3486
|
const body = {
|
|
2996
3487
|
model: params.model,
|
|
2997
3488
|
max_tokens: params.max_tokens,
|
|
2998
|
-
...
|
|
3489
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
2999
3490
|
system: system || void 0,
|
|
3000
3491
|
messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
|
|
3001
3492
|
...tools ? {
|
|
3002
3493
|
tools,
|
|
3003
3494
|
tool_choice: toolChoice
|
|
3004
3495
|
} : {},
|
|
3005
|
-
...outputFormat ? { output_config: {
|
|
3496
|
+
...outputFormat || effort ? { output_config: {
|
|
3497
|
+
...outputFormat ? { format: outputFormat } : {},
|
|
3498
|
+
...effort ? { effort } : {}
|
|
3499
|
+
} } : {},
|
|
3500
|
+
...thinking ? { thinking } : {}
|
|
3006
3501
|
};
|
|
3007
3502
|
return {
|
|
3008
3503
|
body,
|
|
@@ -3031,103 +3526,113 @@ function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
|
|
|
3031
3526
|
* schema matching applies only when `strict: true` is forwarded and
|
|
3032
3527
|
* supported.
|
|
3033
3528
|
*
|
|
3034
|
-
* `response_format: json_object` (
|
|
3035
|
-
*
|
|
3036
|
-
*
|
|
3037
|
-
*
|
|
3038
|
-
*
|
|
3529
|
+
* `response_format: json_object` throws `LLMError('validation')`. Anthropic
|
|
3530
|
+
* has no API-level field that mechanically guarantees JSON output the way
|
|
3531
|
+
* OpenAI's `json_object` mode does; the only way to emulate it was a
|
|
3532
|
+
* system-prompt instruction with no actual enforcement behind it, a
|
|
3533
|
+
* guarantee this adapter no longer pretends to make. Use `jsonSchema`
|
|
3534
|
+
* instead, which maps to a real constraint either way (native
|
|
3535
|
+
* `output_config.format` or a forced tool call).
|
|
3039
3536
|
*/
|
|
3040
3537
|
function fromAnthropic(anthropicClient, options) {
|
|
3041
3538
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3539
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3540
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
3042
3541
|
const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
|
|
3043
|
-
return {
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
if (
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
choices: [{ message: {
|
|
3068
|
-
content: text,
|
|
3069
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3070
|
-
} }],
|
|
3071
|
-
usage: {
|
|
3072
|
-
prompt_tokens: response.usage?.input_tokens,
|
|
3073
|
-
completion_tokens: response.usage?.output_tokens,
|
|
3074
|
-
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
|
|
3542
|
+
return {
|
|
3543
|
+
supportsJsonObjectMode: false,
|
|
3544
|
+
chat: { completions: {
|
|
3545
|
+
async create(params, options$1) {
|
|
3546
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3547
|
+
const response = await anthropicClient.messages.create(body, options$1);
|
|
3548
|
+
let text;
|
|
3549
|
+
let wireToolCalls;
|
|
3550
|
+
if (toolName) {
|
|
3551
|
+
const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
|
|
3552
|
+
if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3553
|
+
if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
|
|
3554
|
+
text = JSON.stringify(toolUse.input);
|
|
3555
|
+
} else {
|
|
3556
|
+
text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
|
|
3557
|
+
const toolUses = response.content.filter((block) => block.type === "tool_use");
|
|
3558
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
|
|
3559
|
+
id: block.id,
|
|
3560
|
+
type: "function",
|
|
3561
|
+
function: {
|
|
3562
|
+
name: block.name,
|
|
3563
|
+
arguments: JSON.stringify(block.input ?? {})
|
|
3564
|
+
}
|
|
3565
|
+
}));
|
|
3075
3566
|
}
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3088
|
-
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3089
|
-
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3090
|
-
blockKinds.set(event.index, kind);
|
|
3091
|
-
if (kind === "json-tool") sawJsonTool = true;
|
|
3092
|
-
else if (!toolName) yield {
|
|
3093
|
-
type: "tool_call_delta",
|
|
3094
|
-
index: event.index,
|
|
3095
|
-
id: event.content_block.id,
|
|
3096
|
-
name: event.content_block.name
|
|
3567
|
+
return {
|
|
3568
|
+
choices: [{ message: {
|
|
3569
|
+
content: text,
|
|
3570
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3571
|
+
} }],
|
|
3572
|
+
usage: {
|
|
3573
|
+
prompt_tokens: response.usage?.input_tokens,
|
|
3574
|
+
completion_tokens: response.usage?.output_tokens,
|
|
3575
|
+
total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0),
|
|
3576
|
+
...response.usage?.output_tokens_details?.thinking_tokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usage.output_tokens_details.thinking_tokens } } : {}
|
|
3577
|
+
}
|
|
3097
3578
|
};
|
|
3098
|
-
}
|
|
3099
|
-
|
|
3100
|
-
|
|
3101
|
-
|
|
3102
|
-
|
|
3103
|
-
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3110
|
-
|
|
3579
|
+
},
|
|
3580
|
+
async *createStream(params, options$1) {
|
|
3581
|
+
const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
3582
|
+
const stream = await rawMessagesCreate({
|
|
3583
|
+
...body,
|
|
3584
|
+
stream: true
|
|
3585
|
+
}, options$1);
|
|
3586
|
+
const blockKinds = new Map();
|
|
3587
|
+
let inputTokens = 0;
|
|
3588
|
+
let sawJsonTool = false;
|
|
3589
|
+
for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
|
|
3590
|
+
else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
|
|
3591
|
+
const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
|
|
3592
|
+
blockKinds.set(event.index, kind);
|
|
3593
|
+
if (kind === "json-tool") sawJsonTool = true;
|
|
3111
3594
|
else if (!toolName) yield {
|
|
3112
3595
|
type: "tool_call_delta",
|
|
3113
3596
|
index: event.index,
|
|
3114
|
-
|
|
3597
|
+
id: event.content_block.id,
|
|
3598
|
+
name: event.content_block.name
|
|
3115
3599
|
};
|
|
3116
|
-
}
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3600
|
+
} else blockKinds.set(event.index, "text");
|
|
3601
|
+
else if (event.type === "content_block_delta") {
|
|
3602
|
+
if (event.delta.type === "text_delta") {
|
|
3603
|
+
if (!toolName) yield {
|
|
3604
|
+
type: "text-delta",
|
|
3605
|
+
delta: event.delta.text
|
|
3606
|
+
};
|
|
3607
|
+
} else if (event.delta.type === "input_json_delta") {
|
|
3608
|
+
const kind = blockKinds.get(event.index);
|
|
3609
|
+
if (kind === "json-tool") yield {
|
|
3610
|
+
type: "text-delta",
|
|
3611
|
+
delta: event.delta.partial_json
|
|
3612
|
+
};
|
|
3613
|
+
else if (!toolName) yield {
|
|
3614
|
+
type: "tool_call_delta",
|
|
3615
|
+
index: event.index,
|
|
3616
|
+
argumentsDelta: event.delta.partial_json
|
|
3617
|
+
};
|
|
3125
3618
|
}
|
|
3126
|
-
}
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
|
|
3130
|
-
|
|
3619
|
+
} else if (event.type === "message_delta") {
|
|
3620
|
+
const outputTokens = event.usage?.output_tokens ?? 0;
|
|
3621
|
+
const thinkingTokens = event.usage?.output_tokens_details?.thinking_tokens;
|
|
3622
|
+
yield {
|
|
3623
|
+
type: "usage",
|
|
3624
|
+
usage: {
|
|
3625
|
+
prompt_tokens: inputTokens,
|
|
3626
|
+
completion_tokens: outputTokens,
|
|
3627
|
+
total_tokens: inputTokens + outputTokens,
|
|
3628
|
+
...thinkingTokens !== void 0 ? { completion_tokens_details: { reasoning_tokens: thinkingTokens } } : {}
|
|
3629
|
+
}
|
|
3630
|
+
};
|
|
3631
|
+
} else if (event.type === "ping") yield { type: "ping" };
|
|
3632
|
+
if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
|
|
3633
|
+
}
|
|
3634
|
+
} }
|
|
3635
|
+
};
|
|
3131
3636
|
}
|
|
3132
3637
|
/**
|
|
3133
3638
|
* Anthropic requires strict role alternation, so the per-wire-message
|
|
@@ -3264,12 +3769,23 @@ function parseToolArguments(text, toolName) {
|
|
|
3264
3769
|
if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
|
|
3265
3770
|
return parsed;
|
|
3266
3771
|
}
|
|
3772
|
+
/**
|
|
3773
|
+
* Parses a wire tool message's `content` into the object Gemini's
|
|
3774
|
+
* `functionResponse.response` expects. Gemini (and the real SDK's
|
|
3775
|
+
* `FunctionResponse.response` type) requires an object, so a result that
|
|
3776
|
+
* parses to something other than a plain JSON object (a string, number,
|
|
3777
|
+
* array, or unparseable text) is wrapped under an `output` key, mirroring
|
|
3778
|
+
* Gemini's own documented convention for non-object function results.
|
|
3779
|
+
*/
|
|
3267
3780
|
function parseToolResult(text) {
|
|
3781
|
+
let parsed;
|
|
3268
3782
|
try {
|
|
3269
|
-
|
|
3783
|
+
parsed = text.trim() ? JSON.parse(text) : "";
|
|
3270
3784
|
} catch {
|
|
3271
|
-
|
|
3785
|
+
parsed = text;
|
|
3272
3786
|
}
|
|
3787
|
+
if (parsed && !Array.isArray(parsed) && typeof parsed === "object") return parsed;
|
|
3788
|
+
return { output: parsed };
|
|
3273
3789
|
}
|
|
3274
3790
|
/**
|
|
3275
3791
|
* Gemini expects the results of everything the model asked for in one turn
|
|
@@ -3298,7 +3814,7 @@ function mergeConsecutiveFunctionResponses(contents) {
|
|
|
3298
3814
|
* `abortSignal` is folded into `config` by the caller (`create`/
|
|
3299
3815
|
* `createStream`), once the request options are available.
|
|
3300
3816
|
*/
|
|
3301
|
-
function buildGeminiRequest(params) {
|
|
3817
|
+
function buildGeminiRequest(params, effortTokenTable, thinkingLevelModels) {
|
|
3302
3818
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3303
3819
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3304
3820
|
const wantsJson = Boolean(params.response_format);
|
|
@@ -3323,51 +3839,37 @@ function buildGeminiRequest(params) {
|
|
|
3323
3839
|
})) }];
|
|
3324
3840
|
config.toolConfig = toGeminiToolConfig(params.tool_choice);
|
|
3325
3841
|
}
|
|
3842
|
+
if (usesGeminiThinkingLevel(params.model, thinkingLevelModels)) {
|
|
3843
|
+
const effortTier = params.reasoning_effort ?? (params.budget_tokens !== void 0 ? budgetTokensToEffort(params.budget_tokens, effortTokenTable) : void 0);
|
|
3844
|
+
if (effortTier !== void 0) config.thinkingConfig = { thinkingLevel: toGeminiThinkingLevel(effortTier, params.model) };
|
|
3845
|
+
} else {
|
|
3846
|
+
const thinkingBudget = params.budget_tokens ?? (params.reasoning_effort ? effortToBudgetTokens(params.reasoning_effort, effortTokenTable) : void 0);
|
|
3847
|
+
if (thinkingBudget !== void 0) config.thinkingConfig = { thinkingBudget };
|
|
3848
|
+
}
|
|
3326
3849
|
return {
|
|
3327
3850
|
model: params.model,
|
|
3328
3851
|
contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
|
|
3329
3852
|
config
|
|
3330
3853
|
};
|
|
3331
3854
|
}
|
|
3332
|
-
|
|
3333
|
-
|
|
3334
|
-
|
|
3335
|
-
|
|
3336
|
-
|
|
3337
|
-
|
|
3338
|
-
|
|
3339
|
-
|
|
3340
|
-
|
|
3341
|
-
|
|
3342
|
-
*
|
|
3343
|
-
* `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
|
|
3344
|
-
* `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
|
|
3345
|
-
* `responseSchema` and `tools` in the same request natively, so both are
|
|
3346
|
-
* set independently here and no special-casing is needed for the
|
|
3347
|
-
* combination, unlike `fromAnthropic`/`fromBedrock`.
|
|
3348
|
-
*
|
|
3349
|
-
* `createStream` calls `generateContentStream` (optional on `GeminiClient`
|
|
3350
|
-
*, required only if the caller sets `stream: true`) and translates each
|
|
3351
|
-
* partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
|
|
3352
|
-
* Gemini's own function-calling API doesn't stream tool-call arguments
|
|
3353
|
-
* incrementally: a `functionCall` part always arrives whole in one chunk,
|
|
3354
|
-
* so each one is emitted as a single, complete `tool_call_delta` (a
|
|
3355
|
-
* one-shot "delta" containing the full arguments) rather than accumulated
|
|
3356
|
-
* fragments, that's a real difference in the underlying API, not
|
|
3357
|
-
* something this adapter can smooth over. `usageMetadata` is (per Gemini's
|
|
3358
|
-
* own behavior) only reliably present on the last chunk, so the `usage`
|
|
3359
|
-
* `WireStreamChunk` is emitted once, after the stream completes, from
|
|
3360
|
-
* whichever chunk's `usageMetadata` was seen last.
|
|
3361
|
-
*/
|
|
3362
|
-
function fromGemini(geminiClient) {
|
|
3855
|
+
function fromGemini(client, options) {
|
|
3856
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
3857
|
+
const thinkingLevelModels = options?.thinkingLevelModels;
|
|
3858
|
+
const resolved = client.models ?? client;
|
|
3859
|
+
if (typeof resolved.generateContent !== "function") throw new LLMError("fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).", "invalid_params", {
|
|
3860
|
+
code: "unsupported_capability",
|
|
3861
|
+
issues: { capability: "generateContent" }
|
|
3862
|
+
});
|
|
3863
|
+
const generateContent = resolved.generateContent.bind(resolved);
|
|
3864
|
+
const generateContentStream = typeof resolved.generateContentStream === "function" ? resolved.generateContentStream.bind(resolved) : void 0;
|
|
3363
3865
|
return { chat: { completions: {
|
|
3364
|
-
async create(params, options) {
|
|
3365
|
-
const request = buildGeminiRequest(params);
|
|
3866
|
+
async create(params, options$1) {
|
|
3867
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
3366
3868
|
request.config = {
|
|
3367
3869
|
...request.config,
|
|
3368
|
-
abortSignal: options.signal
|
|
3870
|
+
abortSignal: options$1.signal
|
|
3369
3871
|
};
|
|
3370
|
-
const response = await
|
|
3872
|
+
const response = await generateContent(request);
|
|
3371
3873
|
const parts = response.candidates?.[0]?.content?.parts ?? [];
|
|
3372
3874
|
const text = parts.map((p) => p.text ?? "").join("");
|
|
3373
3875
|
const functionCalls = parts.filter((p) => p.functionCall);
|
|
@@ -3388,21 +3890,22 @@ function fromGemini(geminiClient) {
|
|
|
3388
3890
|
usage: {
|
|
3389
3891
|
prompt_tokens: response.usageMetadata?.promptTokenCount,
|
|
3390
3892
|
completion_tokens: response.usageMetadata?.candidatesTokenCount,
|
|
3391
|
-
total_tokens: response.usageMetadata?.totalTokenCount
|
|
3893
|
+
total_tokens: response.usageMetadata?.totalTokenCount,
|
|
3894
|
+
...response.usageMetadata?.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: response.usageMetadata.thoughtsTokenCount } } : {}
|
|
3392
3895
|
}
|
|
3393
3896
|
};
|
|
3394
3897
|
},
|
|
3395
|
-
async *createStream(params, options) {
|
|
3396
|
-
if (!
|
|
3898
|
+
async *createStream(params, options$1) {
|
|
3899
|
+
if (!generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
|
|
3397
3900
|
code: "unsupported_capability",
|
|
3398
3901
|
issues: { capability: "generateContentStream" }
|
|
3399
3902
|
});
|
|
3400
|
-
const request = buildGeminiRequest(params);
|
|
3903
|
+
const request = buildGeminiRequest(params, effortTokenTable, thinkingLevelModels);
|
|
3401
3904
|
request.config = {
|
|
3402
3905
|
...request.config,
|
|
3403
|
-
abortSignal: options.signal
|
|
3906
|
+
abortSignal: options$1.signal
|
|
3404
3907
|
};
|
|
3405
|
-
const stream = await
|
|
3908
|
+
const stream = await generateContentStream(request);
|
|
3406
3909
|
let toolCallIndex = 0;
|
|
3407
3910
|
let lastUsage;
|
|
3408
3911
|
for await (const chunk of stream) {
|
|
@@ -3431,7 +3934,8 @@ function fromGemini(geminiClient) {
|
|
|
3431
3934
|
usage: {
|
|
3432
3935
|
prompt_tokens: lastUsage.promptTokenCount,
|
|
3433
3936
|
completion_tokens: lastUsage.candidatesTokenCount,
|
|
3434
|
-
total_tokens: lastUsage.totalTokenCount
|
|
3937
|
+
total_tokens: lastUsage.totalTokenCount,
|
|
3938
|
+
...lastUsage.thoughtsTokenCount !== void 0 ? { completion_tokens_details: { reasoning_tokens: lastUsage.thoughtsTokenCount } } : {}
|
|
3435
3939
|
}
|
|
3436
3940
|
};
|
|
3437
3941
|
}
|
|
@@ -3440,6 +3944,19 @@ function fromGemini(geminiClient) {
|
|
|
3440
3944
|
|
|
3441
3945
|
//#endregion
|
|
3442
3946
|
//#region src/adapters/bedrock.ts
|
|
3947
|
+
/**
|
|
3948
|
+
* Default heuristic for whether a Bedrock model id is a Claude model,
|
|
3949
|
+
* matching AWS's own `anthropic.claude-*`/`us.anthropic.claude-*` naming.
|
|
3950
|
+
* Only used to decide whether a reasoning token budget is worth forwarding
|
|
3951
|
+
* through `additionalModelRequestFields`, not a general capability check,
|
|
3952
|
+
* so a plain substring match is enough, no override hook needed the way
|
|
3953
|
+
* `nativeStructuredOutputModels`/`toolUseSupportedModels` have one: a
|
|
3954
|
+
* false positive here just sends an inert extra field, not a request that
|
|
3955
|
+
* fails outright.
|
|
3956
|
+
*/
|
|
3957
|
+
function isClaudeModel(model) {
|
|
3958
|
+
return model.includes("claude");
|
|
3959
|
+
}
|
|
3443
3960
|
/** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
|
|
3444
3961
|
function toBedrockImageFormat(mimeType) {
|
|
3445
3962
|
switch (assertSupportedImageMimeType(mimeType)) {
|
|
@@ -3506,7 +4023,7 @@ function buildBedrockToolConfig(tools, toolChoiceParam) {
|
|
|
3506
4023
|
* normal, non-forced tool-call handling both `create` and `createStream`
|
|
3507
4024
|
* already do when `toolName` is unset.
|
|
3508
4025
|
*/
|
|
3509
|
-
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
|
|
4026
|
+
function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels) {
|
|
3510
4027
|
const systemMessage = params.messages.find((m) => m.role === "system");
|
|
3511
4028
|
const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
|
|
3512
4029
|
const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
|
|
@@ -3514,8 +4031,8 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3514
4031
|
if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
|
|
3515
4032
|
const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
|
|
3516
4033
|
if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
|
|
4034
|
+
if (params.response_format?.type === "json_object") throw new LLMError("response_format: \"json_object\" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a \"respond with JSON only\" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse's native outputConfig.textFormat on covered models, or a forced tool call otherwise).", "validation");
|
|
3517
4035
|
let toolName;
|
|
3518
|
-
let jsonInstruction;
|
|
3519
4036
|
let toolConfig;
|
|
3520
4037
|
let outputConfig;
|
|
3521
4038
|
if (jsonSchema && isNative) {
|
|
@@ -3540,7 +4057,7 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3540
4057
|
} }],
|
|
3541
4058
|
toolChoice: { tool: { name: toolName } }
|
|
3542
4059
|
};
|
|
3543
|
-
}
|
|
4060
|
+
}
|
|
3544
4061
|
if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
|
|
3545
4062
|
if (jsonSchema && toolConfig && toolUseSupportedModels) {
|
|
3546
4063
|
const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
|
|
@@ -3549,17 +4066,39 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3549
4066
|
issues: { capability: "toolUseSupportedModels" }
|
|
3550
4067
|
});
|
|
3551
4068
|
}
|
|
3552
|
-
|
|
4069
|
+
let additionalModelRequestFields;
|
|
4070
|
+
let effort;
|
|
4071
|
+
if (isClaudeModel(params.model) && (params.budget_tokens !== void 0 || params.reasoning_effort !== void 0)) {
|
|
4072
|
+
const forcedToolChoice = toolConfig?.toolChoice;
|
|
4073
|
+
assertNoForcedToolChoiceWithThinking(forcedToolChoice && "tool" in forcedToolChoice ? `toolChoice forcing the "${forcedToolChoice.tool?.name}" tool` : forcedToolChoice && "any" in forcedToolChoice ? "toolChoice: 'required' (Converse's \"any\" tool_choice)" : void 0);
|
|
4074
|
+
if (supportsManualThinkingBudget(params.model, adaptiveOnlyModels)) {
|
|
4075
|
+
const budgetTokens = params.budget_tokens ?? effortToBudgetTokens(params.reasoning_effort, effortTokenTable);
|
|
4076
|
+
assertValidClaudeBudgetTokens(budgetTokens, params.max_tokens);
|
|
4077
|
+
additionalModelRequestFields = { thinking: {
|
|
4078
|
+
type: "enabled",
|
|
4079
|
+
budget_tokens: budgetTokens
|
|
4080
|
+
} };
|
|
4081
|
+
} else {
|
|
4082
|
+
const effortTier = params.reasoning_effort ?? budgetTokensToEffort(params.budget_tokens, effortTokenTable);
|
|
4083
|
+
additionalModelRequestFields = { thinking: { type: "adaptive" } };
|
|
4084
|
+
effort = toClaudeAdaptiveEffort(effortTier);
|
|
4085
|
+
}
|
|
4086
|
+
}
|
|
4087
|
+
const temperature = additionalModelRequestFields ? void 0 : params.temperature;
|
|
3553
4088
|
const request = {
|
|
3554
4089
|
modelId: params.model,
|
|
3555
4090
|
messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
|
|
3556
|
-
system:
|
|
4091
|
+
system: systemMessage?.content ? [{ text: systemMessage.content }] : void 0,
|
|
3557
4092
|
inferenceConfig: {
|
|
3558
|
-
...
|
|
4093
|
+
...temperature !== void 0 ? { temperature } : {},
|
|
3559
4094
|
maxTokens: params.max_tokens
|
|
3560
4095
|
},
|
|
3561
4096
|
...toolConfig ? { toolConfig } : {},
|
|
3562
|
-
...outputConfig ? { outputConfig
|
|
4097
|
+
...outputConfig || effort ? { outputConfig: {
|
|
4098
|
+
...outputConfig ?? {},
|
|
4099
|
+
...effort ? { effort } : {}
|
|
4100
|
+
} } : {},
|
|
4101
|
+
...additionalModelRequestFields ? { additionalModelRequestFields } : {}
|
|
3563
4102
|
};
|
|
3564
4103
|
return {
|
|
3565
4104
|
request,
|
|
@@ -3567,6 +4106,120 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3567
4106
|
};
|
|
3568
4107
|
}
|
|
3569
4108
|
/**
|
|
4109
|
+
* Distinguishes a real AWS SDK v3 client (`.send(command)`) from a
|
|
4110
|
+
* hand-written `BedrockConverseClient` (`.converse(params)`) purely
|
|
4111
|
+
* structurally, so `fromBedrock` can accept either without the caller
|
|
4112
|
+
* saying which one they're passing. The two shapes don't overlap: nothing
|
|
4113
|
+
* implementing `.converse()` would also need `.send()`.
|
|
4114
|
+
*/
|
|
4115
|
+
function isAwsSendClient(client) {
|
|
4116
|
+
return typeof client.send === "function";
|
|
4117
|
+
}
|
|
4118
|
+
/**
|
|
4119
|
+
* Narrows one raw AWS stream event down to VernLLM's intentionally minimal
|
|
4120
|
+
* `BedrockConverseStreamEvent` union. Returns `undefined` if the event
|
|
4121
|
+
* isn't one of the kinds this adapter models.
|
|
4122
|
+
*
|
|
4123
|
+
* AWS's real `ConverseStreamOutput` type is a strictly larger union than
|
|
4124
|
+
* `BedrockConverseStreamEvent`. On top of every member modeled here, it
|
|
4125
|
+
* also includes a generated `$unknown` member, AWS's forward-compatibility
|
|
4126
|
+
* escape hatch for event kinds added to the service after this SDK version
|
|
4127
|
+
* was generated. A blind type assertion from one union to the other would
|
|
4128
|
+
* compile, but would let `$unknown` (or any other future member) reach
|
|
4129
|
+
* `fromBedrock`'s event-handling loop unnarrowed, as if it were one of the
|
|
4130
|
+
* kinds actually handled there.
|
|
4131
|
+
*
|
|
4132
|
+
* Returning `undefined` for anything unrecognized, filtered out by
|
|
4133
|
+
* `normalizeBedrockEventStream` below, keeps two guarantees. AWS SDK
|
|
4134
|
+
* generated types never leak into `fromBedrock`'s application code, only
|
|
4135
|
+
* this module's own `BedrockConverseStreamEvent` shape does. An event kind
|
|
4136
|
+
* this adapter doesn't yet know about is silently skipped, the same
|
|
4137
|
+
* forward-compatible behavior AWS's own `$unknown` convention implies,
|
|
4138
|
+
* rather than crashing the stream or being misrouted into a handler that
|
|
4139
|
+
* doesn't actually match its shape.
|
|
4140
|
+
*/
|
|
4141
|
+
function normalizeBedrockStreamEvent(raw) {
|
|
4142
|
+
if ("messageStart" in raw) return { messageStart: raw.messageStart };
|
|
4143
|
+
if ("contentBlockStart" in raw) return { contentBlockStart: raw.contentBlockStart };
|
|
4144
|
+
if ("contentBlockDelta" in raw) return { contentBlockDelta: raw.contentBlockDelta };
|
|
4145
|
+
if ("contentBlockStop" in raw) return { contentBlockStop: raw.contentBlockStop };
|
|
4146
|
+
if ("messageStop" in raw) return { messageStop: raw.messageStop };
|
|
4147
|
+
if ("metadata" in raw) return { metadata: raw.metadata };
|
|
4148
|
+
if ("internalServerException" in raw) return { internalServerException: raw.internalServerException };
|
|
4149
|
+
if ("modelStreamErrorException" in raw) return { modelStreamErrorException: raw.modelStreamErrorException };
|
|
4150
|
+
if ("validationException" in raw) return { validationException: raw.validationException };
|
|
4151
|
+
if ("throttlingException" in raw) return { throttlingException: raw.throttlingException };
|
|
4152
|
+
if ("serviceUnavailableException" in raw) return { serviceUnavailableException: raw.serviceUnavailableException };
|
|
4153
|
+
return void 0;
|
|
4154
|
+
}
|
|
4155
|
+
/**
|
|
4156
|
+
* Wraps a raw AWS event stream, narrowing each event through
|
|
4157
|
+
* `normalizeBedrockStreamEvent` and filtering out anything that doesn't
|
|
4158
|
+
* map onto `BedrockConverseStreamEvent`. `fromBedrock`'s event loop only
|
|
4159
|
+
* ever sees the shapes it actually models.
|
|
4160
|
+
*/
|
|
4161
|
+
async function* normalizeBedrockEventStream(rawStream) {
|
|
4162
|
+
for await (const raw of rawStream) {
|
|
4163
|
+
const event = normalizeBedrockStreamEvent(raw);
|
|
4164
|
+
if (event) yield event;
|
|
4165
|
+
}
|
|
4166
|
+
}
|
|
4167
|
+
/**
|
|
4168
|
+
* Adapts a real AWS SDK v3 client (anything with `.send()`, matching
|
|
4169
|
+
* `BedrockRuntimeClient`) into a `BedrockConverseClient`, so `fromBedrock`
|
|
4170
|
+
* can accept either without a hand-written `.converse()`/`.converseStream()`
|
|
4171
|
+
* wrapper. Internally does what that wrapper would: `client.send(new
|
|
4172
|
+
* ConverseCommand(params))`, `client.send(new
|
|
4173
|
+
* ConverseStreamCommand(params))`.
|
|
4174
|
+
*
|
|
4175
|
+
* `@aws-sdk/client-bedrock-runtime` is intentionally not a dependency (not
|
|
4176
|
+
* even a peer dependency) of this package. `vern-llm` otherwise has zero
|
|
4177
|
+
* runtime dependencies, and every other adapter works the same way:
|
|
4178
|
+
* structural typing over whatever client the caller already has. Instead,
|
|
4179
|
+
* `ConverseCommand`/`ConverseStreamCommand` are pulled in with a dynamic
|
|
4180
|
+
* `import()` the first time either method actually runs, and memoized
|
|
4181
|
+
* after that. Nothing is added to `package.json`, static or peer.
|
|
4182
|
+
* Bundlers only pull the AWS SDK in for code paths that actually pass a
|
|
4183
|
+
* raw AWS client to `fromBedrock`; a hand-written `BedrockConverseClient`
|
|
4184
|
+
* stays unaffected. If `@aws-sdk/client-bedrock-runtime` isn't installed,
|
|
4185
|
+
* the failure is a clear `LLMError` naming exactly what's missing, at the
|
|
4186
|
+
* moment it's needed, rather than a silent peer-dependency warning at
|
|
4187
|
+
* install time or a raw "Cannot find module" a caller has to trace back
|
|
4188
|
+
* themselves.
|
|
4189
|
+
*
|
|
4190
|
+
* Also closes two structural gaps between AWS's generated types and
|
|
4191
|
+
* `BedrockConverseClient`. AWS's `ConverseStreamCommandOutput.stream` is
|
|
4192
|
+
* optional, a response may not include it. This throws a clear `LLMError`
|
|
4193
|
+
* instead of letting `undefined` reach `fromBedrock`'s `for await` loop.
|
|
4194
|
+
* AWS's `ConverseStreamOutput` union is larger than
|
|
4195
|
+
* `BedrockConverseStreamEvent`, it includes a generated `$unknown` member.
|
|
4196
|
+
* Every event is narrowed through `normalizeBedrockStreamEvent` before it
|
|
4197
|
+
* reaches application code, instead of being asserted wholesale from one
|
|
4198
|
+
* type to the other.
|
|
4199
|
+
*/
|
|
4200
|
+
function wrapAwsSendClient(client) {
|
|
4201
|
+
let commandsPromise;
|
|
4202
|
+
function loadCommands() {
|
|
4203
|
+
commandsPromise ??= import("@aws-sdk/client-bedrock-runtime").then((mod) => mod, (cause) => {
|
|
4204
|
+
commandsPromise = void 0;
|
|
4205
|
+
throw new LLMError("fromBedrock requires \"@aws-sdk/client-bedrock-runtime\" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.", "validation", { cause });
|
|
4206
|
+
});
|
|
4207
|
+
return commandsPromise;
|
|
4208
|
+
}
|
|
4209
|
+
return {
|
|
4210
|
+
converse: async (params, requestOptions) => {
|
|
4211
|
+
const { ConverseCommand } = await loadCommands();
|
|
4212
|
+
return client.send(new ConverseCommand(params), { abortSignal: requestOptions.signal });
|
|
4213
|
+
},
|
|
4214
|
+
converseStream: async (params, requestOptions) => {
|
|
4215
|
+
const { ConverseStreamCommand } = await loadCommands();
|
|
4216
|
+
const result = await client.send(new ConverseStreamCommand(params), { abortSignal: requestOptions.signal });
|
|
4217
|
+
if (!result.stream) throw new LLMError("Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.", "api", { code: "server_error" });
|
|
4218
|
+
return { stream: normalizeBedrockEventStream(result.stream) };
|
|
4219
|
+
}
|
|
4220
|
+
};
|
|
4221
|
+
}
|
|
4222
|
+
/**
|
|
3570
4223
|
* Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
|
|
3571
4224
|
* interface VernLLM uses for OpenAI/Groq. The Converse API is unified
|
|
3572
4225
|
* across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
|
|
@@ -3574,6 +4227,16 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3574
4227
|
* regardless of which underlying model `modelId` points at, as long as
|
|
3575
4228
|
* that model supports Converse (most current-generation ones do)
|
|
3576
4229
|
*
|
|
4230
|
+
* `bedrockClient` accepts either a hand-written `BedrockConverseClient`
|
|
4231
|
+
* (a `.converse()`/`.converseStream()` wrapper you provide) or a real AWS
|
|
4232
|
+
* SDK v3 client (anything with `.send()`, matching `BedrockRuntimeClient`)
|
|
4233
|
+
* directly, detected structurally. Passing a raw AWS client skips the
|
|
4234
|
+
* hand-written wrapper entirely, internally doing what it would
|
|
4235
|
+
* (`send(new ConverseCommand(...))`, `send(new
|
|
4236
|
+
* ConverseStreamCommand(...))`). See `wrapAwsSendClient` for how that path
|
|
4237
|
+
* is implemented, including why `@aws-sdk/client-bedrock-runtime` stays
|
|
4238
|
+
* out of this package's dependencies either way.
|
|
4239
|
+
*
|
|
3577
4240
|
* `response_format: json_schema`, on a model covered by
|
|
3578
4241
|
* `options.nativeStructuredOutputModels` (opt-in, unset by default), is
|
|
3579
4242
|
* sent as `outputConfig.textFormat`, its own request field, independent of
|
|
@@ -3595,12 +4258,15 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3595
4258
|
* `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
|
|
3596
4259
|
* unsupported model surfaces Bedrock's raw error unchanged.
|
|
3597
4260
|
*
|
|
3598
|
-
* `response_format: json_object` (
|
|
3599
|
-
*
|
|
3600
|
-
*
|
|
3601
|
-
*
|
|
3602
|
-
*
|
|
3603
|
-
*
|
|
4261
|
+
* `response_format: json_object` throws `LLMError('validation')`: Converse
|
|
4262
|
+
* has no field that mechanically guarantees JSON output, and the only way
|
|
4263
|
+
* to emulate it was an unenforced system-prompt instruction, a guarantee
|
|
4264
|
+
* this adapter no longer pretends to make. Use `jsonSchema` instead.
|
|
4265
|
+
* `reasoning_effort` (no Converse equivalent) is converted to a token
|
|
4266
|
+
* budget and forwarded via `additionalModelRequestFields` for Claude
|
|
4267
|
+
* models only; `budget_tokens` is forwarded the same way directly. Both
|
|
4268
|
+
* are silently dropped for non-Claude models, which have no equivalent
|
|
4269
|
+
* field to reach for. See `adapters/internal/reasoningBudget.utils.ts`.
|
|
3604
4270
|
*
|
|
3605
4271
|
* `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
|
|
3606
4272
|
* `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
|
|
@@ -3618,107 +4284,113 @@ function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOut
|
|
|
3618
4284
|
* `create` branch above unwraps it.
|
|
3619
4285
|
*/
|
|
3620
4286
|
function fromBedrock(bedrockClient, options) {
|
|
4287
|
+
const client = isAwsSendClient(bedrockClient) ? wrapAwsSendClient(bedrockClient) : bedrockClient;
|
|
3621
4288
|
const toolUseSupportedModels = options?.toolUseSupportedModels;
|
|
3622
4289
|
const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
|
|
3623
|
-
|
|
3624
|
-
|
|
3625
|
-
|
|
3626
|
-
|
|
3627
|
-
|
|
3628
|
-
|
|
3629
|
-
|
|
3630
|
-
const
|
|
3631
|
-
text
|
|
3632
|
-
|
|
3633
|
-
|
|
3634
|
-
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
const
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3647
|
-
|
|
3648
|
-
|
|
3649
|
-
|
|
3650
|
-
|
|
3651
|
-
|
|
3652
|
-
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
3653
|
-
} }],
|
|
3654
|
-
usage: {
|
|
3655
|
-
prompt_tokens: response.usage?.inputTokens,
|
|
3656
|
-
completion_tokens: response.usage?.outputTokens,
|
|
3657
|
-
total_tokens: response.usage?.totalTokens
|
|
4290
|
+
const effortTokenTable = resolveEffortTokenTable(options?.reasoningEffortTokens);
|
|
4291
|
+
const adaptiveOnlyModels = options?.adaptiveOnlyModels;
|
|
4292
|
+
return {
|
|
4293
|
+
supportsJsonObjectMode: false,
|
|
4294
|
+
chat: { completions: {
|
|
4295
|
+
async create(params, requestOptions) {
|
|
4296
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4297
|
+
const response = await client.converse(request, requestOptions);
|
|
4298
|
+
let text;
|
|
4299
|
+
let wireToolCalls;
|
|
4300
|
+
if (toolName) {
|
|
4301
|
+
const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
|
|
4302
|
+
text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
|
|
4303
|
+
} else {
|
|
4304
|
+
const blocks = response.output?.message?.content ?? [];
|
|
4305
|
+
text = blocks.map((c) => c.text ?? "").join("");
|
|
4306
|
+
const toolUses = blocks.filter((block) => Boolean(block.toolUse));
|
|
4307
|
+
if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
|
|
4308
|
+
const toolUse = block.toolUse;
|
|
4309
|
+
if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
|
|
4310
|
+
return {
|
|
4311
|
+
id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
|
|
4312
|
+
type: "function",
|
|
4313
|
+
function: {
|
|
4314
|
+
name: toolUse.name,
|
|
4315
|
+
arguments: JSON.stringify(toolUse.input ?? {})
|
|
4316
|
+
}
|
|
4317
|
+
};
|
|
4318
|
+
});
|
|
3658
4319
|
}
|
|
3659
|
-
|
|
3660
|
-
|
|
3661
|
-
|
|
3662
|
-
|
|
3663
|
-
|
|
3664
|
-
|
|
3665
|
-
|
|
3666
|
-
|
|
3667
|
-
|
|
3668
|
-
|
|
3669
|
-
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
3670
|
-
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
3671
|
-
if (start?.toolUse) {
|
|
3672
|
-
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
3673
|
-
blockKinds.set(contentBlockIndex, kind);
|
|
3674
|
-
if (kind === "tool_use" && !toolName) yield {
|
|
3675
|
-
type: "tool_call_delta",
|
|
3676
|
-
index: contentBlockIndex,
|
|
3677
|
-
id: start.toolUse.toolUseId,
|
|
3678
|
-
name: start.toolUse.name
|
|
3679
|
-
};
|
|
3680
|
-
} else blockKinds.set(contentBlockIndex, "text");
|
|
3681
|
-
} else if ("contentBlockDelta" in event) {
|
|
3682
|
-
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
3683
|
-
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3684
|
-
type: "text-delta",
|
|
3685
|
-
delta: delta.text
|
|
4320
|
+
return {
|
|
4321
|
+
choices: [{ message: {
|
|
4322
|
+
content: text,
|
|
4323
|
+
...wireToolCalls ? { tool_calls: wireToolCalls } : {}
|
|
4324
|
+
} }],
|
|
4325
|
+
usage: {
|
|
4326
|
+
prompt_tokens: response.usage?.inputTokens,
|
|
4327
|
+
completion_tokens: response.usage?.outputTokens,
|
|
4328
|
+
total_tokens: response.usage?.totalTokens
|
|
4329
|
+
}
|
|
3686
4330
|
};
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
4331
|
+
},
|
|
4332
|
+
async *createStream(params, requestOptions) {
|
|
4333
|
+
if (!client.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
|
|
4334
|
+
code: "unsupported_capability",
|
|
4335
|
+
issues: { capability: "converseStream" }
|
|
4336
|
+
});
|
|
4337
|
+
const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels, effortTokenTable, adaptiveOnlyModels);
|
|
4338
|
+
const { stream } = await client.converseStream(request, requestOptions);
|
|
4339
|
+
const blockKinds = new Map();
|
|
4340
|
+
for await (const event of stream) if ("contentBlockStart" in event) {
|
|
4341
|
+
const { contentBlockIndex, start } = event.contentBlockStart;
|
|
4342
|
+
if (start?.toolUse) {
|
|
4343
|
+
const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
|
|
4344
|
+
blockKinds.set(contentBlockIndex, kind);
|
|
4345
|
+
if (kind === "tool_use" && !toolName) yield {
|
|
4346
|
+
type: "tool_call_delta",
|
|
4347
|
+
index: contentBlockIndex,
|
|
4348
|
+
id: start.toolUse.toolUseId,
|
|
4349
|
+
name: start.toolUse.name
|
|
4350
|
+
};
|
|
4351
|
+
} else blockKinds.set(contentBlockIndex, "text");
|
|
4352
|
+
} else if ("contentBlockDelta" in event) {
|
|
4353
|
+
const { contentBlockIndex, delta } = event.contentBlockDelta;
|
|
4354
|
+
if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
|
|
3690
4355
|
type: "text-delta",
|
|
3691
|
-
delta: delta.
|
|
4356
|
+
delta: delta.text
|
|
3692
4357
|
};
|
|
3693
|
-
else if (
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
3699
|
-
|
|
3700
|
-
|
|
3701
|
-
|
|
3702
|
-
|
|
3703
|
-
|
|
3704
|
-
|
|
3705
|
-
}
|
|
3706
|
-
|
|
3707
|
-
|
|
3708
|
-
|
|
3709
|
-
|
|
3710
|
-
|
|
3711
|
-
|
|
3712
|
-
|
|
3713
|
-
|
|
3714
|
-
|
|
3715
|
-
|
|
3716
|
-
status,
|
|
3717
|
-
code: status >= 500 ? "server_error" : void 0
|
|
4358
|
+
else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
|
|
4359
|
+
const kind = blockKinds.get(contentBlockIndex);
|
|
4360
|
+
if (kind === "json-tool") yield {
|
|
4361
|
+
type: "text-delta",
|
|
4362
|
+
delta: delta.toolUse.input
|
|
4363
|
+
};
|
|
4364
|
+
else if (!toolName) yield {
|
|
4365
|
+
type: "tool_call_delta",
|
|
4366
|
+
index: contentBlockIndex,
|
|
4367
|
+
argumentsDelta: delta.toolUse.input
|
|
4368
|
+
};
|
|
4369
|
+
}
|
|
4370
|
+
} else if ("metadata" in event && event.metadata.usage) yield {
|
|
4371
|
+
type: "usage",
|
|
4372
|
+
usage: {
|
|
4373
|
+
prompt_tokens: event.metadata.usage.inputTokens,
|
|
4374
|
+
completion_tokens: event.metadata.usage.outputTokens,
|
|
4375
|
+
total_tokens: event.metadata.usage.totalTokens
|
|
4376
|
+
}
|
|
4377
|
+
};
|
|
4378
|
+
else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
|
|
4379
|
+
status: 429,
|
|
4380
|
+
code: "provider_rate_limited"
|
|
3718
4381
|
});
|
|
4382
|
+
else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
|
|
4383
|
+
else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
|
|
4384
|
+
const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
|
|
4385
|
+
const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
|
|
4386
|
+
throw new LLMError(detail, "api", {
|
|
4387
|
+
status,
|
|
4388
|
+
code: status >= 500 ? "server_error" : void 0
|
|
4389
|
+
});
|
|
4390
|
+
}
|
|
3719
4391
|
}
|
|
3720
|
-
}
|
|
3721
|
-
}
|
|
4392
|
+
} }
|
|
4393
|
+
};
|
|
3722
4394
|
}
|
|
3723
4395
|
/** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
|
|
3724
4396
|
function toBedrockToolChoice(toolChoice) {
|
|
@@ -3957,6 +4629,22 @@ function fromFetch(config) {
|
|
|
3957
4629
|
//#endregion
|
|
3958
4630
|
//#region src/adapters/openaiCompatible.ts
|
|
3959
4631
|
/**
|
|
4632
|
+
* OpenAI's wire format only understands `reasoning_effort`, not a raw
|
|
4633
|
+
* token budget. When the caller set `reasoningEffort`, it's already on
|
|
4634
|
+
* `params` and passed through unchanged, this function does nothing.
|
|
4635
|
+
* When only `budgetTokens` was set, it's converted to the nearest tier
|
|
4636
|
+
* and `budget_tokens` is dropped, since OpenAI's API would otherwise
|
|
4637
|
+
* silently ignore an unrecognized field.
|
|
4638
|
+
*/
|
|
4639
|
+
function applyReasoningBudget(params, effortTokenTable) {
|
|
4640
|
+
if (params.budget_tokens === void 0) return params;
|
|
4641
|
+
const { budget_tokens,...rest } = params;
|
|
4642
|
+
return rest.reasoning_effort !== void 0 ? rest : {
|
|
4643
|
+
...rest,
|
|
4644
|
+
reasoning_effort: budgetTokensToEffort(budget_tokens, effortTokenTable)
|
|
4645
|
+
};
|
|
4646
|
+
}
|
|
4647
|
+
/**
|
|
3960
4648
|
* Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
|
|
3961
4649
|
* array. Text blocks become `{ type: 'text', text }`; image blocks become
|
|
3962
4650
|
* `{ type: 'image_url', image_url: { url } }` with the base64 payload
|
|
@@ -4022,23 +4710,24 @@ function* toWireStreamChunks(chunk) {
|
|
|
4022
4710
|
function fromOpenAICompatible(client, options = {}) {
|
|
4023
4711
|
const raw = client;
|
|
4024
4712
|
const { supportsStreamUsage = true } = options;
|
|
4713
|
+
const effortTokenTable = resolveEffortTokenTable(options.reasoningEffortTokens);
|
|
4025
4714
|
const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
|
|
4026
4715
|
return { chat: { completions: {
|
|
4027
4716
|
async create(params, options$1) {
|
|
4028
4717
|
const messages = toOpenAIMessages(params);
|
|
4029
|
-
return raw.chat.completions.create({
|
|
4718
|
+
return raw.chat.completions.create(applyReasoningBudget({
|
|
4030
4719
|
...params,
|
|
4031
4720
|
messages
|
|
4032
|
-
}, options$1);
|
|
4721
|
+
}, effortTokenTable), options$1);
|
|
4033
4722
|
},
|
|
4034
4723
|
async *createStream(params, options$1) {
|
|
4035
4724
|
const messages = toOpenAIMessages(params);
|
|
4036
|
-
const stream = await rawCreate({
|
|
4725
|
+
const stream = await rawCreate(applyReasoningBudget({
|
|
4037
4726
|
...params,
|
|
4038
4727
|
messages,
|
|
4039
4728
|
stream: true,
|
|
4040
4729
|
...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
|
|
4041
|
-
}, options$1);
|
|
4730
|
+
}, effortTokenTable), options$1);
|
|
4042
4731
|
for await (const chunk of stream) yield* toWireStreamChunks(chunk);
|
|
4043
4732
|
}
|
|
4044
4733
|
} } };
|
|
@@ -4150,5 +4839,5 @@ const fromAtlasCloud = fromOpenAICompatible;
|
|
|
4150
4839
|
const from01AI = fromOpenAICompatible;
|
|
4151
4840
|
|
|
4152
4841
|
//#endregion
|
|
4153
|
-
export { CircuitBreaker, ConsoleLogger, FallbackExhaustedError, InMemoryCacheAdapter, LLMError, NormalizedCacheAdapter, RateLimiter, SSE_PING, TieredCacheAdapter, VernLLM, defaultEstimateTokens, defaultFallbackOn, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, hasIssues, isFallbackExhaustedError, isLLMError, isToolCallResult, parseSseStream };
|
|
4842
|
+
export { CircuitBreaker, ConsoleLogger, FallbackExhaustedError, InMemoryCacheAdapter, LLMError, NormalizedCacheAdapter, RateLimiter, SSE_PING, TieredCacheAdapter, VernLLM, defaultEstimateTokens, defaultFallbackOn, defineCachedCallParams, defineCallParams, from01AI, fromAnthropic, fromAnyscale, fromAtlasCloud, fromBaseten, fromBedrock, fromCerebras, fromCloudflareWorkersAI, fromDeepInfra, fromDeepSeek, fromFeatherless, fromFetch, fromFireworks, fromFriendli, fromGemini, fromGroq, fromHyperbolic, fromInferenceNet, fromInfermatic, fromLMStudio, fromLambdaLabs, fromLepton, fromMiniMax, fromMistral, fromMoonshot, fromNebius, fromNovita, fromNvidiaNIM, fromOllama, fromOpenAI, fromOpenAICompatible, fromOpenRouter, fromParasail, fromPerplexity, fromSambaNova, fromSiliconFlow, fromSnowflakeCortex, fromStepFun, fromTogether, fromVLLM, fromVercelAIGateway, fromXAI, fromZhipu, hasIssues, isFallbackExhaustedError, isLLMError, isToolCallResult, parseSseStream };
|
|
4154
4843
|
//# sourceMappingURL=index.js.map
|