vern-llm 2.3.0 → 2.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -1,4237 +1,15 @@
1
- "use strict";
2
- //#region rolldown:runtime
3
- var __create = Object.create;
4
- var __defProp = Object.defineProperty;
5
- var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
6
- var __getOwnPropNames = Object.getOwnPropertyNames;
7
- var __getProtoOf = Object.getPrototypeOf;
8
- var __hasOwnProp = Object.prototype.hasOwnProperty;
9
- var __copyProps = (to, from, except, desc) => {
10
- if (from && typeof from === "object" || typeof from === "function") for (var keys = __getOwnPropNames(from), i = 0, n = keys.length, key; i < n; i++) {
11
- key = keys[i];
12
- if (!__hasOwnProp.call(to, key) && key !== except) __defProp(to, key, {
13
- get: ((k) => from[k]).bind(null, key),
14
- enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable
15
- });
16
- }
17
- return to;
18
- };
19
- var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", {
20
- value: mod,
21
- enumerable: true
22
- }) : target, mod));
1
+ "use strict";var e=Object.create,t=Object.defineProperty,n=Object.getOwnPropertyDescriptor,r=Object.getOwnPropertyNames,i=Object.getPrototypeOf,a=Object.prototype.hasOwnProperty,o=(e,i,o,s)=>{if(i&&typeof i==`object`||typeof i==`function`)for(var c=r(i),l=0,u=c.length,d;l<u;l++)d=c[l],!a.call(e,d)&&d!==o&&t(e,d,{get:(e=>i[e]).bind(null,d),enumerable:!(s=n(i,d))||s.enumerable});return e},s=(n,r,a)=>(a=n==null?{}:e(i(n)),o(r||!n||!n.__esModule?t(a,`default`,{value:n,enumerable:!0}):a,n));const c=s(require(`crypto`)),l=new Set([`unknown_tool`,`duplicate_tool_call_id`,`tool_choice_none_violated`,`unexpected_tool_calls`]),u=new Set([`rate_limit_queue_full`,`rate_limit_queue_timeout`,`rate_limit_capacity_exceeded`]),d=new Set([`parse`,`validation`,`invalid_params`,`aborted`]);function f(e,t){return!(d.has(e)||t&&l.has(t)||t&&u.has(t))}function p(e){if(e!==void 0)try{return JSON.stringify(e),e}catch{return`[Unserializable: issues contained a circular reference]`}}function m(e){if(e!==void 0)try{return JSON.parse(JSON.stringify(e))}catch{return`[Unserializable: request body contained a circular reference]`}}const h=new Set([`authorization`,`x-api-key`,`x-goog-api-key`,`api-key`]);function g(e){if(e===void 0)return;let t={};for(let[n,r]of Object.entries(e))h.has(n.toLowerCase())||(t[n]=r);return t}const _=20;function v(e,t=0){if(e!==void 0)return t>=_?[]:e.map(e=>({...e,error:{...e.error,issues:p(e.error.issues),attempts:v(e.error.attempts,t+1)},request:e.request&&{...e.request,body:m(e.request.body),headers:g(e.request.headers)}}))}function y(e,t,n,r,i=Date.now()){return{provider:e,model:t,body:m(n),headers:g(r),startedAt:i}}var b=class extends Error{status;issues;cause;retryAfterMs;code;attempts;constructor(e,t,n={}){super(e),this.type=t,this.name=`LLMError`,this.status=n.status,this.issues=n.issues,this.cause=n.cause,this.retryAfterMs=n.retryAfterMs,this.code=n.code,this.attempts=n.attempts}get retryable(){return f(this.type,this.code)}toSnapshot(){return{message:this.message,type:this.type,status:this.status,issues:p(this.issues),retryAfterMs:this.retryAfterMs,code:this.code,retryable:this.retryable,attempts:v(this.attempts)}}toJSON(){return{name:this.name,message:this.message,type:this.type,status:this.status,issues:p(this.issues),retryAfterMs:this.retryAfterMs,code:this.code,retryable:this.retryable,attempts:v(this.attempts)}}};function x(e){return e instanceof b}function ee(e,t){return e.code===t&&e.issues!==void 0}var S=class{store=new Map;constructor(e=1e3){this.maxSize=e}async get(e){let t=this.store.get(e);return t?Date.now()>=t.expiresAt?(this.store.delete(e),{hit:!1,value:null}):{hit:!0,value:t.value}:{hit:!1,value:null}}async set(e,t,n){this.cleanupExpiredEntries(),this.store.set(e,{value:t,expiresAt:Date.now()+n*1e3}),this.enforceSizeLimit()}async delete(e){this.store.delete(e)}cleanupExpiredEntries(){let e=Date.now();for(let[t,n]of this.store)e>=n.expiresAt&&this.store.delete(t)}enforceSizeLimit(){for(;this.store.size>this.maxSize;){let e=this.store.keys().next().value;if(e===void 0)break;this.store.delete(e)}}},C=class{constructor(e=new S){this.inner=e}normalize(e){return e.toLowerCase().trim().replace(/[^\p{L}\p{N}\s]/gu,` `).replace(/\s+/g,` `).trim()}async resolveKey(e){return this.normalize(e)}async get(e){return this.inner.get(this.normalize(e))}async set(e,t,n){await this.inner.set(this.normalize(e),t,n)}async delete(e){await this.inner.delete?.(this.normalize(e))}},w=class{constructor(e,t,n){this.l1=e,this.l2=t,this.l1Ttl=n}async resolveKey(e){return this.l1.resolveKey?this.l1.resolveKey(e):this.l2.resolveKey?this.l2.resolveKey(e):e}async get(e){let t=await this.l1.get(e);if(t.hit)return t;let n=await this.l2.get(e);return n.hit&&await this.l1.set(e,n.value,this.l1Ttl??60),n}async set(e,t,n){await Promise.all([this.l1.set(e,t,this.l1Ttl??n),this.l2.set(e,t,n)])}async delete(e){await Promise.all([this.l1.delete?.(e),this.l2.delete?.(e)])}};function T(e){return typeof e==`object`&&!!e&&`type`in e&&e.type===`tool_calls`&&Array.isArray(e.toolCalls)}const te=new Set([`unknown_tool`,`duplicate_tool_call_id`,`tool_choice_none_violated`,`unexpected_tool_calls`]),ne=e=>e.type===`parse`||e.type===`validation`||e.type===`aborted`||e.type===`quota_exceeded`||e.code&&te.has(e.code)?`stop`:`next`;var E=class extends b{constructor(e){let t=e[e.length-1]?.error;super(`${e.length} provider${e.length===1?``:`s`} attempted and failed: ${e.map(e=>`${e.provider}(${e.error.type})`).join(` then `)}`,`fallback_exhausted`,{status:t?.status,cause:t,retryAfterMs:t?.retryAfterMs,code:`fallback_exhausted`,attempts:e}),this.attempts=e}get retryable(){let e=this.attempts[this.attempts.length-1]?.error;return e?e.retryable:super.retryable}};function re(e){return e instanceof E}async function ie(e,t,n){if(!e.reserveUsage)return!1;try{return await e.reserveUsage({coalesced:t,signal:n}),!0}catch(e){throw n?.aborted?new b(`LLM request aborted`,`aborted`):new b(e instanceof Error?e.message:`Usage reservation failed`,`quota_exceeded`,{cause:e})}}function ae(e,t,n,r){return async i=>{try{await e.refundUsage?.({coalesced:t,signal:n})}catch(e){r(i,e)}}}async function D(e,t,n,r,i){if(r?.aborted)throw new b(`LLM request aborted`,`aborted`);let a=await ie(e,t,r),o=ae(e,t,r,i);if(r?.aborted)throw a&&await o(`[VernLLM] refundUsage failed after abort`),new b(`LLM request aborted`,`aborted`);let s;try{s=await n()}catch(e){throw a&&await o(`[VernLLM] refundUsage failed`),e}if(r?.aborted)throw a&&await o(`[VernLLM] refundUsage failed after abort`),new b(`LLM request aborted`,`aborted`);return s}async function oe(e,t,n,r){if(n?.aborted)throw new b(`LLM request aborted`,`aborted`);let i=await ie(e,!1,n),a=ae(e,!1,n,r);if(n?.aborted)throw i&&await a(`[VernLLM] refundUsage failed after abort`),new b(`LLM request aborted`,`aborted`);let o;try{o=await t()}catch(e){throw i&&await a(`[VernLLM] refundUsage failed after stream-open failure`),e}let s=o.finalResult.then(e=>e,async e=>{throw i&&await a(`[VernLLM] refundUsage failed after stream error`),e});return s.catch(()=>{}),{chunks:o.chunks,finalResult:s}}function se(e){return typeof e==`string`?e:JSON.stringify(e)??``}function ce(e,t){let n=[];if(t){let t=e;t.type===`tool_calls`?(t.toolCalls.forEach((e,t)=>{n.push({type:`tool_call_delta`,index:t,id:e.id,name:e.name,argsDelta:JSON.stringify(e.arguments??{}),complete:!0})}),t.content&&n.push({type:`text-delta`,delta:t.content})):n.push({type:`text-delta`,delta:se(t.content)})}else n.push({type:`text-delta`,delta:se(e)});return{async*[Symbol.asyncIterator](){for(let e of n)yield e}}}function le(e,t){return{async*[Symbol.asyncIterator](){let n=await e;yield*ce(n,t)}}}var ue=class{inFlight=new Map;constructor(e,t){this.cache=e,this.logger=t}async resolveCacheKey(e){return this.cache.resolveKey?await this.cache.resolveKey(e):e}async deleteCache(e){if(this.cache.delete)try{await this.cache.delete(await this.resolveCacheKey(e))}catch(e){this.logger.warn(`[VernLLM] cache delete failed: ${e instanceof Error?e.message:`unknown`}`)}}logRefundError(e,t){this.logger.error(e,{message:t instanceof Error?t.message:`unknown`})}async getCached(e){try{return await this.cache.get(e)}catch(e){return this.logger.warn(`[VernLLM] cache read failed: ${e instanceof Error?e.message:`unknown`}`),{hit:!1}}}async runCached(e){let t=await this.resolveCacheKey(e.cacheKey),n=t===e.cacheKey?e:{...e,cacheKey:t},r=await this.getCached(t);if(r.hit)return r.value;let i=this.inFlight.get(t);return i?D(n,!0,()=>i,e.signal,(e,t)=>this.logRefundError(e,t)):this.registerTrigger(n)}registerTrigger(e){let t=D(e,!1,()=>this.runAndCache(e),e.signal,(e,t)=>this.logRefundError(e,t));return this.inFlight.set(e.cacheKey,t),t.catch(()=>{}).finally(()=>{this.inFlight.delete(e.cacheKey)}),t}async runAndCache(e){let t=await e.fn();try{await this.cache.set(e.cacheKey,t,e.ttl)}catch(e){this.logger.warn(`[VernLLM] cache write failed: ${e instanceof Error?e.message:`unknown`}`)}return t}async runCachedStream(e,t){let n=await this.resolveCacheKey(e.cacheKey),r=n===e.cacheKey?e:{...e,cacheKey:n},i=await this.getCached(n);if(i.hit){let e=i.value;return{chunks:ce(e,t),finalResult:Promise.resolve(e)}}let a=this.inFlight.get(n);if(a){let n=D(r,!0,()=>a,e.signal,(e,t)=>this.logRefundError(e,t));return n.catch(()=>{}),{chunks:le(n,t),finalResult:n}}return this.registerStreamTrigger(r)}registerStreamTrigger(e){let t,n,r=new Promise((e,r)=>{t=e,n=r});this.inFlight.set(e.cacheKey,r),r.catch(()=>{}).finally(()=>{this.inFlight.delete(e.cacheKey)});let i=oe(e,async()=>{let t=await e.openStream(),n=t.finalResult.then(async t=>{try{await this.cache.set(e.cacheKey,t,e.ttl)}catch(e){this.logger.warn(`[VernLLM] cache write failed: ${e instanceof Error?e.message:`unknown`}`)}return t},e=>{throw e});return{chunks:t.chunks,finalResult:n}},e.signal,(e,t)=>this.logRefundError(e,t));return i.then(e=>{e.finalResult.then(t,n)},e=>{n(e)}),i}};function de(){return{state:`closed`,consecutiveFailures:0,openedAt:0,trialInFlight:!1}}const O=``;var fe=class{threshold;cooldownMs;onStateChange;isolateByModel;sharedBucket=de();bucketsByModel=new Map;constructor(e={}){this.threshold=e.threshold??5,this.cooldownMs=e.cooldownMs??3e4,this.onStateChange=e.onStateChange,this.isolateByModel=e.isolateByModel??!1}lookupBucket(e){if(!this.isolateByModel)return this.sharedBucket;let t=e??O;return this.bucketsByModel.get(t)}ensureBucketFor(e){if(!this.isolateByModel)return this.sharedBucket;let t=e??O,n=this.bucketsByModel.get(t);return n||(n=de(),this.bucketsByModel.set(t,n)),n}transition(e,t,n){if(t===e.state)return;let r=e.state;e.state=t,this.onStateChange?.(r,t,e.consecutiveFailures,n)}assertClosed(e){let t=this.ensureBucketFor(e);if(t.state!==`closed`){if(t.state===`open`){let n=Date.now()-t.openedAt;if(n<this.cooldownMs)throw new b(`Circuit open, provider has failed ${t.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs-n)/1e3)}s.`,`circuit_open`,{code:`circuit_cooling_down`});t.trialInFlight=!0,this.transition(t,`half-open`,e);return}if(t.trialInFlight)throw new b(`Circuit half-open. A trial request is already in flight. Try again shortly.`,`circuit_open`,{code:`circuit_trial_in_flight`});t.trialInFlight=!0}}recordSuccess(e){let t=this.lookupBucket(e);t&&(t.consecutiveFailures=0,t.trialInFlight=!1,this.transition(t,`closed`,e),this.isolateByModel&&t.state===`closed`&&t.consecutiveFailures===0&&this.bucketsByModel.delete(e??O))}recordFailure(e){let t=this.ensureBucketFor(e);if(t.consecutiveFailures+=1,t.trialInFlight=!1,t.state===`half-open`){t.openedAt=Date.now(),this.transition(t,`open`,e);return}t.consecutiveFailures>=this.threshold&&(t.openedAt=Date.now(),this.transition(t,`open`,e))}getState(e){return this.lookupBucket(e)?.state??`closed`}open(e){let t=this.ensureBucketFor(e);t.openedAt=Date.now(),t.trialInFlight=!1,this.transition(t,`open`,e)}close(e){let t=this.ensureBucketFor(e);t.consecutiveFailures=0,t.trialInFlight=!1,this.transition(t,`closed`,e),this.isolateByModel&&t.state===`closed`&&t.consecutiveFailures===0&&this.bucketsByModel.delete(e??O)}};function k(e,t){return n=>{if(e)try{e(n)}catch(e){t.error(`[VernLLM] onEvent failed`,{message:e instanceof Error?e.message:`unknown`})}}}function pe(e,t,n,r,i){if(!e)return;let a=typeof e==`object`?e:void 0,o=a?.onStateChange,s=k(r,i);return new fe({...a,onStateChange:(e,r,a,c)=>{if(s({kind:`circuit_state`,provider:t,model:c??n,from:e,to:r,consecutiveFailures:a}),o)try{o(e,r,a,c)}catch(e){i.error(`[VernLLM] circuitBreaker.onStateChange failed`,{message:e instanceof Error?e.message:`unknown`})}}})}const me=1e4,he=2147483647;function A(e){return!e||e<=0||e===1/0?void 0:e}function j(e){return Math.min(e,he)}async function M(e,t,n){let r=new AbortController,i=A(t),a=i===void 0?void 0:setTimeout(()=>{r.abort()},j(i)),o=n?AbortSignal.any([n,r.signal]):r.signal;try{return await e(o)}catch(e){throw r.signal.aborted&&!n?.aborted&&e instanceof DOMException&&e.name===`AbortError`?new b(`Request timed out`,`timeout`,{code:`request_timeout`}):e}finally{clearTimeout(a)}}function ge(e,t,n,r){let i=A(t);if(i===void 0)return e();let a=!1;return new Promise((t,o)=>{let s=setTimeout(()=>{a=!0,n?.(),o(new b(`No stream chunk received for ${i}ms (idle timeout)`,`timeout`,{code:`idle_timeout`}))},j(i));e().then(e=>{if(clearTimeout(s),a){r?.debug(`[VernLLM] chunk resolved after idle timeout already fired; discarding`);return}a=!0,t(e)},e=>{if(clearTimeout(s),a){r?.debug(`[VernLLM] chunk rejection arrived after idle timeout already fired; discarding`);return}a=!0,o(e)})})}function N(e,t=me){if(!e||typeof e!=`object`)return;let n=e,r=n.headers??n.response?.headers;if(!r||typeof r!=`object`)return;let i=r,a=typeof i.get==`function`?i.get(`Retry-After`):Object.entries(r).find(([e])=>e.toLowerCase()===`retry-after`)?.at(1);if(typeof a!=`string`||a.trim()===``)return;let o=a.trim();if(/^\d+$/.test(o))return Math.max(0,Math.min(Number(o)*1e3,t));let s=Date.parse(o);if(!Number.isNaN(s))return Math.max(0,Math.min(s-Date.now(),t))}function _e(e,t,n=me){let r=Math.min(e*2**t,n);return r/2+Math.random()*(r/2)}async function ve(e,t){if(t?.aborted)throw new b(`Operation aborted`,`aborted`);await new Promise((n,r)=>{let i=()=>{clearTimeout(a),r(new b(`Operation aborted`,`aborted`))},a=setTimeout(()=>{t?.removeEventListener(`abort`,i),n()},e);t?.addEventListener(`abort`,i,{once:!0})})}function P(e){if(!e||typeof e!=`object`)return;let t=e;if(typeof t.status==`number`)return t.status;if(typeof t.statusCode==`number`)return t.statusCode;if(typeof t.$metadata?.httpStatusCode==`number`)return t.$metadata.httpStatusCode}const F=new Set([`ECONNREFUSED`,`ENOTFOUND`,`ECONNRESET`,`ETIMEDOUT`,`EAI_AGAIN`,`EPIPE`,`ECONNABORTED`,`EHOSTUNREACH`,`ENETUNREACH`]),ye=new Set([`fetch failed`,`failed to fetch`,`load failed`,`networkerror when attempting to fetch resource.`]);function be(e){if(!e||typeof e!=`object`)return!1;let t=e;if(typeof t.code==`string`&&F.has(t.code)||typeof t.message==`string`&&ye.has(t.message.toLowerCase()))return!0;if(t.cause&&typeof t.cause==`object`){let e=t.cause;if(typeof e.code==`string`&&F.has(e.code))return!0}return!1}function I(e){try{return JSON.stringify(e,null,2)??String(e)}catch{try{return String(e)}catch{return`[unprintable error]`}}}function L(e){if(e&&typeof e==`object`)try{let t=e;if(t.error!==void 0)return I(t.error);if(typeof t.message==`string`)return t.message}catch{}return I(e)}function R(e){switch(e){case 429:return`provider_rate_limited`;case 401:return`authentication`;case 403:return`authorization`;case 404:return`not_found`;case 413:return`payload_too_large`;default:return e>=500?`server_error`:void 0}}const xe=/\(no body\)/i;function Se(e){return Object.keys(e).length===0}function Ce(e){if(e&&typeof e==`object`){let{error:t,message:n}=e;if(t!=null){let e=typeof t==`string`&&t.trim().length===0,n=typeof t==`object`&&Se(t);if(!e&&!n)return!1}if(typeof n==`string`){let e=n.trim();return e.length===0||xe.test(e)}return!0}return!0}function z(e,t,n){if(t?.aborted)return new b(`LLM request aborted`,`aborted`,{attempts:n});if(e instanceof b)return e.code===void 0&&e.status!==void 0&&(e.code=R(e.status)),e.attempts===void 0&&n!==void 0&&(e.attempts=n),e;let r=P(e),i=N(e);if(r!==void 0){let t=L(e),a=R(r),o=a===void 0,s=Ce(e)?o?`LLM request failed with status ${r} and no error detail from the provider. This usually means a field or value in the request isn't supported by the specific model (for example, a reasoning/thinking parameter the model doesn't accept), rather than a transport or auth problem.`:`LLM request failed with status ${r} and no error detail from the provider.`:`LLM request failed: ${t}`;return new b(s,`api`,{status:r,cause:e,retryAfterMs:i,code:a,attempts:n})}return be(e)?new b(`LLM request failed`,`network`,{cause:e,retryAfterMs:i,code:`connection_failed`,attempts:n}):new b(`LLM request failed`,`unknown`,{cause:e,retryAfterMs:i,attempts:n})}function we(e){try{return JSON.parse(e)}catch{return}}function Te(e){return e.map(e=>({type:`function`,function:{name:e.name,description:e.description,parameters:e.parameters}}))}function Ee(e){return e.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:JSON.stringify(e.arguments??{})}}))}function De(e){return e.map(e=>{let t;try{t=e.function.arguments.trim()?JSON.parse(e.function.arguments):{}}catch{throw new b(`Invalid JSON arguments for tool call "${e.function.name}"`,`parse`,{code:`tool_arguments_parse_failed`})}return{id:e.id,name:e.function.name,arguments:t}})}function B(e){return typeof e==`string`?e:JSON.stringify(e)}var Oe=class{model;defaultMaxTokens;defaultTemperature;defaultReasoningEffort;defaultBudgetTokens;supportsJsonObjectMode;constructor(e){this.model=e.model,this.defaultMaxTokens=e.defaultMaxTokens,this.defaultTemperature=e.defaultTemperature,this.defaultReasoningEffort=e.defaultReasoningEffort,this.defaultBudgetTokens=e.defaultBudgetTokens,this.supportsJsonObjectMode=e.supportsJsonObjectMode}build(e){let{systemPrompt:t,userContent:n,history:r=[],maxTokens:i=this.defaultMaxTokens,model:a=this.model,jsonSchema:o,tools:s,toolChoice:c}=e,l=e.temperature===void 0?this.defaultTemperature:e.temperature,u=e.reasoningEffort===void 0?this.defaultReasoningEffort:e.reasoningEffort,d=e.budgetTokens===void 0?this.defaultBudgetTokens:e.budgetTokens;if(s&&s.length===0)throw new b("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.",`invalid_params`);if(s){let e=new Set,t=new Set;for(let n of s)e.has(n.name)&&t.add(n.name),e.add(n.name);if(t.size)throw new b(`\`tools\` has duplicate name(s): [${[...t].join(`, `)}]. Tool names must be unique.`,`invalid_params`,{code:`duplicate_tool_names`,issues:{names:[...t]}})}if(c&&!s)throw new b("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.",`invalid_params`);if(s&&typeof c==`object`&&!s.some(e=>e.name===c.name))throw new b(`toolChoice names "${c.name}", which is not in \`tools\` ([${s.map(e=>e.name).join(`, `)}]).`,`invalid_params`,{code:`unknown_tool_choice`,issues:{requested:c.name,available:s.map(e=>e.name)}});let f=e.jsonMode,p=f??!s;if(!this.supportsJsonObjectMode&&!o&&f===!0)throw new b('jsonMode: true was set explicitly, but this client does not support `response_format: "json_object"` (see LLMClient.supportsJsonObjectMode). Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output for this mode. Use `jsonSchema` instead, which maps to a real constraint on both.',`invalid_params`);if(!this.supportsJsonObjectMode&&!o&&f===void 0&&e.schema)throw new b('`schema` was provided, which requires JSON output to validate against, but this client does not support `response_format: "json_object"` (see LLMClient.supportsJsonObjectMode) and no `jsonSchema` was set. Neither Anthropic nor Bedrock has a field that mechanically guarantees JSON output without one. Use `jsonSchema` instead, which maps to a real constraint on both and still runs `schema` against its parsed result.',`invalid_params`);let m=!this.supportsJsonObjectMode&&!o&&f===void 0?!1:p,h=m||!!o;if(e.schema&&!h)throw new b(`schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.`,`invalid_params`);let g=this.buildResponseFormat(o,h);this.validateHistory(r);let _={model:a,...l===null?{}:{temperature:l},max_tokens:i,...g?{response_format:g}:{},...u?{reasoning_effort:u}:{},...d==null?{}:{budget_tokens:d},...s?{tools:Te(s)}:{},...s?{tool_choice:this.buildWireToolChoice(c)}:{},messages:[...t?[{role:`system`,content:t}]:[],...r.flatMap(e=>this.turnToWireMessages(e)),{role:`user`,content:n}]};return{useJson:h,model:a,request:_}}validateHistory(e){let t;for(let[n,r]of e.entries()){if(r.role===`tool`){if(t?.role!==`assistant`||!t.toolCalls?.length)throw new b(`history[${n}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`,`invalid_params`);if(!r.toolResults?.length)throw new b(`history[${n}] is a "tool" turn but has no toolResults`,`invalid_params`);let e=new Set(t.toolCalls.map(e=>e.id)),i=r.toolResults.map(e=>e.toolCallId),a=i.filter(t=>!e.has(t));if(a.length)throw new b(`history[${n}].toolResults references unknown toolCallId(s) [${a.join(`, `)}]`,`invalid_params`,{code:`unknown_tool_result_ids`,issues:{historyIndex:n,ids:a}});let o=new Set,s=new Set;for(let e of i)o.has(e)&&s.add(e),o.add(e);if(s.size)throw new b(`history[${n}].toolResults has duplicate toolCallId(s) [${[...s].join(`, `)}]`,`invalid_params`,{code:`duplicate_tool_result_ids`,issues:{historyIndex:n,ids:[...s]}});let c=[...e].filter(e=>!i.includes(e));if(c.length)throw new b(`history[${n}] is missing toolResults for toolCallId(s) [${c.join(`, `)}]`,`invalid_params`,{code:`missing_tool_results`,issues:{historyIndex:n,ids:c}})}else{if(r.role===t?.role)throw new b(`history must alternate user/assistant turns: consecutive "${r.role}" turns at history[${n-1}] and history[${n}]`,`invalid_params`);if(t?.role===`assistant`&&t.toolCalls?.length)throw new b(`history[${n}] follows an assistant tool request without tool results`,`invalid_params`)}t=r}if(t?.role===`assistant`&&t.toolCalls?.length)throw new b(`The last entry in history is an assistant tool request without tool results`,`invalid_params`);if(t?.role===`user`)throw new b(`The last entry in history is a "user" turn, which would collide with the current userContent turn.`,`invalid_params`)}buildWireToolChoice(e){return!e||e===`auto`?`auto`:e===`none`||e===`required`?e:{type:`function`,function:{name:e.name}}}turnToWireMessages(e){return e.role===`tool`?(e.toolResults??[]).map(e=>({role:`tool`,tool_call_id:e.toolCallId,content:typeof e.content==`string`?e.content:JSON.stringify(e.content??null),...e.isError?{is_error:!0}:{}})):e.role===`assistant`&&e.toolCalls?.length?[{role:`assistant`,...e.content===void 0?{}:{content:B(e.content)},tool_calls:Ee(e.toolCalls)}]:e.role===`assistant`?[{role:`assistant`,content:B(e.content===void 0?``:e.content)}]:[{role:e.role,content:e.content??``}]}buildResponseFormat(e,t){return e?{type:`json_schema`,json_schema:{name:e.name,schema:e.schema,strict:e.strict??!0,description:e.description}}:t?{type:`json_object`}:void 0}};function ke(e,t,n){let{requestId:r,model:i,providerName:a,isFallback:o,chunkIdleTimeoutMs:s,streamController:c,logger:l,signal:u}=n,d,f,p=new Promise((e,t)=>{d=e,f=t});p.catch(()=>{});let m=1e4,h=[],g=[],_=!1,v,y=!1,b=e=>{let t=g.shift();if(t){t.resolve({done:!1,value:e});return}h.push(e),h.length>m*2&&(y||(y=!0,l.warn(`[VernLLM] stream chunk buffer exceeded cap (${m}), evicting ${h.length-m} oldest chunk(s); buffered=${h.length}. The chunks iterable was never read (or fell far behind) for this stream.`)),h.splice(0,h.length-m))},x=()=>{_=!0;for(let e of g.splice(0))e.resolve({done:!0,value:void 0})},ee=e=>{_=!0,v=e;for(let t of g.splice(0))t.reject(e)},S={[Symbol.asyncIterator](){return{next(){return h.length?Promise.resolve({done:!1,value:h.shift()}):_?v?Promise.reject(v):Promise.resolve({done:!0,value:void 0}):new Promise((e,t)=>{g.push({resolve:e,reject:t})})}}}},C=new Map,w=``,T;return(async()=>{try{let n=t;for(;!n.done;){let t=n.value;if(t.type!==`ping`){if(t.type===`text-delta`)w+=t.delta,b({type:`text-delta`,delta:t.delta});else if(t.type===`tool_call_delta`){let e=C.get(t.index)??{args:``};e.id??=t.id,e.name??=t.name,e.args+=t.argumentsDelta??``,C.set(t.index,e),b({type:`tool_call_delta`,index:t.index,id:t.id,name:t.name,argsDelta:t.argumentsDelta,complete:t.complete})}else if(t.type===`usage`){let e=t.usage.completion_tokens_details?.reasoning_tokens;T={promptTokens:t.usage.prompt_tokens??0,completionTokens:t.usage.completion_tokens??0,totalTokens:t.usage.total_tokens??0,...e===void 0?{}:{reasoningTokens:e},requestId:r,model:i,provider:a,usedFallback:o},b({type:`usage`,usage:T})}}n=await ge(()=>e.next(),s,()=>c.abort(),l)}}catch(t){try{await e.return?.()}catch{}c.abort();let r=z(t,u);try{n.onStreamFailure(r,T)}catch{}ee(r),f(r);return}x();try{n.onStreamSuccess(T)}catch{}try{let e=C.size?[...C.entries()].sort(([e],[t])=>e-t).map(([,e])=>({id:e.id??``,type:`function`,function:{name:e.name??``,arguments:e.args}})):void 0,t=n.finalize(w,e,T);d(t)}catch(e){f(e)}})(),{chunks:S,finalResult:p}}function Ae(e){return e}var je=class{maxRetries;timeoutMs;chunkIdleTimeoutMs;baseDelayMs;nonRetryableStatus;parseJson;logger;redact;onUsage;onUsageFailure;reportEvent;breaker;limiter;isFallback;requestBuilder;constructor(e,t,n,r){this.providerName=e,this.client=t,this.model=n,this.maxRetries=r.maxRetries,this.timeoutMs=r.timeoutMs,this.chunkIdleTimeoutMs=r.chunkIdleTimeoutMs,this.baseDelayMs=r.baseDelayMs,this.nonRetryableStatus=r.nonRetryableStatus,this.parseJson=r.parseJson??we,this.logger=r.logger,this.redact=r.redact,this.onUsage=r.onUsage,this.onUsageFailure=r.onUsageFailure,this.reportEvent=k(r.onEvent,this.logger),this.breaker=r.breaker,this.limiter=r.limiter,this.isFallback=r.isFallback??!1,this.requestBuilder=new Oe({model:n,defaultMaxTokens:r.defaultMaxTokens,defaultTemperature:r.defaultTemperature,defaultReasoningEffort:r.defaultReasoningEffort,defaultBudgetTokens:r.defaultBudgetTokens,supportsJsonObjectMode:t.supportsJsonObjectMode??!0})}getCircuitState(e){return this.breaker?.getState(e)}get isolateByModel(){return this.breaker?.isolateByModel??!1}openCircuit(e){this.breaker?.open(e)}closeCircuit(e){this.breaker?.close(e)}assertBreakerClosed(e){this.breaker?.assertClosed(e??this.model)}async run(e,t,n){let r=e.model??this.model,i=[];try{return await this.retryWithBackoff((n,r)=>this.executeCall(e,t,n,r),t,r,e.signal,n,i)}catch(n){let a=z(n,e.signal,i.length>0?i:void 0);throw this.countsTowardBreaker(a)&&this.breaker?.recordFailure(r),this.logger.debug(`[VernLLM:${t}] error:\n${this.redactText(L(n))}`),a}}async runStream(e,t,n){let r=e.model??this.model,i=[];try{return await this.retryWithBackoff((n,r)=>this.executeStreamCall(e,t,n,r),t,r,e.signal,n,i)}catch(n){let a=z(n,e.signal,i.length>0?i:void 0);throw this.countsTowardBreaker(a)&&this.breaker?.recordFailure(r),this.logger.debug(`[VernLLM:${t}] stream-open error:\n${this.redactText(L(n))}`),a}}async executeCall(e,t,n,r){let{useJson:i,model:a,request:o}=this.requestBuilder.build(e);r?.(y(this.providerName,a,o,void 0,Date.now()));let s;if(this.limiter){let n=await this.limiter.acquire(this.limiter.estimate(o),e.signal);s=n.release,n.waitedMs>0&&this.reportEvent({kind:`rate_limited`,requestId:t,provider:this.providerName,model:a,waitedMs:n.waitedMs,reason:n.reason??`rpm`})}try{let r=await M(e=>this.client.chat.completions.create(o,{signal:e}),this.timeoutMs,e.signal),c=this.extractUsage(r,t,a);s?.(this.actualTokensFor(c)),s=void 0;let l=r.choices?.[0]?.message?.content,u=r.choices?.[0]?.message?.tool_calls;return this.finalizeResponse(l,u,e,i,a,c,t,n)}finally{s?.()}}redactText(e){return this.redact?this.redact(e):e}redactedOutput(e,t){return this.redactText(e??`[${t?.length??0} tool call(s)]`)}finalizeResponse(e,t,n,r,i,a,o,s){try{let s=e?.trim();if(!s&&!t?.length)throw new b(`Empty LLM response`,`api`,{code:`empty_response`});if(this.logger.debug(`[VernLLM:${o}] output:\n${this.redactedOutput(s,t).slice(0,800)}`),t?.length){if(!n.tools)throw new b("Provider returned tool_calls but no `tools` were sent with this call.",`validation`,{code:`unexpected_tool_calls`});if(n.toolChoice===`none`)throw new b(`Provider returned tool_calls despite toolChoice: 'none'.`,`validation`,{code:`tool_choice_none_violated`});let e=De(t);return this.validateToolCallArguments(e,n.tools),this.breaker?.recordSuccess(i),this.reportUsage(a),{type:`tool_calls`,toolCalls:e,...s?{content:s}:{}}}let c=s??``;if(!r)return this.breaker?.recordSuccess(i),this.reportUsage(a),n.tools?{type:`content`,content:c}:c;let l=this.parseAndValidate(c,n.schema);return this.breaker?.recordSuccess(i),this.reportUsage(a),n.tools?{type:`content`,content:l}:l}catch(e){let t=z(e,n.signal);throw a&&t.type!==`aborted`&&this.reportUsageFailure(a,t,s),t}}async executeStreamCall(e,t,n,r){let{useJson:i,model:a,request:o}=this.requestBuilder.build(e);r?.(y(this.providerName,a,o,void 0,Date.now()));let s=this.client.chat.completions;if(!s.createStream)throw new b(`stream: true requires a client/adapter with createStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`createStream`}});let c=s.createStream.bind(s),l;if(this.limiter){let n=await this.limiter.acquire(this.limiter.estimate(o),e.signal);l=n.release,n.waitedMs>0&&this.reportEvent({kind:`rate_limited`,requestId:t,provider:this.providerName,model:a,waitedMs:n.waitedMs,reason:n.reason??`rpm`})}let u=new AbortController,d=e.signal?AbortSignal.any([e.signal,u.signal]):u.signal;try{let{iterator:r,first:s}=await M(async e=>{let t=c(o,{signal:e})[Symbol.asyncIterator](),n=await t.next();return{iterator:t,first:n}},this.timeoutMs,d);if(s.done)throw new b(`Empty LLM response`,`api`);let f=l,p=ke(r,s,{requestId:t,model:a,providerName:this.providerName,isFallback:this.isFallback,chunkIdleTimeoutMs:e.chunkIdleTimeoutMs??this.chunkIdleTimeoutMs,streamController:u,logger:this.logger,signal:e.signal,onStreamSuccess:e=>{this.breaker?.recordSuccess(a),f?.(this.actualTokensFor(e))},onStreamFailure:(e,t)=>{e.type===`timeout`&&this.breaker?.recordFailure(a),t&&e.type!==`aborted`&&this.reportUsageFailure(t,e,n,!0),f?.(this.actualTokensFor(t))},finalize:(r,o,s)=>this.finalizeResponse(r,o,e,i,a,s,t,n)});return l=void 0,p}finally{l?.()}}validateToolCallArguments(e,t){let n=new Map(t.map(e=>[e.name,e])),r=new Set,i=[];for(let t of e)r.has(t.id)&&i.push({name:t.name,toolCallId:t.id,code:`duplicate_tool_call_id`}),r.add(t.id),n.has(t.name)||i.push({name:t.name,toolCallId:t.id,code:`unknown_tool`});if(i.length>0){let e=i.find(e=>e.code===`unknown_tool`),t=e?`Model requested tool "${e.name}", which was not in the tools offered ([${[...n.keys()].join(`, `)}]).`:`Duplicate tool call id "${i[0].toolCallId}" in the model's response.`,r=i.length>1?`${t} (${i.length} tool call issues total, see error.issues.)`:t;throw new b(r,`validation`,{code:e?`unknown_tool`:`duplicate_tool_call_id`,issues:i})}for(let t of e){let e=n.get(t.name);if(!e?.argumentsSchema)continue;let r=e.argumentsSchema.safeParse(t.arguments);if(!r.success)throw new b(`Arguments for tool call "${t.name}" failed validation`,`validation`,{issues:r.error})}}async retryWithBackoff(e,t,n,r,i,a){let o,s;for(let c=0;c<=this.maxRetries;c++){s=void 0;try{return c>0&&await this.recoverDelay(t,n,c,o,r),i?.(),await e(c,e=>{s=e})}catch(e){o=e;let t=c<this.maxRetries&&this.shouldRetry(e,r);if(!t)break;a?.push({index:c,error:z(e,r).toSnapshot(),request:Ae(s)})}}throw o}extractUsage(e,t,n){if(!e.usage)return;let r=e.usage.completion_tokens_details?.reasoning_tokens;return{promptTokens:e.usage.prompt_tokens??0,completionTokens:e.usage.completion_tokens??0,totalTokens:e.usage.total_tokens??0,...r===void 0?{}:{reasoningTokens:r},requestId:t,model:n,provider:this.providerName,usedFallback:this.isFallback}}actualTokensFor(e){if(e)return e.totalTokens||e.promptTokens+e.completionTokens}reportUsage(e){if(!(!e||!this.onUsage))try{this.onUsage(e)}catch(e){this.logger.error(`[VernLLM] onUsage failed`,{message:e instanceof Error?e.message:`unknown`})}}reportUsageFailure(e,t,n,r=!1){let i=e.totalTokens||e.promptTokens+e.completionTokens,a=r?`mid-stream failure (terminal, no further attempts)`:`attempt ${n+1}/${this.maxRetries+1}`;if(this.logger.warn(`[VernLLM:${e.requestId}] usage failure, ${a}: type=${t.type} tokens=${i}`),this.onUsageFailure)try{this.onUsageFailure(e,t)}catch(e){this.logger.error(`[VernLLM] onUsageFailure failed`,{message:e instanceof Error?e.message:`unknown`})}}parseAndValidate(e,t){let n;try{n=this.parseJson(e)}catch{throw new b(`Invalid JSON response`,`parse`)}if(n==null)throw new b(`Invalid JSON response`,`parse`);if(!t)return n;let r=t.safeParse(n);if(!r.success)throw new b(`Schema validation failed`,`validation`,{issues:r.error});return r.data}async recoverDelay(e,t,n,r,i){let a=N(r),o=a??_e(this.baseDelayMs,n),s=a!==void 0;this.logger.warn(`[VernLLM:${e}] recovery attempt ${n}/${this.maxRetries}, waiting ${Math.ceil(o)}ms`+(s?` (honoring Retry-After)`:``)),this.reportEvent({kind:`retry`,requestId:e,provider:this.providerName,model:t,attempt:n,maxRetries:this.maxRetries,delayMs:o,retryAfterHonored:s,error:z(r,i)}),await ve(o,i)}shouldRetry(e,t){if(t?.aborted||e instanceof b&&!e.retryable)return!1;let n=P(e);return!(n!==void 0&&this.nonRetryableStatus.includes(n))}countsTowardBreaker(e){return e.retryable}};function Me(e){return{debug:V(e,`debug`),warn:V(e,`warn`),error:V(e,`error`)}}function V(e,t){let n=e[t].bind(e);return(...e)=>{try{Pe(n(...e))}catch{}}}function Ne(e){return typeof e?.then==`function`}function Pe(e){Ne(e)&&Promise.resolve(e).catch(()=>{})}var Fe=class{constructor(e){this.debugEnabled=e}debug(e){this.debugEnabled&&console.debug(e)}warn(e){console.warn(e)}error(e,t){console.error(e,t??``)}};function Ie(e){let t=e.messages.reduce((e,t)=>{let n=t.content;if(typeof n==`string`)return e+n.length;if(n==null)return e;try{return e+JSON.stringify(n).length}catch{return e}},0);return Math.ceil(t/4)+(e.max_tokens??0)}var H=class{available;lastRefill=Date.now();constructor(e,t){this.capacity=e,this.refillPerMs=t,this.available=e}refill(){if(this.refillPerMs===0)return;let e=Date.now(),t=e-this.lastRefill;this.available=Math.min(this.capacity,this.available+Math.max(0,t)*this.refillPerMs),this.lastRefill=e}tryTake(e){return this.refill(),this.available<e?!1:(this.available-=e,!0)}msUntilAvailable(e){return this.refill(),this.available>=e?0:this.refillPerMs===0?1/0:(e-this.available)/this.refillPerMs}give(e){this.available=Math.min(this.capacity,this.available+e)}getCapacity(){return this.capacity}};const Le=2147483647;var Re=class{requests;tokens;concurrency;maxQueueMs;maxQueueSize;estimateTokensFn;queue=[];wakeTimer;constructor(e){e.requestsPerMinute&&(this.requests=new H(e.requestsPerMinute,e.requestsPerMinute/6e4)),e.tokensPerMinute&&(this.tokens=new H(e.tokensPerMinute,e.tokensPerMinute/6e4)),e.maxConcurrent&&(this.concurrency=new H(e.maxConcurrent,0)),this.maxQueueMs=e.maxQueueMs??3e4,this.maxQueueSize=e.maxQueueSize??0,this.estimateTokensFn=e.estimateTokens??Ie}estimate(e){return this.estimateTokensFn(e)}async acquire(e,t){if(t?.aborted)throw new b(`LLM request aborted`,`aborted`);if(!Number.isFinite(e)||e<0)throw new b(`estimatedTokens must be a finite, non-negative number, got ${String(e)}`,`invalid_params`);if(this.tokens&&e>this.tokens.getCapacity())throw new b(`estimatedTokens (${e}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`,`rate_limited`,{code:`rate_limit_capacity_exceeded`});if(this.queue.length===0){let n=this.tryAcquireBuckets(e);return n.ok?{release:this.makeRelease(e),waitedMs:0}:this.enqueue(e,n.reason,t)}if(this.maxQueueSize>0&&this.queue.length>=this.maxQueueSize)throw this.queueFullError();return this.enqueue(e,void 0,t)}queueFullError(){return new b(`Rate limit queue is full`,`rate_limited`,{code:`rate_limit_queue_full`})}enqueue(e,t,n){return new Promise((r,i)=>{let a={estimatedTokens:e,enqueuedAt:Date.now(),lastReason:t,resolve:e=>{c(),r(e)},reject:e=>{c(),this.wakeTimer&&(clearTimeout(this.wakeTimer),this.wakeTimer=void 0),this.drain(),i(e)}},o,s=()=>{a.reject(new b(`LLM request aborted`,`aborted`))},c=()=>{o&&clearTimeout(o),n?.removeEventListener(`abort`,s);let e=this.queue.indexOf(a);e!==-1&&this.queue.splice(e,1)};this.maxQueueMs>0&&(o=setTimeout(()=>{a.reject(new b(`Rate limit queue timed out before capacity was available`,`rate_limited`,{code:`rate_limit_queue_timeout`}))},this.maxQueueMs)),n?.addEventListener(`abort`,s,{once:!0}),this.queue.push(a),this.drain()})}tryAcquireBuckets(e){let t=[],n=(e,n)=>e?e.tryTake(n)?(t.push({bucket:e,amount:n}),!0):!1:!0;if(!n(this.concurrency,1))return{ok:!1,reason:`concurrency`};if(!n(this.requests,1)){for(let e of t)e.bucket.give(e.amount);return{ok:!1,reason:`rpm`}}if(!n(this.tokens,e)){for(let e of t)e.bucket.give(e.amount);return{ok:!1,reason:`tpm`}}return{ok:!0}}drain(){for(;this.queue.length>0;){let e=this.queue[0],t=this.tryAcquireBuckets(e.estimatedTokens);if(!t.ok){e.lastReason=t.reason,this.scheduleWake(t.reason,e.estimatedTokens);return}let n=Date.now()-e.enqueuedAt;e.resolve({release:this.makeRelease(e.estimatedTokens),waitedMs:n,reason:e.lastReason})}}scheduleWake(e,t){if(this.wakeTimer)return;let n=e===`rpm`?this.requests?.msUntilAvailable(1):e===`tpm`?this.tokens?.msUntilAvailable(t):void 0;if(n===void 0||!Number.isFinite(n))return;let r=Math.min(Math.max(1,Math.ceil(n)),Le);this.wakeTimer=setTimeout(()=>{this.wakeTimer=void 0,this.drain()},r)}makeRelease(e){let t=!1;return n=>{t||(t=!0,this.concurrency?.give(1),this.tokens&&n!==void 0&&Number.isFinite(n)&&this.tokens.give(e-n),this.drain())}}},ze=class{logger;executors;fallbackOn;reportEvent;cacheOrchestrator;constructor(e){this.logger=Me(e.logger??new Fe(e.debug??!1));let t=e.name??`primary`;this.cacheOrchestrator=new ue(e.cache??new S,this.logger),this.fallbackOn=e.fallbackOn??ne,this.reportEvent=k(e.onEvent,this.logger);let n=e.defaultTemperature===void 0?.2:e.defaultTemperature,r=e.defaultReasoningEffort,i=e.defaultBudgetTokens,a={client:e.client,model:e.model,name:t,maxRetries:e.maxRetries,timeoutMs:e.timeoutMs,chunkIdleTimeoutMs:e.chunkIdleTimeoutMs,baseDelayMs:e.baseDelayMs,defaultMaxTokens:e.defaultMaxTokens,defaultTemperature:n,defaultReasoningEffort:r,defaultBudgetTokens:i,nonRetryableStatus:e.nonRetryableStatus,circuitBreaker:e.circuitBreaker,rateLimit:e.rateLimit},o=Array.isArray(e.fallback)?e.fallback:e.fallback?[e.fallback]:[],s=[a,...o];this.executors=s.map((a,o)=>{let s=o>0,c=a.name??(s?`fallback[${o-1}]`:t),l=pe(a.circuitBreaker,c,a.model,e.onEvent,this.logger);return new je(c,a.client,a.model,{maxRetries:a.maxRetries??e.maxRetries??1,timeoutMs:a.timeoutMs??e.timeoutMs??25e3,chunkIdleTimeoutMs:a.chunkIdleTimeoutMs??e.chunkIdleTimeoutMs??3e4,baseDelayMs:a.baseDelayMs??e.baseDelayMs??500,defaultMaxTokens:a.defaultMaxTokens??e.defaultMaxTokens??1e3,defaultTemperature:a.defaultTemperature===void 0?n:a.defaultTemperature,defaultReasoningEffort:a.defaultReasoningEffort===void 0?r:a.defaultReasoningEffort,defaultBudgetTokens:a.defaultBudgetTokens===void 0?i:a.defaultBudgetTokens,nonRetryableStatus:a.nonRetryableStatus??e.nonRetryableStatus??[400,401,403,404,422],parseJson:e.parseJson,logger:this.logger,redact:e.redact,onUsage:e.onUsage,onUsageFailure:e.onUsageFailure,onEvent:e.onEvent,breaker:l,limiter:a.rateLimit?new Re(a.rateLimit):void 0,isFallback:s})})}logRefundError(e,t){this.logger.error(e,{message:t instanceof Error?t.message:`unknown`})}async runFallbackChain(e,t,n,r=!1){let i=[];for(let a=0;a<this.executors.length;a++){let o=this.executors[a],s=Date.now(),c=0;try{a===0&&r||o.assertBreakerClosed(e.model);let t=await n(o,()=>{c+=1});return{result:t,executor:o,index:a,attemptCount:c}}catch(n){let r=z(n,e.signal);i.push({index:a-1,provider:o.providerName,model:e.model??o.model,error:r.toSnapshot()});let c=a===this.executors.length-1,l=this.fallbackOn(r,{isLastTarget:c}),u=c?`stop`:l;if(u===`stop`)throw i.length>1?new E(i):r;let d=this.executors[a+1];this.reportEvent({kind:`fallback`,requestId:t,from:o.providerName,to:d.providerName,fromIndex:a-1,toIndex:a,error:r,elapsedMs:Date.now()-s})}}throw new b(`No provider targets configured`,`invalid_params`)}async call(e){if(e.signal?.aborted)throw new b(`LLM request aborted`,`aborted`);let t=e.requestId??(0,c.randomUUID)(),n=this.executors.length===1;return n&&this.executors[0].assertBreakerClosed(e.model),e.stream?oe(e,async()=>{let{result:r}=await this.runFallbackChain(e,t,(n,r)=>n.runStream(e,t,r),n);return r},e.signal,(e,t)=>this.logRefundError(e,t)):D(e,!1,async()=>{let{result:r,executor:i,index:a,attemptCount:o}=await this.runFallbackChain(e,t,(n,r)=>n.run(e,t,r),n);return e.meta&&(e.meta.current={provider:i.providerName,model:e.model??i.model,fallbackIndex:a-1,usedFallback:a>0,attempts:o}),r},e.signal,(e,t)=>this.logRefundError(e,t))}runCached(e){return this.cacheOrchestrator.runCached(e)}async deleteCache(e){await this.cacheOrchestrator.deleteCache(e)}async cachedCall(e){let{call:t,...n}=e,r=t;if(r.reserveUsage||r.refundUsage)throw new b("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.",`invalid_params`);if(r.stream){let e=r;return this.cacheOrchestrator.runCachedStream({...n,openStream:()=>this.call(e)},!!r.tools)}return this.runCached({...n,fn:()=>this.call(r)})}getCircuitState(e){let t=this.resolveExecutor(e?.index??0,`getCircuitState`);return this.warnIfModelUnsupported(t.isolateByModel,e?.model,`getCircuitState`),t.getCircuitState(e?.model??t.model)}getCircuitStates(e){return this.executors.map((t,n)=>({provider:t.providerName,index:n,isFallback:n>0,isolateByModel:t.isolateByModel,state:t.getCircuitState(e??t.model)}))}openCircuit(e){let t=this.resolveExecutor(e?.index??0,`openCircuit`);this.warnIfModelUnsupported(t.isolateByModel,e?.model,`openCircuit`),t.openCircuit(e?.model??t.model)}closeCircuit(e){let t=this.resolveExecutor(e?.index??0,`closeCircuit`);this.warnIfModelUnsupported(t.isolateByModel,e?.model,`closeCircuit`),t.closeCircuit(e?.model??t.model)}resolveExecutor(e,t){let n=this.executors[e];if(!n)throw RangeError(`${t}: no target at index ${e} (chain has ${this.executors.length} target${this.executors.length===1?``:`s`})`);return n}warnIfModelUnsupported(e,t,n){t!==void 0&&!e&&this.logger.warn(`[VernLLM] ${n}: \`model: '${t}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`)}};function Be(e){return e}function Ve(e){return e}async function*He(e){let t=new TextDecoder(`utf-8`,{fatal:!0}),n=``;for await(let r of e){let e;try{e=typeof r==`string`?r:t.decode(r,{stream:!0})}catch(e){throw new b(`Invalid UTF-8 in SSE stream`,`parse`,{cause:e})}n=(n+e).replace(/\r\n/g,`
2
+ `).replace(/\r(?!$)/g,`
3
+ `);let i=n.indexOf(`
23
4
 
24
- //#endregion
25
- const crypto = __toESM(require("crypto"));
5
+ `);for(;i!==-1;){let e=n.slice(0,i);n=n.slice(i+2);let t=K(e);if(t===U)return;t!==W&&(yield t),i=n.indexOf(`
26
6
 
27
- //#region src/types/errors.ts
28
- /**
29
- * Tool contract codes: a model or provider response defect, not a
30
- * transient provider fault. Deterministic on the wire request, so
31
- * retrying can't change the outcome and it shouldn't count toward the
32
- * circuit breaker either. Shared by `LLMError.retryable` below and by
33
- * `CallExecutor`'s own retry/breaker accounting, so the two can't drift
34
- * apart.
35
- */
36
- const NON_RETRYABLE_TOOL_CONTRACT_CODES = new Set([
37
- "unknown_tool",
38
- "duplicate_tool_call_id",
39
- "tool_choice_none_violated",
40
- "unexpected_tool_calls"
41
- ]);
42
- /**
43
- * Local rate-limit codes: the call never reached the provider, so it says
44
- * nothing about the provider's health, and retrying either just requeues
45
- * behind the same limit (the two queue codes) or can never succeed at all
46
- * (`rate_limit_capacity_exceeded`). Shared for the same reason as
47
- * {@link NON_RETRYABLE_TOOL_CONTRACT_CODES}.
48
- */
49
- const LOCAL_RATE_LIMIT_CODES = new Set([
50
- "rate_limit_queue_full",
51
- "rate_limit_queue_timeout",
52
- "rate_limit_capacity_exceeded"
53
- ]);
54
- /**
55
- * Types that are never worth retrying on their own: deterministic
56
- * caller-input, model-response, or cancellation failures rather than a
57
- * transient provider fault.
58
- */
59
- const NON_RETRYABLE_TYPES = new Set([
60
- "parse",
61
- "validation",
62
- "invalid_params",
63
- "aborted"
64
- ]);
65
- /**
66
- * Shared retryability rule behind both `LLMError.retryable` and
67
- * `LLMErrorSnapshot.retryable`. Pulled out so the two can't drift apart:
68
- * a snapshot is a point-in-time copy of an error's fields, and this is
69
- * one of them, so it has to be computed the same way in both places.
70
- */
71
- function computeRetryable(type, code) {
72
- if (NON_RETRYABLE_TYPES.has(type)) return false;
73
- if (code && NON_RETRYABLE_TOOL_CONTRACT_CODES.has(code)) return false;
74
- if (code && LOCAL_RATE_LIMIT_CODES.has(code)) return false;
75
- return true;
76
- }
77
- /**
78
- * Returns `issues` unchanged when it can survive `JSON.stringify`.
79
- * Most `issues` values are VernLLM's own structured shapes (see
80
- * `LLMErrorIssuesByCode`) and always safe. The one exception is a
81
- * schema validation failure, where `issues` is a caller supplied
82
- * `SchemaLike` validator's own `error: unknown`, not controlled by
83
- * VernLLM and not guaranteed to be circular free. Rather than silently
84
- * dropping it in that case, this returns a marker string so a reader
85
- * of serialized output can tell "no issues data" apart from "issues
86
- * existed but could not be shown".
87
- */
88
- function safeIssues(issues) {
89
- if (issues === void 0) return void 0;
90
- try {
91
- JSON.stringify(issues);
92
- return issues;
93
- } catch {
94
- return "[Unserializable: issues contained a circular reference]";
95
- }
96
- }
97
- /**
98
- * Depth cap for `safeAttempts`, guarding against a pathological,
99
- * self referential `attempts` array. `attempts` is a public
100
- * `LLMErrorOptions` field, so a caller can construct one by hand; this
101
- * keeps that path bounded the same way a circular `issues` value is
102
- * bounded, rather than assuming well formed input.
103
- */
104
- const MAX_ATTEMPTS_DEPTH = 20;
105
- /**
106
- * Returns a copy of `attempts` with every nested snapshot's `issues`
107
- * re-checked through `safeIssues`, recursively through each snapshot's
108
- * own `attempts`. Needed for two reasons: `safeIssues` returns a safe
109
- * `issues` value by reference, so a shared object can be mutated into a
110
- * circular one after the snapshot was created, and `attempts` is a
111
- * public constructor option, so a caller can hand build a `RetryAttempt`
112
- * (or a whole `LLMErrorSnapshot`) with a circular `issues` and pass it
113
- * in directly, never touching `toSnapshot()` at all. Extra fields on an
114
- * attempt (e.g. `FallbackAttempt`'s `provider`/`model`) are preserved.
115
- */
116
- function safeAttempts(attempts, depth = 0) {
117
- if (attempts === void 0) return void 0;
118
- if (depth >= MAX_ATTEMPTS_DEPTH) return [];
119
- return attempts.map((attempt) => ({
120
- ...attempt,
121
- error: {
122
- ...attempt.error,
123
- issues: safeIssues(attempt.error.issues),
124
- attempts: safeAttempts(attempt.error.attempts, depth + 1)
125
- }
126
- }));
127
- }
128
- var LLMError = class extends Error {
129
- status;
130
- issues;
131
- cause;
132
- retryAfterMs;
133
- /** Stable discriminator within `type`. Absent on errors predating it. */
134
- code;
135
- /** Every attempt made before this error was thrown, in order. Absent when nothing was retried. */
136
- attempts;
137
- constructor(message, type, options = {}) {
138
- super(message);
139
- this.type = type;
140
- this.name = "LLMError";
141
- this.status = options.status;
142
- this.issues = options.issues;
143
- this.cause = options.cause;
144
- this.retryAfterMs = options.retryAfterMs;
145
- this.code = options.code;
146
- this.attempts = options.attempts;
147
- }
148
- /**
149
- * Computed purely from `type`/`code`, independent of any specific call's
150
- * `nonRetryableStatus` list. False for `parse`/`validation`/
151
- * `invalid_params`/`aborted` types (the caller's own input, the model's
152
- * own response, or intentional cancellation, none of which are the
153
- * provider being unhealthy), the tool contract codes, and the local
154
- * rate limit codes. Subclasses (see `FallbackExhaustedError`) may
155
- * override this when `type` alone carries no retry signal.
156
- */
157
- get retryable() {
158
- return computeRetryable(this.type, this.code);
159
- }
160
- /**
161
- * Copies this error's fields into an {@link LLMErrorSnapshot}, for
162
- * recording as a `RetryAttempt`/`FallbackAttempt`. `retryable` is
163
- * captured here since a snapshot has no getter of its own. `cause` is
164
- * not copied, see `LLMErrorSnapshot`'s own doc. `issues` and every
165
- * nested `attempts` entry's own `issues` go through `safeAttempts`,
166
- * since a schema validation failure's `issues` is a caller supplied
167
- * value, not controlled by VernLLM, and `attempts` is itself a public
168
- * constructor option a caller can hand build.
169
- */
170
- toSnapshot() {
171
- return {
172
- message: this.message,
173
- type: this.type,
174
- status: this.status,
175
- issues: safeIssues(this.issues),
176
- retryAfterMs: this.retryAfterMs,
177
- code: this.code,
178
- retryable: this.retryable,
179
- attempts: safeAttempts(this.attempts)
180
- };
181
- }
182
- /**
183
- * Controls what `JSON.stringify(err)` produces. Omits `cause` for the
184
- * same reason `toSnapshot()` does: `cause` is `unknown` and never
185
- * validated by VernLLM, and some SDK errors carry circular structures
186
- * `JSON.stringify` cannot serialize at all. Read `err.cause` directly
187
- * instead. `issues`, including every nested `attempts` entry's own
188
- * `issues`, goes through `safeAttempts` for the same reason: a schema
189
- * validation failure's `issues` is caller supplied and not guaranteed
190
- * circular free. Also includes `message` and `retryable`, which a
191
- * plain property walk would otherwise miss: `message` is
192
- * non-enumerable on `Error`, and `retryable` is a getter, not an own
193
- * property.
194
- */
195
- toJSON() {
196
- return {
197
- name: this.name,
198
- message: this.message,
199
- type: this.type,
200
- status: this.status,
201
- issues: safeIssues(this.issues),
202
- retryAfterMs: this.retryAfterMs,
203
- code: this.code,
204
- retryable: this.retryable,
205
- attempts: safeAttempts(this.attempts)
206
- };
207
- }
208
- };
209
- function isLLMError(err) {
210
- return err instanceof LLMError;
211
- }
212
- /**
213
- * Narrows `err.issues` to the exact shape {@link LLMErrorIssuesByCode} maps
214
- * `code` to, for any code listed there. `code` stays the only discriminator
215
- * VernLLM uses; this just gives that existing check a typed return instead
216
- * of requiring a manual cast of `issues`:
217
- *
218
- * ```ts
219
- * if (isLLMError(err) && hasIssues(err, 'duplicate_tool_names')) {
220
- * console.log(err.issues.names); // string[], no cast needed
221
- * }
222
- * ```
223
- */
224
- function hasIssues(err, code) {
225
- return err.code === code && err.issues !== void 0;
226
- }
7
+ `)}}try{n+=t.decode()}catch(e){throw new b(`Invalid UTF-8 in SSE stream`,`parse`,{cause:e})}n=n.replace(/\r$/,`
8
+ `);let r=n.indexOf(`
227
9
 
228
- //#endregion
229
- //#region src/types/cache.ts
230
- /**
231
- * Trivial default so the package works out of the box with no external deps
232
- * Not shared across processes, swap in Redis/Upstash/etc for production
233
- */
234
- var InMemoryCacheAdapter = class {
235
- store = new Map();
236
- constructor(maxSize = 1e3) {
237
- this.maxSize = maxSize;
238
- }
239
- async get(key) {
240
- const entry = this.store.get(key);
241
- if (!entry) return {
242
- hit: false,
243
- value: null
244
- };
245
- if (Date.now() >= entry.expiresAt) {
246
- this.store.delete(key);
247
- return {
248
- hit: false,
249
- value: null
250
- };
251
- }
252
- return {
253
- hit: true,
254
- value: entry.value
255
- };
256
- }
257
- async set(key, value, ttl) {
258
- this.cleanupExpiredEntries();
259
- this.store.set(key, {
260
- value,
261
- expiresAt: Date.now() + ttl * 1e3
262
- });
263
- this.enforceSizeLimit();
264
- }
265
- async delete(key) {
266
- this.store.delete(key);
267
- }
268
- cleanupExpiredEntries() {
269
- const now = Date.now();
270
- for (const [key, entry] of this.store) if (now >= entry.expiresAt) this.store.delete(key);
271
- }
272
- enforceSizeLimit() {
273
- while (this.store.size > this.maxSize) {
274
- const oldestKey = this.store.keys().next().value;
275
- if (oldestKey === void 0) break;
276
- this.store.delete(oldestKey);
277
- }
278
- }
279
- };
280
- /**
281
- * Normalizes keys before caching to avoid duplicate entries from formatting differences.
282
- */
283
- var NormalizedCacheAdapter = class {
284
- constructor(inner = new InMemoryCacheAdapter()) {
285
- this.inner = inner;
286
- }
287
- normalize(key) {
288
- return key.toLowerCase().trim().replace(/[^\p{L}\p{N}\s]/gu, " ").replace(/\s+/g, " ").trim();
289
- }
290
- async resolveKey(key) {
291
- return this.normalize(key);
292
- }
293
- async get(key) {
294
- return this.inner.get(this.normalize(key));
295
- }
296
- async set(key, value, ttl) {
297
- await this.inner.set(this.normalize(key), value, ttl);
298
- }
299
- async delete(key) {
300
- await this.inner.delete?.(this.normalize(key));
301
- }
302
- };
303
- /**
304
- * Two-tier cache with fast local L1 and shared L2.
305
- * L2 hits are promoted back to L1.
306
- */
307
- var TieredCacheAdapter = class {
308
- constructor(l1, l2, l1Ttl) {
309
- this.l1 = l1;
310
- this.l2 = l2;
311
- this.l1Ttl = l1Ttl;
312
- }
313
- /**
314
- * Forwards to L1's `resolveKey` if it has one, otherwise L2's. L1 is
315
- * preferred since `get()` checks L1 first, so its notion of "the same
316
- * key" is the one that determines whether a lookup can skip L2 entirely.
317
- */
318
- async resolveKey(key) {
319
- if (this.l1.resolveKey) return this.l1.resolveKey(key);
320
- if (this.l2.resolveKey) return this.l2.resolveKey(key);
321
- return key;
322
- }
323
- async get(key) {
324
- const l1Result = await this.l1.get(key);
325
- if (l1Result.hit) return l1Result;
326
- const l2Result = await this.l2.get(key);
327
- if (l2Result.hit) await this.l1.set(key, l2Result.value, this.l1Ttl ?? 60);
328
- return l2Result;
329
- }
330
- async set(key, value, ttl) {
331
- await Promise.all([this.l1.set(key, value, this.l1Ttl ?? ttl), this.l2.set(key, value, ttl)]);
332
- }
333
- async delete(key) {
334
- await Promise.all([this.l1.delete?.(key), this.l2.delete?.(key)]);
335
- }
336
- };
10
+ `);for(;r!==-1;){let e=n.slice(0,r);n=n.slice(r+2);let t=K(e);if(t===U)return;t!==W&&(yield t),r=n.indexOf(`
337
11
 
338
- //#endregion
339
- //#region src/types/tools.ts
340
- /**
341
- * Runtime-safe check for whether a `call()` result is a `tool_calls`
342
- * result. Prefer this over relying on TypeScript's static narrowing
343
- * whenever `params` passed to `call()` wasn't a literal with `tools`
344
- * inlined (see the "note on the overload" in `VernLLM.call`'s docs), in
345
- * that case TS may have typed the result as plain `T` even though it's
346
- * actually a `CallWithToolsResult<T>` at runtime, and this check works
347
- * either way.
348
- */
349
- function isToolCallResult(result) {
350
- return typeof result === "object" && result !== null && "type" in result && result.type === "tool_calls" && Array.isArray(result.toolCalls);
351
- }
352
-
353
- //#endregion
354
- //#region src/types/fallback.ts
355
- /** Tool contract failures are the model ignoring the request, not a sick provider: repeating it elsewhere can't help. */
356
- const TOOL_CONTRACT_CODES = new Set([
357
- "unknown_tool",
358
- "duplicate_tool_call_id",
359
- "tool_choice_none_violated",
360
- "unexpected_tool_calls"
361
- ]);
362
- /**
363
- * The default `fallbackOn` policy. Exported so a caller can wrap rather
364
- * than replace it, e.g. `fallbackOn: (e, ctx) => myCheck(e) ? 'stop' : defaultFallbackOn(e, ctx)`.
365
- */
366
- const defaultFallbackOn = (error) => {
367
- if (error.type === "parse" || error.type === "validation" || error.type === "aborted") return "stop";
368
- if (error.type === "quota_exceeded") return "stop";
369
- if (error.code && TOOL_CONTRACT_CODES.has(error.code)) return "stop";
370
- return "next";
371
- };
372
- /**
373
- * Thrown when the chain gives up, whether because the last target failed
374
- * or `fallbackOn` chose to stop early. Carries each attempt in order so
375
- * an outage across providers stays debuggable without reproducing it.
376
- * Extends `LLMError` so `isLLMError` and any `instanceof LLMError` check
377
- * still passes, inheriting the last failure's `type`/`status`/`retryAfterMs`
378
- * so existing type-based handling, including reading `retryAfterMs` on an
379
- * `'api'`-typed error, keeps working on a fallback-exhausted error too.
380
- */
381
- var FallbackExhaustedError = class extends LLMError {
382
- constructor(attempts) {
383
- const last = attempts[attempts.length - 1]?.error;
384
- super(`${attempts.length} provider${attempts.length === 1 ? "" : "s"} attempted and failed: ${attempts.map((a) => `${a.provider}(${a.error.type})`).join(" then ")}`, "fallback_exhausted", {
385
- status: last?.status,
386
- cause: last,
387
- retryAfterMs: last?.retryAfterMs,
388
- code: "fallback_exhausted",
389
- attempts
390
- });
391
- this.attempts = attempts;
392
- }
393
- /**
394
- * `type: 'fallback_exhausted'` by itself says nothing about whether
395
- * retrying could help; the reason the last target failed does. Defers to
396
- * that attempt's own `retryable` instead of anything about this class's
397
- * own type.
398
- */
399
- get retryable() {
400
- const last = this.attempts[this.attempts.length - 1]?.error;
401
- return last ? last.retryable : super.retryable;
402
- }
403
- };
404
- /** Narrows `err` to {@link FallbackExhaustedError}, for direct access to its `attempts` (`provider`/`model` per failed target) without a manual `instanceof` check. */
405
- function isFallbackExhaustedError(err) {
406
- return err instanceof FallbackExhaustedError;
407
- }
408
-
409
- //#endregion
410
- //#region src/internal/execution/usage.utils.ts
411
- /**
412
- * Calls `params.reserveUsage`, if present, mapping any failure to a
413
- * `quota_exceeded` LLMError (or an aborted error, if the signal fired
414
- * during reservation). Returns whether a reservation was actually made,
415
- * so callers know whether a later refund is needed. Shared by
416
- * `withReservedUsage` and `withReservedUsageForStream`, which differ only
417
- * in whether `coalesced` is caller-supplied or always `false`.
418
- */
419
- async function reserve(params, coalesced, signal) {
420
- if (!params.reserveUsage) return false;
421
- try {
422
- await params.reserveUsage({
423
- coalesced,
424
- signal
425
- });
426
- return true;
427
- } catch (error) {
428
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
429
- throw new LLMError(error instanceof Error ? error.message : "Usage reservation failed", "quota_exceeded", { cause: error });
430
- }
431
- }
432
- /**
433
- * Builds a `(logMessage) => Promise<void>` refund function bound to the
434
- * given hooks/coalesced/signal, reporting (instead of throwing) any error
435
- * the refund hook itself raises, so a broken refund hook never masks the
436
- * original error it was called to clean up after.
437
- */
438
- function makeRefund(params, coalesced, signal, onRefundError) {
439
- return async (logMessage) => {
440
- try {
441
- await params.refundUsage?.({
442
- coalesced,
443
- signal
444
- });
445
- } catch (refundError) {
446
- onRefundError(logMessage, refundError);
447
- }
448
- };
449
- }
450
- /**
451
- * Runs `getResult` after reserving usage, if a `reserveUsage` hook was
452
- * provided. `refundUsage` fires only if a reservation was actually made.
453
- * `onRefundError` is called (instead of throwing) whenever a refund attempt
454
- * itself fails, so a broken refund hook never masks the original error.
455
- */
456
- async function withReservedUsage(params, coalesced, getResult, signal, onRefundError) {
457
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
458
- const reserved = await reserve(params, coalesced, signal);
459
- const refund = makeRefund(params, coalesced, signal, onRefundError);
460
- if (signal?.aborted) {
461
- if (reserved) await refund("[VernLLM] refundUsage failed after abort");
462
- throw new LLMError("LLM request aborted", "aborted");
463
- }
464
- let result;
465
- try {
466
- result = await getResult();
467
- } catch (error) {
468
- if (reserved) await refund("[VernLLM] refundUsage failed");
469
- throw error;
470
- }
471
- if (signal?.aborted) {
472
- if (reserved) await refund("[VernLLM] refundUsage failed after abort");
473
- throw new LLMError("LLM request aborted", "aborted");
474
- }
475
- return result;
476
- }
477
- /**
478
- * Streaming counterpart to `withReservedUsage`. `withReservedUsage` assumes
479
- * `getResult()` settling *is* the operation's final outcome, awaiting it
480
- * synchronously before reserve/refund resolve. Streaming can't satisfy that:
481
- * `call()` must return `{ chunks, finalResult }` as soon as the stream
482
- * opens, well before the real outcome (validation, schema/tool-call checks)
483
- * is known.
484
- *
485
- * Reserves usage before `openStream` runs, same failure mode as the
486
- * non-streaming path if `reserveUsage` itself throws (mapped to
487
- * `quota_exceeded`, nothing opened). If `openStream` itself throws (stream
488
- * never opened), refunds synchronously and rethrows, exactly like
489
- * `withReservedUsage` does today. If it succeeds, returns `{ chunks,
490
- * finalResult }` immediately, refund/report is deferred onto
491
- * `finalResult`'s continuation, since that's the only point the real
492
- * outcome is known. This means `onUsageFailure` (and any refund) can fire
493
- * well after this function itself has returned.
494
- */
495
- async function withReservedUsageForStream(params, openStream, signal, onRefundError) {
496
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
497
- const reserved = await reserve(params, false, signal);
498
- const refund = makeRefund(params, false, signal, onRefundError);
499
- if (signal?.aborted) {
500
- if (reserved) await refund("[VernLLM] refundUsage failed after abort");
501
- throw new LLMError("LLM request aborted", "aborted");
502
- }
503
- let opened;
504
- try {
505
- opened = await openStream();
506
- } catch (error) {
507
- if (reserved) await refund("[VernLLM] refundUsage failed after stream-open failure");
508
- throw error;
509
- }
510
- const finalResult = opened.finalResult.then((value) => value, async (error) => {
511
- if (reserved) await refund("[VernLLM] refundUsage failed after stream error");
512
- throw error;
513
- });
514
- finalResult.catch(() => {});
515
- return {
516
- chunks: opened.chunks,
517
- finalResult
518
- };
519
- }
520
-
521
- //#endregion
522
- //#region src/internal/cache/replay.utils.ts
523
- /**
524
- * Converts an already-known cache value back into a plausible "text" form
525
- * for a one-shot replay chunk: passed through unchanged if it's already a
526
- * string (the `jsonMode: false` case), otherwise `JSON.stringify`'d (the
527
- * `jsonMode: true` case, where the cached value is the *parsed* result, not
528
- * the original raw text). This is a reasonable reconstruction, not a
529
- * byte-identical replay of whatever text the model originally streamed,
530
- * good enough for `for await (const c of chunks)` call sites that don't
531
- * branch on hit vs. miss, which is the only thing a cache-hit replay needs
532
- * to support.
533
- */
534
- function toReplayText(value) {
535
- return typeof value === "string" ? value : JSON.stringify(value) ?? "";
536
- }
537
- /**
538
- * Builds a trivially-exhausted one-shot `chunks` iterable from an
539
- * already-known value, used for a `cachedCall` cache hit, where there's no
540
- * live generation to relay (see `VernLLM.cachedCall`'s docs). No `usage`
541
- * chunk is emitted: a cache hit spent no real tokens, so there's nothing to
542
- * report, matching how non-streaming `cachedCall` never calls `onUsage` on
543
- * a hit either.
544
- *
545
- * `hasTools` must reflect whether the *original* call that produced this
546
- * cached value had `tools` set, that's what determines whether `value` is
547
- * `T` directly or a `CallWithToolsResult<T>` wrapper, and it isn't
548
- * something that can be reliably guessed from the value's shape alone
549
- * (a `schema`-validated `T` could coincidentally look like a
550
- * `CallWithToolsResult`).
551
- */
552
- function buildReplayChunks(value, hasTools) {
553
- const items = [];
554
- if (hasTools) {
555
- const result = value;
556
- if (result.type === "tool_calls") {
557
- result.toolCalls.forEach((toolCall, index) => {
558
- items.push({
559
- type: "tool_call_delta",
560
- index,
561
- id: toolCall.id,
562
- name: toolCall.name,
563
- argsDelta: JSON.stringify(toolCall.arguments ?? {}),
564
- complete: true
565
- });
566
- });
567
- if (result.content) items.push({
568
- type: "text-delta",
569
- delta: result.content
570
- });
571
- } else items.push({
572
- type: "text-delta",
573
- delta: toReplayText(result.content)
574
- });
575
- } else items.push({
576
- type: "text-delta",
577
- delta: toReplayText(value)
578
- });
579
- return { async *[Symbol.asyncIterator]() {
580
- for (const item of items) yield item;
581
- } };
582
- }
583
- /**
584
- * Streaming counterpart to `buildReplayChunks` for a `cachedCall` that
585
- * *joined* an already-in-flight call for the same key rather than
586
- * triggering one itself (see `runCachedStream`'s in-flight-coalescing
587
- * path): there's no live stream to relay (it isn't this call's stream to
588
- * relay, see the joiner-path comment in `runCachedStream`), but there's
589
- * also no value yet, only a pending promise for one. Waits for `promise`,
590
- * then delegates to `buildReplayChunks`. If `promise` rejects, iterating
591
- * `chunks` throws that same error, consistent with how a live stream's
592
- * `chunks` throws on a mid-stream failure.
593
- */
594
- function buildReplayChunksFromPromise(promise, hasTools) {
595
- return { async *[Symbol.asyncIterator]() {
596
- const value = await promise;
597
- yield* buildReplayChunks(value, hasTools);
598
- } };
599
- }
600
-
601
- //#endregion
602
- //#region src/internal/cache/cacheOrchestrator.ts
603
- /**
604
- * Owns cache key resolution, cache reads/writes, and in-flight coalescing
605
- * for concurrent misses on the same key. Doesn't know about `CallExecutor`,
606
- * retries, or providers at all: `fn`/`openStream` are opaque callbacks
607
- * (`VernLLM.cachedCall` passes `() => this.call(...)`), so this class only
608
- * needs the cache adapter and a logger. Extracted from `VernLLM` since
609
- * caching and per-target call mechanics are independent concerns that
610
- * happened to live on the same class.
611
- */
612
- var CacheOrchestrator = class {
613
- inFlight = new Map();
614
- constructor(cache, logger) {
615
- this.cache = cache;
616
- this.logger = logger;
617
- }
618
- /** Resolves a cache key through the adapter when it supports normalization. */
619
- async resolveCacheKey(key) {
620
- return this.cache.resolveKey ? await this.cache.resolveKey(key) : key;
621
- }
622
- /**
623
- * Removes a cached response by key when the configured cache adapter
624
- * supports deletion. Cache invalidation is the caller's responsibility;
625
- * only the application knows when cached data is stale.
626
- */
627
- async deleteCache(key) {
628
- if (!this.cache.delete) return;
629
- try {
630
- await this.cache.delete(await this.resolveCacheKey(key));
631
- } catch (error) {
632
- this.logger.warn(`[VernLLM] cache delete failed: ${error instanceof Error ? error.message : "unknown"}`);
633
- }
634
- }
635
- /** Logs a failed refundUsage attempt via the configured logger. */
636
- logRefundError(logMessage, error) {
637
- this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
638
- }
639
- /**
640
- * Reads from the cache, treating a failed adapter read as a miss rather
641
- * than letting it fail the call. The request still falls through to a
642
- * real provider call, but that fallback is now logged instead of silent.
643
- */
644
- async getCached(key) {
645
- try {
646
- return await this.cache.get(key);
647
- } catch (error) {
648
- this.logger.warn(`[VernLLM] cache read failed: ${error instanceof Error ? error.message : "unknown"}`);
649
- return { hit: false };
650
- }
651
- }
652
- /**
653
- * Internal cache primitive around caller-supplied logic. Concurrent misses
654
- * for the same `cacheKey` share a single in-flight call, avoiding cache
655
- * stampedes.
656
- *
657
- * Backs the public `VernLLM.cachedCall()`, which always composes this
658
- * with `call()` so cached results get the same retry/timeout/
659
- * circuit-breaker guarantees as any other LLM call.
660
- *
661
- * @param params `cacheKey`, `ttl`, `fn` (the work to run on a cache
662
- * miss, typically `() => this.call(...)`), and optional
663
- * `reserveUsage`/`refundUsage`/`signal`. See `InternalCacheParams`.
664
- * @returns The cached value on a hit, or the result of `fn()` on a miss.
665
- */
666
- async runCached(params) {
667
- const resolvedKey = await this.resolveCacheKey(params.cacheKey);
668
- const resolvedParams = resolvedKey === params.cacheKey ? params : {
669
- ...params,
670
- cacheKey: resolvedKey
671
- };
672
- const cached = await this.getCached(resolvedKey);
673
- if (cached.hit) return cached.value;
674
- const existing = this.inFlight.get(resolvedKey);
675
- if (existing) return withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
676
- return this.registerTrigger(resolvedParams);
677
- }
678
- /** Starts the shared fn() call for a cache miss and tracks it in the in-flight map until it settles. */
679
- registerTrigger(params) {
680
- const resultPromise = withReservedUsage(params, false, () => this.runAndCache(params), params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
681
- this.inFlight.set(params.cacheKey, resultPromise);
682
- resultPromise.catch(() => {}).finally(() => {
683
- this.inFlight.delete(params.cacheKey);
684
- });
685
- return resultPromise;
686
- }
687
- /** Runs `fn` and writes its result to the cache. */
688
- async runAndCache(params) {
689
- const result = await params.fn();
690
- try {
691
- await this.cache.set(params.cacheKey, result, params.ttl);
692
- } catch (error) {
693
- this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
694
- }
695
- return result;
696
- }
697
- /**
698
- * Streaming counterpart to `runCached`. Three cases:
699
- *
700
- * - Hit: no live generation to relay. Returns immediately with
701
- * `finalResult` resolved to the cached value and a one-shot `chunks`
702
- * replay built from it, so `for await (const c of chunks)` call sites
703
- * work identically on a hit or a miss. No usage hooks fire, since
704
- * nothing was actually spent.
705
- * - Miss, nothing else in flight for this key: delegates to
706
- * `registerStreamTrigger`, which opens the stream and relays its
707
- * `chunks` live.
708
- * - Miss, but another call for the same key is already in flight: this
709
- * call has no live chunks of its own to relay, so it's treated like a
710
- * delayed hit. `finalResult` shares the trigger's in-flight promise
711
- * (the same in-flight map non-streaming `runCached` uses, so
712
- * streaming and non-streaming calls for the same key coalesce
713
- * against each other too), and `chunks` is a one-shot replay built
714
- * once that promise resolves.
715
- */
716
- async runCachedStream(params, hasTools) {
717
- const resolvedKey = await this.resolveCacheKey(params.cacheKey);
718
- const resolvedParams = resolvedKey === params.cacheKey ? params : {
719
- ...params,
720
- cacheKey: resolvedKey
721
- };
722
- const cached = await this.getCached(resolvedKey);
723
- if (cached.hit) {
724
- const value = cached.value;
725
- return {
726
- chunks: buildReplayChunks(value, hasTools),
727
- finalResult: Promise.resolve(value)
728
- };
729
- }
730
- const existing = this.inFlight.get(resolvedKey);
731
- if (existing) {
732
- const finalResult = withReservedUsage(resolvedParams, true, () => existing, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
733
- finalResult.catch(() => {});
734
- return {
735
- chunks: buildReplayChunksFromPromise(finalResult, hasTools),
736
- finalResult
737
- };
738
- }
739
- return this.registerStreamTrigger(resolvedParams);
740
- }
741
- /**
742
- * Opens the shared stream for a cache miss and tracks its settled value
743
- * in the in-flight map until it resolves or rejects. Writes to the cache
744
- * on success only, matching `runAndCache`.
745
- *
746
- * Registers the in-flight promise synchronously, before anything async
747
- * runs, so a concurrent `cachedCall` for the same key always sees it in
748
- * time to join instead of triggering its own stream. Settlement is
749
- * wired onto the whole `withReservedUsageForStream` call rather than a
750
- * line inside its callback, so any failure point (reserving usage,
751
- * opening the stream, or the stream itself) reliably settles the
752
- * in-flight entry instead of leaving it stuck.
753
- */
754
- registerStreamTrigger(params) {
755
- let resolveInFlight;
756
- let rejectInFlight;
757
- const inFlightResult = new Promise((resolve, reject) => {
758
- resolveInFlight = resolve;
759
- rejectInFlight = reject;
760
- });
761
- this.inFlight.set(params.cacheKey, inFlightResult);
762
- inFlightResult.catch(() => {}).finally(() => {
763
- this.inFlight.delete(params.cacheKey);
764
- });
765
- const streamPromise = withReservedUsageForStream(params, async () => {
766
- const opened = await params.openStream();
767
- const trackedResult = opened.finalResult.then(async (value) => {
768
- try {
769
- await this.cache.set(params.cacheKey, value, params.ttl);
770
- } catch (error) {
771
- this.logger.warn(`[VernLLM] cache write failed: ${error instanceof Error ? error.message : "unknown"}`);
772
- }
773
- return value;
774
- }, (error) => {
775
- throw error;
776
- });
777
- return {
778
- chunks: opened.chunks,
779
- finalResult: trackedResult
780
- };
781
- }, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
782
- streamPromise.then((opened) => {
783
- opened.finalResult.then(resolveInFlight, rejectInFlight);
784
- }, (error) => {
785
- rejectInFlight(error);
786
- });
787
- return streamPromise;
788
- }
789
- };
790
-
791
- //#endregion
792
- //#region src/circuitBreaker.ts
793
- function newBucket() {
794
- return {
795
- state: "closed",
796
- consecutiveFailures: 0,
797
- openedAt: 0,
798
- trialInFlight: false
799
- };
800
- }
801
- /** Key a bucket lookup falls into when the call omitted `model` under `isolateByModel`. */
802
- const UNLABELED_MODEL = "";
803
- /**
804
- * Per retry VernLLM-instance circuit breaker. Tracks consecutive failures across
805
- * calls. Once the threshold is hit, short-circuits new calls with an
806
- * LLMError('circuit_open') instead of hitting the provider, until the
807
- * cooldown elapses and a single trial call is allowed through
808
- */
809
- var CircuitBreaker = class {
810
- threshold;
811
- cooldownMs;
812
- onStateChange;
813
- /** Whether this breaker tracks failures per model instead of one shared circuit. Read by `CallExecutor`/`VernLLM` to report per-target in `getCircuitStates`. */
814
- isolateByModel;
815
- sharedBucket = newBucket();
816
- bucketsByModel = new Map();
817
- constructor(options = {}) {
818
- this.threshold = options.threshold ?? 5;
819
- this.cooldownMs = options.cooldownMs ?? 3e4;
820
- this.onStateChange = options.onStateChange;
821
- this.isolateByModel = options.isolateByModel ?? false;
822
- }
823
- /** Returns the bucket for a model if one already exists, without allocating. */
824
- lookupBucket(model) {
825
- if (!this.isolateByModel) return this.sharedBucket;
826
- const key = model ?? UNLABELED_MODEL;
827
- return this.bucketsByModel.get(key);
828
- }
829
- /** Creates and stores a bucket for a model when the first mutation needs one. */
830
- ensureBucketFor(model) {
831
- if (!this.isolateByModel) return this.sharedBucket;
832
- const key = model ?? UNLABELED_MODEL;
833
- let bucket = this.bucketsByModel.get(key);
834
- if (!bucket) {
835
- bucket = newBucket();
836
- this.bucketsByModel.set(key, bucket);
837
- }
838
- return bucket;
839
- }
840
- /** Every state mutation routes through here, so `onStateChange` fires exactly once per real change. */
841
- transition(bucket, to, model) {
842
- if (to === bucket.state) return;
843
- const from = bucket.state;
844
- bucket.state = to;
845
- this.onStateChange?.(from, to, bucket.consecutiveFailures, model);
846
- }
847
- /**
848
- * Throws if the circuit is open and the cooldown hasn't elapsed, or if
849
- * the circuit is half-open and a trial call is already in flight.
850
- * Otherwise, if the circuit just became eligible for a trial (cooldown
851
- * elapsed, or half-open with no trial currently running), this call
852
- * becomes that trial
853
- */
854
- assertClosed(model) {
855
- const bucket = this.ensureBucketFor(model);
856
- if (bucket.state === "closed") return;
857
- if (bucket.state === "open") {
858
- const elapsed = Date.now() - bucket.openedAt;
859
- if (elapsed < this.cooldownMs) throw new LLMError(`Circuit open, provider has failed ${bucket.consecutiveFailures} times in a row. Retry in ${Math.ceil((this.cooldownMs - elapsed) / 1e3)}s.`, "circuit_open", { code: "circuit_cooling_down" });
860
- bucket.trialInFlight = true;
861
- this.transition(bucket, "half-open", model);
862
- return;
863
- }
864
- if (bucket.trialInFlight) throw new LLMError("Circuit half-open. A trial request is already in flight. Try again shortly.", "circuit_open", { code: "circuit_trial_in_flight" });
865
- bucket.trialInFlight = true;
866
- }
867
- recordSuccess(model) {
868
- const bucket = this.lookupBucket(model);
869
- if (!bucket) return;
870
- bucket.consecutiveFailures = 0;
871
- bucket.trialInFlight = false;
872
- this.transition(bucket, "closed", model);
873
- if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
874
- }
875
- recordFailure(model) {
876
- const bucket = this.ensureBucketFor(model);
877
- bucket.consecutiveFailures += 1;
878
- bucket.trialInFlight = false;
879
- if (bucket.state === "half-open") {
880
- bucket.openedAt = Date.now();
881
- this.transition(bucket, "open", model);
882
- return;
883
- }
884
- if (bucket.consecutiveFailures >= this.threshold) {
885
- bucket.openedAt = Date.now();
886
- this.transition(bucket, "open", model);
887
- }
888
- }
889
- /**
890
- * With `isolateByModel` off (the default), `model` is ignored and the
891
- * one shared circuit's state is returned, unchanged from every version
892
- * before this option existed. With `isolateByModel` on, returns that
893
- * model's own state, `'closed'` for a model never seen yet, same as a
894
- * fresh breaker.
895
- */
896
- getState(model) {
897
- return this.lookupBucket(model)?.state ?? "closed";
898
- }
899
- /**
900
- * Manually opens the circuit, as if `threshold` consecutive failures had
901
- * just happened, e.g. to pull a provider out of rotation ahead of known
902
- * maintenance. Resets the cooldown window from now, same as a real
903
- * threshold-crossing failure would, and clears any in-flight half-open
904
- * trial since it no longer applies once the circuit is (re)opened.
905
- */
906
- open(model) {
907
- const bucket = this.ensureBucketFor(model);
908
- bucket.openedAt = Date.now();
909
- bucket.trialInFlight = false;
910
- this.transition(bucket, "open", model);
911
- }
912
- /**
913
- * Manually closes the circuit and resets its failure count, e.g. once a
914
- * provider is confirmed healthy again without waiting out the cooldown.
915
- * Mirrors `recordSuccess`'s bookkeeping (including dropping the
916
- * per-model bucket under `isolateByModel`, once idle) but without
917
- * requiring an actual successful call first.
918
- */
919
- close(model) {
920
- const bucket = this.ensureBucketFor(model);
921
- bucket.consecutiveFailures = 0;
922
- bucket.trialInFlight = false;
923
- this.transition(bucket, "closed", model);
924
- if (this.isolateByModel && bucket.state === "closed" && bucket.consecutiveFailures === 0) this.bucketsByModel.delete(model ?? UNLABELED_MODEL);
925
- }
926
- };
927
-
928
- //#endregion
929
- //#region src/internal/circuitBreaker.utils.ts
930
- /**
931
- * Builds a `(event) => void` reporter that no-ops when `onEvent` is unset,
932
- * and otherwise calls it, swallowing and logging any error the handler
933
- * throws so a broken `onEvent` can't break the call that triggered it.
934
- * Shared by `buildCircuitBreaker` (which needs to report before any
935
- * executor exists) and `CallExecutor.reportEvent`, kept independent of the
936
- * executor for that reason.
937
- */
938
- function makeEventReporter(onEvent, logger) {
939
- return (event) => {
940
- if (!onEvent) return;
941
- try {
942
- onEvent(event);
943
- } catch (error) {
944
- logger.error("[VernLLM] onEvent failed", { message: error instanceof Error ? error.message : "unknown" });
945
- }
946
- };
947
- }
948
- /**
949
- * Builds the optional circuit breaker for one provider target, wiring its
950
- * `onStateChange` to emit a `circuit_state` event and chain any
951
- * caller-supplied `onStateChange`. Returns `undefined` when
952
- * `circuitBreakerOption` is falsy, matching the option's own semantics.
953
- *
954
- * Lives outside `CallExecutor` (and outside `VernLLM`, once this were
955
- * inlined) because the breaker has to exist *before* the executor it's
956
- * passed into, so its construction can't be an executor concern.
957
- * `onEvent` is called directly rather than through the executor for the
958
- * same reason: nothing executor-shaped exists yet at this point.
959
- *
960
- * Takes the specific fields it needs (rather than a full `VernLLMOptions`)
961
- * so it works identically for the primary target and for each fallback
962
- * target, which carry their own `circuitBreaker` override alongside the
963
- * shared `onEvent`.
964
- */
965
- function buildCircuitBreaker(circuitBreakerOption, providerName, defaultModel, onEvent, logger) {
966
- if (!circuitBreakerOption) return void 0;
967
- const breakerOptions = typeof circuitBreakerOption === "object" ? circuitBreakerOption : void 0;
968
- const userOnStateChange = breakerOptions?.onStateChange;
969
- const reportEvent = makeEventReporter(onEvent, logger);
970
- return new CircuitBreaker({
971
- ...breakerOptions,
972
- onStateChange: (from, to, consecutiveFailures, model) => {
973
- reportEvent({
974
- kind: "circuit_state",
975
- provider: providerName,
976
- model: model ?? defaultModel,
977
- from,
978
- to,
979
- consecutiveFailures
980
- });
981
- if (!userOnStateChange) return;
982
- try {
983
- userOnStateChange(from, to, consecutiveFailures, model);
984
- } catch (error) {
985
- logger.error("[VernLLM] circuitBreaker.onStateChange failed", { message: error instanceof Error ? error.message : "unknown" });
986
- }
987
- }
988
- });
989
- }
990
-
991
- //#endregion
992
- //#region src/internal/execution/retry.utils.ts
993
- /**
994
- * Default cap (ms) for both exponential backoff and honored Retry-After
995
- * values, so a misbehaving/adversarial Retry-After can't stall a caller
996
- * indefinitely
997
- */
998
- const DEFAULT_MAX_DELAY_MS = 1e4;
999
- /**
1000
- * `setTimeout` silently clamps any delay above this (~24.8 days) or
1001
- * `Infinity` down to ~1ms instead of erroring, so a caller passing
1002
- * `Infinity` as "no timeout" gets the opposite of what they asked for.
1003
- * Both timeout helpers below guard against this explicitly.
1004
- */
1005
- const MAX_SETTIMEOUT_MS = 2147483647;
1006
- /**
1007
- * Resolves a timeout value to the number `setTimeout` should actually use,
1008
- * or `undefined` when the timeout should be treated as disabled (0,
1009
- * negative, or `Infinity`). Returning the resolved value directly, rather
1010
- * than a boolean, lets callers narrow `number | undefined` to `number`
1011
- * without an `as number` cast.
1012
- */
1013
- function resolveActiveTimeoutMs(ms) {
1014
- return !ms || ms <= 0 || ms === Infinity ? void 0 : ms;
1015
- }
1016
- /** Caps a timeout at the largest delay `setTimeout` actually honors. */
1017
- function clampTimeoutMs(ms) {
1018
- return Math.min(ms, MAX_SETTIMEOUT_MS);
1019
- }
1020
- /**
1021
- * Runs an async function and cancels it if it takes longer than the given
1022
- * timeout. Creates an internal abort controller that fires after the
1023
- * timeout elapses, and combines it with any external signal the caller
1024
- * passed in so either one can cancel the underlying call. If the internal
1025
- * timeout triggers and the underlying operation aborts, the error is
1026
- * converted into an LLMError with type "timeout". External cancellations
1027
- * continue to propagate as aborted errors. The internal timer is always
1028
- * cleared afterward, whether the function succeeds, fails, or is aborted,
1029
- * so nothing is left running in the background.
1030
- *
1031
- * `timeoutMs` of `Infinity` (or any value beyond what `setTimeout` can
1032
- * represent) disables the timeout rather than firing almost immediately.
1033
- */
1034
- async function withTimeout(fn, timeoutMs, externalSignal) {
1035
- const controller = new AbortController();
1036
- const activeTimeoutMs = resolveActiveTimeoutMs(timeoutMs);
1037
- const timer = activeTimeoutMs === void 0 ? void 0 : setTimeout(() => {
1038
- controller.abort();
1039
- }, clampTimeoutMs(activeTimeoutMs));
1040
- const signal = externalSignal ? AbortSignal.any([externalSignal, controller.signal]) : controller.signal;
1041
- try {
1042
- return await fn(signal);
1043
- } catch (err) {
1044
- if (controller.signal.aborted && !externalSignal?.aborted && err instanceof DOMException && err.name === "AbortError") throw new LLMError("Request timed out", "timeout", { code: "request_timeout" });
1045
- throw err;
1046
- } finally {
1047
- clearTimeout(timer);
1048
- }
1049
- }
1050
- /**
1051
- * Races one `iterator.next()` call against a per-call idle timer, to
1052
- * bound the gap *between* chunks (unlike `withTimeout`, which only bounds
1053
- * opening the stream and its first chunk). Without this, a connection
1054
- * that streams one chunk then hangs would never fail.
1055
- *
1056
- * `timeoutMs` of 0/undefined/`Infinity` disables the check. Otherwise
1057
- * rejects with `LLMError('timeout')` if `next()` doesn't settle in time.
1058
- * The clock resets on every call, so the window is measured from the most
1059
- * recent chunk, not from stream start.
1060
- *
1061
- * `onIdle`, if given, is called the moment the timer fires (before the
1062
- * rejection), so callers can abort the underlying transport instead of
1063
- * just walking away from an unread promise. `logger`, if given, records a
1064
- * debug line if `next()` still settles *after* the idle timeout already
1065
- * rejected. `resolve`/`reject` on an already-settled promise is otherwise
1066
- * a silent no-op, so without this the late chunk (possibly the final
1067
- * usage chunk) would vanish with no trace.
1068
- */
1069
- function withChunkIdleTimeout(next, timeoutMs, onIdle, logger) {
1070
- const activeTimeoutMs = resolveActiveTimeoutMs(timeoutMs);
1071
- if (activeTimeoutMs === void 0) return next();
1072
- let settled = false;
1073
- return new Promise((resolve, reject) => {
1074
- const timer = setTimeout(() => {
1075
- settled = true;
1076
- onIdle?.();
1077
- reject(new LLMError(`No stream chunk received for ${activeTimeoutMs}ms (idle timeout)`, "timeout", { code: "idle_timeout" }));
1078
- }, clampTimeoutMs(activeTimeoutMs));
1079
- next().then((result) => {
1080
- clearTimeout(timer);
1081
- if (settled) {
1082
- logger?.debug("[VernLLM] chunk resolved after idle timeout already fired; discarding");
1083
- return;
1084
- }
1085
- settled = true;
1086
- resolve(result);
1087
- }, (error) => {
1088
- clearTimeout(timer);
1089
- if (settled) {
1090
- logger?.debug("[VernLLM] chunk rejection arrived after idle timeout already fired; discarding");
1091
- return;
1092
- }
1093
- settled = true;
1094
- reject(error);
1095
- });
1096
- });
1097
- }
1098
- /**
1099
- * Looks inside an unknown error value for a Retry-After header and
1100
- * converts it to milliseconds. Checks `.headers` first (fetch-style,
1101
- * Headers-like with `.get()`), then `.response.headers` (axios-style,
1102
- * plain object) since different client libraries surface headers
1103
- * differently. Supports both the delta-seconds form ("30") and the
1104
- * HTTP-date form ("Wed, 21 Oct 2015 07:28:00 GMT"). The result is capped
1105
- * at maxDelayMs. Returns undefined when no usable Retry-After is present
1106
- */
1107
- function extractRetryAfterMs(err, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
1108
- if (!err || typeof err !== "object") return void 0;
1109
- const error = err;
1110
- const headers = error.headers ?? error.response?.headers;
1111
- if (!headers || typeof headers !== "object") return void 0;
1112
- const getter = headers;
1113
- const raw = typeof getter.get === "function" ? getter.get("Retry-After") : Object.entries(headers).find(([name]) => name.toLowerCase() === "retry-after")?.at(1);
1114
- if (typeof raw !== "string" || raw.trim() === "") return void 0;
1115
- const trimmed = raw.trim();
1116
- if (/^\d+$/.test(trimmed)) return Math.max(0, Math.min(Number(trimmed) * 1e3, maxDelayMs));
1117
- const dateMs = Date.parse(trimmed);
1118
- if (!Number.isNaN(dateMs)) return Math.max(0, Math.min(dateMs - Date.now(), maxDelayMs));
1119
- return void 0;
1120
- }
1121
- /**
1122
- * Exponential backoff with jitter, capped at maxDelayMs.
1123
- * Jitter avoids thundering-herd retries when many callers back off in lockstep,
1124
- * the cap prevents unbounded delays when maxRetries is high
1125
- */
1126
- function getBackoffDelay(baseDelayMs, attempt, maxDelayMs = DEFAULT_MAX_DELAY_MS) {
1127
- const exp = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);
1128
- return exp / 2 + Math.random() * (exp / 2);
1129
- }
1130
- /**
1131
- * Pauses execution for the given delay before a retry attempt. If an
1132
- * abort signal is provided and it fires while waiting, the pending
1133
- * timer is cancelled immediately and the wait rejects right away with
1134
- * an aborted error instead of continuing to sit idle until the delay
1135
- * would have finished on its own
1136
- */
1137
- async function waitForRetry(delay, signal) {
1138
- if (signal?.aborted) throw new LLMError("Operation aborted", "aborted");
1139
- await new Promise((resolve, reject) => {
1140
- const onAbort = () => {
1141
- clearTimeout(timer);
1142
- reject(new LLMError("Operation aborted", "aborted"));
1143
- };
1144
- const timer = setTimeout(() => {
1145
- signal?.removeEventListener("abort", onAbort);
1146
- resolve();
1147
- }, delay);
1148
- signal?.addEventListener("abort", onAbort, { once: true });
1149
- });
1150
- }
1151
-
1152
- //#endregion
1153
- //#region src/internal/execution/errors.utils.ts
1154
- /**
1155
- * Looks inside an unknown error value and pulls out an http status code
1156
- * if one is present. Checks the status field first then the status code
1157
- * field since different client libraries use different names for this,
1158
- * falling back to AWS SDK v3's `$metadata.httpStatusCode` (e.g. Bedrock's
1159
- * `ThrottlingException`), which doesn't set either of the other two.
1160
- * Returns undefined when the error is not an object or carries no status
1161
- */
1162
- function extractStatus(err) {
1163
- if (!err || typeof err !== "object") return void 0;
1164
- const error = err;
1165
- if (typeof error.status === "number") return error.status;
1166
- if (typeof error.statusCode === "number") return error.statusCode;
1167
- if (typeof error.$metadata?.httpStatusCode === "number") return error.$metadata.httpStatusCode;
1168
- return void 0;
1169
- }
1170
- /**
1171
- * POSIX/libuv error codes libuv (and so Node's `fetch`/undici) attaches to
1172
- * genuine transport-level failures: connection refused, DNS lookup
1173
- * failure, connection reset mid-request, a connect that never completed,
1174
- * DNS server unreachable, broken pipe, or host/network unreachable.
1175
- * Deliberately narrow: only codes that can only mean "the connection
1176
- * itself failed," not anything that could also indicate an application
1177
- * error.
1178
- */
1179
- const NETWORK_ERROR_CODES = new Set([
1180
- "ECONNREFUSED",
1181
- "ENOTFOUND",
1182
- "ECONNRESET",
1183
- "ETIMEDOUT",
1184
- "EAI_AGAIN",
1185
- "EPIPE",
1186
- "ECONNABORTED",
1187
- "EHOSTUNREACH",
1188
- "ENETUNREACH"
1189
- ]);
1190
- /** `fetch`'s own wording for a transport-level failure, across runtimes/browsers. */
1191
- const NETWORK_ERROR_MESSAGES = new Set([
1192
- "fetch failed",
1193
- "failed to fetch",
1194
- "load failed",
1195
- "networkerror when attempting to fetch resource."
1196
- ]);
1197
- /**
1198
- * Whether `error` is, with reasonable confidence, a transport-level
1199
- * failure (never reached the provider, as opposed to the provider itself
1200
- * responding with an error) rather than some other unexpected exception.
1201
- * Checked via explicit, well-known signals only, so a genuinely unknown
1202
- * error never gets misclassified as a connection failure just because it
1203
- * also lacked an HTTP status.
1204
- */
1205
- function isNetworkError(error) {
1206
- if (!error || typeof error !== "object") return false;
1207
- const err = error;
1208
- if (typeof err.code === "string" && NETWORK_ERROR_CODES.has(err.code)) return true;
1209
- if (typeof err.message === "string" && NETWORK_ERROR_MESSAGES.has(err.message.toLowerCase())) return true;
1210
- if (err.cause && typeof err.cause === "object") {
1211
- const cause = err.cause;
1212
- if (typeof cause.code === "string" && NETWORK_ERROR_CODES.has(cause.code)) return true;
1213
- }
1214
- return false;
1215
- }
1216
- function formatSafely(value) {
1217
- try {
1218
- return JSON.stringify(value, null, 2) ?? String(value);
1219
- } catch {
1220
- try {
1221
- return String(value);
1222
- } catch {
1223
- return "[unprintable error]";
1224
- }
1225
- }
1226
- }
1227
- /**
1228
- * Looks inside an unknown thrown value and pulls out a human-readable
1229
- * description of it. Checks the `error` field first (the provider's raw
1230
- * rejection body, JSON-stringified if possible) then falls back to the
1231
- * message` field. Always returns a safe string, even when the thrown value
1232
- * has hostile properties or cannot be serialized normally.
1233
- */
1234
- function describeError(err) {
1235
- if (err && typeof err === "object") try {
1236
- const error = err;
1237
- if (error.error !== void 0) return formatSafely(error.error);
1238
- if (typeof error.message === "string") return error.message;
1239
- } catch {}
1240
- return formatSafely(err);
1241
- }
1242
- /**
1243
- * Maps an HTTP status to its corresponding `LLMErrorCode`, derived purely
1244
- * from the status itself so it applies the same way regardless of which
1245
- * adapter or client raised the error. Used both when building a fresh
1246
- * `LLMError` and when filling in a `code` on an already-normalized one
1247
- * that doesn't have one yet, so the two paths can't drift apart.
1248
- */
1249
- function codeForStatus(status) {
1250
- switch (status) {
1251
- case 429: return "provider_rate_limited";
1252
- case 401: return "authentication";
1253
- case 403: return "authorization";
1254
- case 404: return "not_found";
1255
- case 413: return "payload_too_large";
1256
- default: return status >= 500 ? "server_error" : void 0;
1257
- }
1258
- }
1259
- /**
1260
- * Converts any thrown value into a well-typed LLMError. `attempts`, when
1261
- * given, is the accumulated record of every attempt made before `error`
1262
- * was thrown; it's passed straight into the constructed error's options
1263
- * rather than assigned onto the error afterward, so `attempts` is always
1264
- * settled once, through the constructor, like every other field on
1265
- * `LLMError`.
1266
- */
1267
- function normalizeError(error, signal, attempts) {
1268
- if (signal?.aborted) return new LLMError("LLM request aborted", "aborted", { attempts });
1269
- if (error instanceof LLMError) {
1270
- if (error.code === void 0 && error.status !== void 0) error.code = codeForStatus(error.status);
1271
- if (error.attempts === void 0 && attempts !== void 0) error.attempts = attempts;
1272
- return error;
1273
- }
1274
- const status = extractStatus(error);
1275
- const retryAfterMs = extractRetryAfterMs(error);
1276
- if (status !== void 0) return new LLMError("LLM request failed", "api", {
1277
- status,
1278
- cause: error,
1279
- retryAfterMs,
1280
- code: codeForStatus(status),
1281
- attempts
1282
- });
1283
- if (isNetworkError(error)) return new LLMError("LLM request failed", "network", {
1284
- cause: error,
1285
- retryAfterMs,
1286
- code: "connection_failed",
1287
- attempts
1288
- });
1289
- return new LLMError("LLM request failed", "unknown", {
1290
- cause: error,
1291
- retryAfterMs,
1292
- attempts
1293
- });
1294
- }
1295
-
1296
- //#endregion
1297
- //#region src/internal/execution/parse.utils.ts
1298
- /** Default `parseJson`: `JSON.parse` wrapped in try/catch, returning `undefined` on failure. */
1299
- function defaultParseJson(content) {
1300
- try {
1301
- return JSON.parse(content);
1302
- } catch {
1303
- return void 0;
1304
- }
1305
- }
1306
-
1307
- //#endregion
1308
- //#region src/internal/execution/wire.utils.ts
1309
- /** Translates app-facing `ToolDefinition[]` into the OpenAI-shaped wire tools array. */
1310
- function toWireTools(tools) {
1311
- return tools.map((tool) => ({
1312
- type: "function",
1313
- function: {
1314
- name: tool.name,
1315
- description: tool.description,
1316
- parameters: tool.parameters
1317
- }
1318
- }));
1319
- }
1320
- /** Translates app-facing `ToolCall[]` (e.g. from a replayed assistant turn) into wire tool_calls. */
1321
- function toWireToolCalls(toolCalls) {
1322
- return toolCalls.map((tc) => ({
1323
- id: tc.id,
1324
- type: "function",
1325
- function: {
1326
- name: tc.name,
1327
- arguments: JSON.stringify(tc.arguments ?? {})
1328
- }
1329
- }));
1330
- }
1331
- /**
1332
- * Parses the provider's wire-shaped `tool_calls` back into VernLLM's
1333
- * `ToolCall[]`. Malformed argument JSON is a `'parse'` error, same
1334
- * convention as malformed JSON response bodies elsewhere in VernLLM.
1335
- */
1336
- function parseWireToolCalls(wireToolCalls) {
1337
- return wireToolCalls.map((wc) => {
1338
- let parsedArgs;
1339
- try {
1340
- parsedArgs = wc.function.arguments.trim() ? JSON.parse(wc.function.arguments) : {};
1341
- } catch {
1342
- throw new LLMError(`Invalid JSON arguments for tool call "${wc.function.name}"`, "parse", { code: "tool_arguments_parse_failed" });
1343
- }
1344
- return {
1345
- id: wc.id,
1346
- name: wc.function.name,
1347
- arguments: parsedArgs
1348
- };
1349
- });
1350
- }
1351
-
1352
- //#endregion
1353
- //#region src/internal/execution/requestBuilder.ts
1354
- /**
1355
- * Builds the wire request object for one call, applying per-instance
1356
- * defaults (model, max tokens, temperature) and per-call overrides.
1357
- * Owns every check that depends only on the caller's own input shape, not
1358
- * on execution: history alternation, duplicate/empty tool lists,
1359
- * `toolChoice` naming a real tool. All deterministic on the call site's
1360
- * own input and never touch the network, so every throw here is
1361
- * `type: 'invalid_params'`, not `'validation'` (which is reserved for the
1362
- * model/provider's own response failing a contract check). Has no
1363
- * knowledge of retry, timeouts, or the breaker, only
1364
- * the three defaults a `FallbackTarget` can override per-target (see the
1365
- * `defaultMaxTokens`/`defaultTemperature` overrides in the fallback
1366
- * design), which is what keeps it separable from `CallExecutor`.
1367
- */
1368
- var RequestBuilder = class {
1369
- model;
1370
- defaultMaxTokens;
1371
- defaultTemperature;
1372
- constructor(options) {
1373
- this.model = options.model;
1374
- this.defaultMaxTokens = options.defaultMaxTokens;
1375
- this.defaultTemperature = options.defaultTemperature;
1376
- }
1377
- /** Applies per-call defaults and shapes params into the client's request object. */
1378
- build(params) {
1379
- const { systemPrompt, userContent, history = [], maxTokens = this.defaultMaxTokens, model = this.model, reasoningEffort, jsonSchema, tools, toolChoice } = params;
1380
- const temperature = params.temperature === void 0 ? this.defaultTemperature : params.temperature;
1381
- if (tools && tools.length === 0) throw new LLMError("`tools` was an empty array. This is almost always a bug (e.g. a filtered tool list that ended up empty). An empty `tools` array still switches on tool-call mode (response shape, jsonMode default, wire format) with nothing for the model to call. Omit `tools` entirely for a normal call, or make sure the array is non-empty.", "invalid_params");
1382
- if (tools) {
1383
- const seen = new Set();
1384
- const duplicates = new Set();
1385
- for (const tool of tools) {
1386
- if (seen.has(tool.name)) duplicates.add(tool.name);
1387
- seen.add(tool.name);
1388
- }
1389
- if (duplicates.size) throw new LLMError(`\`tools\` has duplicate name(s): [${[...duplicates].join(", ")}]. Tool names must be unique.`, "invalid_params", {
1390
- code: "duplicate_tool_names",
1391
- issues: { names: [...duplicates] }
1392
- });
1393
- }
1394
- if (toolChoice && !tools) throw new LLMError("`toolChoice` was set without `tools`. There is nothing for it to choose between. Set `tools`, or remove `toolChoice`.", "invalid_params");
1395
- if (tools && typeof toolChoice === "object" && !tools.some((t) => t.name === toolChoice.name)) throw new LLMError(`toolChoice names "${toolChoice.name}", which is not in \`tools\` ([${tools.map((t) => t.name).join(", ")}]).`, "invalid_params", {
1396
- code: "unknown_tool_choice",
1397
- issues: {
1398
- requested: toolChoice.name,
1399
- available: tools.map((t) => t.name)
1400
- }
1401
- });
1402
- const jsonMode = params.jsonMode ?? (tools ? false : true);
1403
- const useJson = jsonMode || Boolean(jsonSchema);
1404
- if (params.schema && !useJson) throw new LLMError("schema was provided but jsonMode: false disables JSON parsing, so nothing would validate it. Remove jsonMode: false, set jsonSchema, or remove schema.", "invalid_params");
1405
- const responseFormat = this.buildResponseFormat(jsonSchema, useJson);
1406
- this.validateHistory(history);
1407
- const request = {
1408
- model,
1409
- ...temperature !== null ? { temperature } : {},
1410
- max_tokens: maxTokens,
1411
- ...responseFormat ? { response_format: responseFormat } : {},
1412
- ...reasoningEffort ? { reasoning_effort: reasoningEffort } : {},
1413
- ...tools ? { tools: toWireTools(tools) } : {},
1414
- ...tools ? { tool_choice: this.buildWireToolChoice(toolChoice) } : {},
1415
- messages: [
1416
- ...systemPrompt ? [{
1417
- role: "system",
1418
- content: systemPrompt
1419
- }] : [],
1420
- ...history.flatMap((turn) => this.turnToWireMessages(turn)),
1421
- {
1422
- role: "user",
1423
- content: userContent
1424
- }
1425
- ]
1426
- };
1427
- return {
1428
- useJson,
1429
- model,
1430
- request
1431
- };
1432
- }
1433
- /**
1434
- * Validates `history` alternates user/assistant turns, since providers
1435
- * like Anthropic/Gemini reject or mishandle consecutive same-role turns.
1436
- */
1437
- validateHistory(history) {
1438
- let previousTurn;
1439
- for (const [index, turn] of history.entries()) {
1440
- if (turn.role === "tool") {
1441
- if (previousTurn?.role !== "assistant" || !previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] is a "tool" turn, but must immediately follow an "assistant" turn that requested tools`, "invalid_params");
1442
- if (!turn.toolResults?.length) throw new LLMError(`history[${index}] is a "tool" turn but has no toolResults`, "invalid_params");
1443
- const requestedIds = new Set(previousTurn.toolCalls.map((tc) => tc.id));
1444
- const resultIds = turn.toolResults.map((tr) => tr.toolCallId);
1445
- const unknownIds = resultIds.filter((id) => !requestedIds.has(id));
1446
- if (unknownIds.length) throw new LLMError(`history[${index}].toolResults references unknown toolCallId(s) [${unknownIds.join(", ")}]`, "invalid_params", {
1447
- code: "unknown_tool_result_ids",
1448
- issues: {
1449
- historyIndex: index,
1450
- ids: unknownIds
1451
- }
1452
- });
1453
- const seenIds = new Set();
1454
- const duplicateIds = new Set();
1455
- for (const id of resultIds) {
1456
- if (seenIds.has(id)) duplicateIds.add(id);
1457
- seenIds.add(id);
1458
- }
1459
- if (duplicateIds.size) throw new LLMError(`history[${index}].toolResults has duplicate toolCallId(s) [${[...duplicateIds].join(", ")}]`, "invalid_params", {
1460
- code: "duplicate_tool_result_ids",
1461
- issues: {
1462
- historyIndex: index,
1463
- ids: [...duplicateIds]
1464
- }
1465
- });
1466
- const missingIds = [...requestedIds].filter((id) => !resultIds.includes(id));
1467
- if (missingIds.length) throw new LLMError(`history[${index}] is missing toolResults for toolCallId(s) [${missingIds.join(", ")}]`, "invalid_params", {
1468
- code: "missing_tool_results",
1469
- issues: {
1470
- historyIndex: index,
1471
- ids: missingIds
1472
- }
1473
- });
1474
- } else {
1475
- if (turn.role === previousTurn?.role) throw new LLMError(`history must alternate user/assistant turns: consecutive "${turn.role}" turns at history[${index - 1}] and history[${index}]`, "invalid_params");
1476
- if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError(`history[${index}] follows an assistant tool request without tool results`, "invalid_params");
1477
- }
1478
- previousTurn = turn;
1479
- }
1480
- if (previousTurn?.role === "assistant" && previousTurn.toolCalls?.length) throw new LLMError("The last entry in history is an assistant tool request without tool results", "invalid_params");
1481
- if (previousTurn?.role === "user") throw new LLMError("The last entry in history is a \"user\" turn, which would collide with the current userContent turn.", "invalid_params");
1482
- }
1483
- /** Maps VernLLM's app-facing `ToolChoice` onto the OpenAI-shaped wire `tool_choice`. */
1484
- buildWireToolChoice(toolChoice) {
1485
- if (!toolChoice || toolChoice === "auto") return "auto";
1486
- if (toolChoice === "none" || toolChoice === "required") return toolChoice;
1487
- return {
1488
- type: "function",
1489
- function: { name: toolChoice.name }
1490
- };
1491
- }
1492
- /**
1493
- * Expands one `ConversationTurn` into one or more wire messages. Plain
1494
- * user/assistant turns map 1:1. An assistant turn with `toolCalls` maps
1495
- * to an assistant message carrying wire-shaped `tool_calls`. A `'tool'`
1496
- * turn expands into one wire `tool` message per `toolResult`, since
1497
- * OpenAI-shaped wire format wants one message per tool_call_id.
1498
- */
1499
- turnToWireMessages(turn) {
1500
- if (turn.role === "tool") return (turn.toolResults ?? []).map((tr) => ({
1501
- role: "tool",
1502
- tool_call_id: tr.toolCallId,
1503
- content: typeof tr.content === "string" ? tr.content : JSON.stringify(tr.content ?? null),
1504
- ...tr.isError ? { is_error: true } : {}
1505
- }));
1506
- if (turn.role === "assistant" && turn.toolCalls?.length) return [{
1507
- role: "assistant",
1508
- ...turn.content ? { content: turn.content } : {},
1509
- tool_calls: toWireToolCalls(turn.toolCalls)
1510
- }];
1511
- return [{
1512
- role: turn.role,
1513
- content: turn.content ?? ""
1514
- }];
1515
- }
1516
- /**
1517
- * Chooses the response format: a provider-native `jsonSchema` takes
1518
- * priority when supplied (constrains generation directly), otherwise
1519
- * falls back to the looser `json_object` mode when JSON output is
1520
- * requested, or no format at all for plain text responses.
1521
- */
1522
- buildResponseFormat(jsonSchema, useJson) {
1523
- if (jsonSchema) return {
1524
- type: "json_schema",
1525
- json_schema: {
1526
- name: jsonSchema.name,
1527
- schema: jsonSchema.schema,
1528
- strict: jsonSchema.strict ?? true,
1529
- description: jsonSchema.description
1530
- }
1531
- };
1532
- return useJson ? { type: "json_object" } : void 0;
1533
- }
1534
- };
1535
-
1536
- //#endregion
1537
- //#region src/internal/execution/streamAccumulator.ts
1538
- /**
1539
- * The streaming accumulator: wraps the raw `WireStreamChunk` iterator in
1540
- * an async generator that yields translated `StreamChunk`s to the caller
1541
- * live, as they arrive, with no per-chunk timeout and no bound on total
1542
- * duration, and accumulates text/tool-call deltas internally so that
1543
- * `finalize` can produce `finalResult` once the stream completes.
1544
- *
1545
- * Two separate try/catches: the iteration loop's catch handles errors
1546
- * the transport itself throws, which aren't normalized yet, so that
1547
- * happens here, alongside the one `onStreamFailure` call for them. The
1548
- * second catch, around `finalize`, does not re-normalize or re-report,
1549
- * since `finalize`'s caller (`finalizeResponse`) already does both
1550
- * internally.
1551
- */
1552
- function buildStreamResult(iterator, first, options) {
1553
- const { requestId, model, providerName, isFallback, chunkIdleTimeoutMs, streamController, logger, signal } = options;
1554
- let resolveFinal;
1555
- let rejectFinal;
1556
- const finalResult = new Promise((resolve, reject) => {
1557
- resolveFinal = resolve;
1558
- rejectFinal = reject;
1559
- });
1560
- finalResult.catch(() => {});
1561
- const MAX_BUFFERED_CHUNKS = 1e4;
1562
- const buffered = [];
1563
- const pending = [];
1564
- let streamDone = false;
1565
- let streamError;
1566
- let hasLoggedEviction = false;
1567
- const push = (chunk) => {
1568
- const waiter = pending.shift();
1569
- if (waiter) {
1570
- waiter.resolve({
1571
- done: false,
1572
- value: chunk
1573
- });
1574
- return;
1575
- }
1576
- buffered.push(chunk);
1577
- if (buffered.length > MAX_BUFFERED_CHUNKS * 2) {
1578
- if (!hasLoggedEviction) {
1579
- hasLoggedEviction = true;
1580
- logger.warn(`[VernLLM] stream chunk buffer exceeded cap (${MAX_BUFFERED_CHUNKS}), evicting ${buffered.length - MAX_BUFFERED_CHUNKS} oldest chunk(s); buffered=${buffered.length}. The chunks iterable was never read (or fell far behind) for this stream.`);
1581
- }
1582
- buffered.splice(0, buffered.length - MAX_BUFFERED_CHUNKS);
1583
- }
1584
- };
1585
- const finish = () => {
1586
- streamDone = true;
1587
- for (const waiter of pending.splice(0)) waiter.resolve({
1588
- done: true,
1589
- value: void 0
1590
- });
1591
- };
1592
- const fail = (error) => {
1593
- streamDone = true;
1594
- streamError = error;
1595
- for (const waiter of pending.splice(0)) waiter.reject(error);
1596
- };
1597
- const chunks = { [Symbol.asyncIterator]() {
1598
- return { next() {
1599
- if (buffered.length) return Promise.resolve({
1600
- done: false,
1601
- value: buffered.shift()
1602
- });
1603
- if (streamDone) return streamError ? Promise.reject(streamError) : Promise.resolve({
1604
- done: true,
1605
- value: void 0
1606
- });
1607
- return new Promise((resolve, reject) => {
1608
- pending.push({
1609
- resolve,
1610
- reject
1611
- });
1612
- });
1613
- } };
1614
- } };
1615
- const toolCallAcc = new Map();
1616
- let textAcc = "";
1617
- let usage;
1618
- (async () => {
1619
- try {
1620
- let result = first;
1621
- while (!result.done) {
1622
- const wireChunk = result.value;
1623
- if (wireChunk.type === "ping") {} else if (wireChunk.type === "text-delta") {
1624
- textAcc += wireChunk.delta;
1625
- push({
1626
- type: "text-delta",
1627
- delta: wireChunk.delta
1628
- });
1629
- } else if (wireChunk.type === "tool_call_delta") {
1630
- const entry = toolCallAcc.get(wireChunk.index) ?? { args: "" };
1631
- entry.id ??= wireChunk.id;
1632
- entry.name ??= wireChunk.name;
1633
- entry.args += wireChunk.argumentsDelta ?? "";
1634
- toolCallAcc.set(wireChunk.index, entry);
1635
- push({
1636
- type: "tool_call_delta",
1637
- index: wireChunk.index,
1638
- id: wireChunk.id,
1639
- name: wireChunk.name,
1640
- argsDelta: wireChunk.argumentsDelta,
1641
- complete: wireChunk.complete
1642
- });
1643
- } else if (wireChunk.type === "usage") {
1644
- usage = {
1645
- promptTokens: wireChunk.usage.prompt_tokens ?? 0,
1646
- completionTokens: wireChunk.usage.completion_tokens ?? 0,
1647
- totalTokens: wireChunk.usage.total_tokens ?? 0,
1648
- requestId,
1649
- model,
1650
- provider: providerName,
1651
- usedFallback: isFallback
1652
- };
1653
- push({
1654
- type: "usage",
1655
- usage
1656
- });
1657
- }
1658
- result = await withChunkIdleTimeout(() => iterator.next(), chunkIdleTimeoutMs, () => streamController.abort(), logger);
1659
- }
1660
- } catch (error) {
1661
- try {
1662
- await iterator.return?.();
1663
- } catch {}
1664
- streamController.abort();
1665
- const normalized = normalizeError(error, signal);
1666
- try {
1667
- options.onStreamFailure(normalized, usage);
1668
- } catch {}
1669
- fail(normalized);
1670
- rejectFinal(normalized);
1671
- return;
1672
- }
1673
- finish();
1674
- try {
1675
- options.onStreamSuccess(usage);
1676
- } catch {}
1677
- try {
1678
- const wireToolCalls = toolCallAcc.size ? [...toolCallAcc.entries()].sort(([indexA], [indexB]) => indexA - indexB).map(([, entry]) => ({
1679
- id: entry.id ?? "",
1680
- type: "function",
1681
- function: {
1682
- name: entry.name ?? "",
1683
- arguments: entry.args
1684
- }
1685
- })) : void 0;
1686
- const finalized = options.finalize(textAcc, wireToolCalls, usage);
1687
- resolveFinal(finalized);
1688
- } catch (error) {
1689
- rejectFinal(error);
1690
- }
1691
- })();
1692
- return {
1693
- chunks,
1694
- finalResult
1695
- };
1696
- }
1697
-
1698
- //#endregion
1699
- //#region src/internal/execution/callExecutor.ts
1700
- /**
1701
- * Everything one provider target needs to attempt a call: request
1702
- * building, retry with backoff, the per-target breaker, the per-target
1703
- * limiter. Never exported publicly. `VernLLM` holds one per target and
1704
- * owns the fallback loop and caching on top.
1705
- */
1706
- var CallExecutor = class {
1707
- maxRetries;
1708
- timeoutMs;
1709
- chunkIdleTimeoutMs;
1710
- baseDelayMs;
1711
- nonRetryableStatus;
1712
- parseJson;
1713
- logger;
1714
- redact;
1715
- onUsage;
1716
- onUsageFailure;
1717
- reportEvent;
1718
- breaker;
1719
- limiter;
1720
- isFallback;
1721
- requestBuilder;
1722
- constructor(providerName, client, model, options) {
1723
- this.providerName = providerName;
1724
- this.client = client;
1725
- this.model = model;
1726
- this.maxRetries = options.maxRetries;
1727
- this.timeoutMs = options.timeoutMs;
1728
- this.chunkIdleTimeoutMs = options.chunkIdleTimeoutMs;
1729
- this.baseDelayMs = options.baseDelayMs;
1730
- this.nonRetryableStatus = options.nonRetryableStatus;
1731
- this.parseJson = options.parseJson ?? defaultParseJson;
1732
- this.logger = options.logger;
1733
- this.redact = options.redact;
1734
- this.onUsage = options.onUsage;
1735
- this.onUsageFailure = options.onUsageFailure;
1736
- this.reportEvent = makeEventReporter(options.onEvent, this.logger);
1737
- this.breaker = options.breaker;
1738
- this.limiter = options.limiter;
1739
- this.isFallback = options.isFallback ?? false;
1740
- this.requestBuilder = new RequestBuilder({
1741
- model,
1742
- defaultMaxTokens: options.defaultMaxTokens,
1743
- defaultTemperature: options.defaultTemperature
1744
- });
1745
- }
1746
- getCircuitState(model) {
1747
- return this.breaker?.getState(model);
1748
- }
1749
- /** Whether this target's breaker tracks failures per model. `false` if no breaker is configured. */
1750
- get isolateByModel() {
1751
- return this.breaker?.isolateByModel ?? false;
1752
- }
1753
- /** Manually opens this target's circuit breaker, if one is configured. No-op otherwise. */
1754
- openCircuit(model) {
1755
- this.breaker?.open(model);
1756
- }
1757
- /** Manually closes this target's circuit breaker, if one is configured. No-op otherwise. */
1758
- closeCircuit(model) {
1759
- this.breaker?.close(model);
1760
- }
1761
- /**
1762
- * Throws if the breaker is open for this target/model, exactly like the
1763
- * check `run`/`runStream` used to make internally. Exposed so `VernLLM`
1764
- * can gate on it before reserving usage, avoiding a reserve-then-refund
1765
- * round trip on a call that was never going to be attempted. `assertClosed`
1766
- * has a stateful side effect (claiming a half-open trial slot), so it must
1767
- * run exactly once per logical call: `run`/`runStream` no longer call it
1768
- * themselves, this is now the only call site.
1769
- */
1770
- assertBreakerClosed(model) {
1771
- this.breaker?.assertClosed(model ?? this.model);
1772
- }
1773
- /**
1774
- * Runs a single logical call against this target: retry with backoff,
1775
- * normalized error on exhaustion. Mirrors the old `VernLLM.call`'s
1776
- * non-streaming branch, minus cache/usage-reservation and the breaker
1777
- * check, which stay one layer up since they aren't per-target concerns
1778
- * (see `assertBreakerClosed`).
1779
- */
1780
- async run(params, requestId, onAttempt) {
1781
- const model = params.model ?? this.model;
1782
- const attempts = [];
1783
- try {
1784
- return await this.retryWithBackoff((attempt) => this.executeCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
1785
- } catch (error) {
1786
- const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
1787
- if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
1788
- this.logger.debug(`[VernLLM:${requestId}] error:\n${this.redactText(describeError(error))}`);
1789
- throw normalized;
1790
- }
1791
- }
1792
- /** Streaming counterpart to `run`. Mirrors the old streaming branch of `VernLLM.call`. */
1793
- async runStream(params, requestId, onAttempt) {
1794
- const model = params.model ?? this.model;
1795
- const attempts = [];
1796
- try {
1797
- return await this.retryWithBackoff((attempt) => this.executeStreamCall(params, requestId, attempt), requestId, model, params.signal, onAttempt, attempts);
1798
- } catch (error) {
1799
- const normalized = normalizeError(error, params.signal, attempts.length > 0 ? attempts : void 0);
1800
- if (this.countsTowardBreaker(normalized)) this.breaker?.recordFailure(model);
1801
- this.logger.debug(`[VernLLM:${requestId}] stream-open error:\n${this.redactText(describeError(error))}`);
1802
- throw normalized;
1803
- }
1804
- }
1805
- /**
1806
- * Performs a single attempt: builds the request (translating `tools` to
1807
- * wire shape when present), dispatches it with a timeout, and shapes the
1808
- * response into `T` or a `CallWithToolsResult<T>` when `params.tools` was
1809
- * set. Throws on an empty response (no text and no tool_calls) so the
1810
- * retry loop treats it like any other transient failure.
1811
- */
1812
- async executeCall(params, requestId, attempt) {
1813
- const { useJson, model, request } = this.requestBuilder.build(params);
1814
- let release;
1815
- if (this.limiter) {
1816
- const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
1817
- release = acquired.release;
1818
- if (acquired.waitedMs > 0) this.reportEvent({
1819
- kind: "rate_limited",
1820
- requestId,
1821
- provider: this.providerName,
1822
- model,
1823
- waitedMs: acquired.waitedMs,
1824
- reason: acquired.reason ?? "rpm"
1825
- });
1826
- }
1827
- try {
1828
- const response = await withTimeout((attemptSignal) => this.client.chat.completions.create(request, { signal: attemptSignal }), this.timeoutMs, params.signal);
1829
- const usage = this.extractUsage(response, requestId, model);
1830
- release?.(this.actualTokensFor(usage));
1831
- release = void 0;
1832
- const rawContent = response.choices?.[0]?.message?.content;
1833
- const wireToolCalls = response.choices?.[0]?.message?.tool_calls;
1834
- return this.finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt);
1835
- } finally {
1836
- release?.();
1837
- }
1838
- }
1839
- /** Applies `redact` (if configured); otherwise returns `text` unchanged. */
1840
- redactText(text) {
1841
- return this.redact ? this.redact(text) : text;
1842
- }
1843
- /**
1844
- * Applies `redact` (if configured) to whatever the debug log is about
1845
- * to show: real content when there is any, otherwise the tool-call
1846
- * placeholder, which carries no user data and passes through
1847
- * `redact` unchanged in practice but is included for a caller whose
1848
- * `redact` does something structural (e.g. adding a marker) rather
1849
- * than just scrubbing PII.
1850
- */
1851
- redactedOutput(content, wireToolCalls) {
1852
- return this.redactText(content ?? `[${wireToolCalls?.length ?? 0} tool call(s)]`);
1853
- }
1854
- /**
1855
- * Shapes a fully-arrived response (content and/or tool_calls, already
1856
- * extracted from the provider's payload) into `T` or a
1857
- * `CallWithToolsResult<T>`. Reused by the streaming path once it has
1858
- * buffered the full text/tool-call deltas, so there's no separate
1859
- * parsing/validation logic for streaming.
1860
- *
1861
- * Normalizes and reports usage failure on error itself, so every caller
1862
- * gets identical error handling without duplicating it.
1863
- */
1864
- finalizeResponse(rawContent, wireToolCalls, params, useJson, model, usage, requestId, attempt) {
1865
- try {
1866
- const content = rawContent?.trim();
1867
- if (!content && !wireToolCalls?.length) throw new LLMError("Empty LLM response", "api", { code: "empty_response" });
1868
- this.logger.debug(`[VernLLM:${requestId}] output:\n${this.redactedOutput(content, wireToolCalls).slice(0, 800)}`);
1869
- if (wireToolCalls?.length) {
1870
- if (!params.tools) throw new LLMError("Provider returned tool_calls but no `tools` were sent with this call.", "validation", { code: "unexpected_tool_calls" });
1871
- if (params.toolChoice === "none") throw new LLMError("Provider returned tool_calls despite toolChoice: 'none'.", "validation", { code: "tool_choice_none_violated" });
1872
- const toolCalls = parseWireToolCalls(wireToolCalls);
1873
- this.validateToolCallArguments(toolCalls, params.tools);
1874
- this.breaker?.recordSuccess(model);
1875
- this.reportUsage(usage);
1876
- return {
1877
- type: "tool_calls",
1878
- toolCalls,
1879
- ...content ? { content } : {}
1880
- };
1881
- }
1882
- const textContent = content ?? "";
1883
- if (!useJson) {
1884
- this.breaker?.recordSuccess(model);
1885
- this.reportUsage(usage);
1886
- return params.tools ? {
1887
- type: "content",
1888
- content: textContent
1889
- } : textContent;
1890
- }
1891
- const result = this.parseAndValidate(textContent, params.schema);
1892
- this.breaker?.recordSuccess(model);
1893
- this.reportUsage(usage);
1894
- return params.tools ? {
1895
- type: "content",
1896
- content: result
1897
- } : result;
1898
- } catch (error) {
1899
- const normalized = normalizeError(error, params.signal);
1900
- if (usage && normalized.type !== "aborted") this.reportUsageFailure(usage, normalized, attempt);
1901
- throw normalized;
1902
- }
1903
- }
1904
- /**
1905
- * Opens a stream for a single attempt: builds the request exactly like
1906
- * `executeCall`, then requires `createStream` on the client (a clear
1907
- * `validation` error if the adapter doesn't support it). The timeout
1908
- * wraps stream construction and the first `.next()` together, not just
1909
- * construction: calling an `async function*` returns an iterator
1910
- * synchronously without running its body until `.next()` is first
1911
- * invoked, so timing only construction would time an operation that's
1912
- * always instant, not the actual connection. Both are folded into a
1913
- * single `withTimeout` so the same abort signal reaches whatever the
1914
- * adapter's `createStream` uses internally for its first network
1915
- * round-trip.
1916
- *
1917
- * Circuit-breaker success is recorded once the stream fully completes,
1918
- * not on the first chunk arriving, so a connection that opens but then
1919
- * dies mid-stream isn't masked as a success (see `buildStreamResult`).
1920
- */
1921
- async executeStreamCall(params, requestId, attempt) {
1922
- const { useJson, model, request } = this.requestBuilder.build(params);
1923
- const completions = this.client.chat.completions;
1924
- if (!completions.createStream) throw new LLMError("stream: true requires a client/adapter with createStream", "invalid_params", {
1925
- code: "unsupported_capability",
1926
- issues: { capability: "createStream" }
1927
- });
1928
- const createStream = completions.createStream.bind(completions);
1929
- let release;
1930
- if (this.limiter) {
1931
- const acquired = await this.limiter.acquire(this.limiter.estimate(request), params.signal);
1932
- release = acquired.release;
1933
- if (acquired.waitedMs > 0) this.reportEvent({
1934
- kind: "rate_limited",
1935
- requestId,
1936
- provider: this.providerName,
1937
- model,
1938
- waitedMs: acquired.waitedMs,
1939
- reason: acquired.reason ?? "rpm"
1940
- });
1941
- }
1942
- const streamController = new AbortController();
1943
- const combinedExternal = params.signal ? AbortSignal.any([params.signal, streamController.signal]) : streamController.signal;
1944
- try {
1945
- const { iterator, first } = await withTimeout(async (attemptSignal) => {
1946
- const streamIterator = createStream(request, { signal: attemptSignal })[Symbol.asyncIterator]();
1947
- const firstResult = await streamIterator.next();
1948
- return {
1949
- iterator: streamIterator,
1950
- first: firstResult
1951
- };
1952
- }, this.timeoutMs, combinedExternal);
1953
- if (first.done) throw new LLMError("Empty LLM response", "api");
1954
- const releaseAtOpen = release;
1955
- const result = buildStreamResult(iterator, first, {
1956
- requestId,
1957
- model,
1958
- providerName: this.providerName,
1959
- isFallback: this.isFallback,
1960
- chunkIdleTimeoutMs: params.chunkIdleTimeoutMs ?? this.chunkIdleTimeoutMs,
1961
- streamController,
1962
- logger: this.logger,
1963
- signal: params.signal,
1964
- onStreamSuccess: (usage) => {
1965
- this.breaker?.recordSuccess(model);
1966
- releaseAtOpen?.(this.actualTokensFor(usage));
1967
- },
1968
- onStreamFailure: (normalized, usage) => {
1969
- if (normalized.type === "timeout") this.breaker?.recordFailure(model);
1970
- if (usage && normalized.type !== "aborted") this.reportUsageFailure(usage, normalized, attempt, true);
1971
- releaseAtOpen?.(this.actualTokensFor(usage));
1972
- },
1973
- finalize: (textAcc, wireToolCalls, usage) => this.finalizeResponse(textAcc, wireToolCalls, params, useJson, model, usage, requestId, attempt)
1974
- });
1975
- release = void 0;
1976
- return result;
1977
- } finally {
1978
- release?.();
1979
- }
1980
- }
1981
- /**
1982
- * Checks every `ToolCall` against the `tools` that were offered, catching
1983
- * a hallucinated tool name and a duplicate call id before either reaches
1984
- * the application's dispatch table, then runs each tool's
1985
- * `argumentsSchema`, if present.
1986
- *
1987
- * Contract failures (unknown name, duplicate id) are collected across
1988
- * every call and thrown together as one `type: 'validation'` error with
1989
- * `issues: ToolIssue[]`, since retrying a request that already has these
1990
- * errors cannot help (excluded from retry by `type`) and a caller fixing
1991
- * them wants to see every one, not just the first. Schema failures keep
1992
- * the original single-error, `type: 'validation'` shape rather than being
1993
- * folded into the aggregate, since they're a distinct failure kind from
1994
- * the contract failures above.
1995
- */
1996
- validateToolCallArguments(toolCalls, tools) {
1997
- const known = new Map(tools.map((t) => [t.name, t]));
1998
- const seenIds = new Set();
1999
- const toolIssues = [];
2000
- for (const call of toolCalls) {
2001
- if (seenIds.has(call.id)) toolIssues.push({
2002
- name: call.name,
2003
- toolCallId: call.id,
2004
- code: "duplicate_tool_call_id"
2005
- });
2006
- seenIds.add(call.id);
2007
- if (!known.has(call.name)) toolIssues.push({
2008
- name: call.name,
2009
- toolCallId: call.id,
2010
- code: "unknown_tool"
2011
- });
2012
- }
2013
- if (toolIssues.length > 0) {
2014
- const unknownTool = toolIssues.find((i) => i.code === "unknown_tool");
2015
- const primary = unknownTool ? `Model requested tool "${unknownTool.name}", which was not in the tools offered ([${[...known.keys()].join(", ")}]).` : `Duplicate tool call id "${toolIssues[0].toolCallId}" in the model's response.`;
2016
- const message = toolIssues.length > 1 ? `${primary} (${toolIssues.length} tool call issues total, see error.issues.)` : primary;
2017
- throw new LLMError(message, "validation", {
2018
- code: unknownTool ? "unknown_tool" : "duplicate_tool_call_id",
2019
- issues: toolIssues
2020
- });
2021
- }
2022
- for (const call of toolCalls) {
2023
- const definition = known.get(call.name);
2024
- if (!definition?.argumentsSchema) continue;
2025
- const result = definition.argumentsSchema.safeParse(call.arguments);
2026
- if (!result.success) throw new LLMError(`Arguments for tool call "${call.name}" failed validation`, "validation", { issues: result.error });
2027
- }
2028
- }
2029
- /**
2030
- * Runs `fn`, retrying with backoff according to `shouldRetry`. When
2031
- * `attempts` is given, every failed attempt that is actually followed by
2032
- * a retry is recorded, in order. This mirrors `LLMError.attempts`'s
2033
- * contract: every attempt made before this error was thrown. The
2034
- * terminal failure is never pushed since it isn't a prior attempt, it
2035
- * is the error being thrown. `attempts` stays empty when nothing was
2036
- * retried, so no separate bookkeeping is needed at the call sites.
2037
- * Each failure is recorded as a snapshot (`LLMError.toSnapshot()`),
2038
- * not the live `LLMError`, per `RetryAttempt`'s contract.
2039
- */
2040
- async retryWithBackoff(fn, requestId, model, signal, onAttempt, attempts) {
2041
- let lastError;
2042
- for (let attempt = 0; attempt <= this.maxRetries; attempt++) try {
2043
- if (attempt > 0) await this.recoverDelay(requestId, model, attempt, lastError, signal);
2044
- onAttempt?.();
2045
- return await fn(attempt);
2046
- } catch (error) {
2047
- lastError = error;
2048
- const willRetry = attempt < this.maxRetries && this.shouldRetry(error, signal);
2049
- if (!willRetry) break;
2050
- attempts?.push({
2051
- index: attempt,
2052
- error: normalizeError(error, signal).toSnapshot()
2053
- });
2054
- }
2055
- throw lastError;
2056
- }
2057
- /**
2058
- * Pulls `TokenUsage` out of a raw response, if the provider reported it.
2059
- * Extraction doesn't depend on what happens to the response afterward, so
2060
- * a malformed body can still yield usage if the provider's usage block
2061
- * itself came through intact.
2062
- */
2063
- extractUsage(response, requestId, model) {
2064
- if (!response.usage) return void 0;
2065
- return {
2066
- promptTokens: response.usage.prompt_tokens ?? 0,
2067
- completionTokens: response.usage.completion_tokens ?? 0,
2068
- totalTokens: response.usage.total_tokens ?? 0,
2069
- requestId,
2070
- model,
2071
- provider: this.providerName,
2072
- usedFallback: this.isFallback
2073
- };
2074
- }
2075
- /**
2076
- * The token count to reconcile the rate limiter against for a finished
2077
- * attempt: `totalTokens` when reported, otherwise the sum of prompt and
2078
- * completion tokens, matching `reportUsageFailure`'s own fallback below
2079
- * for a hand-rolled client that reports the parts but omits the total.
2080
- */
2081
- actualTokensFor(usage) {
2082
- if (!usage) return void 0;
2083
- return usage.totalTokens || usage.promptTokens + usage.completionTokens;
2084
- }
2085
- /** Reports token usage for a successful call, swallowing and logging any error `onUsage` throws. */
2086
- reportUsage(usage) {
2087
- if (!usage || !this.onUsage) return;
2088
- try {
2089
- this.onUsage(usage);
2090
- } catch (error) {
2091
- this.logger.error("[VernLLM] onUsage failed", { message: error instanceof Error ? error.message : "unknown" });
2092
- }
2093
- }
2094
- /**
2095
- * Reports token usage spent on an attempt that then failed, so it isn't
2096
- * dropped alongside the error. Covers any error thrown after usage
2097
- * extraction, since all of them happen only after a response (real
2098
- * spend) already arrived. Swallows and logs any error `onUsageFailure`
2099
- * itself throws.
2100
- */
2101
- reportUsageFailure(usage, error, attempt, terminal = false) {
2102
- const displayTokens = usage.totalTokens || usage.promptTokens + usage.completionTokens;
2103
- const attemptText = terminal ? "mid-stream failure (terminal, no further attempts)" : `attempt ${attempt + 1}/${this.maxRetries + 1}`;
2104
- this.logger.warn(`[VernLLM:${usage.requestId}] usage failure, ${attemptText}: type=${error.type} tokens=${displayTokens}`);
2105
- if (!this.onUsageFailure) return;
2106
- try {
2107
- this.onUsageFailure(usage, error);
2108
- } catch (hookError) {
2109
- this.logger.error("[VernLLM] onUsageFailure failed", { message: hookError instanceof Error ? hookError.message : "unknown" });
2110
- }
2111
- }
2112
- /** Parses response content as JSON and validates it against `schema` when supplied. */
2113
- parseAndValidate(content, schema) {
2114
- let parsed;
2115
- try {
2116
- parsed = this.parseJson(content);
2117
- } catch {
2118
- throw new LLMError("Invalid JSON response", "parse");
2119
- }
2120
- if (parsed === null || parsed === void 0) throw new LLMError("Invalid JSON response", "parse");
2121
- if (!schema) return parsed;
2122
- const result = schema.safeParse(parsed);
2123
- if (!result.success) throw new LLMError("Schema validation failed", "validation", { issues: result.error });
2124
- return result.data;
2125
- }
2126
- /**
2127
- * Waits out the backoff delay for a retry attempt, honoring a
2128
- * Retry-After header on the failed attempt's error when present.
2129
- * Both Retry-After and plain exponential backoff are capped at the same
2130
- * max delay (see `DEFAULT_MAX_DELAY_MS` in `retry.utils.ts`).
2131
- */
2132
- async recoverDelay(requestId, model, attempt, error, signal) {
2133
- const retryAfterMs = extractRetryAfterMs(error);
2134
- const delay = retryAfterMs ?? getBackoffDelay(this.baseDelayMs, attempt);
2135
- const retryAfterHonored = retryAfterMs !== void 0;
2136
- this.logger.warn(`[VernLLM:${requestId}] recovery attempt ${attempt}/${this.maxRetries}, waiting ${Math.ceil(delay)}ms` + (retryAfterHonored ? " (honoring Retry-After)" : ""));
2137
- this.reportEvent({
2138
- kind: "retry",
2139
- requestId,
2140
- provider: this.providerName,
2141
- model,
2142
- attempt,
2143
- maxRetries: this.maxRetries,
2144
- delayMs: delay,
2145
- retryAfterHonored,
2146
- error: normalizeError(error, signal)
2147
- });
2148
- await waitForRetry(delay, signal);
2149
- }
2150
- /** Decides whether a failed attempt is worth retrying. */
2151
- shouldRetry(error, signal) {
2152
- if (signal?.aborted) return false;
2153
- if (error instanceof LLMError && !error.retryable) return false;
2154
- const status = extractStatus(error);
2155
- return !(status !== void 0 && this.nonRetryableStatus.includes(status));
2156
- }
2157
- /**
2158
- * Decides whether a failed attempt should count toward the circuit
2159
- * breaker's failure threshold. A model hallucinating a tool name,
2160
- * reusing a call id, or a provider ignoring `toolChoice: 'none'` isn't
2161
- * the provider being unhealthy, it's a model/provider response defect
2162
- * that will very likely recur regardless of provider health, so it
2163
- * shouldn't push a healthy provider's circuit toward opening. Same for
2164
- * a caller-input bug or a local rate-limit rejection: neither ever
2165
- * reached the provider at all. This is exactly what `LLMError.retryable`
2166
- * already excludes, so this defers to it directly.
2167
- */
2168
- countsTowardBreaker(error) {
2169
- return error.retryable;
2170
- }
2171
- };
2172
-
2173
- //#endregion
2174
- //#region src/internal/logger.utils.ts
2175
- /**
2176
- * Wraps a `Logger` so a throwing implementation can never break the call
2177
- * it's trying to describe. `logger` is user-supplied (`VernLLMOptions.logger`),
2178
- * so a custom logger that ships to a file, Datadog, etc. can throw for
2179
- * reasons unrelated to VernLLM. Wrap once at construction so every
2180
- * downstream `this.logger.warn(...)` call stays as-is and is safe by
2181
- * construction, instead of guarding each call site individually.
2182
- */
2183
- function createSafeLogger(logger) {
2184
- return {
2185
- debug: safe(logger, "debug"),
2186
- warn: safe(logger, "warn"),
2187
- error: safe(logger, "error")
2188
- };
2189
- }
2190
- function safe(logger, method) {
2191
- const fn = logger[method].bind(logger);
2192
- return (...args) => {
2193
- try {
2194
- swallowRejection(fn(...args));
2195
- } catch {}
2196
- };
2197
- }
2198
- function isPromiseLike(value) {
2199
- return typeof value?.then === "function";
2200
- }
2201
- function swallowRejection(result) {
2202
- if (isPromiseLike(result)) Promise.resolve(result).catch(() => {});
2203
- }
2204
-
2205
- //#endregion
2206
- //#region src/logger.ts
2207
- /**
2208
- * Default logger. `debug` is gated by the `debug` option on VernLLM
2209
- * warn/error always fire since they indicate real problems (retries, cache failures)
2210
- */
2211
- var ConsoleLogger = class {
2212
- constructor(debugEnabled) {
2213
- this.debugEnabled = debugEnabled;
2214
- }
2215
- debug(message) {
2216
- if (this.debugEnabled) console.debug(message);
2217
- }
2218
- warn(message) {
2219
- console.warn(message);
2220
- }
2221
- error(message, meta) {
2222
- console.error(message, meta ?? "");
2223
- }
2224
- };
2225
-
2226
- //#endregion
2227
- //#region src/rateLimit.ts
2228
- /** Default `estimateTokens`: chars/4 over every message's content, plus the requested `max_tokens`. */
2229
- function defaultEstimateTokens(request) {
2230
- const messagesChars = request.messages.reduce((sum, message) => {
2231
- const content = message.content;
2232
- if (typeof content === "string") return sum + content.length;
2233
- if (content === void 0 || content === null) return sum;
2234
- try {
2235
- return sum + JSON.stringify(content).length;
2236
- } catch {
2237
- return sum;
2238
- }
2239
- }, 0);
2240
- return Math.ceil(messagesChars / 4) + (request.max_tokens ?? 0);
2241
- }
2242
- /**
2243
- * A capacity that refills continuously. Used for requests per minute and
2244
- * tokens per minute, where `refillPerMs` is `capacity / 60000`, and for
2245
- * concurrency, where `refillPerMs` is 0 and every release calls
2246
- * `give(1)` instead of relying on the clock.
2247
- */
2248
- var TokenBucket = class {
2249
- available;
2250
- lastRefill = Date.now();
2251
- constructor(capacity, refillPerMs) {
2252
- this.capacity = capacity;
2253
- this.refillPerMs = refillPerMs;
2254
- this.available = capacity;
2255
- }
2256
- refill() {
2257
- if (this.refillPerMs === 0) return;
2258
- const now = Date.now();
2259
- const elapsedMs = now - this.lastRefill;
2260
- this.available = Math.min(this.capacity, this.available + Math.max(0, elapsedMs) * this.refillPerMs);
2261
- this.lastRefill = now;
2262
- }
2263
- /** Refills, then takes `amount` if available. Leaves the bucket untouched if it can't. */
2264
- tryTake(amount) {
2265
- this.refill();
2266
- if (this.available < amount) return false;
2267
- this.available -= amount;
2268
- return true;
2269
- }
2270
- /**
2271
- * Refills, then reports how many ms until this bucket could supply
2272
- * `amount`, assuming nothing else takes from it meanwhile. Returns 0 if
2273
- * it already can, `Infinity` if it never will on its own (a
2274
- * concurrency bucket, `refillPerMs === 0`, only frees via `give`).
2275
- */
2276
- msUntilAvailable(amount) {
2277
- this.refill();
2278
- if (this.available >= amount) return 0;
2279
- if (this.refillPerMs === 0) return Infinity;
2280
- return (amount - this.available) / this.refillPerMs;
2281
- }
2282
- /**
2283
- * Gives capacity back. Not floored at 0: a bad token-usage estimate can
2284
- * push `available` negative, and it self-corrects on the next refill
2285
- * rather than being clamped away immediately. Only ceilinged at
2286
- * `capacity`, so a give can never overfill the bucket.
2287
- */
2288
- give(amount) {
2289
- this.available = Math.min(this.capacity, this.available + amount);
2290
- }
2291
- /** The bucket's ceiling, e.g. so a request that could never fit can fail fast instead of queueing forever. */
2292
- getCapacity() {
2293
- return this.capacity;
2294
- }
2295
- };
2296
- /**
2297
- * `setTimeout` silently clamps any delay above this (~24.8 days) instead
2298
- * of erroring, so an uncapped delay derived from a very small
2299
- * `requestsPerMinute`/`tokensPerMinute` could wrap around to firing
2300
- * almost immediately instead of waiting. Mirrors the same guard in
2301
- * `withTimeout`/`withChunkIdleTimeout`.
2302
- */
2303
- const MAX_WAKE_DELAY_MS = 2147483647;
2304
- /**
2305
- * Per-target rate limiter. Up to three buckets (requests/min, tokens/min,
2306
- * concurrency) behind one FIFO queue, so a large call isn't starved by a
2307
- * stream of small ones. Any bucket omitted from `options` has infinite
2308
- * capacity and never blocks.
2309
- */
2310
- var RateLimiter = class {
2311
- requests;
2312
- tokens;
2313
- concurrency;
2314
- maxQueueMs;
2315
- maxQueueSize;
2316
- estimateTokensFn;
2317
- queue = [];
2318
- /**
2319
- * A single scheduled re-check for the head of the queue when it's
2320
- * blocked on a bucket that refills on its own clock (rpm/tpm), so a
2321
- * queue that nobody calls `acquire`/`release` on again isn't stuck
2322
- * forever waiting for an external trigger to re-drain it. Not needed
2323
- * for a concurrency block, which only clears via `release`.
2324
- */
2325
- wakeTimer;
2326
- constructor(options) {
2327
- if (options.requestsPerMinute) this.requests = new TokenBucket(options.requestsPerMinute, options.requestsPerMinute / 6e4);
2328
- if (options.tokensPerMinute) this.tokens = new TokenBucket(options.tokensPerMinute, options.tokensPerMinute / 6e4);
2329
- if (options.maxConcurrent) this.concurrency = new TokenBucket(options.maxConcurrent, 0);
2330
- this.maxQueueMs = options.maxQueueMs ?? 3e4;
2331
- this.maxQueueSize = options.maxQueueSize ?? 0;
2332
- this.estimateTokensFn = options.estimateTokens ?? defaultEstimateTokens;
2333
- }
2334
- /** Pre-flight token estimate for a request, per the configured (or default) heuristic. */
2335
- estimate(request) {
2336
- return this.estimateTokensFn(request);
2337
- }
2338
- /**
2339
- * Waits for capacity in every configured bucket, then takes from each.
2340
- * The returned `release` gives the concurrency slot back and reconciles
2341
- * the token bucket against real usage; it must run in a `finally` block.
2342
- */
2343
- async acquire(estimatedTokens, signal) {
2344
- if (signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
2345
- if (!Number.isFinite(estimatedTokens) || estimatedTokens < 0) throw new LLMError(`estimatedTokens must be a finite, non-negative number, got ${String(estimatedTokens)}`, "invalid_params");
2346
- if (this.tokens && estimatedTokens > this.tokens.getCapacity()) throw new LLMError(`estimatedTokens (${estimatedTokens}) exceeds the configured tokensPerMinute capacity (${this.tokens.getCapacity()}); this call could never acquire capacity.`, "rate_limited", { code: "rate_limit_capacity_exceeded" });
2347
- if (this.queue.length === 0) {
2348
- const attempt = this.tryAcquireBuckets(estimatedTokens);
2349
- if (attempt.ok) return {
2350
- release: this.makeRelease(estimatedTokens),
2351
- waitedMs: 0
2352
- };
2353
- return this.enqueue(estimatedTokens, attempt.reason, signal);
2354
- }
2355
- if (this.maxQueueSize > 0 && this.queue.length >= this.maxQueueSize) throw this.queueFullError();
2356
- return this.enqueue(estimatedTokens, void 0, signal);
2357
- }
2358
- queueFullError() {
2359
- return new LLMError("Rate limit queue is full", "rate_limited", { code: "rate_limit_queue_full" });
2360
- }
2361
- enqueue(estimatedTokens, initialReason, signal) {
2362
- return new Promise((resolvePromise, rejectPromise) => {
2363
- const waiter = {
2364
- estimatedTokens,
2365
- enqueuedAt: Date.now(),
2366
- lastReason: initialReason,
2367
- resolve: (result) => {
2368
- cleanup();
2369
- resolvePromise(result);
2370
- },
2371
- reject: (error) => {
2372
- cleanup();
2373
- if (this.wakeTimer) {
2374
- clearTimeout(this.wakeTimer);
2375
- this.wakeTimer = void 0;
2376
- }
2377
- this.drain();
2378
- rejectPromise(error);
2379
- }
2380
- };
2381
- let queueTimer;
2382
- const onAbort = () => {
2383
- waiter.reject(new LLMError("LLM request aborted", "aborted"));
2384
- };
2385
- const cleanup = () => {
2386
- if (queueTimer) clearTimeout(queueTimer);
2387
- signal?.removeEventListener("abort", onAbort);
2388
- const index = this.queue.indexOf(waiter);
2389
- if (index !== -1) this.queue.splice(index, 1);
2390
- };
2391
- if (this.maxQueueMs > 0) queueTimer = setTimeout(() => {
2392
- waiter.reject(new LLMError("Rate limit queue timed out before capacity was available", "rate_limited", { code: "rate_limit_queue_timeout" }));
2393
- }, this.maxQueueMs);
2394
- signal?.addEventListener("abort", onAbort, { once: true });
2395
- this.queue.push(waiter);
2396
- this.drain();
2397
- });
2398
- }
2399
- /**
2400
- * Checks and takes from every configured bucket as one atomic unit: if
2401
- * any bucket lacks capacity, whatever was already taken from the
2402
- * earlier ones in this attempt is rolled back before reporting which
2403
- * bucket blocked.
2404
- */
2405
- tryAcquireBuckets(estimatedTokens) {
2406
- const taken = [];
2407
- const take = (bucket, amount) => {
2408
- if (!bucket) return true;
2409
- if (!bucket.tryTake(amount)) return false;
2410
- taken.push({
2411
- bucket,
2412
- amount
2413
- });
2414
- return true;
2415
- };
2416
- if (!take(this.concurrency, 1)) return {
2417
- ok: false,
2418
- reason: "concurrency"
2419
- };
2420
- if (!take(this.requests, 1)) {
2421
- for (const entry of taken) entry.bucket.give(entry.amount);
2422
- return {
2423
- ok: false,
2424
- reason: "rpm"
2425
- };
2426
- }
2427
- if (!take(this.tokens, estimatedTokens)) {
2428
- for (const entry of taken) entry.bucket.give(entry.amount);
2429
- return {
2430
- ok: false,
2431
- reason: "tpm"
2432
- };
2433
- }
2434
- return { ok: true };
2435
- }
2436
- /** Drains the queue head first. Stops at the first waiter that still can't proceed, so no one is starved out of turn. */
2437
- drain() {
2438
- while (this.queue.length > 0) {
2439
- const waiter = this.queue[0];
2440
- const attempt = this.tryAcquireBuckets(waiter.estimatedTokens);
2441
- if (!attempt.ok) {
2442
- waiter.lastReason = attempt.reason;
2443
- this.scheduleWake(attempt.reason, waiter.estimatedTokens);
2444
- return;
2445
- }
2446
- const waitedMs = Date.now() - waiter.enqueuedAt;
2447
- waiter.resolve({
2448
- release: this.makeRelease(waiter.estimatedTokens),
2449
- waitedMs,
2450
- reason: waiter.lastReason
2451
- });
2452
- }
2453
- }
2454
- /**
2455
- * Schedules a one-shot re-check of the queue for whenever the bucket
2456
- * that's currently blocking the head waiter should next have enough
2457
- * capacity. A no-op for a concurrency block (only `release` can clear
2458
- * that) or while a wake is already pending.
2459
- */
2460
- scheduleWake(reason, estimatedTokens) {
2461
- if (this.wakeTimer) return;
2462
- const ms = reason === "rpm" ? this.requests?.msUntilAvailable(1) : reason === "tpm" ? this.tokens?.msUntilAvailable(estimatedTokens) : void 0;
2463
- if (ms === void 0 || !Number.isFinite(ms)) return;
2464
- const delay = Math.min(Math.max(1, Math.ceil(ms)), MAX_WAKE_DELAY_MS);
2465
- this.wakeTimer = setTimeout(() => {
2466
- this.wakeTimer = void 0;
2467
- this.drain();
2468
- }, delay);
2469
- }
2470
- /**
2471
- * Builds the one-shot release closure for an acquired slot. Only the
2472
- * concurrency bucket is given back on release; the requests-per-minute
2473
- * bucket is a real spend that only recovers via its own refill, and the
2474
- * tokens bucket is reconciled against `actualTokens` rather than fully
2475
- * refunded, since real tokens really were spent.
2476
- */
2477
- makeRelease(estimatedTokens) {
2478
- let released = false;
2479
- return (actualTokens) => {
2480
- if (released) return;
2481
- released = true;
2482
- this.concurrency?.give(1);
2483
- if (this.tokens && actualTokens !== void 0 && Number.isFinite(actualTokens)) this.tokens.give(estimatedTokens - actualTokens);
2484
- this.drain();
2485
- };
2486
- }
2487
- };
2488
-
2489
- //#endregion
2490
- //#region src/vernLLM.ts
2491
- /**
2492
- * A resilient layer around an LLM chat completions client. This is VernLLM!
2493
- *
2494
- * Adds retry with backoff and jitter, per-attempt timeouts, an optional
2495
- * circuit breaker, JSON parsing with optional schema validation, usage
2496
- * tracking, and an optional response cache. All configurable, all opt-in
2497
- * beyond sensible defaults.
2498
- */
2499
- var VernLLM = class {
2500
- logger;
2501
- /**
2502
- * One `CallExecutor` per provider target: index 0 is the primary,
2503
- * everything after it is a `fallback` target, in the order declared.
2504
- * Each owns its own request building, retry/timeout, circuit breaker,
2505
- * and rate limiter. `call()` walks this array in `runFallbackChain`,
2506
- * moving to the next entry only when `fallbackOn` says to.
2507
- */
2508
- executors;
2509
- /** Decides whether a failed target is followed by the next one or the chain stops. See `VernLLMOptions['fallbackOn']`. */
2510
- fallbackOn;
2511
- /** Reports a `'fallback'` event when the chain moves to the next target. Shared `onEvent` plumbing, same as every executor's. */
2512
- reportEvent;
2513
- /**
2514
- * Owns cache key resolution, cache reads/writes, and in-flight
2515
- * coalescing for `cachedCall()`. Independent of `executor`: it only
2516
- * ever calls back into `this.call()` as an opaque function.
2517
- */
2518
- cacheOrchestrator;
2519
- /**
2520
- * @param options Client, model, and tunables. Defaults: `maxRetries` 1,
2521
- * `timeoutMs` 25000, `baseDelayMs` 500, `defaultMaxTokens` 1000,
2522
- * `defaultTemperature` 0.2, `cache` an in-memory adapter,
2523
- * `nonRetryableStatus` `[400, 401, 403, 404, 422]`, `debug` false.
2524
- */
2525
- constructor(options) {
2526
- this.logger = createSafeLogger(options.logger ?? new ConsoleLogger(options.debug ?? false));
2527
- const providerName = options.name ?? "primary";
2528
- this.cacheOrchestrator = new CacheOrchestrator(options.cache ?? new InMemoryCacheAdapter(), this.logger);
2529
- this.fallbackOn = options.fallbackOn ?? defaultFallbackOn;
2530
- this.reportEvent = makeEventReporter(options.onEvent, this.logger);
2531
- const primaryDefaultTemperature = options.defaultTemperature === void 0 ? .2 : options.defaultTemperature;
2532
- const primaryTarget = {
2533
- client: options.client,
2534
- model: options.model,
2535
- name: providerName,
2536
- maxRetries: options.maxRetries,
2537
- timeoutMs: options.timeoutMs,
2538
- chunkIdleTimeoutMs: options.chunkIdleTimeoutMs,
2539
- baseDelayMs: options.baseDelayMs,
2540
- defaultMaxTokens: options.defaultMaxTokens,
2541
- defaultTemperature: primaryDefaultTemperature,
2542
- nonRetryableStatus: options.nonRetryableStatus,
2543
- circuitBreaker: options.circuitBreaker,
2544
- rateLimit: options.rateLimit
2545
- };
2546
- const declaredFallbacks = Array.isArray(options.fallback) ? options.fallback : options.fallback ? [options.fallback] : [];
2547
- const targets = [primaryTarget, ...declaredFallbacks];
2548
- this.executors = targets.map((target, i) => {
2549
- const isFallback = i > 0;
2550
- const name = target.name ?? (isFallback ? `fallback[${i - 1}]` : providerName);
2551
- const breaker = buildCircuitBreaker(target.circuitBreaker, name, target.model, options.onEvent, this.logger);
2552
- return new CallExecutor(name, target.client, target.model, {
2553
- maxRetries: target.maxRetries ?? options.maxRetries ?? 1,
2554
- timeoutMs: target.timeoutMs ?? options.timeoutMs ?? 25e3,
2555
- chunkIdleTimeoutMs: target.chunkIdleTimeoutMs ?? options.chunkIdleTimeoutMs ?? 3e4,
2556
- baseDelayMs: target.baseDelayMs ?? options.baseDelayMs ?? 500,
2557
- defaultMaxTokens: target.defaultMaxTokens ?? options.defaultMaxTokens ?? 1e3,
2558
- defaultTemperature: target.defaultTemperature === void 0 ? primaryDefaultTemperature : target.defaultTemperature,
2559
- nonRetryableStatus: target.nonRetryableStatus ?? options.nonRetryableStatus ?? [
2560
- 400,
2561
- 401,
2562
- 403,
2563
- 404,
2564
- 422
2565
- ],
2566
- parseJson: options.parseJson,
2567
- logger: this.logger,
2568
- redact: options.redact,
2569
- onUsage: options.onUsage,
2570
- onUsageFailure: options.onUsageFailure,
2571
- onEvent: options.onEvent,
2572
- breaker,
2573
- limiter: target.rateLimit ? new RateLimiter(target.rateLimit) : void 0,
2574
- isFallback
2575
- });
2576
- });
2577
- }
2578
- /** Logs a failed refundUsage attempt via the configured logger. */
2579
- logRefundError(logMessage, error) {
2580
- this.logger.error(logMessage, { message: error instanceof Error ? error.message : "unknown" });
2581
- }
2582
- /**
2583
- * Walks `this.executors` in order, running `attempt` against each until
2584
- * one succeeds or every target has failed. `run` on a lone target
2585
- * (no `fallback` configured) throws exactly what it throws today: the
2586
- * loop's single iteration path is unchanged from pre-fallback behavior.
2587
- *
2588
- * For streaming, `attempt` is `executor.runStream`, whose own retries
2589
- * only cover *opening* the stream (see `CallExecutor.runStream`). A
2590
- * mid-stream failure surfaces through `finalResult` after this function
2591
- * has already returned, so it's never seen here and never falls over,
2592
- * per the streaming limitation: splicing a second model's output into a
2593
- * response the consumer has already partially rendered would corrupt
2594
- * it.
2595
- */
2596
- async runFallbackChain(params, requestId, attempt, skipBreakerCheckForFirst = false) {
2597
- const attempts = [];
2598
- for (let i = 0; i < this.executors.length; i++) {
2599
- const executor = this.executors[i];
2600
- const startedAt = Date.now();
2601
- let attemptCount = 0;
2602
- try {
2603
- if (!(i === 0 && skipBreakerCheckForFirst)) executor.assertBreakerClosed(params.model);
2604
- const result = await attempt(executor, () => {
2605
- attemptCount += 1;
2606
- });
2607
- return {
2608
- result,
2609
- executor,
2610
- index: i,
2611
- attemptCount
2612
- };
2613
- } catch (error) {
2614
- const normalized = normalizeError(error, params.signal);
2615
- attempts.push({
2616
- index: i - 1,
2617
- provider: executor.providerName,
2618
- model: params.model ?? executor.model,
2619
- error: normalized.toSnapshot()
2620
- });
2621
- const isLast = i === this.executors.length - 1;
2622
- const policyDecision = this.fallbackOn(normalized, { isLastTarget: isLast });
2623
- const decision = isLast ? "stop" : policyDecision;
2624
- if (decision === "stop") throw attempts.length > 1 ? new FallbackExhaustedError(attempts) : normalized;
2625
- const next = this.executors[i + 1];
2626
- this.reportEvent({
2627
- kind: "fallback",
2628
- requestId,
2629
- from: executor.providerName,
2630
- to: next.providerName,
2631
- fromIndex: i - 1,
2632
- toIndex: i,
2633
- error: normalized,
2634
- elapsedMs: Date.now() - startedAt
2635
- });
2636
- }
2637
- }
2638
- throw new LLMError("No provider targets configured", "invalid_params");
2639
- }
2640
- async call(params) {
2641
- if (params.signal?.aborted) throw new LLMError("LLM request aborted", "aborted");
2642
- const requestId = params.requestId ?? (0, crypto.randomUUID)();
2643
- const soleTarget = this.executors.length === 1;
2644
- if (soleTarget) this.executors[0].assertBreakerClosed(params.model);
2645
- if (params.stream) return withReservedUsageForStream(params, async () => {
2646
- const { result } = await this.runFallbackChain(params, requestId, (executor, onAttempt) => executor.runStream(params, requestId, onAttempt), soleTarget);
2647
- return result;
2648
- }, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
2649
- return withReservedUsage(params, false, async () => {
2650
- const { result, executor, index, attemptCount } = await this.runFallbackChain(params, requestId, (target, onAttempt) => target.run(params, requestId, onAttempt), soleTarget);
2651
- if (params.meta) params.meta.current = {
2652
- provider: executor.providerName,
2653
- model: params.model ?? executor.model,
2654
- fallbackIndex: index - 1,
2655
- usedFallback: index > 0,
2656
- attempts: attemptCount
2657
- };
2658
- return result;
2659
- }, params.signal, (logMessage, error) => this.logRefundError(logMessage, error));
2660
- }
2661
- /**
2662
- * Thin delegator kept private on `VernLLM` (rather than only existing on
2663
- * `CacheOrchestrator`) since it's the one caching primitive exercised
2664
- * directly by white-box tests, independent of the public `cachedCall()`
2665
- * surface.
2666
- */
2667
- runCached(params) {
2668
- return this.cacheOrchestrator.runCached(params);
2669
- }
2670
- /**
2671
- * Removes a cached response by key when the configured cache adapter
2672
- * supports deletion. Cache invalidation is the caller's responsibility;
2673
- * only the application knows when cached data is stale.
2674
- *
2675
- * @param key The raw cache key (resolved through the adapter's
2676
- * `resolveKey`, if any, before deletion).
2677
- */
2678
- async deleteCache(key) {
2679
- await this.cacheOrchestrator.deleteCache(key);
2680
- }
2681
- async cachedCall(params) {
2682
- const { call: callParams,...cacheParams } = params;
2683
- const restCallParams = callParams;
2684
- if (restCallParams.reserveUsage || restCallParams.refundUsage) throw new LLMError("`reserveUsage`/`refundUsage` were set inside `call`, where cachedCall ignores them. Move them to the top level of the cachedCall() params, alongside cacheKey/ttl, instead.", "invalid_params");
2685
- if (restCallParams.stream) {
2686
- const streamParams = restCallParams;
2687
- return this.cacheOrchestrator.runCachedStream({
2688
- ...cacheParams,
2689
- openStream: () => this.call(streamParams)
2690
- }, Boolean(restCallParams.tools));
2691
- }
2692
- return this.runCached({
2693
- ...cacheParams,
2694
- fn: () => this.call(restCallParams)
2695
- });
2696
- }
2697
- /**
2698
- * @param target.index Which target to read. Defaults to the primary.
2699
- * @param target.model Which model bucket to read, if the target isolates by model.
2700
- * @returns The breaker state, or `undefined` if that target has no breaker.
2701
- * @throws {RangeError} If `target.index` names no target. Lets a real
2702
- * target with no breaker (`undefined`) stay distinguishable from a
2703
- * target that doesn't exist.
2704
- */
2705
- getCircuitState(target) {
2706
- const executor = this.resolveExecutor(target?.index ?? 0, "getCircuitState");
2707
- this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "getCircuitState");
2708
- return executor.getCircuitState(target?.model ?? executor.model);
2709
- }
2710
- /**
2711
- * @param model Which model bucket to read, for targets that isolate by model.
2712
- * @returns Every target's state, in chain order.
2713
- */
2714
- getCircuitStates(model) {
2715
- return this.executors.map((executor, index) => ({
2716
- provider: executor.providerName,
2717
- index,
2718
- isFallback: index > 0,
2719
- isolateByModel: executor.isolateByModel,
2720
- state: executor.getCircuitState(model ?? executor.model)
2721
- }));
2722
- }
2723
- /**
2724
- * Manually opens a target's breaker, e.g. to pull a provider out of
2725
- * rotation ahead of known maintenance instead of waiting for it to fail.
2726
- *
2727
- * @param target.index Which target to open. Defaults to the primary.
2728
- * @param target.model Which model bucket to open, if the target isolates by model.
2729
- * @throws {RangeError} If `target.index` names no target.
2730
- */
2731
- openCircuit(target) {
2732
- const executor = this.resolveExecutor(target?.index ?? 0, "openCircuit");
2733
- this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "openCircuit");
2734
- executor.openCircuit(target?.model ?? executor.model);
2735
- }
2736
- /**
2737
- * Manually closes a target's breaker, e.g. once a provider is confirmed
2738
- * healthy again without waiting out the cooldown.
2739
- *
2740
- * @param target.index Which target to close. Defaults to the primary.
2741
- * @param target.model Which model bucket to close, if the target isolates by model.
2742
- * @throws {RangeError} If `target.index` names no target.
2743
- */
2744
- closeCircuit(target) {
2745
- const executor = this.resolveExecutor(target?.index ?? 0, "closeCircuit");
2746
- this.warnIfModelUnsupported(executor.isolateByModel, target?.model, "closeCircuit");
2747
- executor.closeCircuit(target?.model ?? executor.model);
2748
- }
2749
- /** Resolves a target index so every circuit-breaker method agrees on what counts as valid. */
2750
- resolveExecutor(index, caller) {
2751
- const executor = this.executors[index];
2752
- if (!executor) throw new RangeError(`${caller}: no target at index ${index} (chain has ${this.executors.length} target${this.executors.length === 1 ? "" : "s"})`);
2753
- return executor;
2754
- }
2755
- /** Warns when `model` can't do anything on this target, so it's never silently ignored. */
2756
- warnIfModelUnsupported(isolateByModel, model, caller) {
2757
- if (model !== void 0 && !isolateByModel) this.logger.warn(`[VernLLM] ${caller}: \`model: '${model}'\` has no effect here. This target's circuitBreaker doesn't have isolateByModel on, so it only tracks one shared circuit regardless of \`model\`. Omit \`model\`, or set \`circuitBreaker.isolateByModel: true\` on this target if per-model tracking is what you want.`);
2758
- }
2759
- };
2760
-
2761
- //#endregion
2762
- //#region src/adapters/internal/sse.ts
2763
- /**
2764
- * Parses a Server-Sent-Events byte/text stream into the JSON payload of
2765
- * each `data:` frame, in arrival order. Generic over transport: works with
2766
- * anything that hands back progressively-arriving `Uint8Array` or `string`
2767
- * chunks via async iteration: native `fetch`'s `response.body` (wrapped
2768
- * to be iterable, see `webStreamToAsyncIterable` in `fetch.ts`), axios's
2769
- * Node `Readable` (already async-iterable, no wrapping needed), etc, so
2770
- * this framing layer doesn't care which transport produced the bytes.
2771
- *
2772
- * Follows the SSE spec's frame-delimiting rules closely enough for LLM
2773
- * streaming responses: frames are separated by a blank line, each frame
2774
- * may carry one or more `data:` lines (joined with `\n` per spec when
2775
- * there's more than one), `:`-prefixed lines are comments and ignored, and
2776
- * other SSE fields (`event:`, `id:`, `retry:`) are ignored since VernLLM
2777
- * only needs the payload. A frame whose data is exactly `[DONE]` (the
2778
- * sentinel several providers, notably OpenAI, send to mark stream end)
2779
- * ends iteration without yielding it.
2780
- *
2781
- * Line endings: `\r\n` and bare `\r` (both legal per the SSE spec, alongside `\n`) are normalized
2782
- * to `\n` before frame splitting. A `\r` at the very end of the currently-buffered text is left
2783
- * alone until either more text arrives (in case it's the first half of a split `\r\n` pair) or the
2784
- * stream ends, so a `\r\n` pair split across two transport chunks is never misread as two blank
2785
- * lines.
2786
- *
2787
- * Malformed JSON in a frame throws `LLMError('parse')`, consistent with
2788
- * how malformed JSON is handled elsewhere in VernLLM.
2789
- */
2790
- async function* parseSseStream(source) {
2791
- const decoder = new TextDecoder("utf-8", { fatal: true });
2792
- let buffer = "";
2793
- for await (const chunk of source) {
2794
- let text;
2795
- try {
2796
- text = typeof chunk === "string" ? chunk : decoder.decode(chunk, { stream: true });
2797
- } catch (cause) {
2798
- throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
2799
- }
2800
- buffer = (buffer + text).replace(/\r\n/g, "\n").replace(/\r(?!$)/g, "\n");
2801
- let boundary$1 = buffer.indexOf("\n\n");
2802
- while (boundary$1 !== -1) {
2803
- const frame = buffer.slice(0, boundary$1);
2804
- buffer = buffer.slice(boundary$1 + 2);
2805
- const event = parseSseFrame(frame);
2806
- if (event === DONE) return;
2807
- if (event !== NO_DATA) yield event;
2808
- boundary$1 = buffer.indexOf("\n\n");
2809
- }
2810
- }
2811
- try {
2812
- buffer += decoder.decode();
2813
- } catch (cause) {
2814
- throw new LLMError("Invalid UTF-8 in SSE stream", "parse", { cause });
2815
- }
2816
- buffer = buffer.replace(/\r$/, "\n");
2817
- let boundary = buffer.indexOf("\n\n");
2818
- while (boundary !== -1) {
2819
- const frame = buffer.slice(0, boundary);
2820
- buffer = buffer.slice(boundary + 2);
2821
- const event = parseSseFrame(frame);
2822
- if (event === DONE) return;
2823
- if (event !== NO_DATA) yield event;
2824
- boundary = buffer.indexOf("\n\n");
2825
- }
2826
- const trailing = buffer.trim();
2827
- if (trailing) {
2828
- const event = parseSseFrame(trailing);
2829
- if (event !== DONE && event !== NO_DATA) yield event;
2830
- }
2831
- }
2832
- const DONE = Symbol("sse-stream-done");
2833
- const NO_DATA = Symbol("sse-frame-no-data");
2834
- /**
2835
- * Sentinel yielded by `parseSseStream` for a comment-only frame (no
2836
- * `data:` payload), the mechanism providers use for SSE keep-alive
2837
- * pings. Exported so a consumer (e.g. `fromFetch`) can react to "still
2838
- * alive" separately from a genuinely empty frame (`NO_DATA`, kept internal).
2839
- */
2840
- const SSE_PING = Symbol("sse-frame-ping");
2841
- /** Extracts and JSON-parses the `data:` payload of one SSE frame (the text between two blank lines). */
2842
- function parseSseFrame(frame) {
2843
- const dataLines = [];
2844
- let sawComment = false;
2845
- for (const line of frame.split("\n")) {
2846
- if (line.startsWith(":")) {
2847
- sawComment = true;
2848
- continue;
2849
- }
2850
- if (!line.startsWith("data:")) continue;
2851
- dataLines.push(line.startsWith("data: ") ? line.slice(6) : line.slice(5));
2852
- }
2853
- if (!dataLines.length) return sawComment ? SSE_PING : NO_DATA;
2854
- const data = dataLines.join("\n");
2855
- if (data === "[DONE]") return DONE;
2856
- try {
2857
- return JSON.parse(data);
2858
- } catch (cause) {
2859
- throw new LLMError(`Invalid JSON in SSE frame: ${data.slice(0, 200)}`, "parse", {
2860
- cause,
2861
- code: "stream_frame_invalid"
2862
- });
2863
- }
2864
- }
2865
-
2866
- //#endregion
2867
- //#region src/adapters/internal/imageFormat.ts
2868
- /**
2869
- * MIME types accepted for `ImageBlock.mimeType` across all adapters. This is
2870
- * the intersection of what Anthropic, Gemini, OpenAI-compatible, and Bedrock
2871
- * Converse all natively support, so a `ContentBlock[]` that validates for
2872
- * one provider validates for all of them.
2873
- */
2874
- const SUPPORTED_IMAGE_MIME_TYPES = [
2875
- "image/png",
2876
- "image/jpeg",
2877
- "image/gif",
2878
- "image/webp"
2879
- ];
2880
- /**
2881
- * Validates an `ImageBlock.mimeType` against the shared supported set.
2882
- * Throws a non-retryable `LLMError('invalid_params')`, since an unsupported
2883
- * mimeType is a bug in the caller's own input, deterministic before any
2884
- * request is built, the same class of failure as every other check in
2885
- * `RequestBuilder`.
2886
- */
2887
- function assertSupportedImageMimeType(mimeType) {
2888
- if (SUPPORTED_IMAGE_MIME_TYPES.includes(mimeType)) return mimeType;
2889
- throw new LLMError(`Unsupported image mimeType "${mimeType}": expected one of ${SUPPORTED_IMAGE_MIME_TYPES.join(", ")}`, "invalid_params");
2890
- }
2891
-
2892
- //#endregion
2893
- //#region src/adapters/internal/nativeStructuredOutput.ts
2894
- /** Resolves whether `model` is covered by a caller-supplied allow-list/predicate. */
2895
- function supportsNativeStructuredOutput(model, override) {
2896
- if (!override) return false;
2897
- return Array.isArray(override) ? override.includes(model) : override(model);
2898
- }
2899
-
2900
- //#endregion
2901
- //#region src/adapters/anthropic.ts
2902
- /**
2903
- * Translates a VernLLM `ContentBlock[]` (our provider-agnostic multimodal
2904
- * shape) into Anthropic's native content-block array: text blocks pass
2905
- * through as-is, image blocks become `{ type: 'image', source: { type:
2906
- * 'base64', media_type, data } }`.
2907
- */
2908
- function toAnthropicContent(blocks) {
2909
- return blocks.map((block) => block.type === "image" ? {
2910
- type: "image",
2911
- source: {
2912
- type: "base64",
2913
- media_type: assertSupportedImageMimeType(block.mimeType),
2914
- data: block.data
2915
- }
2916
- } : {
2917
- type: "text",
2918
- text: block.text
2919
- });
2920
- }
2921
- /**
2922
- * Asserts a caller-supplied JSON Schema is an object schema before it's
2923
- * used as Anthropic's `Tool.input_schema`, which (like every other
2924
- * provider's function-calling API) requires `type: 'object'`. VernLLM's own
2925
- * public `tools`/`jsonSchema` APIs accept freeform `Record<string,
2926
- * unknown>` JSON Schema, so nothing upstream guarantees this at compile
2927
- * time; this is the runtime check that stands in for that, so a schema
2928
- * missing (or mistyping) `type: 'object'` fails loudly and immediately
2929
- * instead of being silently forwarded to Anthropic malformed.
2930
- */
2931
- function assertObjectSchema(schema, toolName) {
2932
- if (schema.type !== "object") throw new LLMError(`Tool "${toolName}"'s schema must have "type": "object" (Anthropic requires object-shaped tool parameters).`, "validation");
2933
- return schema;
2934
- }
2935
- /**
2936
- * Translates VernLLM's OpenAI-shaped wire `tool_choice` into Anthropic's
2937
- * `{ type: 'auto' | 'any' | 'none' | 'tool', name? }` shape. `'required'`
2938
- * maps to `'any'` (Anthropic's "must call some tool" equivalent).
2939
- */
2940
- function toAnthropicToolChoice(toolChoice) {
2941
- if (!toolChoice || toolChoice === "auto") return { type: "auto" };
2942
- if (toolChoice === "none") return { type: "none" };
2943
- if (toolChoice === "required") return { type: "any" };
2944
- return {
2945
- type: "tool",
2946
- name: toolChoice.function.name
2947
- };
2948
- }
2949
- /**
2950
- * Maps VernLLM's OpenAI-shaped wire `tools`/`tool_choice` into Anthropic's
2951
- * `tools`/`tool_choice` shape. Shared by the two call sites that build real
2952
- * (non-schema-forced) tool definitions: the plain tools-only branch, and
2953
- * the native-structured-output branch, which sends real tools alongside
2954
- * `output_config` rather than instead of it.
2955
- */
2956
- function buildAnthropicTools(tools, toolChoiceParam) {
2957
- return {
2958
- tools: tools.map((t) => ({
2959
- name: t.function.name,
2960
- description: t.function.description,
2961
- input_schema: assertObjectSchema(t.function.parameters, t.function.name)
2962
- })),
2963
- toolChoice: toAnthropicToolChoice(toolChoiceParam)
2964
- };
2965
- }
2966
- /**
2967
- * Builds the Anthropic-shaped request body from VernLLM's wire params,
2968
- * shared between `create` and `createStream` so both go through identical
2969
- * translation (system prompt, message shaping, and the jsonSchema →
2970
- * forced-single-tool mapping all happen exactly once, not once per entry
2971
- * point).
2972
- *
2973
- * Returns `toolName` alongside the body: when set, the model was forced to
2974
- * call a single synthetic tool standing in for `jsonSchema` output (the
2975
- * legacy path, for models without native structured-output support), and
2976
- * both `create` and `createStream` need to know this so they can unwrap
2977
- * that tool call back into plain text content instead of treating it like
2978
- * a real tool call. On the native path (model supports `output_config`),
2979
- * `toolName` is `undefined`: the schema-conforming JSON already arrives as
2980
- * ordinary text content, nothing to unwrap, and any real tool calls in
2981
- * `params.tools` are left for the normal, non-forced tool-call handling
2982
- * both `create` and `createStream` already do when `toolName` is unset.
2983
- */
2984
- function buildAnthropicRequestBody(params, nativeStructuredOutputModels) {
2985
- const systemMessage = params.messages.find((m) => m.role === "system");
2986
- const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
2987
- const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
2988
- const schemaName = jsonSchema?.name.trim();
2989
- if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
2990
- const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
2991
- if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Anthropic model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`, "validation");
2992
- let toolName;
2993
- let jsonInstruction;
2994
- let outputFormat;
2995
- let tools;
2996
- let toolChoice;
2997
- if (jsonSchema && isNative) {
2998
- outputFormat = {
2999
- type: "json_schema",
3000
- schema: jsonSchema.schema
3001
- };
3002
- if (params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
3003
- } else if (jsonSchema && schemaName) {
3004
- const { schema, description, strict } = jsonSchema;
3005
- toolName = schemaName;
3006
- tools = [{
3007
- name: toolName,
3008
- description,
3009
- input_schema: assertObjectSchema(schema, toolName),
3010
- strict
3011
- }];
3012
- toolChoice = {
3013
- type: "tool",
3014
- name: toolName
3015
- };
3016
- } else if (params.response_format?.type === "json_object") jsonInstruction = "Respond with valid JSON only, no prose or markdown fences.";
3017
- if (!jsonSchema && params.tools?.length) ({tools, toolChoice} = buildAnthropicTools(params.tools, params.tool_choice));
3018
- const system = [systemMessage?.content, jsonInstruction].filter(Boolean).join("\n\n");
3019
- const body = {
3020
- model: params.model,
3021
- max_tokens: params.max_tokens,
3022
- ...params.temperature !== void 0 ? { temperature: params.temperature } : {},
3023
- system: system || void 0,
3024
- messages: mergeConsecutiveToolResults$1(conversationMessages.map((m) => toAnthropicMessage(m))),
3025
- ...tools ? {
3026
- tools,
3027
- tool_choice: toolChoice
3028
- } : {},
3029
- ...outputFormat ? { output_config: { format: outputFormat } } : {}
3030
- };
3031
- return {
3032
- body,
3033
- toolName
3034
- };
3035
- }
3036
- /**
3037
- * Wraps an Anthropic SDK client so it satisfies the same `LLMClient`
3038
- * interface VernLLM uses for OpenAI/Groq.
3039
- *
3040
- * `response_format: json_schema`, on a model covered by
3041
- * `options.nativeStructuredOutputModels`, is sent as `output_config.format`,
3042
- * its own request field, independent of `tools`/`tool_choice`, so it can be
3043
- * combined with real, caller-supplied `tools` in the same request. Only
3044
- * `type` and `schema` are sent on this path, the real Anthropic API's
3045
- * `output_config.format` has no `name`/`description`/`strict` fields.
3046
- *
3047
- * On any other model (the default, since `nativeStructuredOutputModels` is
3048
- * opt-in), `response_format: json_schema` is mapped to Anthropic's forced
3049
- * tool-use instead: a single tool is defined with `input_schema` set to
3050
- * the caller's schema, `description` forwarded when provided, and `strict`
3051
- * forwarded when set, and `tool_choice` forces the model to call it. This
3052
- * legacy path cannot be combined with real `tools` (both would need the
3053
- * same `tools`/`tool_choice` field), and a call that tries throws
3054
- * `LLMError('validation')` before reaching the API. Provider-constrained
3055
- * schema matching applies only when `strict: true` is forwarded and
3056
- * supported.
3057
- *
3058
- * `response_format: json_object` (no schema to build a tool from) falls
3059
- * back to a system-prompt instruction, since there's nothing to constrain
3060
- * generation against. Unlike `jsonSchema`, this combines with real `tools`
3061
- * freely on every model: it's a prompt nudge, not a request field, so
3062
- * there's nothing for it to collide with.
3063
- */
3064
- function fromAnthropic(anthropicClient, options) {
3065
- const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
3066
- const rawMessagesCreate = anthropicClient.messages.create.bind(anthropicClient.messages);
3067
- return { chat: { completions: {
3068
- async create(params, options$1) {
3069
- const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels);
3070
- const response = await anthropicClient.messages.create(body, options$1);
3071
- let text;
3072
- let wireToolCalls;
3073
- if (toolName) {
3074
- const toolUse = response.content.find((block) => block.type === "tool_use" && block.name === toolName);
3075
- if (!toolUse) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
3076
- if (!toolUse.input || typeof toolUse.input !== "object" || Array.isArray(toolUse.input)) throw new LLMError(`Anthropic returned invalid structured output for tool "${toolName}". Expected an object.`, "validation");
3077
- text = JSON.stringify(toolUse.input);
3078
- } else {
3079
- text = response.content.filter((block) => block.type === "text").map((block) => block.text ?? "").join("");
3080
- const toolUses = response.content.filter((block) => block.type === "tool_use");
3081
- if (toolUses.length) wireToolCalls = toolUses.map((block) => ({
3082
- id: block.id,
3083
- type: "function",
3084
- function: {
3085
- name: block.name,
3086
- arguments: JSON.stringify(block.input ?? {})
3087
- }
3088
- }));
3089
- }
3090
- return {
3091
- choices: [{ message: {
3092
- content: text,
3093
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3094
- } }],
3095
- usage: {
3096
- prompt_tokens: response.usage?.input_tokens,
3097
- completion_tokens: response.usage?.output_tokens,
3098
- total_tokens: (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0)
3099
- }
3100
- };
3101
- },
3102
- async *createStream(params, options$1) {
3103
- const { body, toolName } = buildAnthropicRequestBody(params, nativeStructuredOutputModels);
3104
- const stream = await rawMessagesCreate({
3105
- ...body,
3106
- stream: true
3107
- }, options$1);
3108
- const blockKinds = new Map();
3109
- let inputTokens = 0;
3110
- let sawJsonTool = false;
3111
- for await (const event of stream) if (event.type === "message_start") inputTokens = event.message.usage?.input_tokens ?? 0;
3112
- else if (event.type === "content_block_start") if (event.content_block.type === "tool_use") {
3113
- const kind = event.content_block.name === toolName ? "json-tool" : "tool_use";
3114
- blockKinds.set(event.index, kind);
3115
- if (kind === "json-tool") sawJsonTool = true;
3116
- else if (!toolName) yield {
3117
- type: "tool_call_delta",
3118
- index: event.index,
3119
- id: event.content_block.id,
3120
- name: event.content_block.name
3121
- };
3122
- } else blockKinds.set(event.index, "text");
3123
- else if (event.type === "content_block_delta") {
3124
- if (event.delta.type === "text_delta") {
3125
- if (!toolName) yield {
3126
- type: "text-delta",
3127
- delta: event.delta.text
3128
- };
3129
- } else if (event.delta.type === "input_json_delta") {
3130
- const kind = blockKinds.get(event.index);
3131
- if (kind === "json-tool") yield {
3132
- type: "text-delta",
3133
- delta: event.delta.partial_json
3134
- };
3135
- else if (!toolName) yield {
3136
- type: "tool_call_delta",
3137
- index: event.index,
3138
- argumentsDelta: event.delta.partial_json
3139
- };
3140
- }
3141
- } else if (event.type === "message_delta") {
3142
- const outputTokens = event.usage?.output_tokens ?? 0;
3143
- yield {
3144
- type: "usage",
3145
- usage: {
3146
- prompt_tokens: inputTokens,
3147
- completion_tokens: outputTokens,
3148
- total_tokens: inputTokens + outputTokens
3149
- }
3150
- };
3151
- } else if (event.type === "ping") yield { type: "ping" };
3152
- if (toolName && !sawJsonTool) throw new LLMError(`Anthropic did not return the required structured output tool "${toolName}".`, "validation");
3153
- }
3154
- } } };
3155
- }
3156
- /**
3157
- * Anthropic requires strict role alternation, so the per-wire-message
3158
- * mapping above (one `{role:'user', content:[tool_result]}` per VernLLM
3159
- * wire tool message) needs merging back together when an assistant turn
3160
- * requested more than one tool: multiple consecutive user turns would
3161
- * violate that alternation, and Anthropic's API rejects it outright. This
3162
- * merges any run of tool-result-only user messages into one, with all
3163
- * their tool_result blocks combined, the shape Anthropic expects for "here
3164
- * are the results of everything you just asked for."
3165
- */
3166
- function mergeConsecutiveToolResults$1(messages) {
3167
- const isToolResultOnly = (m) => m.role === "user" && Array.isArray(m.content) && m.content.length > 0 && m.content.every((b) => b.type === "tool_result");
3168
- const merged = [];
3169
- for (const m of messages) {
3170
- const prev = merged.at(-1);
3171
- if (isToolResultOnly(m) && prev && isToolResultOnly(prev)) prev.content.push(...m.content);
3172
- else merged.push(m);
3173
- }
3174
- return merged;
3175
- }
3176
- /**
3177
- * Translates one VernLLM wire message (OpenAI-shaped: plain user/assistant
3178
- * turns, an assistant turn with `tool_calls`, or a `tool` turn) into
3179
- * Anthropic's `{ role: 'user' | 'assistant', content }` shape.
3180
- */
3181
- function toAnthropicMessage(m) {
3182
- if (m.role === "tool") return {
3183
- role: "user",
3184
- content: [{
3185
- type: "tool_result",
3186
- tool_use_id: m.tool_call_id,
3187
- content: m.content,
3188
- ...m.is_error ? { is_error: true } : {}
3189
- }]
3190
- };
3191
- if (m.role === "assistant" && m.tool_calls?.length) {
3192
- const blocks = [];
3193
- if (m.content) blocks.push({
3194
- type: "text",
3195
- text: m.content
3196
- });
3197
- for (const tc of m.tool_calls) {
3198
- let input;
3199
- try {
3200
- input = tc.function.arguments.trim() ? JSON.parse(tc.function.arguments) : {};
3201
- } catch (cause) {
3202
- throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
3203
- }
3204
- if (input === null || Array.isArray(input) || typeof input !== "object") throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) arguments must be a JSON object.`, "validation");
3205
- blocks.push({
3206
- type: "tool_use",
3207
- id: tc.id,
3208
- name: tc.function.name,
3209
- input
3210
- });
3211
- }
3212
- return {
3213
- role: "assistant",
3214
- content: blocks
3215
- };
3216
- }
3217
- return {
3218
- role: m.role,
3219
- content: Array.isArray(m.content) ? toAnthropicContent(m.content) : m.content ?? ""
3220
- };
3221
- }
3222
-
3223
- //#endregion
3224
- //#region src/adapters/gemini.ts
3225
- /**
3226
- * Translates a VernLLM `ContentBlock[]` into Gemini's native `parts` array:
3227
- * text blocks become `{ text }`, image blocks become inline data parts
3228
- * (`{ inlineData: { mimeType, data } }`), Gemini's shape for embedding raw
3229
- * base64 image bytes directly in the request.
3230
- */
3231
- function toGeminiParts(blocks) {
3232
- return blocks.map((block) => block.type === "image" ? { inlineData: {
3233
- mimeType: assertSupportedImageMimeType(block.mimeType),
3234
- data: block.data
3235
- } } : { text: block.text });
3236
- }
3237
- /** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Gemini's `functionCallingConfig`. */
3238
- function toGeminiToolConfig(toolChoice) {
3239
- if (!toolChoice || toolChoice === "auto") return { functionCallingConfig: { mode: "AUTO" } };
3240
- if (toolChoice === "none") return { functionCallingConfig: { mode: "NONE" } };
3241
- if (toolChoice === "required") return { functionCallingConfig: { mode: "ANY" } };
3242
- return { functionCallingConfig: {
3243
- mode: "ANY",
3244
- allowedFunctionNames: [toolChoice.function.name]
3245
- } };
3246
- }
3247
- /**
3248
- * Translates one VernLLM wire message into a Gemini `contents` entry.
3249
- * Gemini has no separate 'tool' role: a prior assistant tool request
3250
- * becomes a `'model'` turn with `functionCall` parts, and its result
3251
- * becomes a `'user'` turn with `functionResponse` parts.
3252
- */
3253
- function toGeminiContent(m) {
3254
- if (m.role === "tool") return {
3255
- role: "user",
3256
- parts: [{ functionResponse: {
3257
- name: m.tool_call_id,
3258
- response: parseToolResult(m.content)
3259
- } }]
3260
- };
3261
- if (m.role === "assistant" && m.tool_calls?.length) {
3262
- const parts = [];
3263
- if (typeof m.content === "string" && m.content) parts.push({ text: m.content });
3264
- parts.push(...m.tool_calls.map((tc) => ({ functionCall: {
3265
- name: tc.function.name,
3266
- args: parseToolArguments(tc.function.arguments, tc.function.name)
3267
- } })));
3268
- return {
3269
- role: "model",
3270
- parts
3271
- };
3272
- }
3273
- return {
3274
- role: m.role === "assistant" ? "model" : "user",
3275
- parts: Array.isArray(m.content) ? toGeminiParts(m.content) : [{ text: m.content ?? "" }]
3276
- };
3277
- }
3278
- function parseToolArguments(text, toolName) {
3279
- let parsed;
3280
- try {
3281
- parsed = text.trim() ? JSON.parse(text) : {};
3282
- } catch (cause) {
3283
- throw new LLMError(`Tool call "${toolName}" arguments are not valid JSON.`, "parse", {
3284
- cause,
3285
- code: "tool_arguments_parse_failed"
3286
- });
3287
- }
3288
- if (!parsed || Array.isArray(parsed) || typeof parsed !== "object") throw new LLMError(`Tool call "${toolName}" arguments must be a JSON object.`, "validation");
3289
- return parsed;
3290
- }
3291
- function parseToolResult(text) {
3292
- try {
3293
- return text.trim() ? JSON.parse(text) : "";
3294
- } catch {
3295
- return text;
3296
- }
3297
- }
3298
- /**
3299
- * Gemini expects the results of everything the model asked for in one turn
3300
- * to arrive together as multiple `functionResponse` parts on a single
3301
- * `'user'` entry, not as separate consecutive `'user'` entries. The
3302
- * per-wire-message mapping above produces one `'user'` entry per VernLLM
3303
- * wire tool message, so when an assistant turn requested more than one
3304
- * tool, this merges the resulting run of functionResponse-only `'user'`
3305
- * entries back into one.
3306
- */
3307
- function mergeConsecutiveFunctionResponses(contents) {
3308
- const isFunctionResponseOnly = (c) => c.role === "user" && c.parts.length > 0 && c.parts.every((p) => "functionResponse" in p);
3309
- const merged = [];
3310
- for (const c of contents) {
3311
- const prev = merged.at(-1);
3312
- if (isFunctionResponseOnly(c) && prev && isFunctionResponseOnly(prev)) prev.parts.push(...c.parts);
3313
- else merged.push(c);
3314
- }
3315
- return merged;
3316
- }
3317
- /**
3318
- * Builds the Gemini-shaped request from VernLLM's wire params, shared
3319
- * between `create` and `createStream` so both go through identical
3320
- * translation (contents shaping, `responseSchema`/`responseMimeType`
3321
- * mapping, and tool/toolConfig translation all happen exactly once).
3322
- * `abortSignal` is folded into `config` by the caller (`create`/
3323
- * `createStream`), once the request options are available.
3324
- */
3325
- function buildGeminiRequest(params) {
3326
- const systemMessage = params.messages.find((m) => m.role === "system");
3327
- const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
3328
- const wantsJson = Boolean(params.response_format);
3329
- const config = {
3330
- ...params.temperature !== void 0 ? { temperature: params.temperature } : {},
3331
- maxOutputTokens: params.max_tokens,
3332
- ...systemMessage ? { systemInstruction: { parts: [{ text: systemMessage.content }] } } : {}
3333
- };
3334
- if (wantsJson) config.responseMimeType = "application/json";
3335
- if (params.response_format?.type === "json_schema") {
3336
- const { schema, description } = params.response_format.json_schema;
3337
- config.responseSchema = {
3338
- ...schema,
3339
- ...description ? { description } : {}
3340
- };
3341
- }
3342
- if (params.tools?.length) {
3343
- config.tools = [{ functionDeclarations: params.tools.map((t) => ({
3344
- name: t.function.name,
3345
- description: t.function.description,
3346
- parameters: t.function.parameters
3347
- })) }];
3348
- config.toolConfig = toGeminiToolConfig(params.tool_choice);
3349
- }
3350
- return {
3351
- model: params.model,
3352
- contents: mergeConsecutiveFunctionResponses(conversationMessages.map((m) => toGeminiContent(m))),
3353
- config
3354
- };
3355
- }
3356
- /**
3357
- * Wraps a Gemini client so it satisfies the `LLMClient` interface VernLLM
3358
- * uses for OpenAI-compatible APIs. Gemini's shape differs on nearly every
3359
- * axis: a `contents` array instead of `messages`, a separate
3360
- * `systemInstruction` field instead of a `system` role message,
3361
- * `generationConfig` instead of top-level `temperature`/`max_tokens`, and
3362
- * native JSON Schema support via `responseMimeType: 'application/json'` +
3363
- * `responseSchema`. `reasoning_effort` has no equivalent. Gemini's thinking
3364
- * models use a token budget, not an effort tier, so it's dropped, same as
3365
- * Anthropic.
3366
- *
3367
- * `tools` maps to Gemini's native `functionDeclarations`/`functionCall`;
3368
- * `tool_choice` maps to `toolConfig.functionCallingConfig`. Gemini accepts
3369
- * `responseSchema` and `tools` in the same request natively, so both are
3370
- * set independently here and no special-casing is needed for the
3371
- * combination, unlike `fromAnthropic`/`fromBedrock`.
3372
- *
3373
- * `createStream` calls `generateContentStream` (optional on `GeminiClient`
3374
- *, required only if the caller sets `stream: true`) and translates each
3375
- * partial response into `WireStreamChunk`s. Unlike OpenAI/Anthropic,
3376
- * Gemini's own function-calling API doesn't stream tool-call arguments
3377
- * incrementally: a `functionCall` part always arrives whole in one chunk,
3378
- * so each one is emitted as a single, complete `tool_call_delta` (a
3379
- * one-shot "delta" containing the full arguments) rather than accumulated
3380
- * fragments, that's a real difference in the underlying API, not
3381
- * something this adapter can smooth over. `usageMetadata` is (per Gemini's
3382
- * own behavior) only reliably present on the last chunk, so the `usage`
3383
- * `WireStreamChunk` is emitted once, after the stream completes, from
3384
- * whichever chunk's `usageMetadata` was seen last.
3385
- */
3386
- function fromGemini(geminiClient) {
3387
- return { chat: { completions: {
3388
- async create(params, options) {
3389
- const request = buildGeminiRequest(params);
3390
- request.config = {
3391
- ...request.config,
3392
- abortSignal: options.signal
3393
- };
3394
- const response = await geminiClient.generateContent(request);
3395
- const parts = response.candidates?.[0]?.content?.parts ?? [];
3396
- const text = parts.map((p) => p.text ?? "").join("");
3397
- const functionCalls = parts.filter((p) => p.functionCall);
3398
- let wireToolCalls;
3399
- if (functionCalls.length) wireToolCalls = functionCalls.map((p) => ({
3400
- id: p.functionCall.name,
3401
- type: "function",
3402
- function: {
3403
- name: p.functionCall.name,
3404
- arguments: JSON.stringify(p.functionCall.args ?? {})
3405
- }
3406
- }));
3407
- return {
3408
- choices: [{ message: {
3409
- content: text,
3410
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3411
- } }],
3412
- usage: {
3413
- prompt_tokens: response.usageMetadata?.promptTokenCount,
3414
- completion_tokens: response.usageMetadata?.candidatesTokenCount,
3415
- total_tokens: response.usageMetadata?.totalTokenCount
3416
- }
3417
- };
3418
- },
3419
- async *createStream(params, options) {
3420
- if (!geminiClient.generateContentStream) throw new LLMError("stream: true requires a Gemini client with generateContentStream", "invalid_params", {
3421
- code: "unsupported_capability",
3422
- issues: { capability: "generateContentStream" }
3423
- });
3424
- const request = buildGeminiRequest(params);
3425
- request.config = {
3426
- ...request.config,
3427
- abortSignal: options.signal
3428
- };
3429
- const stream = await geminiClient.generateContentStream(request);
3430
- let toolCallIndex = 0;
3431
- let lastUsage;
3432
- for await (const chunk of stream) {
3433
- const parts = chunk.candidates?.[0]?.content?.parts ?? [];
3434
- for (const part of parts) {
3435
- if (part.text) yield {
3436
- type: "text-delta",
3437
- delta: part.text
3438
- };
3439
- if (part.functionCall) {
3440
- yield {
3441
- type: "tool_call_delta",
3442
- index: toolCallIndex,
3443
- id: part.functionCall.name,
3444
- name: part.functionCall.name,
3445
- argumentsDelta: JSON.stringify(part.functionCall.args ?? {}),
3446
- complete: true
3447
- };
3448
- toolCallIndex++;
3449
- }
3450
- }
3451
- if (chunk.usageMetadata) lastUsage = chunk.usageMetadata;
3452
- }
3453
- if (lastUsage) yield {
3454
- type: "usage",
3455
- usage: {
3456
- prompt_tokens: lastUsage.promptTokenCount,
3457
- completion_tokens: lastUsage.candidatesTokenCount,
3458
- total_tokens: lastUsage.totalTokenCount
3459
- }
3460
- };
3461
- }
3462
- } } };
3463
- }
3464
-
3465
- //#endregion
3466
- //#region src/adapters/bedrock.ts
3467
- /** Maps a `ContentBlock` image MIME type, already validated, to Converse's `format` enum. */
3468
- function toBedrockImageFormat(mimeType) {
3469
- switch (assertSupportedImageMimeType(mimeType)) {
3470
- case "image/png": return "png";
3471
- case "image/jpeg": return "jpeg";
3472
- case "image/gif": return "gif";
3473
- case "image/webp": return "webp";
3474
- }
3475
- }
3476
- /**
3477
- * Decodes base64 image data into the raw `Uint8Array` bytes Converse's
3478
- * `image.source.bytes` expects (unlike Anthropic/Gemini/OpenAI, which all
3479
- * take base64 strings directly). Uses `Buffer`, since this adapter, like
3480
- * the rest of the package, targets Node.
3481
- */
3482
- function decodeBase64(data) {
3483
- return new Uint8Array(Buffer.from(data, "base64"));
3484
- }
3485
- /**
3486
- * Translates a VernLLM `ContentBlock[]` into Converse's native content-block
3487
- * array: text blocks pass through as `{ text }`, image blocks become
3488
- * `{ image: { format, source: { bytes } } }` with the base64 payload decoded
3489
- * to raw bytes, since Converse doesn't accept base64 strings directly.
3490
- */
3491
- function toBedrockContent(blocks) {
3492
- return blocks.map((block) => block.type === "image" ? { image: {
3493
- format: toBedrockImageFormat(block.mimeType),
3494
- source: { bytes: decodeBase64(block.data) }
3495
- } } : { text: block.text });
3496
- }
3497
- /**
3498
- * Maps VernLLM's OpenAI-shaped wire `tools`/`tool_choice` into Converse's
3499
- * `toolConfig` shape. Shared by the two call sites that build real
3500
- * (non-schema-forced) tool definitions: the plain tools-only branch, and
3501
- * the native-structured-output branch, which sends real tools alongside
3502
- * `outputConfig` rather than instead of it.
3503
- */
3504
- function buildBedrockToolConfig(tools, toolChoiceParam) {
3505
- return {
3506
- tools: tools.map((t) => ({ toolSpec: {
3507
- name: t.function.name,
3508
- description: t.function.description,
3509
- inputSchema: { json: t.function.parameters }
3510
- } })),
3511
- toolChoice: toBedrockToolChoice(toolChoiceParam)
3512
- };
3513
- }
3514
- /**
3515
- * Builds the Converse-shaped request from VernLLM's wire params, shared
3516
- * between `create` and `createStream` so both go through identical
3517
- * translation (system prompt, message shaping, the jsonSchema →
3518
- * forced-single-tool mapping, and the `toolUseSupportedModels` preflight
3519
- * check all happen exactly once).
3520
- *
3521
- * Returns `toolName` alongside the request: when set, the model was forced
3522
- * to call a single synthetic tool standing in for `jsonSchema` output (the
3523
- * legacy path, for models not covered by `nativeStructuredOutputModels`),
3524
- * and both `create` and `createStream` need to know this so they can
3525
- * unwrap that tool call back into plain text content instead of treating
3526
- * it like a real tool call. On the native path (model covered by
3527
- * `nativeStructuredOutputModels`), `toolName` is `undefined`: the
3528
- * schema-conforming JSON already arrives as ordinary text content, nothing
3529
- * to unwrap, and any real tool calls in `params.tools` are left for the
3530
- * normal, non-forced tool-call handling both `create` and `createStream`
3531
- * already do when `toolName` is unset.
3532
- */
3533
- function buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels) {
3534
- const systemMessage = params.messages.find((m) => m.role === "system");
3535
- const conversationMessages = params.messages.filter((m) => m.role === "user" || m.role === "assistant" || m.role === "tool");
3536
- const jsonSchema = params.response_format?.type === "json_schema" ? params.response_format.json_schema : void 0;
3537
- const schemaName = jsonSchema?.name.trim();
3538
- if (jsonSchema && !schemaName) throw new LLMError("json_schema.name must not be empty.", "validation");
3539
- const isNative = Boolean(jsonSchema) && supportsNativeStructuredOutput(params.model, nativeStructuredOutputModels);
3540
- if (jsonSchema && params.tools?.length && !isNative) throw new LLMError(`Bedrock model "${params.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`, "validation");
3541
- let toolName;
3542
- let jsonInstruction;
3543
- let toolConfig;
3544
- let outputConfig;
3545
- if (jsonSchema && isNative) {
3546
- const { schema, description } = jsonSchema;
3547
- outputConfig = { textFormat: {
3548
- type: "json_schema",
3549
- structure: { jsonSchema: {
3550
- schema: JSON.stringify(schema),
3551
- name: schemaName,
3552
- description
3553
- } }
3554
- } };
3555
- } else if (jsonSchema && schemaName) {
3556
- const { schema, description, strict } = jsonSchema;
3557
- toolName = schemaName;
3558
- toolConfig = {
3559
- tools: [{ toolSpec: {
3560
- name: toolName,
3561
- description,
3562
- inputSchema: { json: schema },
3563
- strict
3564
- } }],
3565
- toolChoice: { tool: { name: toolName } }
3566
- };
3567
- } else if (params.response_format?.type === "json_object") jsonInstruction = "Respond with valid JSON only, no prose or markdown fences.";
3568
- if (params.tools?.length && !toolName) toolConfig = buildBedrockToolConfig(params.tools, params.tool_choice);
3569
- if (jsonSchema && toolConfig && toolUseSupportedModels) {
3570
- const isSupported = Array.isArray(toolUseSupportedModels) ? toolUseSupportedModels.includes(params.model) : toolUseSupportedModels(params.model);
3571
- if (!isSupported) throw new LLMError(`Bedrock model "${params.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`, "invalid_params", {
3572
- code: "unsupported_capability",
3573
- issues: { capability: "toolUseSupportedModels" }
3574
- });
3575
- }
3576
- const systemParts = [systemMessage?.content, jsonInstruction].filter((s) => Boolean(s));
3577
- const request = {
3578
- modelId: params.model,
3579
- messages: mergeConsecutiveToolResults(conversationMessages.map((m) => toBedrockMessage(m))),
3580
- system: systemParts.length ? systemParts.map((text) => ({ text })) : void 0,
3581
- inferenceConfig: {
3582
- ...params.temperature !== void 0 ? { temperature: params.temperature } : {},
3583
- maxTokens: params.max_tokens
3584
- },
3585
- ...toolConfig ? { toolConfig } : {},
3586
- ...outputConfig ? { outputConfig } : {}
3587
- };
3588
- return {
3589
- request,
3590
- toolName
3591
- };
3592
- }
3593
- /**
3594
- * Wraps a Bedrock Converse-API client so it satisfies the `LLMClient`
3595
- * interface VernLLM uses for OpenAI/Groq. The Converse API is unified
3596
- * across Bedrock's model families (Anthropic, Titan, Llama, Mistral, etc.),
3597
- * so unlike raw per-model Bedrock invocation, this one adapter works
3598
- * regardless of which underlying model `modelId` points at, as long as
3599
- * that model supports Converse (most current-generation ones do)
3600
- *
3601
- * `response_format: json_schema`, on a model covered by
3602
- * `options.nativeStructuredOutputModels` (opt-in, unset by default), is
3603
- * sent as `outputConfig.textFormat`, its own request field, independent of
3604
- * `toolConfig`, so it can be combined with real, caller-supplied `tools`
3605
- * in the same request. Matches the real Converse API's shape exactly: the
3606
- * schema is nested under `structure.jsonSchema` and JSON-encoded as a
3607
- * string, not the parsed object `toolConfig`'s tool schemas use, and there
3608
- * is no `strict` field on this path.
3609
- *
3610
- * On any other model (the default), `response_format: json_schema` is
3611
- * mapped to Converse's `toolConfig` instead: a single tool is defined from
3612
- * the schema, description, and strictness settings, and `toolChoice`
3613
- * forces the model to call it. This legacy path cannot be combined with
3614
- * real `tools` (both would need the same `toolConfig`), and a call that
3615
- * tries throws `LLMError('validation')` before reaching the API.
3616
- * Provider-constrained schema matching applies only when `strict: true` is
3617
- * forwarded and supported. Native tool support varies by model family;
3618
- * pass `toolUseSupportedModels` to preflight-check it (see
3619
- * `BedrockAdapterOptions`), otherwise a `jsonSchema` call to an
3620
- * unsupported model surfaces Bedrock's raw error unchanged.
3621
- *
3622
- * `response_format: json_object` (no schema to build a tool from) and
3623
- * `reasoning_effort` (no Converse equivalent) fall back to a system-prompt
3624
- * instruction and are dropped respectively. Unlike `jsonSchema`,
3625
- * `json_object` combines with real `tools` freely on every model: it's a
3626
- * prompt nudge, not a request field, so there's nothing for it to collide
3627
- * with.
3628
- *
3629
- * `tools` alone maps to Converse's native `toolConfig`/`toolUse`/
3630
- * `toolResult`; `tool_choice` maps to `toolConfig.toolChoice`.
3631
- *
3632
- * `createStream` calls `converseStream` (optional on `BedrockConverseClient`
3633
- *, required only if the caller sets `stream: true`) and translates its
3634
- * `contentBlockStart`/`contentBlockDelta`/`metadata` events into
3635
- * `WireStreamChunk`s. Content blocks are tracked by `contentBlockIndex`,
3636
- * same as `fromAnthropic`'s block-index tracking (Converse's streaming
3637
- * shape is structurally close to Anthropic's own, both being tool-use-aware
3638
- * content-block streams), including the same `json-tool` unwrapping: a
3639
- * `jsonSchema`-forced tool's `toolUse.input` deltas are re-emitted as
3640
- * `text-delta`, not `tool_call_delta`, so the accumulated result lands in
3641
- * `finalizeResponse`'s `content` path exactly like the non-streaming
3642
- * `create` branch above unwraps it.
3643
- */
3644
- function fromBedrock(bedrockClient, options) {
3645
- const toolUseSupportedModels = options?.toolUseSupportedModels;
3646
- const nativeStructuredOutputModels = options?.nativeStructuredOutputModels;
3647
- return { chat: { completions: {
3648
- async create(params, requestOptions) {
3649
- const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
3650
- const response = await bedrockClient.converse(request, requestOptions);
3651
- let text;
3652
- let wireToolCalls;
3653
- if (toolName) {
3654
- const toolUseBlock = response.output?.message?.content?.find((block) => block.toolUse?.name === toolName);
3655
- text = toolUseBlock?.toolUse ? JSON.stringify(toolUseBlock.toolUse.input) : "";
3656
- } else {
3657
- const blocks = response.output?.message?.content ?? [];
3658
- text = blocks.map((c) => c.text ?? "").join("");
3659
- const toolUses = blocks.filter((block) => Boolean(block.toolUse));
3660
- if (toolUses.length) wireToolCalls = toolUses.map((block, i) => {
3661
- const toolUse = block.toolUse;
3662
- if (!toolUse.name) throw new LLMError(`Bedrock returned a toolUse block without a name at index ${i}.`, "validation");
3663
- return {
3664
- id: toolUse.toolUseId ?? `${toolUse.name}_${i}`,
3665
- type: "function",
3666
- function: {
3667
- name: toolUse.name,
3668
- arguments: JSON.stringify(toolUse.input ?? {})
3669
- }
3670
- };
3671
- });
3672
- }
3673
- return {
3674
- choices: [{ message: {
3675
- content: text,
3676
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3677
- } }],
3678
- usage: {
3679
- prompt_tokens: response.usage?.inputTokens,
3680
- completion_tokens: response.usage?.outputTokens,
3681
- total_tokens: response.usage?.totalTokens
3682
- }
3683
- };
3684
- },
3685
- async *createStream(params, requestOptions) {
3686
- if (!bedrockClient.converseStream) throw new LLMError("stream: true requires a Bedrock client with converseStream", "invalid_params", {
3687
- code: "unsupported_capability",
3688
- issues: { capability: "converseStream" }
3689
- });
3690
- const { request, toolName } = buildBedrockRequest(params, toolUseSupportedModels, nativeStructuredOutputModels);
3691
- const { stream } = await bedrockClient.converseStream(request, requestOptions);
3692
- const blockKinds = new Map();
3693
- for await (const event of stream) if ("contentBlockStart" in event) {
3694
- const { contentBlockIndex, start } = event.contentBlockStart;
3695
- if (start?.toolUse) {
3696
- const kind = start.toolUse.name === toolName ? "json-tool" : "tool_use";
3697
- blockKinds.set(contentBlockIndex, kind);
3698
- if (kind === "tool_use" && !toolName) yield {
3699
- type: "tool_call_delta",
3700
- index: contentBlockIndex,
3701
- id: start.toolUse.toolUseId,
3702
- name: start.toolUse.name
3703
- };
3704
- } else blockKinds.set(contentBlockIndex, "text");
3705
- } else if ("contentBlockDelta" in event) {
3706
- const { contentBlockIndex, delta } = event.contentBlockDelta;
3707
- if (delta && "text" in delta && delta.text !== void 0 && !toolName) yield {
3708
- type: "text-delta",
3709
- delta: delta.text
3710
- };
3711
- else if (delta && "toolUse" in delta && delta.toolUse?.input !== void 0) {
3712
- const kind = blockKinds.get(contentBlockIndex);
3713
- if (kind === "json-tool") yield {
3714
- type: "text-delta",
3715
- delta: delta.toolUse.input
3716
- };
3717
- else if (!toolName) yield {
3718
- type: "tool_call_delta",
3719
- index: contentBlockIndex,
3720
- argumentsDelta: delta.toolUse.input
3721
- };
3722
- }
3723
- } else if ("metadata" in event && event.metadata.usage) yield {
3724
- type: "usage",
3725
- usage: {
3726
- prompt_tokens: event.metadata.usage.inputTokens,
3727
- completion_tokens: event.metadata.usage.outputTokens,
3728
- total_tokens: event.metadata.usage.totalTokens
3729
- }
3730
- };
3731
- else if ("throttlingException" in event) throw new LLMError(event.throttlingException.message ?? "Bedrock throttled the request mid-stream", "api", {
3732
- status: 429,
3733
- code: "provider_rate_limited"
3734
- });
3735
- else if ("validationException" in event) throw new LLMError(event.validationException.message ?? "Bedrock rejected the request mid-stream", "validation");
3736
- else if ("internalServerException" in event || "serviceUnavailableException" in event || "modelStreamErrorException" in event) {
3737
- const detail = "internalServerException" in event && event.internalServerException.message || "serviceUnavailableException" in event && event.serviceUnavailableException.message || "modelStreamErrorException" in event && event.modelStreamErrorException.message || "Bedrock reported a mid-stream error";
3738
- const status = "modelStreamErrorException" in event && event.modelStreamErrorException.originalStatusCode || "serviceUnavailableException" in event && 503 || 500;
3739
- throw new LLMError(detail, "api", {
3740
- status,
3741
- code: status >= 500 ? "server_error" : void 0
3742
- });
3743
- }
3744
- }
3745
- } } };
3746
- }
3747
- /** Maps VernLLM's OpenAI-shaped wire `tool_choice` onto Converse's `toolChoice`. */
3748
- function toBedrockToolChoice(toolChoice) {
3749
- if (!toolChoice || toolChoice === "auto") return { auto: {} };
3750
- if (toolChoice === "required") return { any: {} };
3751
- if (toolChoice === "none") throw new LLMError("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.", "invalid_params", {
3752
- code: "unsupported_capability",
3753
- issues: { capability: "toolChoice: 'none'" }
3754
- });
3755
- return { tool: { name: toolChoice.function.name } };
3756
- }
3757
- /**
3758
- * Translates one VernLLM wire message into Converse's
3759
- * `{ role: 'user' | 'assistant', content }` shape.
3760
- */
3761
- function toBedrockMessage(m) {
3762
- if (m.role === "tool") return {
3763
- role: "user",
3764
- content: [{ toolResult: {
3765
- toolUseId: m.tool_call_id,
3766
- content: [{ text: m.content }],
3767
- status: m.is_error ? "error" : "success"
3768
- } }]
3769
- };
3770
- if (m.role === "assistant" && m.tool_calls?.length) {
3771
- const blocks = [];
3772
- if (m.content) blocks.push({ text: m.content });
3773
- for (const tc of m.tool_calls) {
3774
- let input;
3775
- if (!tc.function.arguments.trim()) input = {};
3776
- else try {
3777
- input = JSON.parse(tc.function.arguments);
3778
- } catch (cause) {
3779
- throw new LLMError(`Assistant tool call "${tc.function.name}" (${tc.id}) has arguments that are not valid JSON.`, "validation", { cause });
3780
- }
3781
- blocks.push({ toolUse: {
3782
- toolUseId: tc.id,
3783
- name: tc.function.name,
3784
- input
3785
- } });
3786
- }
3787
- return {
3788
- role: "assistant",
3789
- content: blocks
3790
- };
3791
- }
3792
- return {
3793
- role: m.role,
3794
- content: Array.isArray(m.content) ? toBedrockContent(m.content) : [{ text: m.content ?? "" }]
3795
- };
3796
- }
3797
- /**
3798
- * Converse expects the results of everything the model asked for in one
3799
- * turn to arrive together as multiple `toolResult` content blocks on a
3800
- * single `'user'` message, not as separate consecutive `'user'` messages.
3801
- * The per-wire-message mapping above produces one `'user'` message per
3802
- * VernLLM wire tool message, so when an assistant turn requested more than
3803
- * one tool, this merges the resulting run of toolResult-only `'user'`
3804
- * messages back into one.
3805
- */
3806
- function mergeConsecutiveToolResults(messages) {
3807
- const isToolResultOnly = (m) => m.role === "user" && m.content.length > 0 && m.content.every((b) => "toolResult" in b);
3808
- const merged = [];
3809
- for (const m of messages) {
3810
- const prev = merged.at(-1);
3811
- if (isToolResultOnly(m) && prev && isToolResultOnly(prev)) prev.content.push(...m.content);
3812
- else merged.push(m);
3813
- }
3814
- return merged;
3815
- }
3816
-
3817
- //#endregion
3818
- //#region src/adapters/fetch.ts
3819
- /**
3820
- * Wraps a WHATWG `ReadableStream` (what `response.body` is) so it can be
3821
- * consumed with `for await`. Implemented via `getReader()` rather than
3822
- * relying on `ReadableStream` having a native `Symbol.asyncIterator`,
3823
- * that support varies across runtimes/versions, and this works everywhere
3824
- * a `ReadableStream` does.
3825
- */
3826
- async function* webStreamToAsyncIterable(stream) {
3827
- const reader = stream.getReader();
3828
- try {
3829
- for (;;) {
3830
- const { done, value } = await reader.read();
3831
- if (done) return;
3832
- if (value) yield value;
3833
- }
3834
- } finally {
3835
- try {
3836
- await reader.cancel();
3837
- } catch {}
3838
- reader.releaseLock();
3839
- }
3840
- }
3841
- /** Default `requestStream`: native `fetch`, with the same error/`.status` contract non-streaming errors get. */
3842
- async function defaultRequestStream(url, init) {
3843
- const res = await fetch(url, init);
3844
- if (!res.ok) {
3845
- const body = await res.text().catch(() => "");
3846
- const err = new Error(`Fetch adapter stream request failed (${res.status}): ${body.slice(0, 500)}`);
3847
- err.status = res.status;
3848
- err.headers = res.headers;
3849
- throw err;
3850
- }
3851
- if (!res.body) throw new Error("Fetch adapter stream request received a response with no body.");
3852
- return webStreamToAsyncIterable(res.body);
3853
- }
3854
- /** Builds the shared `{ method, headers, body? }` request-init for both `create` and `createStream`. */
3855
- async function buildRequestInit(config, params, requestBody) {
3856
- const url = typeof config.url === "function" ? config.url(params) : config.url;
3857
- const headers = typeof config.headers === "function" ? await config.headers() : config.headers;
3858
- const method = config.method ?? "POST";
3859
- const supportsBody = !["GET", "HEAD"].includes(method.toUpperCase());
3860
- return {
3861
- url,
3862
- method,
3863
- headers: supportsBody ? {
3864
- "Content-Type": "application/json",
3865
- ...headers
3866
- } : { ...headers },
3867
- ...supportsBody ? { body: JSON.stringify(requestBody) } : {}
3868
- };
3869
- }
3870
- /**
3871
- * A fetch-based escape hatch for providers with no SDK, or where pulling one
3872
- * in isnt worth it. You supply the URL, headers, and two small mapping
3873
- * functions; this handles the HTTP call and slots the result into the same
3874
- * `LLMClient` shape every other adapter produces, so retries, timeouts,
3875
- * the circuit breaker, and JSON/schema handling all still work unmodified
3876
- *
3877
- * Non-2xx responses throw an error with `.status` set to the HTTP status
3878
- * code, so VernLLMs `nonRetryableStatus` handling (e.g. failing fast on
3879
- * 401/403) applies here too
3880
- *
3881
- * Tool calling works the same way as every other adapter: `mapRequest`
3882
- * receives the full `ChatRequest`, including `tools`/`toolChoice`, so it can
3883
- * translate them into whatever shape the provider's wire format expects
3884
- * (typically an OpenAI-`function`-wrapped `tools` array plus a `tool_choice`
3885
- * field). On the way back, `mapResponse` may return a `toolCalls` array
3886
- * (id/name/JSON-encoded-arguments-string per call) alongside or instead of
3887
- * `content`; VernLLM parses and (if `argumentsSchema` was set) validates
3888
- * those arguments the same way it does for every other adapter. For
3889
- * `stream: true`, tool-call deltas go through the existing
3890
- * `mapStreamEvent` seam via `WireStreamChunk`'s `tool_call_delta` variant,
3891
- * no separate config is needed for streaming vs non-streaming tool calls.
3892
- *
3893
-
3894
- * `createStream` requires `mapStreamEvent` (there's no non-streaming
3895
- * response to fall back on, unlike the other three optional streaming
3896
- * seams). It opens the request via `requestStream` (defaults to native
3897
- * `fetch`), splits the raw bytes into individual events via
3898
- * `parseStreamFrames` (defaults to SSE framing, see `parseSseStream`),
3899
- * and translates each event into `WireStreamChunk`(s) via
3900
- * `mapStreamEvent`. Both seams are overridable per-config for providers
3901
- * that don't fit the SSE-over-fetch default. If a custom `request`
3902
- * transport is configured, `requestStream` must be configured too,
3903
- * `requestStream` never silently falls back to `request` (see
3904
- * `createStream`'s own comment for why), so a `stream: true` call with
3905
- * `request` set but no `requestStream` throws a clear
3906
- * `LLMError('validation')` instead of quietly using unrelated native
3907
- * `fetch`.
3908
- */
3909
- function fromFetch(config) {
3910
- return { chat: { completions: {
3911
- async create(params, options) {
3912
- const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
3913
- const request = config.request ?? fetch;
3914
- const res = await request(url, {
3915
- method,
3916
- headers,
3917
- body,
3918
- signal: options.signal
3919
- });
3920
- if (!res.ok) {
3921
- const responseBody = await res.text().catch(() => "");
3922
- const err = new Error(`Fetch adapter request failed (${res.status}): ${responseBody.slice(0, 500)}`);
3923
- err.status = res.status;
3924
- err.headers = res.headers;
3925
- throw err;
3926
- }
3927
- const json = await res.json();
3928
- const { content, usage, toolCalls } = config.mapResponse(json);
3929
- const wireToolCalls = toolCalls?.length ? toolCalls.map((tc) => ({
3930
- id: tc.id,
3931
- type: "function",
3932
- function: {
3933
- name: tc.name,
3934
- arguments: tc.arguments
3935
- }
3936
- })) : void 0;
3937
- return {
3938
- choices: [{ message: {
3939
- content,
3940
- ...wireToolCalls ? { tool_calls: wireToolCalls } : {}
3941
- } }],
3942
- usage: usage ? {
3943
- prompt_tokens: usage.promptTokens,
3944
- completion_tokens: usage.completionTokens,
3945
- total_tokens: usage.totalTokens
3946
- } : void 0
3947
- };
3948
- },
3949
- async *createStream(params, options) {
3950
- if (!config.mapStreamEvent) throw new LLMError("stream: true requires mapStreamEvent to be configured on fromFetch", "invalid_params", {
3951
- code: "unsupported_capability",
3952
- issues: { capability: "mapStreamEvent" }
3953
- });
3954
- if (config.request && !config.requestStream) throw new LLMError("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.", "invalid_params", {
3955
- code: "unsupported_capability",
3956
- issues: { capability: "requestStream" }
3957
- });
3958
- const { url, method, headers, body } = await buildRequestInit(config, params, config.mapRequest(params));
3959
- const requestStream = config.requestStream ?? defaultRequestStream;
3960
- const parseFrames = config.parseStreamFrames ?? parseSseStream;
3961
- const byteStream = await requestStream(url, {
3962
- method,
3963
- headers,
3964
- body,
3965
- signal: options.signal
3966
- });
3967
- for await (const event of parseFrames(byteStream)) {
3968
- if (event === SSE_PING) {
3969
- yield { type: "ping" };
3970
- continue;
3971
- }
3972
- const wireChunks = config.mapStreamEvent(event);
3973
- if (!wireChunks) continue;
3974
- if (Array.isArray(wireChunks)) yield* wireChunks;
3975
- else yield wireChunks;
3976
- }
3977
- }
3978
- } } };
3979
- }
3980
-
3981
- //#endregion
3982
- //#region src/adapters/openaiCompatible.ts
3983
- /**
3984
- * Translates a VernLLM `ContentBlock[]` into OpenAI's wire-level content
3985
- * array. Text blocks become `{ type: 'text', text }`; image blocks become
3986
- * `{ type: 'image_url', image_url: { url } }` with the base64 payload
3987
- * inlined as a `data:` URL, since our `ContentBlock` shape (`{ type:
3988
- * 'image', data, mimeType }`) is provider-agnostic and doesn't itself match
3989
- * OpenAI's wire format.
3990
- */
3991
- function toOpenAIContent(blocks) {
3992
- return blocks.map((block) => block.type === "image" ? {
3993
- type: "image_url",
3994
- image_url: { url: `data:${assertSupportedImageMimeType(block.mimeType)};base64,${block.data}` }
3995
- } : {
3996
- type: "text",
3997
- text: block.text
3998
- });
3999
- }
4000
- /**
4001
- * Translates VernLLM's provider-agnostic `messages` (the one part of a
4002
- * request that isn't a pure passthrough for OpenAI-compatible clients) into
4003
- * OpenAI's native wire shape. Shared between `create` and `createStream` so
4004
- * both go through identical message translation.
4005
- */
4006
- function toOpenAIMessages(params) {
4007
- return params.messages.map((m) => {
4008
- if (m.role === "user" && Array.isArray(m.content)) return {
4009
- ...m,
4010
- content: toOpenAIContent(m.content)
4011
- };
4012
- if (m.role === "tool") {
4013
- const { is_error: _isError,...openAIToolMessage } = m;
4014
- return openAIToolMessage;
4015
- }
4016
- return m;
4017
- });
4018
- }
4019
- /**
4020
- * Translates one OpenAI-shaped SSE chunk into zero or more `WireStreamChunk`s.
4021
- * A single chunk can carry a text delta, one or more tool-call argument
4022
- * deltas (each keyed by `index`, OpenAI's own convention for streaming
4023
- * parallel tool calls, mirrored directly by VernLLM's `tool_call_delta`
4024
- * shape so accumulation composes without translation), and/or a final
4025
- * usage block (present only when `stream_options.include_usage` is set,
4026
- * which this adapter always sets).
4027
- */
4028
- function* toWireStreamChunks(chunk) {
4029
- const delta = chunk.choices?.[0]?.delta;
4030
- if (delta?.content) yield {
4031
- type: "text-delta",
4032
- delta: delta.content
4033
- };
4034
- if (delta?.tool_calls?.length) for (const toolCall of delta.tool_calls) yield {
4035
- type: "tool_call_delta",
4036
- index: toolCall.index,
4037
- id: toolCall.id,
4038
- name: toolCall.function?.name,
4039
- argumentsDelta: toolCall.function?.arguments
4040
- };
4041
- if (chunk.usage) yield {
4042
- type: "usage",
4043
- usage: chunk.usage
4044
- };
4045
- }
4046
- function fromOpenAICompatible(client, options = {}) {
4047
- const raw = client;
4048
- const { supportsStreamUsage = true } = options;
4049
- const rawCreate = raw.chat.completions.create.bind(raw.chat.completions);
4050
- return { chat: { completions: {
4051
- async create(params, options$1) {
4052
- const messages = toOpenAIMessages(params);
4053
- return raw.chat.completions.create({
4054
- ...params,
4055
- messages
4056
- }, options$1);
4057
- },
4058
- async *createStream(params, options$1) {
4059
- const messages = toOpenAIMessages(params);
4060
- const stream = await rawCreate({
4061
- ...params,
4062
- messages,
4063
- stream: true,
4064
- ...supportsStreamUsage ? { stream_options: { include_usage: true } } : {}
4065
- }, options$1);
4066
- for await (const chunk of stream) yield* toWireStreamChunks(chunk);
4067
- }
4068
- } } };
4069
- }
4070
- /**
4071
- * Named alias for the OpenAI SDK itself. A raw `new OpenAI(...)` instance
4072
- * structurally matches most of `LLMClient`, but newer `openai` SDK major
4073
- * versions have widened `ChatCompletionContentPart` (e.g. adding a `file`
4074
- * variant) in ways that no longer structurally satisfy VernLLM's
4075
- * provider-agnostic `ContentBlock[]` on `userContent`, so passing the SDK
4076
- * instance directly can fail to typecheck depending on the installed
4077
- * `openai` version. Wrapping with `fromOpenAI()` (a plain alias of
4078
- * `fromOpenAICompatible()`) sidesteps that by translating through
4079
- * `unknown` at the boundary, and also picks up multimodal image
4080
- * translation and `createStream` wiring that a raw client doesn't have.
4081
- * See Migration Notes for details.
4082
- */
4083
- const fromOpenAI = fromOpenAICompatible;
4084
- /** Groqs SDK matches the OpenAI wire format */
4085
- const fromGroq = fromOpenAICompatible;
4086
- /**
4087
- * Mistrals `chat.completions`-shaped client (or their OpenAI-compat
4088
- * endpoint). Mistral supports `stream_options.include_usage` (added after
4089
- * an earlier period where it returned a 422 for unrecognized fields, per
4090
- * Mistral's changelog and streaming docs), so this is a plain alias like
4091
- * the others, `supportsStreamUsage` defaults to `true`.
4092
- */
4093
- const fromMistral = fromOpenAICompatible;
4094
- /** DeepSeeks API is OpenAI-compatible */
4095
- const fromDeepSeek = fromOpenAICompatible;
4096
- /** Cerebras inference API is OpenAI-compatible */
4097
- const fromCerebras = fromOpenAICompatible;
4098
- /** Together AIs API is OpenAI-compatible */
4099
- const fromTogether = fromOpenAICompatible;
4100
- /** Fireworks AIs API is OpenAI-compatible */
4101
- const fromFireworks = fromOpenAICompatible;
4102
- /**
4103
- * Ollama exposes an OpenAI-compatible endpoint at `/v1/chat/completions`
4104
- * (as opposed to its native `/api/chat` format, which differs). Point an
4105
- * OpenAI SDK instances `baseURL` at your Ollama server and pass it here:
4106
- * this does not talk to Ollamas native API directly.
4107
- */
4108
- const fromOllama = fromOpenAICompatible;
4109
- /** OpenRouter's API is OpenAI-compatible */
4110
- const fromOpenRouter = fromOpenAICompatible;
4111
- /** Perplexity's API is OpenAI-compatible */
4112
- const fromPerplexity = fromOpenAICompatible;
4113
- /** DeepInfra's API is OpenAI-compatible */
4114
- const fromDeepInfra = fromOpenAICompatible;
4115
- /** Novita's API is OpenAI-compatible */
4116
- const fromNovita = fromOpenAICompatible;
4117
- /** Hyperbolic's API is OpenAI-compatible */
4118
- const fromHyperbolic = fromOpenAICompatible;
4119
- /** Moonshot's (Kimi) API is OpenAI-compatible */
4120
- const fromMoonshot = fromOpenAICompatible;
4121
- /** Zhipu's (GLM) API is OpenAI-compatible */
4122
- const fromZhipu = fromOpenAICompatible;
4123
- /**
4124
- * LM Studio exposes an OpenAI-compatible endpoint at `/v1/chat/completions`.
4125
- * Point an OpenAI SDK instance's `baseURL` at your local LM Studio server.
4126
- */
4127
- const fromLMStudio = fromOpenAICompatible;
4128
- /**
4129
- * vLLM's OpenAI-compatible server mode exposes `/v1/chat/completions`.
4130
- * Point an OpenAI SDK instance's `baseURL` at your vLLM server.
4131
- */
4132
- const fromVLLM = fromOpenAICompatible;
4133
- /** xAI's Grok API is OpenAI-compatible */
4134
- const fromXAI = fromOpenAICompatible;
4135
- /** NVIDIA NIM's hosted and self-hosted endpoints are OpenAI-compatible */
4136
- const fromNvidiaNIM = fromOpenAICompatible;
4137
- /** Vercel AI Gateway is OpenAI-compatible */
4138
- const fromVercelAIGateway = fromOpenAICompatible;
4139
- /** Cloudflare Workers AI exposes an OpenAI-compatible endpoint */
4140
- const fromCloudflareWorkersAI = fromOpenAICompatible;
4141
- /** Nebius AI Studio is OpenAI-compatible */
4142
- const fromNebius = fromOpenAICompatible;
4143
- /** SambaNova Cloud's API is OpenAI-compatible */
4144
- const fromSambaNova = fromOpenAICompatible;
4145
- /** Baseten's model hosting exposes an OpenAI-compatible endpoint */
4146
- const fromBaseten = fromOpenAICompatible;
4147
- /** Featherless AI's API is OpenAI-compatible */
4148
- const fromFeatherless = fromOpenAICompatible;
4149
- /** Friendli AI's serving endpoint is OpenAI-compatible */
4150
- const fromFriendli = fromOpenAICompatible;
4151
- /** SiliconFlow's API is OpenAI-compatible */
4152
- const fromSiliconFlow = fromOpenAICompatible;
4153
- /** Parasail's inference API is OpenAI-compatible */
4154
- const fromParasail = fromOpenAICompatible;
4155
- /** StepFun's API is OpenAI-compatible */
4156
- const fromStepFun = fromOpenAICompatible;
4157
- /** MiniMax's API is OpenAI-compatible */
4158
- const fromMiniMax = fromOpenAICompatible;
4159
- /** Lambda Labs' Inference API is OpenAI-compatible */
4160
- const fromLambdaLabs = fromOpenAICompatible;
4161
- /** Snowflake Cortex's LLM endpoint is OpenAI-compatible */
4162
- const fromSnowflakeCortex = fromOpenAICompatible;
4163
- /** Anyscale Endpoints' API is OpenAI-compatible */
4164
- const fromAnyscale = fromOpenAICompatible;
4165
- /** Lepton AI's inference API is OpenAI-compatible */
4166
- const fromLepton = fromOpenAICompatible;
4167
- /** Inference.net's API is OpenAI-compatible */
4168
- const fromInferenceNet = fromOpenAICompatible;
4169
- /** Infermatic's API is OpenAI-compatible */
4170
- const fromInfermatic = fromOpenAICompatible;
4171
- /** AtlasCloud's inference API is OpenAI-compatible */
4172
- const fromAtlasCloud = fromOpenAICompatible;
4173
- /** 01.AI's (Yi models) API is OpenAI-compatible */
4174
- const from01AI = fromOpenAICompatible;
4175
-
4176
- //#endregion
4177
- exports.CircuitBreaker = CircuitBreaker
4178
- exports.ConsoleLogger = ConsoleLogger
4179
- exports.FallbackExhaustedError = FallbackExhaustedError
4180
- exports.InMemoryCacheAdapter = InMemoryCacheAdapter
4181
- exports.LLMError = LLMError
4182
- exports.NormalizedCacheAdapter = NormalizedCacheAdapter
4183
- exports.RateLimiter = RateLimiter
4184
- exports.SSE_PING = SSE_PING
4185
- exports.TieredCacheAdapter = TieredCacheAdapter
4186
- exports.VernLLM = VernLLM
4187
- exports.defaultEstimateTokens = defaultEstimateTokens
4188
- exports.defaultFallbackOn = defaultFallbackOn
4189
- exports.from01AI = from01AI
4190
- exports.fromAnthropic = fromAnthropic
4191
- exports.fromAnyscale = fromAnyscale
4192
- exports.fromAtlasCloud = fromAtlasCloud
4193
- exports.fromBaseten = fromBaseten
4194
- exports.fromBedrock = fromBedrock
4195
- exports.fromCerebras = fromCerebras
4196
- exports.fromCloudflareWorkersAI = fromCloudflareWorkersAI
4197
- exports.fromDeepInfra = fromDeepInfra
4198
- exports.fromDeepSeek = fromDeepSeek
4199
- exports.fromFeatherless = fromFeatherless
4200
- exports.fromFetch = fromFetch
4201
- exports.fromFireworks = fromFireworks
4202
- exports.fromFriendli = fromFriendli
4203
- exports.fromGemini = fromGemini
4204
- exports.fromGroq = fromGroq
4205
- exports.fromHyperbolic = fromHyperbolic
4206
- exports.fromInferenceNet = fromInferenceNet
4207
- exports.fromInfermatic = fromInfermatic
4208
- exports.fromLMStudio = fromLMStudio
4209
- exports.fromLambdaLabs = fromLambdaLabs
4210
- exports.fromLepton = fromLepton
4211
- exports.fromMiniMax = fromMiniMax
4212
- exports.fromMistral = fromMistral
4213
- exports.fromMoonshot = fromMoonshot
4214
- exports.fromNebius = fromNebius
4215
- exports.fromNovita = fromNovita
4216
- exports.fromNvidiaNIM = fromNvidiaNIM
4217
- exports.fromOllama = fromOllama
4218
- exports.fromOpenAI = fromOpenAI
4219
- exports.fromOpenAICompatible = fromOpenAICompatible
4220
- exports.fromOpenRouter = fromOpenRouter
4221
- exports.fromParasail = fromParasail
4222
- exports.fromPerplexity = fromPerplexity
4223
- exports.fromSambaNova = fromSambaNova
4224
- exports.fromSiliconFlow = fromSiliconFlow
4225
- exports.fromSnowflakeCortex = fromSnowflakeCortex
4226
- exports.fromStepFun = fromStepFun
4227
- exports.fromTogether = fromTogether
4228
- exports.fromVLLM = fromVLLM
4229
- exports.fromVercelAIGateway = fromVercelAIGateway
4230
- exports.fromXAI = fromXAI
4231
- exports.fromZhipu = fromZhipu
4232
- exports.hasIssues = hasIssues
4233
- exports.isFallbackExhaustedError = isFallbackExhaustedError
4234
- exports.isLLMError = isLLMError
4235
- exports.isToolCallResult = isToolCallResult
4236
- exports.parseSseStream = parseSseStream
12
+ `)}let i=n.trim();if(i){let e=K(i);e!==U&&e!==W&&(yield e)}}const U=Symbol(`sse-stream-done`),W=Symbol(`sse-frame-no-data`),G=Symbol(`sse-frame-ping`);function K(e){let t=[],n=!1;for(let r of e.split(`
13
+ `)){if(r.startsWith(`:`)){n=!0;continue}if(!r.startsWith(`data:`))continue;t.push(r.startsWith(`data: `)?r.slice(6):r.slice(5))}if(!t.length)return n?G:W;let r=t.join(`
14
+ `);if(r===`[DONE]`)return U;try{return JSON.parse(r)}catch(e){throw new b(`Invalid JSON in SSE frame: ${r.slice(0,200)}`,`parse`,{cause:e,code:`stream_frame_invalid`})}}const Ue=[`image/png`,`image/jpeg`,`image/gif`,`image/webp`];function q(e){if(Ue.includes(e))return e;throw new b(`Unsupported image mimeType "${e}": expected one of ${Ue.join(`, `)}`,`invalid_params`)}function J(e,t){return t?Array.isArray(t)?t.includes(e):t(e):!1}const Y={minimal:1024,low:4096,medium:16e3,high:32e3};function X(e){if(!e)return Y;let t={...Y,...e};if(!(t.minimal<t.low&&t.low<t.medium&&t.medium<t.high))throw new b(`reasoningEffortTokens must keep tiers in strictly ascending order (minimal < low < medium < high), got ${JSON.stringify(t)}. An out-of-order override doesn't just misrank tiers, it can make some of them unreachable.`,`invalid_params`);return t}function Z(e,t=Y){return t[e]}function Q(e,t=Y){return e<=t.minimal?`minimal`:e<=t.low?`low`:e<=t.medium?`medium`:`high`}function We(e){let t=/opus-(\d+)(?:-(\d+))?/.exec(e);if(!t)return null;let n=t[2],r=n===void 0||n.length>=8?0:Number(n);return[Number(t[1]),r]}function Ge(e){let t=We(e);if(t){let[e,n]=t;return e>4||e===4&&n>=7}return[`sonnet-5`,`fable-5`,`mythos`].some(t=>e.includes(t))}function Ke(e,t){return Ge(e)?!0:t?Array.isArray(t)?t.includes(e):t(e):!1}function qe(e,t){return!Ke(e,t)}function Je(e,t){if(e<1024)throw new b(`budgetTokens (${e}) is below Anthropic's minimum of 1024. Raise budgetTokens, or use a reasoningEffort tier of 'low' or above with the default conversion table.`,`invalid_params`);if(e>=t)throw new b(`budgetTokens (${e}) must be less than maxTokens (${t}); the thinking budget and the reply share the same max_tokens ceiling on Anthropic. Raise maxTokens, or lower budgetTokens/reasoningEffort.`,`invalid_params`)}function Ye(e){if(e)throw new b(`budgetTokens/reasoningEffort was set alongside ${e}. Anthropic rejects thinking combined with a tool_choice that forces tool use, the model has to be able to reply with plain text for thinking to run. Use toolChoice: 'auto' (or omit toolChoice) for this call, or drop budgetTokens/reasoningEffort for it.`,`invalid_params`)}function Xe(e){return e===`minimal`?`low`:e}function Ze(e,t){return $e(t,e.toUpperCase())}function Qe(e){let t=/gemini-\d+\.(\d+)/.exec(e);return t?Number(t[1]):0}function $e(e,t){if(!e.includes(`pro`))return t;let n=et(e);if(n===null||n<3)return t;let r=Qe(e);return r===0?t===`HIGH`?`HIGH`:`LOW`:t===`MINIMAL`?`LOW`:t}function et(e){let t=/gemini-(\d+)/.exec(e);return t?Number(t[1]):null}function tt(e){let t=et(e);return t!==null&&t>=3}function nt(e,t){return tt(e)?!0:t?Array.isArray(t)?t.includes(e):t(e):!1}function rt(e){return e.map(e=>e.type===`image`?{type:`image`,source:{type:`base64`,media_type:q(e.mimeType),data:e.data}}:{type:`text`,text:e.text})}function it(e,t){if(e.type!==`object`)throw new b(`Tool "${t}"'s schema must have "type": "object" (Anthropic requires object-shaped tool parameters).`,`validation`);return e}function at(e){return!e||e===`auto`?{type:`auto`}:e===`none`?{type:`none`}:e===`required`?{type:`any`}:{type:`tool`,name:e.function.name}}function ot(e,t){return{tools:e.map(e=>({name:e.function.name,description:e.function.description,input_schema:it(e.function.parameters,e.function.name)})),toolChoice:at(t)}}function st(e,t,n,r){let i=e.messages.find(e=>e.role===`system`),a=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),o=e.response_format?.type===`json_schema`?e.response_format.json_schema:void 0,s=o?.name.trim();if(o&&!s)throw new b(`json_schema.name must not be empty.`,`validation`);let c=!!o&&J(e.model,t);if(o&&e.tools?.length&&!c)throw new b(`Anthropic model "${e.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there, which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromAnthropic's \`nativeStructuredOutputModels\` option once you've confirmed it supports Anthropic's \`output_config.format\`.`,`validation`);if(e.response_format?.type===`json_object`)throw new b('response_format: "json_object" is not supported on Anthropic. Unlike OpenAI, Anthropic has no API-level field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a "respond with JSON only" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real API-level constraint (Anthropic\'s native output_config.format on covered models, or a forced single tool call otherwise).',`validation`);let l,u,d,f;if(o&&c)u={type:`json_schema`,schema:o.schema},e.tools?.length&&({tools:d,toolChoice:f}=ot(e.tools,e.tool_choice));else if(o&&s){let{schema:e,description:t,strict:n}=o;l=s,d=[{name:l,description:t,input_schema:it(e,l),strict:n}],f={type:`tool`,name:l}}!o&&e.tools?.length&&({tools:d,toolChoice:f}=ot(e.tools,e.tool_choice));let p,m;if(e.budget_tokens!==void 0||e.reasoning_effort!==void 0)if(Ye(f?.type===`tool`?`toolChoice forcing the "${f.name}" tool`:f?.type===`any`?`toolChoice: 'required' (Anthropic's "any" tool_choice)`:void 0),qe(e.model,r)){let t=e.budget_tokens??Z(e.reasoning_effort,n);Je(t,e.max_tokens),p={type:`enabled`,budget_tokens:t}}else{let t=e.reasoning_effort??Q(e.budget_tokens,n);p={type:`adaptive`},m=Xe(t)}let h=i?.content,g=p?void 0:e.temperature,_={model:e.model,max_tokens:e.max_tokens,...g===void 0?{}:{temperature:g},system:h||void 0,messages:lt(a.map(e=>ut(e))),...d?{tools:d,tool_choice:f}:{},...u||m?{output_config:{...u?{format:u}:{},...m?{effort:m}:{}}}:{},...p?{thinking:p}:{}};return{body:_,toolName:l}}function ct(e,t){let n=t?.nativeStructuredOutputModels,r=X(t?.reasoningEffortTokens),i=t?.adaptiveOnlyModels,a=e.messages.create.bind(e.messages);return{supportsJsonObjectMode:!1,chat:{completions:{async create(t,a){let{body:o,toolName:s}=st(t,n,r,i),c=await e.messages.create(o,a),l,u;if(s){let e=c.content.find(e=>e.type===`tool_use`&&e.name===s);if(!e)throw new b(`Anthropic did not return the required structured output tool "${s}".`,`validation`);if(!e.input||typeof e.input!=`object`||Array.isArray(e.input))throw new b(`Anthropic returned invalid structured output for tool "${s}". Expected an object.`,`validation`);l=JSON.stringify(e.input)}else{l=c.content.filter(e=>e.type===`text`).map(e=>e.text??``).join(``);let e=c.content.filter(e=>e.type===`tool_use`);e.length&&(u=e.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:JSON.stringify(e.input??{})}})))}return{choices:[{message:{content:l,...u?{tool_calls:u}:{}}}],usage:{prompt_tokens:c.usage?.input_tokens,completion_tokens:c.usage?.output_tokens,total_tokens:(c.usage?.input_tokens??0)+(c.usage?.output_tokens??0),...c.usage?.output_tokens_details?.thinking_tokens===void 0?{}:{completion_tokens_details:{reasoning_tokens:c.usage.output_tokens_details.thinking_tokens}}}}},async*createStream(e,t){let{body:o,toolName:s}=st(e,n,r,i),c=await a({...o,stream:!0},t),l=new Map,u=0,d=!1;for await(let e of c)if(e.type===`message_start`)u=e.message.usage?.input_tokens??0;else if(e.type===`content_block_start`)if(e.content_block.type===`tool_use`){let t=e.content_block.name===s?`json-tool`:`tool_use`;l.set(e.index,t),t===`json-tool`?d=!0:s||(yield{type:`tool_call_delta`,index:e.index,id:e.content_block.id,name:e.content_block.name})}else l.set(e.index,`text`);else if(e.type===`content_block_delta`){if(e.delta.type===`text_delta`)s||(yield{type:`text-delta`,delta:e.delta.text});else if(e.delta.type===`input_json_delta`){let t=l.get(e.index);t===`json-tool`?yield{type:`text-delta`,delta:e.delta.partial_json}:s||(yield{type:`tool_call_delta`,index:e.index,argumentsDelta:e.delta.partial_json})}}else if(e.type===`message_delta`){let t=e.usage?.output_tokens??0,n=e.usage?.output_tokens_details?.thinking_tokens;yield{type:`usage`,usage:{prompt_tokens:u,completion_tokens:t,total_tokens:u+t,...n===void 0?{}:{completion_tokens_details:{reasoning_tokens:n}}}}}else e.type===`ping`&&(yield{type:`ping`});if(s&&!d)throw new b(`Anthropic did not return the required structured output tool "${s}".`,`validation`)}}}}}function lt(e){let t=e=>e.role===`user`&&Array.isArray(e.content)&&e.content.length>0&&e.content.every(e=>e.type===`tool_result`),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.content.push(...r.content):n.push(r)}return n}function ut(e){if(e.role===`tool`)return{role:`user`,content:[{type:`tool_result`,tool_use_id:e.tool_call_id,content:e.content,...e.is_error?{is_error:!0}:{}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];e.content&&t.push({type:`text`,text:e.content});for(let n of e.tool_calls){let e;try{e=n.function.arguments.trim()?JSON.parse(n.function.arguments):{}}catch(e){throw new b(`Assistant tool call "${n.function.name}" (${n.id}) has arguments that are not valid JSON.`,`validation`,{cause:e})}if(e===null||Array.isArray(e)||typeof e!=`object`)throw new b(`Assistant tool call "${n.function.name}" (${n.id}) arguments must be a JSON object.`,`validation`);t.push({type:`tool_use`,id:n.id,name:n.function.name,input:e})}return{role:`assistant`,content:t}}return{role:e.role,content:Array.isArray(e.content)?rt(e.content):e.content??``}}function dt(e){return e.map(e=>e.type===`image`?{inlineData:{mimeType:q(e.mimeType),data:e.data}}:{text:e.text})}function ft(e){return!e||e===`auto`?{functionCallingConfig:{mode:`AUTO`}}:e===`none`?{functionCallingConfig:{mode:`NONE`}}:e===`required`?{functionCallingConfig:{mode:`ANY`}}:{functionCallingConfig:{mode:`ANY`,allowedFunctionNames:[e.function.name]}}}function pt(e){if(e.role===`tool`)return{role:`user`,parts:[{functionResponse:{name:e.tool_call_id,response:ht(e.content)}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];return typeof e.content==`string`&&e.content&&t.push({text:e.content}),t.push(...e.tool_calls.map(e=>({functionCall:{name:e.function.name,args:mt(e.function.arguments,e.function.name)}}))),{role:`model`,parts:t}}return{role:e.role===`assistant`?`model`:`user`,parts:Array.isArray(e.content)?dt(e.content):[{text:e.content??``}]}}function mt(e,t){let n;try{n=e.trim()?JSON.parse(e):{}}catch(e){throw new b(`Tool call "${t}" arguments are not valid JSON.`,`parse`,{cause:e,code:`tool_arguments_parse_failed`})}if(!n||Array.isArray(n)||typeof n!=`object`)throw new b(`Tool call "${t}" arguments must be a JSON object.`,`validation`);return n}function ht(e){let t;try{t=e.trim()?JSON.parse(e):``}catch{t=e}return t&&!Array.isArray(t)&&typeof t==`object`?t:{output:t}}function gt(e){let t=e=>e.role===`user`&&e.parts.length>0&&e.parts.every(e=>`functionResponse`in e),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.parts.push(...r.parts):n.push(r)}return n}function _t(e,t,n){let r=e.messages.find(e=>e.role===`system`),i=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),a=!!e.response_format,o={...e.temperature===void 0?{}:{temperature:e.temperature},maxOutputTokens:e.max_tokens,...r?{systemInstruction:{parts:[{text:r.content}]}}:{}};if(a&&(o.responseMimeType=`application/json`),e.response_format?.type===`json_schema`){let{schema:t,description:n}=e.response_format.json_schema;o.responseSchema={...t,...n?{description:n}:{}}}if(e.tools?.length&&(o.tools=[{functionDeclarations:e.tools.map(e=>({name:e.function.name,description:e.function.description,parameters:e.function.parameters}))}],o.toolConfig=ft(e.tool_choice)),nt(e.model,n)){let n=e.reasoning_effort??(e.budget_tokens===void 0?void 0:Q(e.budget_tokens,t));n!==void 0&&(o.thinkingConfig={thinkingLevel:Ze(n,e.model)})}else{let n=e.budget_tokens??(e.reasoning_effort?Z(e.reasoning_effort,t):void 0);n!==void 0&&(o.thinkingConfig={thinkingBudget:n})}return{model:e.model,contents:gt(i.map(e=>pt(e))),config:o}}function vt(e,t){let n=X(t?.reasoningEffortTokens),r=t?.thinkingLevelModels,i=e.models??e;if(typeof i.generateContent!=`function`)throw new b(`fromGemini requires a client with generateContent: pass ai.models, or the whole ai client (fromGemini(ai)).`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`generateContent`}});let a=i.generateContent.bind(i),o=typeof i.generateContentStream==`function`?i.generateContentStream.bind(i):void 0;return{chat:{completions:{async create(e,t){let i=_t(e,n,r);i.config={...i.config,abortSignal:t.signal};let o=await a(i),s=o.candidates?.[0]?.content?.parts??[],c=s.map(e=>e.text??``).join(``),l=s.filter(e=>e.functionCall),u;return l.length&&(u=l.map(e=>({id:e.functionCall.name,type:`function`,function:{name:e.functionCall.name,arguments:JSON.stringify(e.functionCall.args??{})}}))),{choices:[{message:{content:c,...u?{tool_calls:u}:{}}}],usage:{prompt_tokens:o.usageMetadata?.promptTokenCount,completion_tokens:o.usageMetadata?.candidatesTokenCount,total_tokens:o.usageMetadata?.totalTokenCount,...o.usageMetadata?.thoughtsTokenCount===void 0?{}:{completion_tokens_details:{reasoning_tokens:o.usageMetadata.thoughtsTokenCount}}}}},async*createStream(e,t){if(!o)throw new b(`stream: true requires a Gemini client with generateContentStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`generateContentStream`}});let i=_t(e,n,r);i.config={...i.config,abortSignal:t.signal};let a=await o(i),s=0,c;for await(let e of a){let t=e.candidates?.[0]?.content?.parts??[];for(let e of t)e.text&&(yield{type:`text-delta`,delta:e.text}),e.functionCall&&(yield{type:`tool_call_delta`,index:s,id:e.functionCall.name,name:e.functionCall.name,argumentsDelta:JSON.stringify(e.functionCall.args??{}),complete:!0},s++);e.usageMetadata&&(c=e.usageMetadata)}c&&(yield{type:`usage`,usage:{prompt_tokens:c.promptTokenCount,completion_tokens:c.candidatesTokenCount,total_tokens:c.totalTokenCount,...c.thoughtsTokenCount===void 0?{}:{completion_tokens_details:{reasoning_tokens:c.thoughtsTokenCount}}}})}}}}}function yt(e){return e.includes(`claude`)}function bt(e){switch(q(e)){case`image/png`:return`png`;case`image/jpeg`:return`jpeg`;case`image/gif`:return`gif`;case`image/webp`:return`webp`}}function xt(e){return new Uint8Array(Buffer.from(e,`base64`))}function St(e){return e.map(e=>e.type===`image`?{image:{format:bt(e.mimeType),source:{bytes:xt(e.data)}}}:{text:e.text})}function Ct(e,t){return{tools:e.map(e=>({toolSpec:{name:e.function.name,description:e.function.description,inputSchema:{json:e.function.parameters}}})),toolChoice:At(t)}}function wt(e,t,n,r,i){let a=e.messages.find(e=>e.role===`system`),o=e.messages.filter(e=>e.role===`user`||e.role===`assistant`||e.role===`tool`),s=e.response_format?.type===`json_schema`?e.response_format.json_schema:void 0,c=s?.name.trim();if(s&&!c)throw new b(`json_schema.name must not be empty.`,`validation`);let l=!!s&&J(e.model,n);if(s&&e.tools?.length&&!l)throw new b(`Bedrock model "${e.model}" is not covered by nativeStructuredOutputModels, so \`jsonSchema\` is emulated as a forced single tool call there (via \`toolConfig\`), which collides with the \`tools\` you also provided. Either drop \`tools\` or \`jsonSchema\` for this call, or pass this model in fromBedrock's \`nativeStructuredOutputModels\` option once you've confirmed it supports Converse's \`outputConfig.textFormat\`.`,`validation`);if(e.response_format?.type===`json_object`)throw new b('response_format: "json_object" is not supported on Bedrock. Converse has no field that mechanically guarantees valid JSON output for this mode, so it used to be emulated by injecting a "respond with JSON only" instruction into the system prompt, a guarantee this adapter can no longer make. Use `jsonSchema` instead, which maps to a real constraint (Converse\'s native outputConfig.textFormat on covered models, or a forced tool call otherwise).',`validation`);let u,d,f;if(s&&l){let{schema:e,description:t}=s;f={textFormat:{type:`json_schema`,structure:{jsonSchema:{schema:JSON.stringify(e),name:c,description:t}}}}}else if(s&&c){let{schema:e,description:t,strict:n}=s;u=c,d={tools:[{toolSpec:{name:u,description:t,inputSchema:{json:e},strict:n}}],toolChoice:{tool:{name:u}}}}if(e.tools?.length&&!u&&(d=Ct(e.tools,e.tool_choice)),s&&d&&t){let n=Array.isArray(t)?t.includes(e.model):t(e.model);if(!n)throw new b(`Bedrock model "${e.model}" is not listed in toolUseSupportedModels, but this call requires Converse tool use (either jsonSchema emulated as a forced tool call, or real \`tools\` sent alongside native structured output).`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`toolUseSupportedModels`}})}let p,m;if(yt(e.model)&&(e.budget_tokens!==void 0||e.reasoning_effort!==void 0)){let t=d?.toolChoice;if(Ye(t&&`tool`in t?`toolChoice forcing the "${t.tool?.name}" tool`:t&&`any`in t?`toolChoice: 'required' (Converse's "any" tool_choice)`:void 0),qe(e.model,i)){let t=e.budget_tokens??Z(e.reasoning_effort,r);Je(t,e.max_tokens),p={thinking:{type:`enabled`,budget_tokens:t}}}else{let t=e.reasoning_effort??Q(e.budget_tokens,r);p={thinking:{type:`adaptive`}},m=Xe(t)}}let h=p?void 0:e.temperature,g={modelId:e.model,messages:Mt(o.map(e=>jt(e))),system:a?.content?[{text:a.content}]:void 0,inferenceConfig:{...h===void 0?{}:{temperature:h},maxTokens:e.max_tokens},...d?{toolConfig:d}:{},...f||m?{outputConfig:{...f??{},...m?{effort:m}:{}}}:{},...p?{additionalModelRequestFields:p}:{}};return{request:g,toolName:u}}function Tt(e){return typeof e.send==`function`}function Et(e){if(`messageStart`in e)return{messageStart:e.messageStart};if(`contentBlockStart`in e)return{contentBlockStart:e.contentBlockStart};if(`contentBlockDelta`in e)return{contentBlockDelta:e.contentBlockDelta};if(`contentBlockStop`in e)return{contentBlockStop:e.contentBlockStop};if(`messageStop`in e)return{messageStop:e.messageStop};if(`metadata`in e)return{metadata:e.metadata};if(`internalServerException`in e)return{internalServerException:e.internalServerException};if(`modelStreamErrorException`in e)return{modelStreamErrorException:e.modelStreamErrorException};if(`validationException`in e)return{validationException:e.validationException};if(`throttlingException`in e)return{throttlingException:e.throttlingException};if(`serviceUnavailableException`in e)return{serviceUnavailableException:e.serviceUnavailableException}}async function*Dt(e){for await(let t of e){let e=Et(t);e&&(yield e)}}function Ot(e){let t;function n(){return t??=import(`@aws-sdk/client-bedrock-runtime`).then(e=>e,e=>{throw t=void 0,new b(`fromBedrock requires "@aws-sdk/client-bedrock-runtime" to be installed to use a raw AWS SDK client (it is not a dependency of vern-llm itself). Install it, or pass your own object with .converse()/.converseStream() methods instead.`,`validation`,{cause:e})}),t}return{converse:async(t,r)=>{let{ConverseCommand:i}=await n();return e.send(new i(t),{abortSignal:r.signal})},converseStream:async(t,r)=>{let{ConverseStreamCommand:i}=await n(),a=await e.send(new i(t),{abortSignal:r.signal});if(!a.stream)throw new b(`Bedrock ConverseStreamCommand response did not include a stream. This can happen if the request or the model doesn't actually support Converse streaming.`,`api`,{code:`server_error`});return{stream:Dt(a.stream)}}}}function kt(e,t){let n=Tt(e)?Ot(e):e,r=t?.toolUseSupportedModels,i=t?.nativeStructuredOutputModels,a=X(t?.reasoningEffortTokens),o=t?.adaptiveOnlyModels;return{supportsJsonObjectMode:!1,chat:{completions:{async create(e,t){let{request:s,toolName:c}=wt(e,r,i,a,o),l=await n.converse(s,t),u,d;if(c){let e=l.output?.message?.content?.find(e=>e.toolUse?.name===c);u=e?.toolUse?JSON.stringify(e.toolUse.input):``}else{let e=l.output?.message?.content??[];u=e.map(e=>e.text??``).join(``);let t=e.filter(e=>!!e.toolUse);t.length&&(d=t.map((e,t)=>{let n=e.toolUse;if(!n.name)throw new b(`Bedrock returned a toolUse block without a name at index ${t}.`,`validation`);return{id:n.toolUseId??`${n.name}_${t}`,type:`function`,function:{name:n.name,arguments:JSON.stringify(n.input??{})}}}))}return{choices:[{message:{content:u,...d?{tool_calls:d}:{}}}],usage:{prompt_tokens:l.usage?.inputTokens,completion_tokens:l.usage?.outputTokens,total_tokens:l.usage?.totalTokens}}},async*createStream(e,t){if(!n.converseStream)throw new b(`stream: true requires a Bedrock client with converseStream`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`converseStream`}});let{request:s,toolName:c}=wt(e,r,i,a,o),{stream:l}=await n.converseStream(s,t),u=new Map;for await(let e of l)if(`contentBlockStart`in e){let{contentBlockIndex:t,start:n}=e.contentBlockStart;if(n?.toolUse){let e=n.toolUse.name===c?`json-tool`:`tool_use`;u.set(t,e),e===`tool_use`&&!c&&(yield{type:`tool_call_delta`,index:t,id:n.toolUse.toolUseId,name:n.toolUse.name})}else u.set(t,`text`)}else if(`contentBlockDelta`in e){let{contentBlockIndex:t,delta:n}=e.contentBlockDelta;if(n&&`text`in n&&n.text!==void 0&&!c)yield{type:`text-delta`,delta:n.text};else if(n&&`toolUse`in n&&n.toolUse?.input!==void 0){let e=u.get(t);e===`json-tool`?yield{type:`text-delta`,delta:n.toolUse.input}:c||(yield{type:`tool_call_delta`,index:t,argumentsDelta:n.toolUse.input})}}else if(`metadata`in e&&e.metadata.usage)yield{type:`usage`,usage:{prompt_tokens:e.metadata.usage.inputTokens,completion_tokens:e.metadata.usage.outputTokens,total_tokens:e.metadata.usage.totalTokens}};else if(`throttlingException`in e)throw new b(e.throttlingException.message??`Bedrock throttled the request mid-stream`,`api`,{status:429,code:`provider_rate_limited`});else if(`validationException`in e)throw new b(e.validationException.message??`Bedrock rejected the request mid-stream`,`validation`);else if(`internalServerException`in e||`serviceUnavailableException`in e||`modelStreamErrorException`in e){let t=`internalServerException`in e&&e.internalServerException.message||`serviceUnavailableException`in e&&e.serviceUnavailableException.message||`modelStreamErrorException`in e&&e.modelStreamErrorException.message||`Bedrock reported a mid-stream error`,n=`modelStreamErrorException`in e&&e.modelStreamErrorException.originalStatusCode||`serviceUnavailableException`in e&&503||500;throw new b(t,`api`,{status:n,code:n>=500?`server_error`:void 0})}}}}}}function At(e){if(!e||e===`auto`)return{auto:{}};if(e===`required`)return{any:{}};if(e===`none`)throw new b("'none' is not supported by fromBedrock: Bedrock Converse has no `tool_choice` equivalent to forbidding tool use while tools are still offered. Omit `tools` entirely for this call instead.",`invalid_params`,{code:`unsupported_capability`,issues:{capability:`toolChoice: 'none'`}});return{tool:{name:e.function.name}}}function jt(e){if(e.role===`tool`)return{role:`user`,content:[{toolResult:{toolUseId:e.tool_call_id,content:[{text:e.content}],status:e.is_error?`error`:`success`}}]};if(e.role===`assistant`&&e.tool_calls?.length){let t=[];e.content&&t.push({text:e.content});for(let n of e.tool_calls){let e;if(!n.function.arguments.trim())e={};else try{e=JSON.parse(n.function.arguments)}catch(e){throw new b(`Assistant tool call "${n.function.name}" (${n.id}) has arguments that are not valid JSON.`,`validation`,{cause:e})}t.push({toolUse:{toolUseId:n.id,name:n.function.name,input:e}})}return{role:`assistant`,content:t}}return{role:e.role,content:Array.isArray(e.content)?St(e.content):[{text:e.content??``}]}}function Mt(e){let t=e=>e.role===`user`&&e.content.length>0&&e.content.every(e=>`toolResult`in e),n=[];for(let r of e){let e=n.at(-1);t(r)&&e&&t(e)?e.content.push(...r.content):n.push(r)}return n}async function*Nt(e){let t=e.getReader();try{for(;;){let{done:e,value:n}=await t.read();if(e)return;n&&(yield n)}}finally{try{await t.cancel()}catch{}t.releaseLock()}}async function Pt(e,t){let n=await fetch(e,t);if(!n.ok){let e=await n.text().catch(()=>``),t=Error(`Fetch adapter stream request failed (${n.status}): ${e.slice(0,500)}`);throw t.status=n.status,t.headers=n.headers,t}if(!n.body)throw Error(`Fetch adapter stream request received a response with no body.`);return Nt(n.body)}async function Ft(e,t,n){let r=typeof e.url==`function`?e.url(t):e.url,i=typeof e.headers==`function`?await e.headers():e.headers,a=e.method??`POST`,o=![`GET`,`HEAD`].includes(a.toUpperCase());return{url:r,method:a,headers:o?{"Content-Type":`application/json`,...i}:{...i},...o?{body:JSON.stringify(n)}:{}}}function It(e){return{chat:{completions:{async create(t,n){let{url:r,method:i,headers:a,body:o}=await Ft(e,t,e.mapRequest(t)),s=e.request??fetch,c=await s(r,{method:i,headers:a,body:o,signal:n.signal});if(!c.ok){let e=await c.text().catch(()=>``),t=Error(`Fetch adapter request failed (${c.status}): ${e.slice(0,500)}`);throw t.status=c.status,t.headers=c.headers,t}let l=await c.json(),{content:u,usage:d,toolCalls:f}=e.mapResponse(l),p=f?.length?f.map(e=>({id:e.id,type:`function`,function:{name:e.name,arguments:e.arguments}})):void 0;return{choices:[{message:{content:u,...p?{tool_calls:p}:{}}}],usage:d?{prompt_tokens:d.promptTokens,completion_tokens:d.completionTokens,total_tokens:d.totalTokens}:void 0}},async*createStream(t,n){if(!e.mapStreamEvent)throw new b(`stream: true requires mapStreamEvent to be configured on fromFetch`,`invalid_params`,{code:`unsupported_capability`,issues:{capability:`mapStreamEvent`}});if(e.request&&!e.requestStream)throw new b("`stream: true` requires `requestStream` to be configured on fromFetch when a custom `request` transport is set. `requestStream` does not fall back to `request` (it needs an async-iterable byte stream, which `RequestLike`'s buffered `ResponseLike` has no way to provide), without it, `stream: true` would silently use plain native `fetch` instead of your configured transport. Add a `requestStream` that opens the same connection your `request` does, or omit `request` if native `fetch` is fine for both.",`invalid_params`,{code:`unsupported_capability`,issues:{capability:`requestStream`}});let{url:r,method:i,headers:a,body:o}=await Ft(e,t,e.mapRequest(t)),s=e.requestStream??Pt,c=e.parseStreamFrames??He,l=await s(r,{method:i,headers:a,body:o,signal:n.signal});for await(let t of c(l)){if(t===G){yield{type:`ping`};continue}let n=e.mapStreamEvent(t);if(!n)continue;Array.isArray(n)?yield*n:yield n}}}}}}function Lt(e,t){if(e.budget_tokens===void 0)return e;let{budget_tokens:n,...r}=e;return r.reasoning_effort===void 0?{...r,reasoning_effort:Q(n,t)}:r}function Rt(e){return e.map(e=>e.type===`image`?{type:`image_url`,image_url:{url:`data:${q(e.mimeType)};base64,${e.data}`}}:{type:`text`,text:e.text})}function zt(e){return e.messages.map(e=>{if(e.role===`user`&&Array.isArray(e.content))return{...e,content:Rt(e.content)};if(e.role===`tool`){let{is_error:t,...n}=e;return n}return e})}function*Bt(e){let t=e.choices?.[0]?.delta;if(t?.content&&(yield{type:`text-delta`,delta:t.content}),t?.tool_calls?.length)for(let e of t.tool_calls)yield{type:`tool_call_delta`,index:e.index,id:e.id,name:e.function?.name,argumentsDelta:e.function?.arguments};e.usage&&(yield{type:`usage`,usage:e.usage})}function $(e,t={}){let n=e,{supportsStreamUsage:r=!0}=t,i=X(t.reasoningEffortTokens),a=n.chat.completions.create.bind(n.chat.completions);return{chat:{completions:{async create(e,t){let r=zt(e);return n.chat.completions.create(Lt({...e,messages:r},i),t)},async*createStream(e,t){let n=zt(e),o=await a(Lt({...e,messages:n,stream:!0,...r?{stream_options:{include_usage:!0}}:{}},i),t);for await(let e of o)yield*Bt(e)}}}}}const Vt=$,Ht=$,Ut=$,Wt=$,Gt=$,Kt=$,qt=$,Jt=$,Yt=$,Xt=$,Zt=$,Qt=$,$t=$,en=$,tn=$,nn=$,rn=$,an=$,on=$,sn=$,cn=$,ln=$,un=$,dn=$,fn=$,pn=$,mn=$,hn=$,gn=$,_n=$,vn=$,yn=$,bn=$,xn=$,Sn=$,Cn=$,wn=$,Tn=$;exports.CircuitBreaker=fe,exports.ConsoleLogger=Fe,exports.FallbackExhaustedError=E,exports.InMemoryCacheAdapter=S,exports.LLMError=b,exports.NormalizedCacheAdapter=C,exports.RateLimiter=Re,exports.SSE_PING=G,exports.TieredCacheAdapter=w,exports.VernLLM=ze,exports.defaultEstimateTokens=Ie,exports.defaultFallbackOn=ne,exports.defineCachedCallParams=Ve,exports.defineCallParams=Be,exports.from01AI=Tn,exports.fromAnthropic=ct,exports.fromAnyscale=bn,exports.fromAtlasCloud=wn,exports.fromBaseten=dn,exports.fromBedrock=kt,exports.fromCerebras=Gt,exports.fromCloudflareWorkersAI=cn,exports.fromDeepInfra=Zt,exports.fromDeepSeek=Wt,exports.fromFeatherless=fn,exports.fromFetch=It,exports.fromFireworks=qt,exports.fromFriendli=pn,exports.fromGemini=vt,exports.fromGroq=Ht,exports.fromHyperbolic=$t,exports.fromInferenceNet=Sn,exports.fromInfermatic=Cn,exports.fromLMStudio=nn,exports.fromLambdaLabs=vn,exports.fromLepton=xn,exports.fromMiniMax=_n,exports.fromMistral=Ut,exports.fromMoonshot=en,exports.fromNebius=ln,exports.fromNovita=Qt,exports.fromNvidiaNIM=on,exports.fromOllama=Jt,exports.fromOpenAI=Vt,exports.fromOpenAICompatible=$,exports.fromOpenRouter=Yt,exports.fromParasail=hn,exports.fromPerplexity=Xt,exports.fromSambaNova=un,exports.fromSiliconFlow=mn,exports.fromSnowflakeCortex=yn,exports.fromStepFun=gn,exports.fromTogether=Kt,exports.fromVLLM=rn,exports.fromVercelAIGateway=sn,exports.fromXAI=an,exports.fromZhipu=tn,exports.hasIssues=ee,exports.isFallbackExhaustedError=re,exports.isLLMError=x,exports.isToolCallResult=T,exports.parseSseStream=He;
4237
15
  //# sourceMappingURL=index.cjs.map